Files
leonarski_fandClaude Opus 5.5 da98643940 Refinement: a U per atom tried wherever affordable, its case on the evidence
The fixed bound of 18 working reflections per atom on trying a U per atom is
replaced by the bound Hamilton's test itself sets: a U per atom is tried where
the working reflections outnumber the parameters of the model with it counted
unrestrained (nine per atom, the positions counted as Merritt (2012) counts
them) - the end of the range of restraint weights at which the test can be
taken at all. And where the tests are inconclusive and the free set decides,
a model with more displacement parameters needs its R-free lower outright,
not within the paired tie the incumbent is otherwise given: the burden of
the evidence is on the parameters it adds.

On the eight sets the choice turns on (CPU): a U per atom is taken at
1.2-1.4 A with 16-31 reflections per atom and at 2.0 A with 12, where it
lowers R-free; TLS is kept at 2.25 and 2.8 A with 6-8 reflections per atom
(the bound) and at 1.9 A with 13, where the U per atom raised R-free by 0.003
and only -LL free preferred it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
2026-11-18 17:35:31 +01:00

5067 lines
269 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "Refinement.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <functional>
#include <set>
#include <future>
#include <limits>
#include <map>
#include <mutex>
#include <optional>
#include <cmath>
#include <numeric>
#include <random>
#include <thread>
#include <tuple>
#include <unordered_map>
#include <Eigen/Dense>
#include <ceres/gradient_problem.h>
#include <ceres/gradient_problem_solver.h>
#include <ceres/jet.h>
#include <ceres/rotation.h>
#include "gemmi/fourier.hpp" // get_size_for_hkl, get_f_phi_on_grid
#include "gemmi/it92.hpp"
#include "gemmi/modify.hpp" // remove_hydrogens
#include "gemmi/scaling.hpp"
#include "gemmi/stats.hpp" // calculate_data_statistics
#include "../../common/JFJochException.h"
#include "../../common/JFJochMath.h"
#include "../../common/Logger.h"
#include "../../common/ParallelFor.h"
#include "ClashRelief.h"
#include "ModelFFT.h"
#include "ModelGrid.h" // IonFormFactor, AnisoRadiusB
#include "ModelScaling.h"
#include "PortableRandom.h"
#include "MonomerLibrary.h"
#include "RefinementADP.h"
#include "RefinementMath.h"
#include "RefinementMinimiser.h"
#include "RefinementNcs.h"
#include "RefinementXray.h"
#include "SideChainField.h"
#ifdef JFJOCH_USE_CUDA
#include "../../common/CUDAWrapper.h"
#include "RefinementGPU.h"
#include "RefinementNewtonGPU.h"
#endif
namespace {
constexpr double B_FLOOR = 2.0; // A^2: no atom's B goes below it
// nor above twice the model's largest B, within these bounds (the GPU's gather is reserved for it)
constexpr double B_CEILING_MIN = 100.0, B_CEILING_MAX = 300.0;
// The B the ceiling is set from is at least this times the data's Wilson B (WilsonB).
constexpr double B_CEILING_WILSON = 2.5;
// The ceiling of B for a model whose highest B is `b_max`: twice it, within the bounds - the bounds lifted for a
// model that comes with B above them (a structure refined at 4-5 A, B 400-600 A^2), which the ceiling would
// otherwise flatten (the gather is reserved for it).
double BCeiling(double b_max) {
return std::clamp(2 * b_max, B_CEILING_MIN, std::max(B_CEILING_MAX, 1.5 * b_max));
}
// A B tensor (8 pi^2 U) inside the bounds: its smallest eigenvalue raised to `floor` by a multiple of I, then the
// whole tensor scaled down so that its largest is at most `ceiling`. The ceiling comes last because the GPU's
// gather is reserved for atoms no wider than it.
gemmi::SMat33<double> BoundedB(gemmi::SMat33<double> b, double floor, double ceiling) {
const std::array<double, 3> ev = b.calculate_eigenvalues();
const double lowest = std::min({ev[0], ev[1], ev[2]});
if (!(lowest >= floor))
b = b.added_kI(floor - (std::isfinite(lowest) ? lowest : 0.0));
const std::array<double, 3> ev2 = b.calculate_eigenvalues();
const double top = std::max({ev2[0], ev2[1], ev2[2]});
if (top > ceiling)
b = b.scaled(ceiling / top);
return b;
}
// How far below the smallest B of the model a macro-cycle's B refinement may take an atom; the blur of the
// cycle is chosen for that.
constexpr double B_STEP_DOWN = 10.0;
constexpr double REPULSION_MARGIN = 0.5; // A: pairs this much beyond their contact distance are listed too
// The geometry the weight is chosen within (Afonine et al. 2012 select among weights by R-free with the
// geometry held to a target in the same way).
constexpr double MAX_RMSD_BONDS = 0.02, MAX_RMSD_ANGLES = 2.0;
// The minimiser's iterations per macro-cycle where the solvent build refines the solvent alone.
constexpr int SOLVENT_ITERATIONS = 10;
// The free set decides the weights and the displacement model alone, and only where both of its measures agree: a
// candidate is preferred to the incumbent where its -ln L of the free reflections is lower and its R-free not higher
// than the incumbent's by more than the standard error of the difference. The two are scored on the same free
// reflections, so the difference is paired: R-free is a sum of one term per free twin group (FreeRTerms), and the
// standard error is that of the sum of the groups' differences.
// As PDB-REDO decides on more than one free measure, Joosten, Long, Murshudov & Perrakis (2014) IUCrJ 1, 213-220
bool FreeSetPrefers(const std::vector<float> &r_terms, double nll_free, const std::vector<float> &r_terms_b,
double nll_free_b) {
const size_t n = r_terms.size();
double sum = 0, sum2 = 0;
for (size_t k = 0; k < n; k++) {
const double d = r_terms[k] - r_terms_b[k];
sum += d;
sum2 += d * d;
}
const double se = n > 1 ? std::sqrt(std::max(sum2 - sum * sum / n, 0.0) * n / (n - 1)) : 0.0;
return nll_free < nll_free_b && sum < se;
}
// The upper alpha point of the F distribution with d1 and d2 degrees of freedom, by Paulson's normal approximation of
// F^(1/3) (z the normal deviate of alpha) - accurate to a few percent at the counts a displacement model has.
// Following Paulson (1942) Ann. Math. Statist. 13, 233-235
double FQuantile(double d1, double d2, double z) {
const double u = 2.0 / (9.0 * d1), v = 2.0 / (9.0 * d2), a = 1.0 - v, c = 1.0 - u;
const double q = a * a - z * z * v;
const double x = (a * c + std::sqrt(std::max(a * a * c * c - q * (c * c - z * z * u), 0.0))) / q;
return x * x * x;
}
// Of two displacement models, `large` with more parameters than `small`: the fraction of Hamilton's R-ratio tests it
// passes over the range of weights its restraints may be credited at - R_small / R_large above
// sqrt(b / (n - m) F(b, n - m, 0.05) + 1), n the working reflections, m the larger model's displacement parameters
// (the trials hold the positions), b the extra ones. Each model's restraints are credited as w R observations, w from
// 0 up to the bound at which they take every parameter they restrain off its count - Merritt's N_atoms / R for a B
// per atom (as his 3 N_atoms / R for the positions), 6 N_atoms / R for a U per atom; the two weights are scanned
// independently, six values each.
// Following Hamilton (1965) Acta Cryst. 18, 502-510, as Merritt (2012) Acta Cryst. D68, 468-477 and PDB-REDO
// (Joosten et al. 2012 Acta Cryst. D68, 484-496) apply it to the displacement model
double HamiltonPassFraction(const RefinementWeightTrial &large, const RefinementWeightTrial &small, int n_work) {
int pass = 0, tests = 0;
for (int i = 0; i <= 5; i++)
for (int j = 0; j <= 5; j++, tests++) {
const double m_small = small.adp_parameters - small.adp_restrained * i / 5.0;
const double m_large = large.adp_parameters - large.adp_restrained * j / 5.0;
const double b = m_large - m_small, dof = n_work - m_large;
pass += b <= 0 || (dof > 0 && small.r_work / large.r_work > std::sqrt(b / dof * FQuantile(b, dof, 1.6449) + 1.0));
}
return static_cast<double>(pass) / tests;
}
// The displacement model: of the larger and the smaller, the larger where it passes more than this fraction of the
// tests (HamiltonPassFraction), the smaller where it passes fewer than the other, and the free set's choice between
// (as PDB-REDO; Joosten et al. 2012).
constexpr double HAMILTON_LARGER = 0.95, HAMILTON_SMALLER = 0.30;
// Every other step is taken on the work-set target (WorkTarget) where it lowers it by more than this, relative.
constexpr double TARGET_TOLERANCE = 1e-4;
// The rounds of RefineOnce's weight and displacement-model choice (4b.): the first from the model as given, each
// after it from the model the one before ended with.
// The minimiser (RefinementMinimiser): a solve stops where an iteration lowers the target by less than this,
// relative; the most a position (A) or a B (A^2) moves in the first trial of a line search.
constexpr double MINIMISER_F_TOLERANCE = 1e-7;
constexpr double MAX_STEP_XYZ = 0.3, MAX_STEP_B = 10.0;
// The loosest NCS restraint a trajectory step tries (A); looser than that, the copies are refined independently.
constexpr double NCS_SIGMA_MAX = 0.4;
// The figure of merit of the likelihood, <cos(dphi)>, at X = 2 Fo D Fc / var: I1(X)/I0(X) for an acentric
// reflection, tanh(X / 2) for a centric one.
// Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255
double Fom(double x, bool centric) {
double log_i0 = 0, ratio = 0;
refine_math::BesselI0I1(x, log_i0, ratio);
return centric ? std::tanh(0.5 * x) : ratio;
}
std::string Joined(const std::vector<std::string> &names) {
std::string out;
for (const std::string &n : names)
out += (out.empty() ? "" : " ") + n;
return out;
}
std::vector<gemmi::Atom *> AtomList(gemmi::Model &model) {
std::vector<gemmi::Atom *> atoms;
for (gemmi::Chain &ch : model.chains)
for (gemmi::Residue &r : ch.residues)
for (gemmi::Atom &a : r.atoms)
atoms.push_back(&a);
return atoms;
}
// The charge an atom scatters with (IonFormFactor): a single-atom ion's, a metal or a halide, as its dictionary gives
// it (MG 2+, NA 1+, CL 1-); 0 for every other atom, whatever the file says (a carboxylate's O1-, a lysine's N1+).
signed char ScatteringCharge(const gemmi::Residue &r, const gemmi::Atom &a) {
if (!(a.element.is_metal() || a.element == gemmi::El::F || a.element == gemmi::El::Cl ||
a.element == gemmi::El::Br || a.element == gemmi::El::I))
return 0;
const auto cc = EmbeddedMonomerLibrary().monomers.find(r.name);
if (cc == EmbeddedMonomerLibrary().monomers.end() || cc->second.atoms.size() != 1)
return 0;
return static_cast<signed char>(std::lround(cc->second.atoms[0].charge));
}
// The model as refinement takes it: the first model only, without hydrogens and atoms of unknown element, every B
// inside the allowed range - an anisotropic atom's equivalent isotropic B, its U kept (Context::TakeAniso() takes it) -
// and the single-atom ions charged and nothing else (ScatteringCharge).
gemmi::Structure PrepareModel(const gemmi::Structure &placed, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg) {
gemmi::Structure st = placed;
st.models.resize(1);
st.cell = cell;
st.spacegroup_hm = sg.xhm();
gemmi::remove_hydrogens(st.models[0]);
for (gemmi::Chain &ch : st.models[0].chains)
for (gemmi::Residue &r : ch.residues) {
std::erase_if(r.atoms, [](const gemmi::Atom &a) { return a.element == gemmi::El::X; });
for (gemmi::Atom &a : r.atoms) {
a.charge = ScatteringCharge(r, a);
if (a.aniso.nonzero())
a.b_iso = static_cast<float>(gemmi::u_to_b() * a.aniso.trace() / 3.0);
a.b_iso = static_cast<float>(std::max<double>(a.b_iso, B_FLOOR));
}
}
gemmi::remove_empty_children(st);
st.setup_cell_images();
return st;
}
// What every stage works on: the model, the reflections, the engine and the row model of the cycle.
struct Context {
gemmi::Structure st;
std::vector<gemmi::Atom *> atoms;
gemmi::UnitCell cell;
const gemmi::SpaceGroup *sg = nullptr;
double d_min = 0;
RefinementReflections refl;
std::unique_ptr<RefinementEngine> engine;
std::unique_ptr<BulkSolventMask> mask;
#ifdef JFJOCH_USE_CUDA
std::shared_ptr<ModelScaleGPUPool> scale_gpu; // shared by the slots on the engine's card
#endif
size_t nthreads = 1;
// The threads of the stages and of the engine and the mask from now on.
void SetThreads(size_t n) {
nthreads = n;
engine->SetThreads(n);
mask->SetThreads(n);
}
// Where the trials run side by side on the CPU (RunTrials), the threads of the slots that have no trial left,
// which the slots still running take up at their next evaluation (TakeSpareThreads); null elsewhere.
std::atomic<int> *spare_threads = nullptr;
void TakeSpareThreads() {
if (spare_threads == nullptr)
return;
if (const int n = spare_threads->exchange(0); n > 0)
SetThreads(nthreads + n);
}
// the likelihood fit's: all of the run's, also in a trial's slot - the slots refit at the same time, the GPU
// idles meanwhile, and the bins they queue on the shared worker pool are fitted by whichever thread is free
size_t ml_threads = 1;
int device = -1; // the engine's GPU
size_t engine_bytes = 0; // and what it holds there, beside what it shares with the engines like it
size_t card_bytes = 0; // on the same card, and a lane of working buffers (RefinementGPUBytes)
size_t lane_bytes = 0;
XrayRowModel rows;
std::vector<std::complex<float>> fcalc, fmask;
double k_sol = 0, b_sol = 0;
// where the last Refit()'s solvent grid ended (before MoveOverallB()): the next one's fine pass is around it
// (negative: the next one searches the whole box)
double grid_k_sol = -1, grid_b_sol = -1;
double k_overall = 0;
gemmi::SMat33<double> b_star{0, 0, 0, 0, 0, 0};
double b_floor = B_FLOOR; // the cycle's lowest allowed B
double b_ceiling = B_CEILING_MAX;
// Riding hydrogens: per atom the parent it rides on (-1: none), and where it sits from the parent.
std::vector<int> rider_parent;
std::vector<int> rider_start, rider_index; // SetRiderParents()
std::vector<gemmi::Position> rider_offset;
int evaluations = 0;
const std::atomic<bool> *cancel = nullptr; // RefinementSettings::cancel
// The anisotropic atoms (empty: none), which the engine reads (RefinementEngine::SetAniso), and per atom how far
// its B stays inside the bounds so that its U does: its smallest eigenvalue above b_floor, its largest below
// b_ceiling (in B, A^2).
std::vector<AtomAniso> aniso;
std::vector<double> b_lo, b_hi;
double BLow(size_t i) const { return b_floor + (b_lo.empty() ? 0.0 : b_lo[i]); }
double BHigh(size_t i) const { return std::max(BLow(i), b_ceiling - (b_hi.empty() ? 0.0 : b_hi[i])); }
// The anisotropy of `a` (empty: none) for the atoms as they are, and the bounds of their B with it.
void SetAniso(const std::vector<AtomAniso> &a) {
aniso = a;
b_lo.clear();
b_hi.clear();
if (aniso.empty())
return;
b_lo.assign(atoms.size(), 0.0);
b_hi.assign(atoms.size(), 0.0);
for (size_t i = 0; i < aniso.size(); i++)
if (aniso[i].on) {
const std::array<float, 6> &d = aniso[i].dev;
const gemmi::SMat33<double> dev{d[0], d[1], d[2], d[3], d[4], d[5]};
const std::array<double, 3> ev = dev.calculate_eigenvalues();
b_lo[i] = std::max(0.0, -gemmi::u_to_b() * std::min({ev[0], ev[1], ev[2]}));
b_hi[i] = std::max(0.0, gemmi::u_to_b() * std::max({ev[0], ev[1], ev[2]}));
}
}
// TLS bodies (empty: none) and per atom the part of its B they hold as they were last applied (8 pi^2 tr(U_TLS) / 3,
// and what bringing U inside the bounds added): the rest of its B is its residual.
std::vector<TlsBody> tls;
std::vector<double> tls_b;
// The bodies' U at the atoms as they are now, each atom's residual B kept: its anisotropy the traceless part of
// U_TLS, its B the residual plus the trace.
void ApplyTls() {
if (tls.empty())
return;
if (aniso.size() != atoms.size())
aniso.assign(atoms.size(), AtomAniso());
tls_b.resize(atoms.size(), 0.0);
for (const TlsBody &body : tls)
for (int i : body.atoms) {
const gemmi::SMat33<double> u = body.U(atoms[i]->pos);
const double residual = atoms[i]->b_iso - tls_b[i];
// the total U inside the bounds
const gemmi::SMat33<double> b = BoundedB(u.scaled(gemmi::u_to_b()).added_kI(residual), b_floor, b_ceiling);
const double t = b.trace() / 3.0;
atoms[i]->b_iso = static_cast<float>(t);
// what the bodies hold of B, the bounds' share included: the residual stays as it was
tls_b[i] = atoms[i]->b_iso - residual;
aniso[i].on = true;
aniso[i].dev = {static_cast<float>((b.u11 - t) / gemmi::u_to_b()), static_cast<float>((b.u22 - t) / gemmi::u_to_b()),
static_cast<float>((b.u33 - t) / gemmi::u_to_b()), static_cast<float>(b.u12 / gemmi::u_to_b()),
static_cast<float>(b.u13 / gemmi::u_to_b()), static_cast<float>(b.u23 / gemmi::u_to_b())};
}
MoveRiders();
SetAniso(aniso);
}
// The U of every atom (AnisoU for the anisotropic, B / (8 pi^2) I for the others), in A^2.
std::vector<std::array<double, 6>> UOfAtoms() const {
std::vector<std::array<double, 6>> u(atoms.size());
for (size_t i = 0; i < atoms.size(); i++) {
const double b = atoms[i]->b_iso / gemmi::u_to_b();
if (!aniso.empty() && aniso[i].on) {
const gemmi::SMat33<float> m = AnisoU(aniso[i], atoms[i]->b_iso);
u[i] = {m.u11, m.u22, m.u33, m.u12, m.u13, m.u23};
} else
u[i] = {b, b, b, 0, 0, 0};
}
return u;
}
// The anisotropic U the atoms of the model bring (PrepareModel() keeps it) as their AtomAniso, taken off them.
void TakeAniso() {
std::vector<AtomAniso> a(atoms.size());
bool any = false;
for (size_t i = 0; i < atoms.size(); i++) {
gemmi::Atom &at = *atoms[i];
if (!at.aniso.nonzero())
continue;
// its widest axis no wider than the refinement allows
const double top = AnisoRadiusB(at.aniso.scaled(gemmi::u_to_b()));
if (top > b_ceiling) {
at.aniso = at.aniso.scaled(static_cast<float>(b_ceiling / top));
at.b_iso = static_cast<float>(gemmi::u_to_b() * at.aniso.trace() / 3.0);
}
const double t = at.aniso.trace() / 3.0;
a[i].on = true;
a[i].dev = {static_cast<float>(at.aniso.u11 - t), static_cast<float>(at.aniso.u22 - t),
static_cast<float>(at.aniso.u33 - t), at.aniso.u12, at.aniso.u13, at.aniso.u23};
at.aniso = {0, 0, 0, 0, 0, 0};
any = true;
}
SetAniso(any ? a : std::vector<AtomAniso>());
}
bool Cancelled() const { return cancel != nullptr && cancel->load(std::memory_order_relaxed); }
gemmi::Model &Model() { return st.models[0]; }
// The riders' parents, and per parent its riders in index order (rider_start, rider_index: CSR over the atoms).
void SetRiderParents(const std::vector<int> &parents) {
rider_parent = parents;
rider_start.assign(atoms.size() + 1, 0);
for (int p : rider_parent)
if (p >= 0)
rider_start[p + 1]++;
for (size_t i = 0; i < atoms.size(); i++)
rider_start[i + 1] += rider_start[i];
rider_index.resize(rider_start.back());
std::vector<int> fill(rider_start.begin(), rider_start.end() - 1);
for (size_t i = 0; i < rider_parent.size(); i++)
if (rider_parent[i] >= 0)
rider_index[fill[rider_parent[i]]++] = static_cast<int>(i);
}
// The hydrogens moved with their parents, at the offsets they were last placed at, and given their B.
void MoveRiders() {
const int n = static_cast<int>(rider_parent.size());
ParallelChunks(n, ThreadsForWork(n, nthreads, 16384), [&](int lo, int hi) {
for (int i = lo; i < hi; i++)
if (rider_parent[i] >= 0) {
atoms[i]->pos = atoms[rider_parent[i]]->pos + rider_offset[i];
atoms[i]->b_iso = atoms[rider_parent[i]]->b_iso;
}
});
}
// What the target says of a hydrogen is said of its parent: each parent takes its riders in index order, as
// the serial walk over the atoms added them (a rider has no riders of its own).
void FoldRiders(std::vector<std::array<double, 4>> &g) const {
const int n = static_cast<int>(rider_parent.size());
ParallelChunks(n, ThreadsForWork(n, nthreads, 16384), [&](int lo, int hi) {
for (int i = lo; i < hi; i++)
for (int r = rider_start[i]; r < rider_start[i + 1]; r++)
for (int c = 0; c < 4; c++)
g[i][c] += g[rider_index[r]][c];
});
}
std::vector<std::array<double, 4>> XyzB() const {
std::vector<std::array<double, 4>> out(atoms.size());
for (size_t i = 0; i < atoms.size(); i++)
out[i] = {atoms[i]->pos.x, atoms[i]->pos.y, atoms[i]->pos.z, atoms[i]->b_iso};
return out;
}
std::vector<gemmi::Position> Positions() const {
std::vector<gemmi::Position> out;
for (const gemmi::Atom *a : atoms)
out.push_back(a->pos);
return out;
}
void SetPositions(const std::vector<gemmi::Position> &p) {
for (size_t i = 0; i < atoms.size(); i++)
atoms[i]->pos = p[i];
}
std::vector<double> B() const {
std::vector<double> out;
for (const gemmi::Atom *a : atoms)
out.push_back(a->b_iso);
return out;
}
void SetB(const std::vector<double> &b) {
for (size_t i = 0; i < atoms.size(); i++)
atoms[i]->b_iso = static_cast<float>(b[i]);
}
// What Refit() fitted, to start another context from without fitting it again.
struct Fit {
double b_floor = B_FLOOR;
double k_sol = 0, b_sol = 0, k_overall = 0;
gemmi::SMat33<double> b_star{0, 0, 0, 0, 0, 0};
XrayRowModel rows;
double grid_k_sol = -1, grid_b_sol = -1;
};
Fit Fitted() const { return {b_floor, k_sol, b_sol, k_overall, b_star, rows, grid_k_sol, grid_b_sol}; }
// Refit() as it was where `fit` was taken, for the atoms as they were there.
void Restore(const Fit &fit) {
b_floor = fit.b_floor;
engine->SetBlur(RefinementBlur(cell, *sg, d_min, b_floor));
k_sol = fit.k_sol;
b_sol = fit.b_sol;
k_overall = fit.k_overall;
b_star = fit.b_star;
rows = fit.rows;
grid_k_sol = fit.grid_k_sol;
grid_b_sol = fit.grid_b_sol;
engine->SetRowModel(rows);
}
// The next Refit() from nothing an earlier one found: the whole solvent box, the likelihood from the moments.
void ForgetStarts() {
grid_k_sol = grid_b_sol = -1;
rows.bin_d.clear();
}
// A new macro-cycle: the blur for the B the cycle may reach, the bulk-solvent mask of the model as it is,
// the scale and the solvent fitted to the working set, and the likelihood's parameters from the free set.
// Returns the R-factors of the model as it stands.
RFactors Refit() {
const RefinementStage stage("refit");
double b_min = b_ceiling;
for (size_t i = 0; i < atoms.size(); i++)
b_min = std::min<double>(b_min, atoms[i]->b_iso - (b_lo.empty() ? 0.0 : b_lo[i]));
b_floor = std::max(B_FLOOR, b_min - B_STEP_DOWN);
engine->SetBlur(RefinementBlur(cell, *sg, d_min, b_floor));
{
const RefinementStage s1("refit.fcalc");
engine->Fcalc(Model(), fcalc);
}
{
const RefinementStage s2("refit.fmask");
// the GPU engine masks on the same grid as BulkSolventMask (bit for bit but at points on an atom's
// radius, ModelMaskGPU.h)
if (engine->OnGPU())
engine->Fmask(Model(), fmask);
else
mask->Fmask(Model(), refl.RowHkl(), fmask);
}
std::optional<RefinementStage> s3("refit.scale");
// the per-reflection loops on the threads, each element on its own
const int n_refl = static_cast<int>(refl.Size());
const size_t refl_threads = ThreadsForWork(refl.Size(), nthreads);
gemmi::AsuData<std::complex<float>> fc, fm;
gemmi::AsuData<gemmi::ValueSigma<float>> fo;
fc.unit_cell_ = fm.unit_cell_ = fo.unit_cell_ = cell;
fc.spacegroup_ = fm.spacegroup_ = fo.spacegroup_ = sg;
fc.v.resize(refl.Size());
fm.v.resize(refl.Size());
ParallelChunks(n_refl, refl_threads, [&](int lo, int hi) {
for (int m = lo; m < hi; m++) {
fc.v[m] = {refl.hkl[m], fcalc[m]};
fm.v[m] = {refl.hkl[m], fmask[m]};
}
});
fo.v.reserve(refl.Size());
for (size_t m = 0; m < refl.Size(); m++)
if (!refl.free[m])
fo.v.push_back({refl.hkl[m], {refl.fo[m], 1.0f}});
gemmi::Scaling<float> scaling(cell, sg);
scaling.use_solvent = true;
// the scale of the score: by robust least squares (ModelScaleBox::robust)
auto fit_scale = [&]() {
scaling.points.clear();
scaling.prepare_points(fc, fo, &fm);
#ifdef JFJOCH_USE_CUDA
if (scale_gpu)
scale_gpu->Fit(scaling, SolventBMax(Model()), nthreads, true, grid_k_sol, grid_b_sol);
else
#endif
FitModelScale(scaling, {.b_hi = SolventBMax(Model()), .robust = true, .k_start = grid_k_sol,
.b_start = grid_b_sol}, nthreads);
k_sol = grid_k_sol = scaling.k_sol;
b_sol = grid_b_sol = scaling.b_sol;
k_overall = scaling.k_overall;
b_star = scaling.b_star;
rows.k_total.resize(refl.Size());
rows.fmask.resize(refl.Rows());
ParallelChunks(n_refl, refl_threads, [&](int lo, int hi) {
for (int m = lo; m < hi; m++) {
rows.k_total[m] = static_cast<float>(scaling.get_overall_scale_factor(refl.hkl[m]));
rows.fmask[m] = static_cast<float>(scaling.get_solvent_scale(cell.calculate_stol_sq(refl.hkl[m]))) *
fmask[m];
}
});
for (size_t m = refl.Size(); m < refl.Rows(); m++) {
const gemmi::Miller &h = refl.mate_hkl[m - refl.Size()];
rows.fmask[m] = static_cast<float>(scaling.get_solvent_scale(cell.calculate_stol_sq(h))) * fmask[m];
}
};
fit_scale();
if (refl.Twinned()) {
// The scale and the solvent inside the twin sum: the fraction for the untwinned fit, then the scale
// fitted again against Fo detwinned by the model as it stands, Fo |F_model(h)| / |F_twin(h)| - which
// the twinned model at that scale meets as the untwinned one meets Fo - and the fraction again.
rows.twin_fractions = FitTwinFractions(refl, rows, fcalc);
const std::vector<std::complex<float>> twinned = TwinnedModel(refl, rows, fcalc);
fo.v.clear();
for (size_t m = 0; m < refl.Size(); m++) {
if (refl.free[m])
continue;
const double own = std::abs(rows.k_total[m] * (fcalc[m] + rows.fmask[m]));
const double tw = std::abs(twinned[m]);
fo.v.push_back({refl.hkl[m], {static_cast<float>(tw > 0 ? refl.fo[m] * own / tw : refl.fo[m]), 1.0f}});
}
fit_scale();
rows.twin_fractions = FitTwinFractions(refl, rows, fcalc);
}
s3.reset();
const std::vector<std::complex<float>> fmodel = TwinnedModel(refl, rows, fcalc);
{
const RefinementStage s4("refit.ml");
EstimateLikelihoodParameters(refl, fmodel, rows, ml_threads);
}
engine->SetRowModel(rows);
return ComputeRFactors(refl, fmodel);
}
// The overall isotropic B of the last Refit() moved into the atoms, and into the solvent's B so that F_model stays
// as it is (but where an atom meets a bound of B): left in the scale, it and the atoms' B drift against each
// other as the B factors are refined - and the B restraints, in proportion to B, depend on which holds it.
// As phenix.refine moves the overall B into the atoms (Afonine et al. (2012) Acta Cryst. D68, 352-367)
void MoveOverallB() {
const gemmi::Mat33 b = cell.orth.mat.multiply(b_star.as_mat33()).multiply(cell.orth.mat.transpose());
const double b_iso = (b[0][0] + b[1][1] + b[2][2]) / 3.0;
double b_min = b_ceiling;
for (size_t i = 0; i < atoms.size(); i++)
if (rider_parent.empty() || rider_parent[i] < 0) {
const double lo = b_lo.empty() ? 0.0 : b_lo[i], hi = b_hi.empty() ? 0.0 : b_hi[i];
atoms[i]->b_iso = static_cast<float>(std::clamp(atoms[i]->b_iso + b_iso, B_FLOOR + lo,
std::max(B_FLOOR + lo, b_ceiling - hi)));
b_min = std::min<double>(b_min, atoms[i]->b_iso - lo);
}
MoveRiders();
b_floor = std::max(B_FLOOR, b_min - B_STEP_DOWN);
engine->SetBlur(RefinementBlur(cell, *sg, d_min, b_floor));
// b_iso I in Cartesian is b_iso F F^T in the reciprocal basis
const gemmi::Mat33 f = cell.frac.mat.multiply(cell.frac.mat.transpose());
b_star = {b_star.u11 - b_iso * f[0][0], b_star.u22 - b_iso * f[1][1], b_star.u33 - b_iso * f[2][2],
b_star.u12 - b_iso * f[0][1], b_star.u13 - b_iso * f[0][2], b_star.u23 - b_iso * f[1][2]};
b_sol += b_iso;
for (size_t m = 0; m < refl.Size(); m++) {
rows.k_total[m] *= static_cast<float>(std::exp(b_iso * refl.inv_d2[m] / 4.0));
rows.fmask[m] *= static_cast<float>(std::exp(-b_iso * refl.inv_d2[m] / 4.0));
}
for (size_t m = refl.Size(); m < refl.Rows(); m++)
rows.fmask[m] *= static_cast<float>(std::exp(-b_iso * cell.calculate_1_d2(refl.mate_hkl[m - refl.Size()]) / 4.0));
engine->SetRowModel(rows);
}
// Every term of the score of the atoms as they are, at the scale, the solvent and the likelihood parameters
// of the last Refit().
RefinementScore Score() {
const std::vector<std::complex<float>> fmodel = FModel();
RefinementScore sc;
sc.k_overall = k_overall;
sc.k_sol = k_sol;
sc.b_sol = b_sol;
sc.mask_spacing = mask->Spacing();
// b_star is in the reciprocal basis (exp(-h^T b_star h / 4)); in Cartesian B = O b_star O^T
const gemmi::Mat33 b = cell.orth.mat.multiply(b_star.as_mat33()).multiply(cell.orth.mat.transpose());
sc.b_iso = (b[0][0] + b[1][1] + b[2][2]) / 3.0;
sc.b_aniso = {b[0][0] - sc.b_iso, b[1][1] - sc.b_iso, b[2][2] - sc.b_iso, b[0][1], b[0][2], b[1][2]};
const RFactors r = ComputeRFactors(refl, fmodel);
sc.r_work = r.r_work;
sc.r_free = r.r_free;
sc.n_work = r.n_work;
sc.n_free = r.n_free;
sc.nll_work = sc.nll_free = 0;
double fom_sum = 0;
int fom_n = 0;
for (size_t m = 0; m < refl.Size(); m++) {
double dfc = 0;
const double fc = std::abs(fmodel[m]);
const double nll = refine_math::AmplitudeNegLogLikelihood<double>(refl.fo[m], fc, rows.d[m], rows.var[m],
refl.centric[m], dfc);
(refl.free[m] ? sc.nll_free : sc.nll_work) += nll;
// the figure of merit of the same likelihood, over the working reflections
const double x = 2.0 * refl.fo[m] * rows.d[m] * fc / rows.var[m];
if (!refl.free[m]) {
fom_sum += Fom(x, refl.centric[m]);
fom_n++;
}
}
sc.fom = fom_n > 0 ? fom_sum / fom_n : NAN;
sc.twin_fraction = 0;
for (float a : rows.twin_fractions)
sc.twin_fraction += a;
for (int p : rider_parent)
sc.hydrogens += p >= 0;
constexpr int SHELLS = 20;
double s_lo = refl.inv_d2[0], s_hi = refl.inv_d2[0];
for (double s2 : refl.inv_d2) {
s_lo = std::min(s_lo, s2);
s_hi = std::max(s_hi, s2);
}
std::vector<std::array<double, 4>> sums(SHELLS, {0, 0, 0, 0});
sc.shells.resize(SHELLS);
for (int k = 0; k < SHELLS; k++) {
sc.shells[k].d_max = 1.0 / std::sqrt(s_lo + (s_hi - s_lo) * k / SHELLS);
sc.shells[k].d_min = 1.0 / std::sqrt(s_lo + (s_hi - s_lo) * (k + 1) / SHELLS);
}
for (size_t m = 0; m < refl.Size(); m++) {
const int k = std::min(SHELLS - 1, static_cast<int>((refl.inv_d2[m] - s_lo) / (s_hi - s_lo) * SHELLS));
const int f = refl.free[m] ? 2 : 0;
sums[k][f] += std::fabs(refl.fo[m] - std::abs(fmodel[m]));
sums[k][f + 1] += refl.fo[m];
(refl.free[m] ? sc.shells[k].n_free : sc.shells[k].n_work)++;
}
for (int k = 0; k < SHELLS; k++) {
if (sums[k][1] > 0)
sc.shells[k].r_work = sums[k][0] / sums[k][1];
if (sums[k][3] > 0)
sc.shells[k].r_free = sums[k][2] / sums[k][3];
}
return sc;
}
// The X-ray target's curvature in each atom's B, up to one factor (CalibrateBCurvature): the likelihood's
// curvature in |F_model| of the cycle, 2 (k_total D)^2 / var, summed in resolution shells, against the atom's own
// part of it, (s^2 / 4)^2 f(s)^2 exp(-B s^2 / 2). A hydrogen's is its parent's.
std::vector<double> XrayBCurvatureShape() const { return XrayCurvatureShape(rows, false); }
// The same for each of an atom's x, y and z: its own part of the likelihood's curvature in |F_model| is then
// (2 pi)^2 s^2 / 3 f(s)^2 exp(-B s^2 / 2), the reflections taken as spread evenly over directions - the X-ray
// block of the normal matrix as a multiple of I per atom. `r`: the row model the engine has (blurred or not).
std::vector<double> XrayXyzCurvatureShape(const XrayRowModel &r) const { return XrayCurvatureShape(r, true); }
std::vector<double> XrayCurvatureShape(const XrayRowModel &rows, bool xyz) const {
constexpr int SHELLS = 32;
double s2_max = 0;
for (double s2 : refl.inv_d2)
s2_max = std::max(s2_max, s2);
std::vector<double> w(SHELLS, 0), s2_sum(SHELLS, 0);
std::vector<int> n(SHELLS, 0);
for (size_t m = 0; m < refl.Size(); m++) {
if (refl.free[m])
continue;
const int k = std::min(SHELLS - 1, static_cast<int>(refl.inv_d2[m] / s2_max * SHELLS));
const double kd = rows.k_total[m] * rows.d[m];
w[k] += 2.0 * kd * kd / rows.var[m];
s2_sum[k] += refl.inv_d2[m];
n[k]++;
}
std::map<std::pair<int, signed char>, std::vector<double>> f2; // per element and charge, per shell
std::vector<double> shape(atoms.size(), 0.0);
for (size_t i = 0; i < atoms.size(); i++) {
const gemmi::Atom &a = *atoms[i];
std::vector<double> &f = f2[{static_cast<int>(a.element.elem), a.charge}];
if (f.empty()) {
const auto &coef = IonFormFactor<double>(a.element.elem, a.charge);
f.resize(SHELLS, 0);
for (int k = 0; k < SHELLS; k++)
if (n[k] > 0)
f[k] = std::pow(coef.calculate_sf(s2_sum[k] / n[k] / 4.0), 2);
}
for (int k = 0; k < SHELLS; k++)
if (n[k] > 0) {
const double s2 = s2_sum[k] / n[k];
if (xyz)
shape[i] += w[k] * f[k] * a.occ * a.occ * std::exp(-a.b_iso * s2 / 2.0) * s2 * (4.0 * PI * PI / 3.0);
else
shape[i] += w[k] * f[k] * a.occ * a.occ * std::exp(-a.b_iso * s2 / 2.0) * s2 * s2 / 16.0;
}
}
for (size_t i = 0; i < rider_parent.size(); i++)
if (rider_parent[i] >= 0)
shape[rider_parent[i]] += shape[i];
return shape;
}
// The factor that turns XrayBCurvatureShape() into the X-ray target's curvature: its second derivative along a
// random +-1 direction of the B factors of `params`, by central differences of the gradient, over what the shape
// predicts for it.
double CalibrateBCurvature(const std::vector<int> &params, const std::vector<double> &shape) {
constexpr double EPS = 0.5; // A^2
const std::vector<double> b0 = B();
std::mt19937 rng(7);
std::vector<double> v(params.size());
for (double &x : v)
x = rng() % 2 == 0 ? 1.0 : -1.0;
std::array<std::vector<std::array<double, 4>>, 2> g;
for (int side = 0; side < 2; side++) {
for (size_t k = 0; k < params.size(); k++)
atoms[params[k]]->b_iso = static_cast<float>(b0[params[k]] + (side == 0 ? EPS : -EPS) * v[k]);
MoveRiders();
engine->Evaluate(Model(), &g[side]);
FoldRiders(g[side]);
}
SetB(b0);
MoveRiders();
double num = 0, den = 0;
for (size_t k = 0; k < params.size(); k++) {
num += v[k] * (g[0][params[k]][3] - g[1][params[k]][3]) / (2 * EPS);
den += shape[params[k]];
}
return den > 0 ? std::max(num / den, 0.0) : 0.0;
}
// The factor that turns XrayXyzCurvatureShape() into the X-ray target's curvature, as CalibrateBCurvature() does
// for B: along a random +-1 direction of the positions of `params`, by central differences of the gradient.
// Two evaluations, counted.
double CalibrateXyzCurvature(const std::vector<int> &params, const std::vector<double> &shape) {
constexpr double EPS = 0.01; // A
const std::vector<gemmi::Position> p0 = Positions();
std::mt19937 rng(7);
std::vector<gemmi::Vec3> v(params.size());
for (gemmi::Vec3 &x : v) { // one draw per statement: the order of evaluation of arguments is unspecified
const double sx = rng() % 2 == 0 ? 1.0 : -1.0;
const double sy = rng() % 2 == 0 ? 1.0 : -1.0;
const double sz = rng() % 2 == 0 ? 1.0 : -1.0;
x = gemmi::Vec3(sx, sy, sz);
}
std::array<std::vector<std::array<double, 4>>, 2> g;
for (int side = 0; side < 2; side++) {
for (size_t k = 0; k < params.size(); k++)
atoms[params[k]]->pos = p0[params[k]] + gemmi::Position(v[k] * (side == 0 ? EPS : -EPS));
MoveRiders();
engine->Evaluate(Model(), &g[side]);
evaluations++;
FoldRiders(g[side]);
}
SetPositions(p0);
MoveRiders();
double num = 0, den = 0;
for (size_t k = 0; k < params.size(); k++) {
const int i = params[k];
num += (v[k].x * (g[0][i][0] - g[1][i][0]) + v[k].y * (g[0][i][1] - g[1][i][1]) +
v[k].z * (g[0][i][2] - g[1][i][2])) / (2 * EPS);
den += 3 * shape[i];
}
return den > 0 ? std::max(num / den, 0.0) : 0.0;
}
// F_model with the cycle's scale and solvent, for the atoms as they are now.
std::vector<std::complex<float>> FModel() {
engine->Fcalc(Model(), fcalc);
return TwinnedModel(refl, rows, fcalc);
}
};
ceres::GradientProblemSolver::Options SolverOptions(int iterations) {
ceres::GradientProblemSolver::Options options;
options.line_search_direction_type = ceres::LBFGS;
options.max_num_iterations = iterations;
options.function_tolerance = 1e-7;
options.gradient_tolerance = 1e-10;
options.parameter_tolerance = 1e-9;
options.logging_type = ceres::SILENT;
return options;
}
// SolverOptions(iterations) that stop at the next iteration, where the minimiser stands, once ctx is cancelled.
class CancelCheck : public ceres::IterationCallback {
public:
explicit CancelCheck(const Context &ctx) : ctx_(ctx) {}
ceres::CallbackReturnType operator()(const ceres::IterationSummary &) override {
return ctx_.Cancelled() ? ceres::SOLVER_TERMINATE_SUCCESSFULLY : ceres::SOLVER_CONTINUE;
}
ceres::GradientProblemSolver::Options Options(int iterations) const {
ceres::GradientProblemSolver::Options options = SolverOptions(iterations);
options.callbacks.push_back(const_cast<CancelCheck *>(this));
return options;
}
private:
const Context &ctx_;
};
// The positions or the B factors of the refinable atoms, against w times the X-ray target plus a restraint set
// (minimised by RefinementMinimiser).
class AtomTarget {
public:
enum class Mode { Xyz, B };
// `topo`: its atoms on special positions move only within their site's subspace (SpecialPosition).
AtomTarget(Context &ctx, Mode mode, const std::vector<int> &param_atoms, double weight, int restraint_set,
const RefinementTopology &topo)
: ctx_(ctx), mode_(mode), param_atoms_(param_atoms), weight_(weight), restraint_set_(restraint_set), topo_(topo) {}
// The parameters of the atoms as they are: x y z per atom, or B per atom.
std::vector<double> Get() const {
std::vector<double> p;
for (int i : param_atoms_) {
const gemmi::Atom &a = *ctx_.atoms[i];
if (mode_ == Mode::Xyz) {
p.push_back(a.pos.x);
p.push_back(a.pos.y);
p.push_back(a.pos.z);
} else
p.push_back(a.b_iso);
}
return p;
}
// The atoms set to the parameters, their riders with them.
void Set(const double *p) const {
const int n = static_cast<int>(param_atoms_.size());
ParallelChunks(n, ThreadsForWork(n, ctx_.nthreads, 16384), [&](int lo, int hi) {
for (int k = lo; k < hi; k++) {
gemmi::Atom &a = *ctx_.atoms[param_atoms_[k]];
if (mode_ == Mode::Xyz) {
a.pos = gemmi::Position(p[3 * k], p[3 * k + 1], p[3 * k + 2]);
if (const SpecialPosition *sp = Special(param_atoms_[k]))
a.pos = sp->OnSite(a.pos); // where the shifts, all within its subspace, left it by rounding
} else
a.b_iso = static_cast<float>(std::clamp(p[k], ctx_.BLow(param_atoms_[k]), ctx_.BHigh(param_atoms_[k])));
}
});
ctx_.MoveRiders();
}
// the most a parameter moves in the first trial of a line search
double MaxStep() const { return mode_ == Mode::Xyz ? MAX_STEP_XYZ : MAX_STEP_B; }
bool Evaluate(const double *p, double *cost, double *gradient) const {
const RefinementStage stage(mode_ == Mode::Xyz ? "eval.xyz" : "eval.b");
ctx_.TakeSpareThreads();
Set(p);
ctx_.evaluations++;
std::vector<std::array<double, 4>> &gx = gx_, &gr = gr_;
double xray = 0, restraints = 0;
{
const RefinementStage s1("eval.engine");
ctx_.engine->SetXyzGradient(mode_ == Mode::Xyz); // B alone reads only the B derivatives
xray = ctx_.engine->EvaluateWithRestraints(ctx_.Model(), restraint_set_, gradient != nullptr ? &gx : nullptr,
gradient != nullptr ? &gr : nullptr, restraints);
ctx_.engine->SetXyzGradient(true);
}
*cost = weight_ * xray + restraints;
if (gradient == nullptr)
return std::isfinite(*cost);
ctx_.FoldRiders(gx);
const int n = static_cast<int>(param_atoms_.size());
ParallelChunks(n, ThreadsForWork(n, ctx_.nthreads, 16384), [&](int lo, int hi) {
for (int k = lo; k < hi; k++) {
const int i = param_atoms_[k];
if (mode_ == Mode::Xyz) {
gemmi::Vec3 g(weight_ * gx[i][0] + gr[i][0], weight_ * gx[i][1] + gr[i][1],
weight_ * gx[i][2] + gr[i][2]);
if (const SpecialPosition *sp = Special(i))
g = sp->projector.multiply(g);
gradient[3 * k] = g.x;
gradient[3 * k + 1] = g.y;
gradient[3 * k + 2] = g.z;
} else {
// held at a bound, B does not move further out: by the whole target, restraints included
const double g = weight_ * gx[i][3] + gr[i][3];
const bool held = (p[k] <= ctx_.BLow(i) && g > 0) || (p[k] >= ctx_.BHigh(i) && g < 0);
gradient[k] = held ? 0.0 : g;
}
}
});
return std::isfinite(*cost);
}
Mode GetMode() const { return mode_; }
double Weight() const { return weight_; }
const std::vector<int> &Atoms() const { return param_atoms_; }
private:
const SpecialPosition *Special(int atom) const {
const int k = topo_.special_index.empty() ? -1 : topo_.special_index[atom];
return k >= 0 ? &topo_.special[k] : nullptr;
}
Context &ctx_;
Mode mode_;
const std::vector<int> &param_atoms_;
double weight_;
int restraint_set_;
const RefinementTopology &topo_;
// the gradients per atom, kept from evaluation to evaluation (7 MB each on a large model: made anew, their
// pages fault in every time)
mutable std::vector<std::array<double, 4>> gx_, gr_;
};
// The X-ray target's derivatives in U of every atom, with what the target says of a riding hydrogen's B (its parent's
// equivalent isotropic B, 8 pi^2 tr(U) / 3) said of its parent's U.
std::vector<std::array<double, 6>> FoldedUGradient(const Context &ctx, const std::vector<std::array<double, 4>> &gx) {
std::vector<std::array<double, 6>> g = ctx.engine->UGradient();
g.resize(ctx.atoms.size(), {0, 0, 0, 0, 0, 0});
for (size_t r = 0; r < ctx.rider_parent.size(); r++)
if (const int p = ctx.rider_parent[r]; p >= 0)
for (int c = 0; c < 3; c++)
g[p][c] += gx[r][3] * gemmi::u_to_b() / 3.0;
return g;
}
// The displacements of the anisotropic atoms against w times the X-ray target. Mode::U: a U per atom of
// `param_atoms`, as 8 pi^2 U (in the units of B), plus the U restraints; its smallest eigenvalue held at the cycle's
// floor and its diagonal under the ceiling. Mode::Tls: the parameters of each of ctx.tls, in the units of B at the
// body's radius (T, L r^2, S r, each times 8 pi^2), the atoms' residual B held; X-ray only.
class AdpTarget {
public:
enum class Mode { U, Tls };
AdpTarget(Context &ctx, Mode mode, const std::vector<int> &param_atoms, double weight, const URestraints *restraints)
: ctx_(ctx), mode_(mode), param_atoms_(param_atoms), weight_(weight), restraints_(restraints) {
for (const TlsBody &body : ctx.tls) {
double r2 = 0;
for (int i : body.atoms)
r2 += body.origin.dist_sq(ctx.atoms[i]->pos);
radius_.push_back(std::max(1.0, std::sqrt(r2 / std::max<size_t>(body.atoms.size(), 1))));
}
}
std::vector<double> Get() const {
std::vector<double> p;
if (mode_ == Mode::U) {
const std::vector<std::array<double, 6>> u = ctx_.UOfAtoms();
for (int i : param_atoms_)
for (int c = 0; c < 6; c++)
p.push_back(gemmi::u_to_b() * u[i][c]);
} else
for (size_t b = 0; b < ctx_.tls.size(); b++) {
const std::array<double, TlsBody::PARAMETERS> q = ctx_.tls[b].Get();
for (int k = 0; k < TlsBody::PARAMETERS; k++)
p.push_back(q[k] * Scale(b, k));
}
return p;
}
void Set(const double *p) const {
if (mode_ == Mode::U) {
const int n = static_cast<int>(param_atoms_.size());
ParallelChunks(n, ThreadsForWork(n, ctx_.nthreads, 16384), [&](int lo, int hi) {
for (int k = lo; k < hi; k++)
SetU(param_atoms_[k], p + 6 * k);
});
ctx_.MoveRiders();
ctx_.SetAniso(ctx_.aniso);
} else {
for (size_t b = 0; b < ctx_.tls.size(); b++) {
std::array<double, TlsBody::PARAMETERS> q{};
for (int k = 0; k < TlsBody::PARAMETERS; k++)
q[k] = p[b * TlsBody::PARAMETERS + k] / Scale(b, k);
ctx_.tls[b].Set(q);
}
ctx_.ApplyTls();
}
}
double MaxStep() const { return MAX_STEP_B; }
// The parameter k of TLS body `body` over its value in the body's own units.
double ParamScale(size_t body, int k) const { return Scale(body, k); }
// Mode::U: the U restraints' gradient alone at p, as Evaluate() makes it (zero where there are none).
void RestraintGradient(const double *p, double *gradient) const {
Set(p);
std::fill(gradient, gradient + 6 * param_atoms_.size(), 0.0);
if (restraints_ == nullptr)
return;
std::vector<std::array<double, 6>> gr(ctx_.atoms.size(), {0, 0, 0, 0, 0, 0});
EvaluateURestraints(*restraints_, ctx_.Positions(), ctx_.UOfAtoms(), &gr);
for (size_t k = 0; k < param_atoms_.size(); k++)
for (int c = 0; c < 6; c++)
gradient[6 * k + c] = gr[param_atoms_[k]][c] / gemmi::u_to_b();
}
bool Evaluate(const double *p, double *cost, double *gradient) const {
const RefinementStage stage(mode_ == Mode::U ? "eval.u" : "eval.tls");
ctx_.TakeSpareThreads();
Set(p);
ctx_.evaluations++;
std::vector<std::array<double, 4>> gx;
const double xray = ctx_.engine->Evaluate(ctx_.Model(), gradient != nullptr ? &gx : nullptr);
double restraints = 0;
std::vector<std::array<double, 6>> gr;
if (mode_ == Mode::U && restraints_ != nullptr) {
if (gradient != nullptr)
gr.assign(ctx_.atoms.size(), {0, 0, 0, 0, 0, 0});
restraints = EvaluateURestraints(*restraints_, ctx_.Positions(), ctx_.UOfAtoms(), gradient != nullptr ? &gr : nullptr);
}
*cost = weight_ * xray + restraints;
if (gradient == nullptr)
return std::isfinite(*cost);
const std::vector<std::array<double, 6>> gu = FoldedUGradient(ctx_, gx);
if (mode_ == Mode::U) {
for (size_t k = 0; k < param_atoms_.size(); k++)
for (int c = 0; c < 6; c++)
gradient[6 * k + c] = (weight_ * gu[param_atoms_[k]][c] + (gr.empty() ? 0.0 : gr[param_atoms_[k]][c])) /
gemmi::u_to_b();
} else {
for (size_t b = 0; b < ctx_.tls.size(); b++) {
const std::array<double, TlsBody::PARAMETERS> d = ctx_.tls[b].Gradient(ctx_.atoms, gu);
for (int k = 0; k < TlsBody::PARAMETERS; k++)
gradient[b * TlsBody::PARAMETERS + k] = weight_ * d[k] / Scale(b, k);
}
}
return std::isfinite(*cost);
}
private:
double Scale(size_t body, int k) const {
const double r = radius_[body];
return gemmi::u_to_b() * (k < 6 ? 1.0 : k < 12 ? r * r : r);
}
// Atom i's U from 8 pi^2 U = p[0..5]: its smallest eigenvalue raised to the cycle's floor and its largest brought
// down to the ceiling where they are outside, then its B and its anisotropy.
void SetU(int i, const double *p) const {
const gemmi::SMat33<double> b = BoundedB({p[0], p[1], p[2], p[3], p[4], p[5]}, ctx_.b_floor, ctx_.b_ceiling);
const double t = b.trace() / 3.0;
ctx_.atoms[i]->b_iso = static_cast<float>(t);
AtomAniso &a = ctx_.aniso[i];
a.on = true;
a.dev = {static_cast<float>((b.u11 - t) / gemmi::u_to_b()), static_cast<float>((b.u22 - t) / gemmi::u_to_b()),
static_cast<float>((b.u33 - t) / gemmi::u_to_b()), static_cast<float>(b.u12 / gemmi::u_to_b()),
static_cast<float>(b.u13 / gemmi::u_to_b()), static_cast<float>(b.u23 / gemmi::u_to_b())};
}
Context &ctx_;
Mode mode_;
const std::vector<int> &param_atoms_;
double weight_;
const URestraints *restraints_;
std::vector<double> radius_;
};
// Each element at least a thousandth of the median (a parameter the target does not see still takes a finite step).
std::vector<double> FlooredDiagonal(std::vector<double> d) {
if (d.empty())
return d;
std::vector<double> sorted = d;
std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end());
const double floor = std::max(1e-3 * sorted[sorted.size() / 2], 1e-12);
for (double &v : d)
v = std::max(v, floor);
return d;
}
// The minimisers' initial inverse Hessian (RefinementMinimiser::SetInverseHessian) is the Gauss-Newton normal matrix of
// the target, as REFMAC refines on its sparse normal matrix: the X-ray target's curvature per atom on the diagonal and
// the restraints' J^T J in full, solved for each direction by a fixed number of conjugate-gradient iterations - not to a
// tolerance, whose iteration count, and with it the direction, a rounding difference can flip.
// Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255
constexpr int NORMAL_CG_ITERATIONS = 20;
// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q)
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then z = H^-1 q by
// NORMAL_CG_ITERATIONS conjugate gradients (solve: ConjugateGradients on the CPU, or on the card).
template <typename Target>
void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations,
const std::function<void(const std::vector<double> &, const std::vector<double> &)> &at,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &solve) {
std::vector<double> p = target.Get();
if (p.empty())
return;
minimiser.SetInverseHessian([&](const std::vector<double> &x, std::vector<double> &q) {
at(x, q);
std::vector<double> z;
solve(q, z);
q.swap(z);
});
minimiser.Minimise([&](const std::vector<double> &x, double &f, std::vector<double> &g) {
return target.Evaluate(x.data(), &f, g.data());
},
p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); });
minimiser.SetInverseHessian(nullptr);
target.Set(p.data()); // where the minimiser ended, not where its last line search probed
}
// The conjugate gradients on the CPU: hv = H v (product) and z = M^-1 r for a preconditioner M of H (precondition).
std::function<void(const std::vector<double> &, std::vector<double> &)>
NormalSolve(const Context &ctx, const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
return [&ctx, product, precondition](const std::vector<double> &q, std::vector<double> &z) {
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
}
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
// a GPU, on its card (RestraintNormalMatrixGPU).
class RestraintNormalMatrix {
public:
RestraintNormalMatrix(const Context &ctx, const std::vector<int> &param_atoms, bool xyz)
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(ctx.nthreads),
param_of_(ctx.atoms.size(), -1) {
for (size_t k = 0; k < param_atoms.size(); k++)
param_of_[param_atoms[k]] = static_cast<int>(k);
#ifdef JFJOCH_USE_CUDA
if (ctx.engine->OnGPU())
gpu_ = std::make_unique<RestraintNormalMatrixGPU>(ctx.device, param_atoms, ctx.atoms.size(), xyz);
#endif
}
void At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb) {
const RefinementStage stage("normal.jacobian");
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->At(set, xyzb);
#endif
jac_ = RestraintGaussNewton(set, xyzb);
const size_t rows = jac_.start.size() - 1;
row_of_.resize(jac_.atom.size());
param_.resize(jac_.atom.size());
for (size_t e = 0; e < jac_.atom.size(); e++)
param_[e] = param_of_[jac_.atom[e]];
for (size_t r = 0; r < rows; r++)
for (int e = jac_.start[r]; e < jac_.start[r + 1]; e++)
row_of_[e] = static_cast<int>(r);
// per parameter atom its entries (CSR), for the transpose as a gather
entry_start_.assign(n_params_ + 1, 0);
for (int a : jac_.atom)
if (param_of_[a] >= 0)
entry_start_[param_of_[a] + 1]++;
for (size_t k = 0; k < n_params_; k++)
entry_start_[k + 1] += entry_start_[k];
entry_.resize(entry_start_.back());
std::vector<int> fill(entry_start_.begin(), entry_start_.end() - 1);
for (size_t e = 0; e < jac_.atom.size(); e++)
if (const int k = param_of_[jac_.atom[e]]; k >= 0)
entry_[fill[k]++] = static_cast<int>(e);
t_.resize(rows);
}
// hv = J^T J v: J v per row, then per parameter atom the sum over its entries
void Multiply(const std::vector<double> &v, std::vector<double> &hv) {
const RefinementStage stage("normal.product");
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->Multiply(v, hv);
#endif
const int rows = static_cast<int>(t_.size());
ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) {
for (int r = lo; r < hi; r++) {
double dot = 0;
for (int e = jac_.start[r]; e < jac_.start[r + 1]; e++)
if (const int k = param_[e]; k >= 0)
for (int c = 0; c < per_; c++)
dot += jac_.j[e][c0_ + c] * v[per_ * k + c];
t_[r] = dot;
}
});
hv.assign(per_ * n_params_, 0.0);
const int n = static_cast<int>(n_params_);
ParallelChunks(n, ThreadsForWork(n, nthreads_, 16384), [&](int lo, int hi) {
for (int k = lo; k < hi; k++)
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) {
const int e = entry_[q];
for (int c = 0; c < per_; c++)
hv[per_ * k + c] += t_[row_of_[e]] * jac_.j[e][c0_ + c];
}
});
}
#ifdef JFJOCH_USE_CUDA
// The conjugate gradients on the card, the vectors kept there (RestraintNormalMatrixGPU::Solve)
RestraintNormalMatrixGPU *GPU() { return gpu_.get(); }
#endif
std::vector<double> Diagonal() const {
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->Diagonal();
#endif
std::vector<double> d(per_ * n_params_, 0.0);
for (size_t k = 0; k < n_params_; k++)
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++)
for (int c = 0; c < per_; c++)
d[per_ * k + c] += jac_.j[entry_[q]][c0_ + c] * jac_.j[entry_[q]][c0_ + c];
return d;
}
private:
int per_, c0_;
size_t n_params_, nthreads_;
std::vector<int> param_of_;
RestraintJacobian jac_;
std::vector<int> row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none)
std::vector<double> t_;
#ifdef JFJOCH_USE_CUDA
std::unique_ptr<RestraintNormalMatrixGPU> gpu_;
#endif
};
// The positions' preconditioner: per residue the dense block of H over its atoms' x y z - the restraints within the
// residue (J^T J of the rows whose atoms it holds, at the point given) and the X-ray diagonal `h` - by its Cholesky
// factor. What couples residues (peptide bonds, NCS, contacts) is left to the conjugate gradients; the stiff bonds
// and angles within a residue, which a diagonal or a block per atom leaves to them too, are solved here.
class ResidueBlocks {
public:
ResidueBlocks(Context &ctx, const std::vector<int> &param_atoms, const RestraintSet &set, const std::vector<double> &h) {
std::vector<int> residue_of; // per atom, in model order
for (const gemmi::Chain &ch : ctx.Model().chains)
for (const gemmi::Residue &r : ch.residues) {
residue_of.insert(residue_of.end(), r.atoms.size(), static_cast<int>(params_.size()));
params_.emplace_back();
}
std::vector<int> local(param_atoms.size()), group(param_atoms.size()), param_of(ctx.atoms.size(), -1);
for (size_t k = 0; k < param_atoms.size(); k++) {
group[k] = residue_of[param_atoms[k]];
local[k] = static_cast<int>(params_[group[k]].size());
params_[group[k]].push_back(static_cast<int>(k));
param_of[param_atoms[k]] = static_cast<int>(k);
}
std::vector<Eigen::MatrixXd> block(params_.size());
for (size_t g = 0; g < params_.size(); g++) {
const int m = 3 * static_cast<int>(params_[g].size());
block[g] = Eigen::MatrixXd::Zero(m, m);
for (size_t l = 0; l < params_[g].size(); l++)
for (int c = 0; c < 3; c++)
block[g](3 * l + c, 3 * l + c) = h[3 * params_[g][l] + c];
}
const RestraintJacobian jac = RestraintGaussNewton(set, ctx.XyzB());
for (size_t row = 0; row + 1 < jac.start.size(); row++)
for (int e1 = jac.start[row]; e1 < jac.start[row + 1]; e1++)
for (int e2 = jac.start[row]; e2 < jac.start[row + 1]; e2++) {
const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]];
if (k1 < 0 || k2 < 0 || group[k1] != group[k2])
continue;
Eigen::MatrixXd &b = block[group[k1]];
for (int a = 0; a < 3; a++)
for (int c = 0; c < 3; c++)
b(3 * local[k1] + a, 3 * local[k2] + c) += jac.j[e1][a] * jac.j[e2][c];
}
llt_.resize(params_.size());
ParallelFor(static_cast<int>(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); });
}
// The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and the lower triangle of its
// inverse, row by row.
void Inverses(std::vector<int> &start, std::vector<int> &element, std::vector<double> &inverse) const {
start.assign(1, 0);
element.clear();
inverse.clear();
for (size_t g = 0; g < params_.size(); g++) {
for (int k : params_[g])
for (int c = 0; c < 3; c++)
element.push_back(3 * k + c);
start.push_back(static_cast<int>(element.size()));
const int m = 3 * static_cast<int>(params_[g].size());
const Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m));
for (int i = 0; i < m; i++)
for (int k = 0; k <= i; k++)
inverse.push_back(w(i, k));
}
}
void Apply(const std::vector<double> &r, std::vector<double> &z, size_t nthreads) const {
z.resize(r.size());
ParallelFor(static_cast<int>(params_.size()), nthreads, [&](int g) {
const std::vector<int> &p = params_[g];
Eigen::VectorXd rg(3 * p.size());
for (size_t l = 0; l < p.size(); l++)
for (int c = 0; c < 3; c++)
rg(3 * l + c) = r[3 * p[l] + c];
const Eigen::VectorXd zg = llt_[g].solve(rg);
for (size_t l = 0; l < p.size(); l++)
for (int c = 0; c < 3; c++)
z[3 * p[l] + c] = zg(3 * l + c);
});
}
private:
std::vector<std::vector<int>> params_; // per residue its parameter atoms (their index among the parameters)
std::vector<Eigen::LLT<Eigen::MatrixXd>> llt_;
};
// The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`,
// the Jacobian at the point of each direction. The diagonal is raised per atom, once per call, to at least |g_i| over the
// step bound (g_i the atom's part of the first direction's gradient): where nothing else curves the target (a high-B
// side chain turning about a single bond, which the restraints leave free and the X-ray target hardly sees) the step
// is then held to the bound by the atom's own curvature, instead of one such atom scaling down the whole step.
// Preconditioned per residue for the positions (ResidueBlocks), by the diagonal for the B factors.
// The floor a trust region per atom, after Levenberg (1944) Q. Appl. Math. 2, 164-168 and Marquardt (1963) J. SIAM
// 11, 431-441
void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &minimiser, const std::vector<double> &xray,
const RestraintSet &set, int iterations) {
const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz;
const int per = xyz ? 3 : 1;
std::vector<double> x_at = target.Get();
const size_t n = x_at.size();
std::vector<double> h(n);
for (size_t k = 0; k < n; k++)
h[k] = xray[target.Atoms()[k / per]];
RestraintNormalMatrix normal(ctx, target.Atoms(), xyz);
target.Set(x_at.data());
normal.At(set, ctx.XyzB());
const std::vector<double> restraint_diagonal = normal.Diagonal();
std::vector<double> diagonal;
std::unique_ptr<ResidueBlocks> residues;
auto at = [&](const std::vector<double> &x, const std::vector<double> &q) {
if (diagonal.empty()) { // the first direction: the floor, then the preconditioner with it
for (size_t a = 0; a < n / per; a++) {
double g2 = 0;
for (int c = 0; c < per; c++)
g2 += q[per * a + c] * q[per * a + c];
for (int c = 0; c < per; c++)
h[per * a + c] = std::max(h[per * a + c], std::sqrt(g2) / target.MaxStep());
}
diagonal.resize(n);
for (size_t k = 0; k < n; k++)
diagonal[k] = restraint_diagonal[k] + h[k];
diagonal = FlooredDiagonal(diagonal);
if (xyz)
residues = std::make_unique<ResidueBlocks>(ctx, target.Atoms(), set, h);
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) { // the blocks, or the diagonal as blocks of one
std::vector<int> start, element;
std::vector<double> inverse;
if (residues)
residues->Inverses(start, element, inverse);
else {
for (size_t k = 0; k < n; k++) {
start.push_back(static_cast<int>(k));
element.push_back(static_cast<int>(k));
inverse.push_back(1.0 / diagonal[k]);
}
start.push_back(static_cast<int>(n));
}
normal.GPU()->SetSolve(h, start, element, inverse);
}
#endif
}
if (x != x_at) {
x_at = x;
target.Set(x.data());
normal.At(set, ctx.XyzB());
}
};
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
normal.Multiply(v, hv);
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] += h[k] * v[k]; });
};
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
if (residues)
residues->Apply(r, z, ctx.nthreads);
else {
z.resize(r.size());
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
}
};
auto solve = [&](const std::vector<double> &q, std::vector<double> &z) {
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) {
const RefinementStage stage("normal.solve");
return normal.GPU()->Solve(q, z, NORMAL_CG_ITERATIONS);
}
#endif
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, solve);
}
// Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their
// gradient (they are quadratic in U); the diagonal raised to the trust floor as MinimiseAtoms() raises it;
// preconditioned by `diagonal`.
void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, std::vector<double> xray,
std::vector<double> diagonal, int iterations) {
const size_t n = xray.size();
std::vector<double> x_at, g_at(n), xp(n), gp(n);
bool first = true;
auto at = [&](const std::vector<double> &x, const std::vector<double> &q) {
if (first) { // the trust floor per atom, as MinimiseAtoms() takes it
first = false;
for (size_t a = 0; a < n / 6; a++) {
double g2 = 0;
for (int c = 0; c < 6; c++)
g2 += q[6 * a + c] * q[6 * a + c];
const double floor = std::sqrt(g2) / target.MaxStep();
for (int c = 0; c < 6; c++) {
diagonal[6 * a + c] += std::max(floor - xray[6 * a + c], 0.0);
xray[6 * a + c] = std::max(xray[6 * a + c], floor);
}
}
}
if (x != x_at) {
x_at = x;
target.RestraintGradient(x.data(), g_at.data());
}
};
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
double v_max = 0;
for (double e : v)
v_max = std::max(v_max, std::fabs(e));
hv.assign(n, 0.0);
if (v_max > 0) {
const double t = 1.0 / v_max;
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { xp[k] = x_at[k] + t * v[k]; });
target.RestraintGradient(xp.data(), gp.data());
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] = (gp[k] - g_at[k]) / t; });
}
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] += xray[k] * v[k]; });
};
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
z.resize(n);
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's
// parameters (linear in them), C the X-ray curvature of its U per element (`xray_b`: of its B, a third of it to each);
// one dense block per body, its diagonal raised to the trust floor, preconditioned by its diagonal.
void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, const std::vector<double> &xray_b,
int iterations) {
constexpr int P = TlsBody::PARAMETERS;
const size_t nb = ctx.tls.size();
std::vector<std::array<double, P * P>> blocks(nb);
for (size_t b = 0; b < nb; b++) {
std::array<double, P * P> &m = blocks[b];
m.fill(0.0);
for (int i : ctx.tls[b].atoms) {
if (!ctx.rider_parent.empty() && ctx.rider_parent[i] >= 0)
continue; // its curvature is its parent's already
const std::array<std::array<double, 6>, P> a = ctx.tls[b].Derivatives(ctx.atoms[i]->pos);
const double c = gemmi::u_to_b() * gemmi::u_to_b() * xray_b[i] / 3.0;
for (int k = 0; k < P; k++)
for (int l = 0; l < P; l++) {
double s = 0;
for (int e = 0; e < 6; e++)
s += a[k][e] * a[l][e];
m[k * P + l] += c * s / (target.ParamScale(b, k) * target.ParamScale(b, l));
}
}
}
std::vector<double> diagonal(nb * P);
for (size_t b = 0; b < nb; b++)
for (int k = 0; k < P; k++)
diagonal[b * P + k] = blocks[b][k * P + k];
diagonal = FlooredDiagonal(diagonal);
// the trust floor per parameter, as MinimiseAtoms() takes it per atom
bool first = true;
auto at = [&](const std::vector<double> &, const std::vector<double> &q) {
if (!first)
return;
first = false;
for (size_t b = 0; b < nb; b++)
for (int k = 0; k < P; k++) {
const double floor = std::fabs(q[b * P + k]) / target.MaxStep();
diagonal[b * P + k] += std::max(floor - blocks[b][k * P + k], 0.0);
blocks[b][k * P + k] = std::max(blocks[b][k * P + k], floor);
}
};
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
hv.assign(v.size(), 0.0);
for (size_t b = 0; b < nb; b++)
for (int k = 0; k < P; k++)
for (int l = 0; l < P; l++)
hv[b * P + k] += blocks[b][k * P + l] * v[b * P + l];
};
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
z.resize(r.size());
for (size_t k = 0; k < r.size(); k++)
z[k] = r[k] / diagonal[k];
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis
// rotation about its centroid, scaled by its rms radius so that a unit moves a typical atom by an angstrom as a
// unit of translation does, then a translation.
class RigidBodiesTarget : public ceres::FirstOrderFunction {
public:
RigidBodiesTarget(Context &ctx, const std::vector<std::vector<int>> &bodies)
: ctx_(ctx), bodies_(bodies), base_(ctx.Positions()) {
for (const std::vector<int> &b : bodies) {
gemmi::Position c;
for (int i : b)
c += base_[i];
c *= 1.0 / static_cast<double>(b.size());
double r2 = 0;
for (int i : b)
r2 += c.dist_sq(base_[i]);
centre_.push_back(c);
radius_.push_back(std::max(1.0, std::sqrt(r2 / static_cast<double>(b.size()))));
}
}
int NumParameters() const override { return static_cast<int>(6 * bodies_.size()); }
bool Evaluate(const double *q, double *cost, double *gradient) const override {
const RefinementStage stage("eval.rigid");
Place(q);
ctx_.evaluations++;
std::vector<std::array<double, 4>> g;
*cost = ctx_.engine->Evaluate(ctx_.Model(), gradient != nullptr ? &g : nullptr);
if (gradient == nullptr)
return std::isfinite(*cost);
using Jet = ceres::Jet<double, 3>;
for (size_t b = 0; b < bodies_.size(); b++) {
const Jet aa[3] = {Jet(q[6 * b] / radius_[b], 0), Jet(q[6 * b + 1] / radius_[b], 1),
Jet(q[6 * b + 2] / radius_[b], 2)};
double dq[6] = {0, 0, 0, 0, 0, 0};
for (int i : bodies_[b]) {
const gemmi::Position d = base_[i] - centre_[b];
const Jet p[3] = {Jet(d.x), Jet(d.y), Jet(d.z)};
Jet rp[3];
ceres::AngleAxisRotatePoint(aa, p, rp);
for (int c = 0; c < 3; c++) {
for (int k = 0; k < 3; k++)
dq[k] += g[i][c] * rp[c].v[k] / radius_[b];
dq[3 + c] += g[i][c];
}
}
std::copy(dq, dq + 6, gradient + 6 * b);
}
return std::isfinite(*cost);
}
void Place(const double *q) const {
for (size_t b = 0; b < bodies_.size(); b++) {
const double aa[3] = {q[6 * b] / radius_[b], q[6 * b + 1] / radius_[b], q[6 * b + 2] / radius_[b]};
for (int i : bodies_[b]) {
const gemmi::Position d = base_[i] - centre_[b];
const double p[3] = {d.x, d.y, d.z};
double rp[3];
ceres::AngleAxisRotatePoint(aa, p, rp);
ctx_.atoms[i]->pos = gemmi::Position(rp[0] + centre_[b].x + q[6 * b + 3], rp[1] + centre_[b].y + q[6 * b + 4],
rp[2] + centre_[b].z + q[6 * b + 5]);
}
}
}
private:
Context &ctx_;
const std::vector<std::vector<int>> &bodies_;
std::vector<gemmi::Position> base_;
std::vector<gemmi::Position> centre_;
std::vector<double> radius_;
};
// One B shift per group of atoms (a residue), against the X-ray target.
class GroupBTarget : public ceres::FirstOrderFunction {
public:
GroupBTarget(Context &ctx, const std::vector<std::vector<int>> &groups)
: ctx_(ctx), groups_(groups), base_(ctx.B()) {}
int NumParameters() const override { return static_cast<int>(groups_.size()); }
bool Evaluate(const double *q, double *cost, double *gradient) const override {
const RefinementStage stage("eval.group_b");
Place(q);
ctx_.evaluations++;
std::vector<std::array<double, 4>> g;
*cost = ctx_.engine->Evaluate(ctx_.Model(), gradient != nullptr ? &g : nullptr);
if (gradient != nullptr)
for (size_t k = 0; k < groups_.size(); k++) {
gradient[k] = 0;
for (int i : groups_[k]) {
const double b = base_[i] + q[k];
if (b > ctx_.BLow(i) && b < ctx_.BHigh(i))
gradient[k] += g[i][3];
}
}
return std::isfinite(*cost);
}
void Place(const double *q) const {
for (size_t k = 0; k < groups_.size(); k++)
for (int i : groups_[k])
ctx_.atoms[i]->b_iso = static_cast<float>(std::clamp(base_[i] + q[k], ctx_.BLow(i), ctx_.BHigh(i)));
}
private:
Context &ctx_;
const std::vector<std::vector<int>> &groups_;
std::vector<double> base_;
};
// The placement stage: rigid bodies, then group B.
void RefinePlacement(Context &ctx, bool group_b, RefinementResult &result) {
const RefinementStage stage("placement");
std::vector<std::vector<int>> bodies, groups;
size_t i = 0;
for (const gemmi::Chain &ch : ctx.Model().chains) {
std::vector<int> body;
for (const gemmi::Residue &r : ch.residues) {
std::vector<int> group;
for (size_t k = 0; k < r.atoms.size(); k++, i++) {
group.push_back(static_cast<int>(i));
// an atom on a special position stays on it
if (!r.is_water() && ctx.st.cell.is_special_position(r.atoms[k].pos, SPECIAL_POSITION_TOLERANCE) == 0)
body.push_back(static_cast<int>(i));
}
groups.push_back(group);
}
// A chain of a few atoms (an ion, a ligand of its own chain) stays where it is.
if (body.size() >= 50)
bodies.push_back(body);
}
// Each step is kept only where it lowers the -ln L of the working reflections, the target it minimised (the free
// set decides the weights and the displacement model alone).
ctx.Refit();
double before = ctx.Score().nll_work;
if (!bodies.empty()) {
const std::vector<gemmi::Position> pos0 = ctx.Positions();
std::vector<double> q(6 * bodies.size(), 0.0);
auto target = std::make_unique<RigidBodiesTarget>(ctx, bodies);
RigidBodiesTarget *rb = target.get();
ceres::GradientProblem problem(std::move(target));
ceres::GradientProblemSolver::Summary summary;
const CancelCheck cancel_check(ctx);
ceres::Solve(cancel_check.Options(30), problem, q.data(), &summary);
rb->Place(q.data());
ctx.Refit();
const double after = ctx.Score().nll_work;
if (after < before) {
result.rigid_bodies = static_cast<int>(bodies.size());
before = after;
} else {
ctx.SetPositions(pos0);
ctx.Refit();
before = ctx.Score().nll_work;
}
}
if (group_b) {
const std::vector<double> b0 = ctx.B();
std::vector<double> q(groups.size(), 0.0);
auto target = std::make_unique<GroupBTarget>(ctx, groups);
GroupBTarget *gb = target.get();
ceres::GradientProblem problem(std::move(target));
ceres::GradientProblemSolver::Summary summary;
const CancelCheck cancel_check(ctx);
ceres::Solve(cancel_check.Options(30), problem, q.data(), &summary);
gb->Place(q.data());
ctx.Refit();
result.group_b = ctx.Score().nll_work < before;
if (!result.group_b)
ctx.SetB(b0);
}
}
// The X-ray weight that balances the two gradients: the norm of the geometry's gradient (taken on the model
// shaken by 0.1 A, where a refined model's geometry would have none) over the norm of the X-ray target's.
// Following Brunger, Karplus & Petsko (1989) Acta Cryst. A45, 50-61, as phenix.refine starts its weight search
// (Afonine et al. (2012) Acta Cryst. D68, 352-367)
double BalancedWeight(Context &ctx, const std::vector<int> &refinable) {
std::vector<std::array<double, 4>> gx, gr;
ctx.engine->Evaluate(ctx.Model(), &gx);
std::vector<std::array<double, 4>> xyzb = ctx.XyzB();
std::mt19937 rng(1);
for (std::array<double, 4> &a : xyzb)
for (int c = 0; c < 3; c++)
a[c] += PortableNormal(rng, 0.0, 0.1);
ctx.engine->EvaluateRestraints(0, xyzb, &gr);
double nx = 0, ng = 0;
for (int i : refinable)
for (int c = 0; c < 3; c++) {
nx += gx[i][c] * gx[i][c];
ng += gr[i][c] * gr[i][c];
}
return nx > 0 ? std::sqrt(ng / nx) : 1.0;
}
// The B factors' X-ray weight that balances the two gradients, as BalancedWeight() does for the positions: the B
// similarity's gradient on the model with its B shaken by 20% (a flat B, as a model may come, gives none) over the
// X-ray target's.
double BalancedADPWeight(Context &ctx, const RestraintSet &b_restraints, const std::vector<int> &refinable) {
std::vector<std::array<double, 4>> gx;
ctx.engine->Evaluate(ctx.Model(), &gx);
ctx.FoldRiders(gx);
std::vector<std::array<double, 4>> xyzb = ctx.XyzB();
std::mt19937 rng(2);
for (std::array<double, 4> &a : xyzb)
a[3] *= std::exp(PortableNormal(rng, 0.0, 0.2));
std::vector<std::array<double, 4>> gr(xyzb.size(), {0, 0, 0, 0});
EvaluateRestraints(b_restraints, xyzb, &gr);
double nx = 0, ng = 0;
for (int i : refinable) {
nx += gx[i][3] * gx[i][3];
ng += gr[i][3] * gr[i][3];
}
return nx > 0 ? std::sqrt(ng / nx) : 1.0;
}
// A phase in degrees as an MTZ 'P' column holds it: in [0, 360), also after the rounding to float.
float MtzPhase(double deg) {
const auto f = static_cast<float>(deg - 360.0 * std::floor(deg / 360.0));
return f < 360.0f ? f : 0.0f;
}
// A reflection of the data's resolution range that the data do not have: F_model on the scale of the map
// coefficients (the overall anisotropy divided out), and the D of the observed reflection nearest in resolution.
struct MissingReflection {
gemmi::Miller hkl;
std::complex<double> f_model;
double d = 0;
};
// The reflections 2mFo - DFc is filled at, with D F_model - the reflection's expected value given the model - as
// phenix.refine (fill_missing_f_obs) and BUSTER fill their maps. F_model is computed on the CPU, its bulk solvent
// and overall scale those of the last Refit(), `b_aniso` the anisotropic part of the scale.
std::vector<MissingReflection> MissingReflections(Context &ctx, const gemmi::SMat33<double> &b_aniso) {
if (ctx.refl.Size() == 0)
return {};
const auto [s_lo, s_hi] = std::minmax_element(ctx.refl.inv_d2.begin(), ctx.refl.inv_d2.end());
const std::set<gemmi::Miller> observed(ctx.refl.hkl.begin(), ctx.refl.hkl.end());
const gemmi::ReciprocalAsu asu(ctx.sg);
const gemmi::GroupOps gops = ctx.sg->operations();
RefinementReflections missing;
const double q_max = std::sqrt(*s_hi);
const int h_max = static_cast<int>(ctx.cell.a * q_max) + 1, k_max = static_cast<int>(ctx.cell.b * q_max) + 1,
l_max = static_cast<int>(ctx.cell.c * q_max) + 1;
for (int h = -h_max; h <= h_max; h++)
for (int k = -k_max; k <= k_max; k++)
for (int l = -l_max; l <= l_max; l++) {
const gemmi::Miller hkl{{h, k, l}};
const double s = ctx.cell.calculate_1_d2(hkl);
if (s < *s_lo || s > *s_hi || !asu.is_in(hkl) || gops.is_systematically_absent(hkl)
|| observed.count(hkl) > 0)
continue;
missing.hkl.push_back(hkl);
missing.inv_d2.push_back(s);
}
if (missing.hkl.empty())
return {};
std::unique_ptr<RefinementEngine> engine = MakeRefinementEngineCPU(ctx.cell, *ctx.sg, ctx.d_min, missing, ctx.nthreads);
engine->SetBlur(RefinementBlur(ctx.cell, *ctx.sg, ctx.d_min, ctx.b_floor));
engine->SetAniso(&ctx.aniso);
// the atoms and the solvent side by side: neither reads the other
std::vector<std::complex<float>> fcalc, fmask;
std::future<void> atoms = std::async(std::launch::async, [&] {
const RefinementStage s("missing.fcalc");
engine->Fcalc(ctx.Model(), fcalc);
});
{
const RefinementStage s("missing.fmask");
ctx.mask->Fmask(ctx.Model(), missing.hkl, fmask);
}
atoms.get();
std::vector<std::pair<double, double>> d_by_s; // (1/d^2, D) of the observed reflections, by resolution
for (size_t m = 0; m < ctx.refl.Size(); m++)
d_by_s.emplace_back(ctx.refl.inv_d2[m], ctx.rows.d[m]);
std::sort(d_by_s.begin(), d_by_s.end());
std::vector<MissingReflection> out;
for (size_t i = 0; i < missing.hkl.size(); i++) {
const double s = missing.inv_d2[i];
auto it = std::lower_bound(d_by_s.begin(), d_by_s.end(), std::make_pair(s, -1.0));
if (it == d_by_s.end() || (it != d_by_s.begin() && s - std::prev(it)->first < it->first - s))
--it;
const double k = ctx.k_overall * std::exp(-0.25 * (ctx.b_star.r_u_r(missing.hkl[i]) - b_aniso.r_u_r(missing.hkl[i])));
const std::complex<double> f_model = k * (std::complex<double>(fcalc[i])
+ ctx.k_sol * std::exp(-ctx.b_sol * s / 4.0) * std::complex<double>(fmask[i]));
out.push_back({missing.hkl[i], f_model, it->second});
}
return out;
}
// The model's map coefficients and its difference map, weighted by the refinement's own likelihood: m its figure
// of merit and D its D (2mFo - DFc, mFo - DFc). With a twin law, Fo is detwinned by the model in proportion to its
// own share of the twinned intensity, Fo |F_model(h)| / |F_twin(h)|, and Fc is the model's own |F_model(h)|.
// Following Read (1986) Acta Cryst. A42, 140-149
// The coefficients written to the file have the overall anisotropy of the fit (the anisotropic part of k_total)
// divided out, as phenix.refine's are: left in, it stretches every atom along the data's weak direction. The maps
// the refinement reads itself (result.map, result.difference_map) keep the observed scale. FWT/PHWT is filled at
// the reflections the data do not have (MissingReflections), FWT_no_fill/PHWT_no_fill is the same map without
// them, and the difference map is never filled.
void MakeMaps(Context &ctx, const std::vector<MergedReflection> &merged, double wavelength_A, RefinementResult &result) {
const RefinementStage stage("maps");
std::optional<RefinementStage> sub("maps.refit");
ctx.Refit();
const std::vector<std::complex<float>> fmodel = ctx.FModel();
const gemmi::GroupOps gops = ctx.sg->operations();
sub.emplace("maps.rows");
std::vector<double> fom(ctx.refl.Size());
double fom_sum = 0;
int fom_n = 0;
for (size_t m = 0; m < ctx.refl.Size(); m++) {
const double x = 2.0 * ctx.refl.fo[m] * ctx.rows.d[m] * std::abs(fmodel[m]) / ctx.rows.var[m];
fom[m] = Fom(x, ctx.refl.centric[m]);
if (!ctx.refl.free[m]) {
fom_sum += fom[m];
fom_n++;
}
}
result.mean_fom = fom_n > 0 ? fom_sum / fom_n : 0;
auto maps = std::make_shared<ModelMaps>();
MtzTable &mtz = maps->coefficients;
mtz.space_group_name = ctx.sg->xhm();
mtz.unit_cell = UnitCell{static_cast<float>(ctx.cell.a), static_cast<float>(ctx.cell.b), static_cast<float>(ctx.cell.c),
static_cast<float>(ctx.cell.alpha), static_cast<float>(ctx.cell.beta),
static_cast<float>(ctx.cell.gamma)};
mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
{"refinement", "refinement", "refinement", static_cast<float>(wavelength_A)}};
const std::pair<const char *, char> columns[] = {{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"FP", 'F'}, {"FC", 'F'},
{"PHIC", 'P'}, {"FWT", 'F'}, {"PHWT", 'P'}, {"FWT_no_fill", 'F'},
{"PHWT_no_fill", 'P'}, {"DELFWT", 'F'}, {"PHDELWT", 'P'},
{"FOM", 'W'}, {"FREE", 'I'}};
for (const auto &[label, type] : columns)
mtz.columns.push_back({label, type, type == 'H' ? 0 : 1, {}});
gemmi::AsuData<std::complex<float>> diff, fwt_data;
diff.unit_cell_ = fwt_data.unit_cell_ = ctx.cell;
diff.spacegroup_ = fwt_data.spacegroup_ = ctx.sg;
const gemmi::SMat33<double> b_aniso = AnisotropicPart(ctx.b_star, ctx.cell);
for (size_t m = 0; m < ctx.refl.Size(); m++) {
double fo = ctx.refl.fo[m], fc = std::abs(fmodel[m]);
if (ctx.refl.Twinned()) {
const double own = std::abs(ctx.rows.k_total[m] * (ctx.fcalc[m] + ctx.rows.fmask[m]));
if (fc > 0)
fo *= own / fc;
fc = own;
}
const double mw = fom[m], d = ctx.rows.d[m];
const double fwt = ctx.refl.centric[m] ? mw * fo : 2 * mw * fo - d * fc;
const double delfwt = mw * fo - d * fc;
const double phi = std::arg(fmodel[m]);
const double phi_deg = phi * 180.0 / PI;
const double k_aniso = std::exp(-0.25 * b_aniso.r_u_r(ctx.refl.hkl[m]));
const double fwt_file = fwt / k_aniso, delfwt_file = delfwt / k_aniso;
// A negative coefficient is written as its magnitude on the phase turned by 180 degrees: an MTZ amplitude
// column is non-negative by convention, and not every reader accepts otherwise.
const float row[] = {static_cast<float>(ctx.refl.hkl[m][0]), static_cast<float>(ctx.refl.hkl[m][1]),
static_cast<float>(ctx.refl.hkl[m][2]), static_cast<float>(fo), static_cast<float>(fc),
MtzPhase(phi_deg), static_cast<float>(std::abs(fwt_file)),
MtzPhase(fwt_file < 0 ? phi_deg + 180.0 : phi_deg), static_cast<float>(std::abs(fwt_file)),
MtzPhase(fwt_file < 0 ? phi_deg + 180.0 : phi_deg), static_cast<float>(std::abs(delfwt_file)),
MtzPhase(delfwt_file < 0 ? phi_deg + 180.0 : phi_deg),
static_cast<float>(mw), ctx.refl.free[m] ? 0.0f : 1.0f};
for (size_t c = 0; c < mtz.columns.size(); c++)
mtz.columns[c].values.push_back(row[c]);
diff.v.push_back({ctx.refl.hkl[m], std::polar(static_cast<float>(delfwt), static_cast<float>(phi))});
fwt_data.v.push_back({ctx.refl.hkl[m], std::polar(static_cast<float>(fwt), static_cast<float>(phi))});
}
result.maps = maps;
sub.emplace("maps.anom");
// The reflections the data do not have, the difference map and the 2mFo-DFc map beside the anomalous one below:
// none reads another.
std::future<std::vector<MissingReflection>> missing = std::async(std::launch::async, [&ctx, &b_aniso] {
const RefinementStage s("maps.missing");
return MissingReflections(ctx, b_aniso);
});
const std::array<int, 3> size = gemmi::get_size_for_hkl(diff, {{0, 0, 0}}, 3.0);
std::future<void> difference_map = std::async(std::launch::async, [&diff, &size, &result] {
const RefinementStage s("maps.diff_map");
result.difference_map = std::make_shared<gemmi::Grid<float>>(MapFromFPhi(gemmi::get_f_phi_on_grid<float>(diff, size, true)));
result.difference_map_rms = gemmi::calculate_data_statistics(result.difference_map->data).rms;
});
std::future<void> map = std::async(std::launch::async, [&fwt_data, &size, &result] {
const RefinementStage s("maps.map");
result.map = std::make_shared<gemmi::Grid<float>>(MapFromFPhi(gemmi::get_f_phi_on_grid<float>(fwt_data, size, true)));
result.map_rms = gemmi::calculate_data_statistics(result.map->data).rms;
});
// The anomalous difference map: F(+) - F(-) on the model's phase turned back by 90 degrees, read on the +
// rows of an anomalous merge only (as model validation reads it).
// Following ANODE, Thorn & Sheldrick (2011) J. Appl. Cryst. 44, 1285-1287
const gemmi::ReciprocalAsu asu(ctx.sg);
std::map<gemmi::Miller, float> danom;
for (const MergedReflection &r : merged) {
if (!std::isfinite(r.F_plus) || !std::isfinite(r.F_minus))
continue;
const gemmi::Miller h{{r.h, r.k, r.l}};
if (gops.is_reflection_centric(h))
continue;
const auto [hasu, plus] = asu.to_asu_sign(h, gops);
if (plus)
danom[hasu] = r.F_plus - r.F_minus;
}
if (!danom.empty()) {
gemmi::AsuData<std::complex<float>> anom;
anom.unit_cell_ = ctx.cell;
anom.spacegroup_ = ctx.sg;
// Its coefficients in the maps file too (ANOM/PHANOM, as model validation writes them), missing (NaN) where
// the reflection has no Bijvoet pair.
const float nan = std::numeric_limits<float>::quiet_NaN();
MtzTable::Column anom_col{"ANOM", 'F', 1, {}}, phanom_col{"PHANOM", 'P', 1, {}};
for (size_t m = 0; m < ctx.refl.Size(); m++) {
const auto it = danom.find(ctx.refl.hkl[m]);
if (it == danom.end()) {
anom_col.values.push_back(nan);
phanom_col.values.push_back(nan);
continue;
}
const std::complex<float> c = it->second * std::polar(1.0f, static_cast<float>(std::arg(fmodel[m]) - PI / 2));
anom.v.push_back({ctx.refl.hkl[m], c});
anom_col.values.push_back(std::abs(c));
phanom_col.values.push_back(MtzPhase(std::arg(c) * 180.0 / PI));
}
if (!anom.v.empty()) {
mtz.columns.push_back(std::move(anom_col));
mtz.columns.push_back(std::move(phanom_col));
result.anomalous_map = std::make_shared<gemmi::Grid<float>>(
MapFromFPhi(gemmi::get_f_phi_on_grid<float>(anom, gemmi::get_size_for_hkl(anom, {{0, 0, 0}}, 3.0), true)));
result.anomalous_map_rms = gemmi::calculate_data_statistics(result.anomalous_map->data).rms;
}
}
// The filled reflections after the observed ones: FWT/PHWT and the model's FC/PHIC, every other column missing.
sub.emplace("maps.fill");
const float nan = std::numeric_limits<float>::quiet_NaN();
for (const MissingReflection &r : missing.get()) {
const double phi_deg = std::arg(r.f_model) * 180.0 / PI;
const double k_aniso = std::exp(-0.25 * b_aniso.r_u_r(r.hkl));
const float row[] = {static_cast<float>(r.hkl[0]), static_cast<float>(r.hkl[1]), static_cast<float>(r.hkl[2]), nan,
static_cast<float>(std::abs(r.f_model) * k_aniso), MtzPhase(phi_deg),
static_cast<float>(r.d * std::abs(r.f_model)), MtzPhase(phi_deg)};
for (size_t c = 0; c < mtz.columns.size(); c++)
mtz.columns[c].values.push_back(c < std::size(row) ? row[c] : nan);
}
sub.emplace("maps.join");
difference_map.get();
map.get();
}
// The fractions of the twin laws, together.
double FractionSum(const XrayRowModel &rows) {
double sum = 0;
for (float a : rows.twin_fractions)
sum += a;
return sum;
}
// law=fraction per twin law, ';'-separated.
std::string DomainText(const RefinementReflections &refl, const XrayRowModel &rows) {
std::string out;
for (size_t d = 0; d < refl.twin_laws.size(); d++)
out += fmt::format("{}{}={:.3f}", out.empty() ? "" : ";", refl.twin_laws[d].as_hkl().triplet('h'),
d < rows.twin_fractions.size() ? rows.twin_fractions[d] : 0.0f);
return out;
}
// A domain added to the adopted law is kept only at this fraction or more, on top of its null.
// As REFMAC5 drops twin domains of small fraction, Murshudov et al. (2011) Acta Cryst. D67, 355-367
constexpr double TWIN_DOMAIN_MIN = 0.05;
// Each observed reflection's twin group, named by its lowest observed row (itself where there is no twin law).
std::vector<int> TwinGroups(const RefinementReflections &refl) {
const size_t n = refl.Size();
std::vector<int> group(n);
for (size_t m = 0; m < n; m++) {
group[m] = static_cast<int>(m);
for (size_t d = 0; d < refl.Laws(); d++)
if (refl.mates[d][m] < static_cast<int>(n))
group[m] = std::min(group[m], refl.mates[d][m]);
}
return group;
}
// R-free of the model as the last Fcalc left it, as one term per free twin group, in the order of the groups (the
// members of a group share their F_calc, so their terms are summed into one, as TwinNull sums them): the terms
// FreeSetPrefers pairs.
std::vector<float> FreeRTerms(const Context &ctx) {
const std::vector<std::complex<float>> fmodel = TwinnedModel(ctx.refl, ctx.rows, ctx.fcalc);
const std::vector<int> group = TwinGroups(ctx.refl);
double fo = 0;
std::vector<double> sum(ctx.refl.Size(), 0.0);
for (size_t m = 0; m < ctx.refl.Size(); m++)
if (ctx.refl.free[m]) {
fo += ctx.refl.fo[m];
sum[group[m]] += std::fabs(ctx.refl.fo[m] - std::abs(fmodel[m]));
}
std::vector<float> terms;
for (size_t m = 0; m < ctx.refl.Size(); m++)
if (ctx.refl.free[m] && group[m] == static_cast<int>(m))
terms.push_back(static_cast<float>(sum[m] / fo));
return terms;
}
// The null of a twinned refinement, for the model as the last Refit() left it: its R-free at the same fraction with
// every reflection's mate replaced by an unrelated observed one of the same resolution and the same set (free with
// free, work with work, as a law's twin pairs are). The reflections of each set are put in order of 1/d^2 with the two
// members of a twin pair next to each other, and paired TWIN_NULL_SHIFT apart - past the reflection's own pair, and no
// further, as the free set is sparse and a wider step takes the mate to another resolution, which a null pays for.
// What the averaging of two intensities buys on its own, whatever they are.
constexpr size_t TWIN_NULL_SHIFT = 2;
// The twinned refinement is taken only where the law beats its null on the free set by this many standard errors -
// or by TWIN_NULL_Z_DATA where the data's own intensity statistics read twinned, a second, independent piece of
// evidence (AdoptTwinLaw).
constexpr double TWIN_NULL_Z = 3.0;
constexpr double TWIN_NULL_Z_DATA = 2.0;
// A twinned refinement stops after its first ladder where the law is ahead of its null by less than this many
// standard errors there (RefineOnce, 1a.): 1.5 under the lowest bar a law is taken at (TWIN_NULL_Z_DATA). From the
// ladder's end to the refinement's, the law's lead rose by 1.35 at most (on 19 refinements with a law, 3 twinned: those
// 8.6-17.7 at the ladder's end, the others under 2.4).
constexpr double TWIN_STOP_Z = 0.5;
// `first`: the first law whose mates are replaced - the laws before it keep theirs (the null of the domains added to
// them). With several laws a twin group holds up to Laws() + 1 reflections, kept next to each other, and law first + k
// is paired (Laws() + 1) (k + 1) apart: past the reflection's own group, and no two laws drawing the same partner.
void TwinNull(const Context &ctx, RefinementResult &result, size_t first = 0) {
RefinementReflections null = ctx.refl;
const size_t n = null.Size();
const std::vector<int> group = TwinGroups(ctx.refl);
for (size_t d = first; d < null.Laws(); d++)
std::iota(null.mates[d].begin(), null.mates[d].end(), 0);
for (const bool free : {false, true}) {
std::vector<int> order;
for (size_t m = 0; m < n; m++)
if ((null.free[m] != 0) == free)
order.push_back(static_cast<int>(m));
// a twin group is ordered at the 1/d^2 of its first member
std::stable_sort(order.begin(), order.end(), [&](int a, int b) {
const int pa = group[a], pb = group[b];
return std::make_pair(null.inv_d2[pa], pa) < std::make_pair(null.inv_d2[pb], pb);
});
for (size_t d = first; d < null.Laws(); d++) {
const size_t shift = (null.Laws() + 1) * (d - first + 1);
for (size_t r = 0; r < order.size(); r++) {
// in blocks of twice the shift, the first half with the second
const size_t p = r % (2 * shift) < shift ? r + shift : r - shift;
null.mates[d][order[r]] = order[p < order.size() ? p : r];
}
}
}
// at the law's fraction: averaging any two intensities lifts the weak reflections' model amplitudes off zero
// by itself (the fraction the unrelated mates ask for, reported beside it, is about 0)
XrayRowModel rows = ctx.rows;
{
const std::vector<float> a = FitTwinFractions(null, rows, ctx.fcalc);
result.twin_fraction_null = 0;
for (size_t d = first; d < a.size(); d++)
result.twin_fraction_null += a[d];
}
// TwinnedModel with each unrelated mate on its own scale k_total: TwinnedModel puts a mate on the scale of the
// reflection it is the mate of, as the twin domain's reflection at the same spot is measured, but an unrelated
// mate lies in another direction, where the anisotropic scale can differ many times over (22 times in amplitude
// at 2 A, on a model whose U carried the anisotropy opposite to k_total), and on the reflection's scale it put
// the null 0.06 in R-free above the untwinned refinement - a law "ahead" of its null by 7.9 standard errors at a
// fraction of 0.04.
std::vector<std::complex<float>> f_null(n);
double own_fraction = 1.0;
for (float a : rows.twin_fractions)
own_fraction -= a;
for (size_t m = 0; m < n; m++) {
const std::complex<double> u = std::complex<double>(ctx.fcalc[m]) + std::complex<double>(rows.fmask[m]);
double i_model = own_fraction * rows.k_total[m] * rows.k_total[m] * std::norm(u);
for (size_t d = 0; d < null.Laws(); d++) {
const int j = null.mates[d][m];
const double k = d < first ? rows.k_total[m] : rows.k_total[j];
i_model += rows.twin_fractions[d] * k * k *
std::norm(std::complex<double>(ctx.fcalc[j]) + std::complex<double>(rows.fmask[j]));
}
const double amplitude = std::sqrt(std::max(i_model, 0.0));
f_null[m] = std::norm(u) > 0 ? std::complex<float>(u * (amplitude / std::abs(u)))
: std::complex<float>(static_cast<float>(amplitude), 0);
}
result.twin_r_free_null = ComputeRFactors(null, f_null).r_free;
// How far the laws beat their null on the free reflections, paired: the mean of |Fo - |F_null|| - |Fo - |F_twin||
// over its standard error. The members of a free twin group share their F_calc, so their terms are summed into
// one: the standard error counts twin groups, not reflections.
const std::vector<std::complex<float>> f_law = TwinnedModel(ctx.refl, ctx.rows, ctx.fcalc);
std::vector<double> sum_group(n, 0.0);
for (size_t m = 0; m < n; m++)
if (ctx.refl.free[m])
sum_group[group[m]] += std::fabs(ctx.refl.fo[m] - std::abs(f_null[m]))
- std::fabs(ctx.refl.fo[m] - std::abs(f_law[m]));
double sum = 0, sum2 = 0;
int n_groups = 0;
for (size_t m = 0; m < n; m++)
if (ctx.refl.free[m] && group[m] == static_cast<int>(m)) {
sum += sum_group[m];
sum2 += sum_group[m] * sum_group[m];
n_groups++;
}
if (n_groups > 1) {
const double mean = sum / n_groups, var = (sum2 - n_groups * mean * mean) / (n_groups - 1);
result.twin_null_z = var > 0 ? mean / std::sqrt(var / n_groups) : 0.0;
}
}
// The engine for the model as it is now: on the GPU where it is wanted and takes the case within its budget,
// on the CPU otherwise. `restraint_records`, `restraint_slots`: what its restraint sets hold at most (0: the GPU
// engine's estimate per atom).
void MakeEngine(Context &ctx, bool gpu, const RefinementSettings &settings, Logger &logger, size_t restraint_records = 0,
size_t restraint_slots = 0, size_t normal_bytes = 0) {
const RefinementStage stage("engine");
ctx.engine.reset();
#ifdef JFJOCH_USE_CUDA
ctx.scale_gpu.reset();
if (gpu && get_gpu_count() > 0) {
try {
const int device = get_gpu();
const size_t card = get_gpu_total_memory(device);
const size_t usable = card / 5 * 4;
const size_t budget = usable > settings.gpu_resident_bytes ? usable - settings.gpu_resident_bytes : 0;
RefinementGPUBytes bytes;
ctx.engine = MakeRefinementEngineGPU(device, ctx.cell, *ctx.sg, ctx.d_min, ctx.refl, ctx.Model(),
ctx.b_ceiling, budget, &bytes, settings.nthreads, restraint_records,
restraint_slots, normal_bytes);
ctx.device = device;
ctx.engine_bytes = bytes.engine;
ctx.card_bytes = bytes.card;
ctx.lane_bytes = bytes.lane;
if (ctx.engine) {
ctx.scale_gpu = std::make_shared<ModelScaleGPUPool>(device, ctx.refl.Size(), 1);
logger.Debug("Refinement: GPU engine {:.2f} GB, {:.2f} GB shared with the slots on its card and {:.2f} GB "
"per lane, of a {:.2f} GB budget (scale pool {:.2f} GB)", bytes.engine / 1e9,
bytes.card / 1e9, bytes.lane / 1e9, budget / 1e9,
ModelScaleGPUPool::DeviceBytes(ctx.refl.Size(), 1, true) / 1e9);
} else
logger.Info("Refinement: on the CPU - the GPU cannot take this case or {:.1f} GB is over its budget",
(bytes.engine + bytes.card + bytes.lane) / 1e9);
} catch (const JFJochException &e) {
logger.Warning("Refinement: the GPU failed ({}), on the CPU", e.what());
ctx.engine.reset();
ctx.scale_gpu.reset();
}
}
#else
(void) gpu;
(void) logger;
#endif
if (!ctx.engine)
ctx.engine = MakeRefinementEngineCPU(ctx.cell, *ctx.sg, ctx.d_min, ctx.refl, settings.nthreads);
ctx.engine->SetAniso(&ctx.aniso);
}
// The riding hydrogens placed on their parents as they are, and the offsets and B they ride with.
void PlaceRiders(Context &ctx, RefinementTopology &topo) {
const RefinementStage stage("riders");
PlaceRidingHydrogens(topo);
ctx.rider_offset.assign(ctx.atoms.size(), gemmi::Position());
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (ctx.rider_parent[i] >= 0) {
ctx.rider_offset[i] = ctx.atoms[i]->pos - ctx.atoms[ctx.rider_parent[i]]->pos;
ctx.atoms[i]->b_iso = ctx.atoms[ctx.rider_parent[i]]->b_iso;
}
}
// The geometry and the repulsions of the model as it is, and where `ncs_sigma` is not 0 the NCS restraints at it.
RestraintSet RestraintsNow(Context &ctx, const RefinementTopology &topo, const NcsCopies *ncs = nullptr,
double ncs_sigma = 0) {
const RefinementStage stage("repulsions");
RestraintSet set = topo.geometry;
set.Append(topo.metal_bonds);
set.Append(BuildRepulsions(topo, ctx.Model(), ctx.cell, *ctx.sg, REPULSION_MARGIN, ctx.nthreads));
if (ncs != nullptr && ncs_sigma > 0)
set.Append(NcsRestraintsAt(*ncs, ncs_sigma));
set.Finalize(ctx.atoms.size());
return set;
}
// The B similarity of the model as it is: bonded and angle-related pairs, and atoms in contact.
RestraintSet BRestraintsNow(Context &ctx, const RefinementTopology &topo) {
const RefinementStage stage("b_restraints");
RestraintSet set = topo.adp;
set.Append(BuildBContacts(topo, ctx.Model(), ctx.cell, *ctx.sg, ctx.nthreads));
set.Finalize(ctx.atoms.size());
return set;
}
// Where a weight trial runs: a model with its topology and its engine.
struct TrialSlot {
Context *ctx;
RefinementTopology *topo;
};
// A slot of its own beside the main one.
struct OwnSlot {
Context ctx;
RefinementTopology topo;
};
// The scale pools of the slots' cards: one per card, shared by the slots there (a fit waits for the one before it).
struct ScalePools {
#ifdef JFJOCH_USE_CUDA
std::mutex m;
std::map<int, std::shared_ptr<ModelScaleGPUPool>> pool;
std::shared_ptr<ModelScaleGPUPool> Get(int device, size_t points) {
const std::lock_guard lock(m);
std::shared_ptr<ModelScaleGPUPool> &p = pool[device];
if (!p)
p = std::make_shared<ModelScaleGPUPool>(device, points, 1);
return p;
}
#endif
};
// A copy of the model as it was before the topology was made (the same topology comes out of it, hydrogens
// included), with an engine of the main one's kind - a GPU's on `device`, like the main one's and sharing with the
// engines on that card what they share (RefinementGPUBytes), and the card's scale pool. Null where the card cannot
// make one.
std::unique_ptr<OwnSlot> MakeOwnSlot(const gemmi::Structure &before_topology, const Context &main, int device,
bool own_lane, size_t nthreads, const RefinementSettings &settings,
ScalePools &pools) {
auto slot = std::make_unique<OwnSlot>();
Context &ctx = slot->ctx;
ctx.st = before_topology;
ctx.cell = main.cell;
ctx.sg = main.sg;
ctx.d_min = main.d_min;
ctx.refl = main.refl;
ctx.nthreads = nthreads;
ctx.ml_threads = settings.nthreads;
ctx.cancel = main.cancel;
ctx.b_ceiling = main.b_ceiling;
ctx.mask = std::make_unique<BulkSolventMask>(ctx.cell, *ctx.sg, ctx.d_min, nthreads);
slot->topo = BuildRefinementTopology(ctx.st, settings.riding_hydrogens, settings.dictionary);
ctx.atoms = AtomList(ctx.Model());
ctx.SetRiderParents(slot->topo.rider_parent);
ctx.TakeAniso(); // each trial sets the anisotropy it starts from
#ifdef JFJOCH_USE_CUDA
if (main.engine->OnGPU()) {
try {
ctx.engine = MakeRefinementEngineGPU(*main.engine, device, nthreads, own_lane);
ctx.scale_gpu = pools.Get(device, ctx.refl.Size());
} catch (const std::exception &) {
return nullptr;
}
ctx.device = device;
}
#else
(void) device;
(void) own_lane;
(void) pools;
#endif
if (!ctx.engine)
ctx.engine = MakeRefinementEngineCPU(ctx.cell, *ctx.sg, ctx.d_min, ctx.refl, nthreads);
ctx.engine->SetAniso(&ctx.aniso);
ctx.engine->SetRestraints(1, slot->topo.adp);
return slot;
}
// Where a slot beside the main one goes: its GPU (-1: the CPU), and whether its engine has a lane of working buffers
// of its own there.
struct SlotPlace {
int device = -1;
bool own_lane = false;
};
// The places of the slots beside the main one, `wanted` of them at most: the GPUs in turn from the main engine's,
// each taking engines while they fit in its budget - so the first slots are on cards of their own, where RunTrials
// puts the long trials. A slot takes its engine's own part and a lane of its own where both fit, else its engine
// alone on a lane it shares (one evaluation at a time: lanes side by side keep the card busier, more slots overlap
// more host work); the first one on another card also the part the engines there share and a scale pool. On the
// CPU, the main one and those beside it are no more than the threads. `busiest`: what the main engine and the slots
// hold of the card that holds the most of them (0 on the CPU).
std::vector<SlotPlace> OwnSlotPlaces(const Context &main, int wanted, const RefinementSettings &settings,
size_t &busiest) {
busiest = 0;
#ifdef JFJOCH_USE_CUDA
if (main.engine->OnGPU()) {
const int n = get_gpu_count();
// the robust fit's buffers included
const size_t pool = ModelScaleGPUPool::DeviceBytes(main.refl.Size(), 1, true);
std::vector<size_t> used(n, 0);
used[main.device] = main.engine_bytes + main.lane_bytes + main.card_bytes + pool;
std::vector<SlotPlace> places;
for (int k = 1; k <= wanted; k++) {
const int d = (main.device + k) % n;
const size_t usable = get_gpu_total_memory(d) / 5 * 4;
const size_t budget = usable > settings.gpu_resident_bytes ? usable - settings.gpu_resident_bytes : 0;
const size_t first = used[d] == 0 ? main.card_bytes + pool : 0;
if (used[d] + first + main.engine_bytes + main.lane_bytes <= budget) {
used[d] += first + main.engine_bytes + main.lane_bytes;
places.push_back({d, true});
} else if (first == 0 && used[d] + main.engine_bytes <= budget) {
used[d] += main.engine_bytes;
places.push_back({d, false});
}
}
busiest = *std::max_element(used.begin(), used.end());
return places;
}
#endif
// on the CPU, no more trials at once than threads: each slot is a model, a topology and an engine of its own
const int beside = std::min(wanted, static_cast<int>(settings.nthreads) - 1);
return std::vector<SlotPlace>(std::max(beside, 0));
}
// The weights of one trial: the positions' X-ray weight, the B factors' (0: held) and the NCS restraints' sigma (0:
// none).
struct TrialWeights {
double w = 0, adp = 0, ncs = 0;
// A member of the population (RefineOnce, 2a.): the positions refined on the data blurred by B_h per macro-cycle
// (A^2; empty or 0: not blurred) - the likelihood's variance raised by exp(B_h s^2 / 2), which is Fo and F_model
// both blurred by exp(-B_h s^2 / 4): the landscape of a lower resolution, which a position 1-2 A off still
// descends - with the X-ray weight balanced again for it; and before the first macro-cycle every refinable atom
// displaced by up to `kick` (A) in each coordinate, at random from `seed`.
std::vector<double> blur;
double kick = 0;
unsigned seed = 0;
};
// A displacement model a trial refines (RefinementSettings::adp_hypotheses): the B factors as they are (isotropic, or
// the shapes of the anisotropic atoms held), TLS bodies with a residual B per atom, or a U per atom.
struct AdpHypothesis {
enum class Kind { Isotropic, Tls, Anisotropic };
Kind kind = Kind::Isotropic;
std::string name;
std::vector<TlsBody> bodies; // Tls, with the part of each atom's B they hold at the start (Context::tls_b)
std::vector<double> tls_b;
std::vector<std::pair<int, int>> pairs; // Anisotropic: the pairs of the U restraints
int segments = 0; // Tls: the TLSMD cut (segments per chain at most), 0 the model's own
};
// One weight trial: `cycles` macro-cycles from pos0 and b0, of the positions at X-ray weight w and, where adp_w is
// not 0, the B factors at X-ray weight adp_w. `fit`: what was fitted at pos0 and b0 already, null to fit it here.
struct TrialOutcome {
RefinementWeightTrial trial;
std::vector<gemmi::Position> pos;
std::vector<double> b;
std::vector<AtomAniso> aniso;
std::vector<TlsBody> tls;
std::vector<double> tls_b;
Context::Fit fit; // the scale, the solvent and the likelihood's parameters fitted where it ended
std::string line; // what the log says of it
bool ran = false; // false: left out, the refinement cancelled before it
int evaluations = 0;
};
// The work-set target's terms of the model as it stands (RefinementWeightTrial), at the last Refit()'s fit: what every
// decision but the weights' and the displacement model's is made on (WorkTarget).
void MeasureWorkTarget(Context &ctx, const RefinementTopology &topo, const NcsCopies *ncs, double ncs_sigma,
RefinementWeightTrial &t) {
const RefinementScore sc = ctx.Score();
t.nll_work = sc.nll_work;
t.nll_free = sc.nll_free;
t.free_r_terms = FreeRTerms(ctx);
const std::vector<std::array<double, 4>> xyzb = ctx.XyzB();
t.e_geometry = EvaluateRestraints(RestraintsNow(ctx, topo, ncs, ncs_sigma), xyzb, nullptr);
t.e_b = EvaluateRestraints(BRestraintsNow(ctx, topo), xyzb, nullptr);
}
// The work-set target of a trial at the X-ray weights w (positions) and adp (B, 0: held), its NCS restraints at its own
// sigma: what a trial at those weights minimises, the X-ray term of the positions and the geometry, and that of the B
// factors and the B similarity where they are refined.
double WorkTarget(const RefinementWeightTrial &t, double w, double adp) {
return w * t.nll_work + t.e_geometry + (adp > 0 ? adp * t.nll_work + t.e_b : 0.0);
}
// The X-ray target's gradient on the positions, its squared norm over `refinable`, for the atoms as they are.
double XrayGradientNorm(Context &ctx, const std::vector<int> &refinable) {
std::vector<std::array<double, 4>> g;
ctx.engine->Evaluate(ctx.Model(), &g);
double n = 0;
for (int i : refinable)
for (int c = 0; c < 3; c++)
n += g[i][c] * g[i][c];
return n;
}
// The X-ray curvature per atom in each of x, y, z at weight w, for the row model `rows` the engine has; `scale`
// calibrated by `calibrate` (from the shape) at the first call of the trial (negative: not yet).
std::vector<double> XyzCurvature(Context &ctx, const XrayRowModel &rows, double w, double &scale,
const std::function<double(const std::vector<double> &)> &calibrate) {
std::vector<double> shape = ctx.XrayXyzCurvatureShape(rows);
if (scale < 0)
scale = calibrate(shape);
for (double &x : shape)
x *= w * scale;
return shape;
}
// The calibrations of the X-ray curvature in x, y, z (CalibrateXyzCurvature) of the trials RunTrials() runs together:
// trials that start from the same model with the same B and displacement model calibrate the same, so the first to
// need one makes it and the others wait for it.
class XyzCalibrations {
public:
// whether the B factors are refined (MoveOverallB), the displacement hypothesis, the kick, its seed and the first
// macro-cycle's blur
using Key = std::tuple<bool, const void *, double, unsigned, double>;
double Get(const Key &key, const std::function<double()> &calibrate) {
std::unique_lock<std::mutex> lock(m_);
if (auto it = made_.find(key); it != made_.end()) {
const std::shared_future<double> f = it->second;
lock.unlock();
return f.get();
}
std::promise<double> p;
made_.emplace(key, p.get_future().share());
lock.unlock();
try {
const double scale = calibrate();
p.set_value(scale);
return scale;
} catch (...) {
p.set_exception(std::current_exception()); // to those waiting for it
throw;
}
}
private:
std::mutex m_;
std::map<Key, std::shared_future<double>> made_;
};
TrialOutcome RunTrial(TrialSlot slot, XyzCalibrations &calibrations, const std::vector<int> &refinable, const std::vector<gemmi::Position> &pos0,
const std::vector<double> &b0, const std::vector<AtomAniso> &aniso0, const Context::Fit *fit,
const TrialWeights &tw, int cycles, const RefinementSettings &settings, const NcsCopies *ncs,
const AdpHypothesis *hyp = nullptr, bool hold_positions = false) {
Context &ctx = *slot.ctx;
RefinementTopology &topo = *slot.topo;
const double w = tw.w, adp_w = tw.adp, ncs_sigma = tw.ncs;
const RefinementStage stage(hyp != nullptr ? "rung.adp" : adp_w > 0 ? "rung.b" : "rung.xyz");
const int evaluations_before = ctx.evaluations;
ctx.SetPositions(pos0);
if (tw.kick > 0) {
std::mt19937 rng(tw.seed);
for (int i : refinable)
if (!ctx.atoms[i]->is_hydrogen()) {
// One draw per statement: the order of a call's arguments is unspecified.
const double dx = PortableUniform(rng, -tw.kick, tw.kick);
const double dy = PortableUniform(rng, -tw.kick, tw.kick);
const double dz = PortableUniform(rng, -tw.kick, tw.kick);
ctx.atoms[i]->pos += gemmi::Position(dx, dy, dz);
}
}
ctx.SetB(b0);
ctx.SetAniso(aniso0);
ctx.tls.clear();
ctx.tls_b.clear();
const AdpHypothesis::Kind kind = hyp != nullptr ? hyp->kind : AdpHypothesis::Kind::Isotropic;
// The displacement model of the hypothesis: its TLS bodies applied to their atoms, or a U of its own for every
// refinable atom (the isotropic ones from their B) with the U restraints.
std::vector<int> u_atoms;
URestraints u_restraints;
if (kind == AdpHypothesis::Kind::Tls) {
PlaceRiders(ctx, topo);
ctx.tls = hyp->bodies;
ctx.tls_b = hyp->tls_b;
// the bodies' U bounded by the start's floor, not by the one the slot's last trial left
ctx.b_floor = fit != nullptr ? fit->b_floor : B_FLOOR;
ctx.ApplyTls();
} else if (kind == AdpHypothesis::Kind::Anisotropic) {
if (ctx.aniso.size() != ctx.atoms.size())
ctx.aniso.assign(ctx.atoms.size(), AtomAniso());
std::vector<char> refines_u(ctx.atoms.size(), 0);
for (int i : refinable)
if (!ctx.atoms[i]->is_hydrogen()) {
ctx.aniso[i].on = true;
refines_u[i] = 1;
u_atoms.push_back(i);
}
ctx.SetAniso(ctx.aniso);
u_restraints = MakeURestraints(hyp->pairs, refines_u);
}
// the positions and the B factors in turn, each with a memory of its own kept from macro-cycle to macro-cycle
const AtomTarget xyz(ctx, AtomTarget::Mode::Xyz, refinable, w, 0, topo),
b(ctx, AtomTarget::Mode::B, refinable, adp_w, 1, topo);
const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr);
const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints);
RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads);
// the X-ray curvature's calibrations (CalibrateXyzCurvature, CalibrateBCurvature), once per trial - that in x, y,
// z shared with the trials that start alike
double xyz_scale = -1, b_scale = -1;
auto calibrate_xyz = [&](const std::vector<double> &shape) {
const XyzCalibrations::Key key{adp_w > 0, hyp, tw.kick, tw.seed, tw.blur.empty() ? 0.0 : tw.blur[0]};
const double scale = calibrations.Get(key, [&]() { return ctx.CalibrateXyzCurvature(refinable, shape); });
ctx.MoveRiders(); // the riders where the calibration leaves them, wherever it ran
return scale;
};
for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) {
PlaceRiders(ctx, topo);
if (cycle == 0 && fit != nullptr)
ctx.Restore(*fit);
else {
// a trial starts from nothing the slot's previous trial left
if (cycle == 0)
ctx.ForgetStarts();
ctx.Refit();
}
if (adp_w > 0)
ctx.MoveOverallB();
if (!hold_positions) {
const RestraintSet set = RestraintsNow(ctx, topo, ncs, ncs_sigma);
{
const RefinementStage s1("set_restraints");
ctx.engine->SetRestraints(0, set);
}
const RefinementStage s2("minimise.xyz");
const double b_h = cycle < static_cast<int>(tw.blur.size()) ? tw.blur[cycle] : 0.0;
if (b_h > 0) {
// the continuation of Piela, Kostrowicki & Scheraga (1989) J. Phys. Chem. 93, 3339-3346, on the
// resolution, as Jack & Levitt (1978) Acta Cryst. A34, 931-935 refined low resolution first: the
// data blurred (TrialWeights::blur), the weight keeping the gradients' balance
const double n0 = XrayGradientNorm(ctx, refinable);
XrayRowModel blurred = ctx.rows;
for (size_t m = 0; m < ctx.refl.Size(); m++)
blurred.var[m] *= static_cast<float>(std::exp(b_h * ctx.refl.inv_d2[m] / 2.0));
ctx.engine->SetRowModel(blurred);
const double nh = XrayGradientNorm(ctx, refinable);
const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo);
MinimiseAtoms(ctx, xyz_h, xyz_minimiser, XyzCurvature(ctx, blurred, xyz_h.Weight(), xyz_scale, calibrate_xyz),
set, settings.xyz_iterations);
ctx.engine->SetRowModel(ctx.rows);
} else
MinimiseAtoms(ctx, xyz, xyz_minimiser, XyzCurvature(ctx, ctx.rows, w, xyz_scale, calibrate_xyz), set,
settings.xyz_iterations);
}
// the X-ray curvature in B for the atoms as they are, which TLS, U and B take a share of
auto b_curvature = [&]() {
std::vector<double> c = ctx.XrayBCurvatureShape();
if (b_scale < 0)
b_scale = ctx.CalibrateBCurvature(refinable, c);
for (double &x : c)
x *= b_scale;
return c;
};
if (kind == AdpHypothesis::Kind::Tls) {
ctx.ApplyTls(); // the bodies' U at the atoms as they are
const RefinementStage s4("minimise.tls");
MinimiseTls(ctx, tls_target, tls_minimiser, b_curvature(), settings.b_iterations);
}
if (adp_w > 0 && kind == AdpHypothesis::Kind::Anisotropic) {
const std::vector<double> xray_b = b_curvature();
// each U's six elements take a third of the curvature of its B; and the restraints' curvature in 8 pi^2 U:
// per pair 2 / sigma^2 of the similarity and about a third of the rigid bond's, per isotropic atom
// 2 / sigma^2 - the preconditioner's diagonal
std::vector<double> restraint(xray_b.size(), 0.0);
const double u2 = gemmi::u_to_b() * gemmi::u_to_b();
const double per_pair = (2.0 / (URestraints::SIGMA_SIMILAR * URestraints::SIGMA_SIMILAR) +
2.0 / (3.0 * URestraints::SIGMA_RIGID * URestraints::SIGMA_RIGID)) / u2;
for (const auto &[i, j] : u_restraints.pairs) {
restraint[i] += per_pair;
restraint[j] += per_pair;
}
for (int i : u_restraints.isotropic)
restraint[i] += 2.0 / (URestraints::SIGMA_ISOTROPIC * URestraints::SIGMA_ISOTROPIC) / u2;
std::vector<double> h, diagonal;
for (int i : u_atoms) {
h.insert(h.end(), 6, adp_w * xray_b[i] / 3.0);
diagonal.insert(diagonal.end(), 6, adp_w * xray_b[i] / 3.0 + restraint[i]);
}
const RefinementStage s3("minimise.u");
MinimiseU(ctx, u_target, b_minimiser, h, FlooredDiagonal(diagonal), settings.b_iterations);
} else if (adp_w > 0) {
const RestraintSet b_set = BRestraintsNow(ctx, topo);
ctx.engine->SetRestraints(1, b_set);
std::vector<double> xray_b = b_curvature();
for (double &x : xray_b)
x *= adp_w;
const RefinementStage s3("minimise.b");
MinimiseAtoms(ctx, b, b_minimiser, xray_b, b_set, settings.b_iterations);
}
}
PlaceRiders(ctx, topo);
TrialOutcome out;
RefinementWeightTrial &t = out.trial;
t.weight = w;
t.adp_weight = adp_w;
t.ncs_sigma = ncs_sigma;
const RFactors rf = ctx.Refit();
t.r_work = rf.r_work;
t.r_free = rf.r_free;
{
const RefinementStage s1("measure");
t.geometry = MeasureGeometry(topo.geometry,
BuildRepulsions(topo, ctx.Model(), ctx.cell, *ctx.sg, 0.0, ctx.nthreads), ctx.XyzB(),
&topo.metal_bonds);
}
t.within_bounds = t.geometry.rmsd_bonds <= MAX_RMSD_BONDS && t.geometry.rmsd_angles <= MAX_RMSD_ANGLES;
if (ncs != nullptr && !ncs->Empty())
t.ncs_rmsd = NcsRmsd(*ncs, ctx.XyzB());
if (hyp != nullptr)
t.adp = hyp->name;
// the displacement parameters: a B per refinable atom, the TLS bodies' 20 each beside it, or a U per atom
int atoms_b = 0;
for (int i : refinable)
atoms_b += !ctx.atoms[i]->is_hydrogen();
t.adp_parameters = kind == AdpHypothesis::Kind::Anisotropic ? 6 * static_cast<int>(u_atoms.size())
: kind == AdpHypothesis::Kind::Tls ? atoms_b + 20 * static_cast<int>(hyp->bodies.size())
: atoms_b;
// of them the restrained ones: the B or the U of every atom (not the TLS bodies')
t.adp_restrained = kind == AdpHypothesis::Kind::Anisotropic ? 6 * static_cast<int>(u_atoms.size()) : atoms_b;
MeasureWorkTarget(ctx, topo, ncs, ncs_sigma, t);
out.line = fmt::format("{}X-ray weight {:.3g}, {} {}{}: R-work {:.4f}, R-free {:.4f}{}, target {:.6g}, rmsd bonds "
"{:.4f} A, angles {:.2f} deg{}{}",
std::string(tw.blur.empty() ? "" : "blurred, ") + (hyp != nullptr ? hyp->name + ", " : ""), w,
kind == AdpHypothesis::Kind::Anisotropic ? "U" : kind == AdpHypothesis::Kind::Tls ? "residual B" : "B",
adp_w > 0 ? fmt::format("refined at weight {:.3g}", adp_w) : std::string("held"),
ncs == nullptr || ncs->Empty() ? std::string()
: ncs_sigma > 0 ? fmt::format(", NCS at {:.3g} A (copies {:.3f} A apart)", ncs_sigma, t.ncs_rmsd)
: fmt::format(", no NCS (copies {:.3f} A apart)", t.ncs_rmsd),
rf.r_work, rf.r_free, fmt::format(", -LL free {:.1f}", t.nll_free), WorkTarget(t, w, adp_w),
t.geometry.rmsd_bonds, t.geometry.rmsd_angles,
t.geometry.metal_bonds > 0 ? fmt::format(", metal bonds {:.3f} A", t.geometry.rmsd_metal)
: std::string(),
t.within_bounds ? "" : " (geometry out of bounds)");
out.pos = ctx.Positions();
out.b = ctx.B();
out.aniso = ctx.aniso;
out.tls = ctx.tls;
out.tls_b = ctx.tls_b;
out.fit = ctx.Fitted();
out.ran = true;
out.evaluations = ctx.evaluations - evaluations_before;
return out;
}
// The trials (w, adp_w) side by side, each slot on a thread of its own (on its GPU), returned in the order given: the
// longest first, one to each slot, then each slot takes the next one as it is done. Every trial starts from pos0 and
// b0 and takes nothing from the one before it in its slot, so the outcome is the same for any number of slots and
// whichever slot runs it.
std::vector<TrialOutcome> RunTrials(const std::vector<TrialSlot> &trial_slots,
const std::vector<TrialWeights> &trials,
const std::vector<int> &refinable, const std::vector<gemmi::Position> &pos0,
const std::vector<double> &b0, const std::vector<AtomAniso> &aniso0,
const Context::Fit *fit, int cycles, const RefinementSettings &settings,
const NcsCopies *ncs, const std::vector<const AdpHypothesis *> &hyps = {},
bool hold_positions = false) {
std::vector<TrialOutcome> out(trials.size());
std::vector<std::exception_ptr> errors(trial_slots.size());
// The threads shared among the slots that run (the outcome does not depend on how many each has). On the CPU
// every thread is dealt out, and a slot that has no trial left gives its threads to those still running.
const size_t running = std::min(trial_slots.size(), trials.size());
const bool on_cpu = !trial_slots[0].ctx->engine->OnGPU();
const size_t nthreads =
settings.nthreads + (settings.spare_arm_threads != nullptr ? settings.spare_arm_threads->load() : 0);
std::atomic<int> spare_threads{0};
for (size_t s = 0; s < running; s++) {
const size_t share = nthreads / running + (on_cpu && s < nthreads % running ? 1 : 0);
trial_slots[s].ctx->SetThreads(std::max<size_t>(1, share));
if (on_cpu)
trial_slots[s].ctx->spare_threads = &spare_threads;
}
// The longest go to the slots first - TLS bodies, then a U per atom, then the B factors refined, then held - so
// that two of them share a card only where the cards are fewer than they are: the slots are on the cards in turn
// (OwnSlotPlaces), a card's second slot only after every card's first.
auto length = [&](size_t i) {
const AdpHypothesis::Kind kind = hyps.empty() ? AdpHypothesis::Kind::Isotropic : hyps[i]->kind;
return kind == AdpHypothesis::Kind::Tls ? 3 : kind == AdpHypothesis::Kind::Anisotropic ? 2 : trials[i].adp > 0;
};
std::vector<size_t> order(trials.size());
std::iota(order.begin(), order.end(), 0);
std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) { return length(a) > length(b); });
std::atomic<size_t> next{trial_slots.size()};
XyzCalibrations calibrations;
auto run_slot = [&](size_t s) {
try {
#ifdef JFJOCH_USE_CUDA
const ScopedGpuPin pin(trial_slots[s].ctx->device >= 0 ? trial_slots[s].ctx->device : get_gpu());
#endif
for (size_t j = s; j < trials.size(); j = next++)
if (const size_t i = order[j]; !trial_slots[s].ctx->Cancelled())
out[i] = RunTrial(trial_slots[s], calibrations, refinable, pos0, b0, aniso0, fit, trials[i], cycles,
settings, ncs, hyps.empty() ? nullptr : hyps[i], hold_positions);
} catch (...) {
errors[s] = std::current_exception();
}
Context &ctx = *trial_slots[s].ctx;
if (ctx.spare_threads != nullptr) {
ctx.spare_threads = nullptr;
spare_threads += static_cast<int>(ctx.nthreads);
}
};
std::vector<std::thread> threads;
for (size_t s = 1; s < std::min(trial_slots.size(), trials.size()); s++)
threads.emplace_back(run_slot, s);
run_slot(0);
for (std::thread &t : threads)
t.join();
for (const std::exception_ptr &e : errors)
if (e)
std::rethrow_exception(e);
return out;
}
// The 2mFo-DFc map of a population (RefineOnce, 2a.): every member's coefficients, at the scale, the solvent and the
// likelihood parameters fitted where it ended, averaged over the members - the averaged kick map of Praznikar et al.
// (2009) Acta Cryst. D65, 921-931 - with the free reflections left out, so that what is chosen on the map is chosen
// without them. Leaves ctx at the last member.
gemmi::Grid<float> PopulationMap(Context &ctx, const std::vector<TrialOutcome> &members) {
gemmi::AsuData<std::complex<float>> fwt;
fwt.unit_cell_ = ctx.cell;
fwt.spacegroup_ = ctx.sg;
for (size_t m = 0; m < ctx.refl.Size(); m++)
fwt.v.push_back({ctx.refl.hkl[m], {0.0f, 0.0f}});
int n = 0;
for (const TrialOutcome &o : members) {
if (!o.ran)
continue;
ctx.SetPositions(o.pos);
ctx.SetB(o.b);
ctx.SetAniso(o.aniso);
ctx.Restore(o.fit);
const std::vector<std::complex<float>> fmodel = ctx.FModel();
for (size_t m = 0; m < ctx.refl.Size(); m++) {
if (ctx.refl.free[m])
continue;
double fo = ctx.refl.fo[m], fc = std::abs(fmodel[m]);
if (ctx.refl.Twinned()) {
const double own = std::abs(ctx.rows.k_total[m] * (ctx.fcalc[m] + ctx.rows.fmask[m]));
if (fc > 0)
fo *= own / fc;
fc = own;
}
const double d = ctx.rows.d[m];
const double mw = Fom(2.0 * fo * d * fc / ctx.rows.var[m], ctx.refl.centric[m]);
const double coef = ctx.refl.centric[m] ? mw * fo : 2 * mw * fo - d * fc;
fwt.v[m].value += std::polar(static_cast<float>(coef), static_cast<float>(std::arg(fmodel[m])));
}
n++;
}
for (gemmi::HklValue<std::complex<float>> &v : fwt.v)
v.value /= static_cast<float>(n);
const std::array<int, 3> size = gemmi::get_size_for_hkl(fwt, {{0, 0, 0}}, 3.0);
return MapFromFPhi(gemmi::get_f_phi_on_grid<float>(fwt, size, true));
}
// Per atom the residue it belongs to, numbered through the model in the atoms' order (AtomList).
std::vector<int> ResidueOfAtom(gemmi::Model &model) {
std::vector<int> out;
int r = 0;
for (gemmi::Chain &ch : model.chains)
for (gemmi::Residue &res : ch.residues)
out.insert(out.end(), res.atoms.size(), r++);
return out;
}
} // namespace
namespace {
// The solvent build inside the refinement (SolventBuild.h): the atom list of a refinement is fixed - its engines are
// made for it - so the waters a round may add are there from the start, as placeholders: one water each in a chain
// of their own at the end of the model, at occupancy 0, which puts no density, no mask and no contact anywhere and is
// not refined, until a round puts one at a peak. A water taken out goes to occupancy 0 the same way, and a site
// relabelled keeps its place in the list.
const char *const PLACEHOLDER_CHAIN = "~";
// As many placeholders as a fifteenth of the model's heavy atoms, at least 64: the depositions' builds add up to a
// tenth (2R8S, 1HR2), most of it in the first round.
int PlaceholderCount(const gemmi::Model &model) {
int heavy = 0;
for (const gemmi::Chain &ch : model.chains)
for (const gemmi::Residue &r : ch.residues)
for (const gemmi::Atom &a : r.atoms)
heavy += !a.is_hydrogen();
return std::max(64, heavy / 15);
}
void AddPlaceholders(gemmi::Structure &st, int n) {
gemmi::Chain ch(PLACEHOLDER_CHAIN);
for (int k = 0; k < n; k++) {
gemmi::Residue r;
r.name = "HOH";
r.seqid = gemmi::SeqId(k + 1, ' ');
r.het_flag = 'H';
r.entity_type = gemmi::EntityType::Water;
gemmi::Atom a;
a.name = "O";
a.element = gemmi::El::O;
a.occ = 0;
a.b_iso = 20;
r.atoms.push_back(a);
ch.residues.push_back(r);
}
st.models[0].chains.push_back(ch);
}
// The placeholders of a model (the last `n` atoms) and where those put in go: the water chain (PickSolvent's) and the
// number they take there.
struct SolventPool {
int first = 0, n = 0;
std::vector<std::string> chain; // empty: not put in
std::vector<gemmi::SeqId> seqid;
std::map<std::string, int> last; // per chain name, the last residue number taken
int Free() const {
for (int k = 0; k < n; k++)
if (chain[k].empty())
return k;
return -1;
}
};
// The model as the solvent build and the output see it: the placeholders put in moved into their chains (the last of
// the name) under their numbers, those not put in and the waters taken out (occupancy 0) left out. Each atom's serial
// is its index in `st`.
gemmi::Structure SolventView(const gemmi::Structure &st, const SolventPool &pool) {
gemmi::Structure view = st;
gemmi::Model &model = view.models[0];
int index = 0;
for (gemmi::Chain &ch : model.chains)
for (gemmi::Residue &r : ch.residues)
for (gemmi::Atom &a : r.atoms)
a.serial = index++;
std::vector<gemmi::Residue> moved;
std::vector<std::string> into;
for (gemmi::Chain &ch : model.chains)
if (ch.name == PLACEHOLDER_CHAIN)
for (size_t k = 0; k < ch.residues.size(); k++)
if (!pool.chain[k].empty() && ch.residues[k].atoms[0].occ > 0) {
moved.push_back(ch.residues[k]);
moved.back().seqid = pool.seqid[k];
into.push_back(pool.chain[k]);
}
std::erase_if(model.chains, [](const gemmi::Chain &ch) { return ch.name == PLACEHOLDER_CHAIN; });
for (size_t k = 0; k < moved.size(); k++) {
gemmi::Chain *target = nullptr;
for (gemmi::Chain &ch : model.chains)
if (ch.name == into[k])
target = &ch;
if (target == nullptr) {
model.chains.emplace_back(into[k]);
target = &model.chains.back();
}
// a water the subchain and entity of the waters already there, where there are any; else none, for the
// writer to give the chain's residues subchains of their own (setup_entities), not the placeholders'
moved[k].subchain.clear();
moved[k].entity_id.clear();
for (const gemmi::Residue &w : target->residues)
if (w.is_water() && moved[k].is_water()) {
moved[k].subchain = w.subchain;
moved[k].entity_id = w.entity_id;
}
target->residues.push_back(moved[k]);
}
// In a chain that took any, the waters after its other residues (a site relabelled from a water put in), so that
// the writer's subchains are each one run of residues.
for (gemmi::Chain &ch : model.chains)
if (std::find(into.begin(), into.end(), ch.name) != into.end())
std::stable_partition(ch.residues.begin(), ch.residues.end(),
[](const gemmi::Residue &r) { return !r.is_water(); });
for (gemmi::Chain &ch : model.chains) {
for (gemmi::Residue &r : ch.residues)
if (r.is_water())
std::erase_if(r.atoms, [](const gemmi::Atom &a) { return !(a.occ > 0); });
std::erase_if(ch.residues, [](const gemmi::Residue &r) { return r.atoms.empty(); });
}
return view;
}
// TLS bodies with their atoms (the refinable ones but hydrogens), about their centroid, T = t I with t half of what
// the lowest B among them holds, L = S = 0: so they leave every B as it is (tls_b), the residual taking the rest.
AdpHypothesis TlsHypothesis(const Context &ctx, std::string name, std::vector<std::vector<int>> groups) {
AdpHypothesis h;
h.kind = AdpHypothesis::Kind::Tls;
h.name = std::move(name);
h.tls_b.assign(ctx.atoms.size(), 0.0);
for (std::vector<int> &atoms : groups) {
if (atoms.empty())
continue;
TlsBody body;
body.id = std::to_string(h.bodies.size() + 1);
double b_min = ctx.b_ceiling;
for (int i : atoms) {
body.origin += ctx.atoms[i]->pos;
b_min = std::min<double>(b_min, ctx.atoms[i]->b_iso);
}
body.origin /= static_cast<double>(atoms.size());
const double t = 0.5 * b_min / gemmi::u_to_b();
body.T = {t, t, t, 0, 0, 0};
for (int i : atoms)
h.tls_b[i] = gemmi::u_to_b() * t;
body.atoms = std::move(atoms);
h.bodies.push_back(std::move(body));
}
return h;
}
// The displacement models the refinement chooses among once its weights are chosen, from the model as it stands: the
// TLS bodies of the model's own groups (`input`, where it has them); one body per chain of at least MIN_TLS_ATOMS
// refinable atoms, and the chains cut into 4, 8 and 16 TLSMD segments of at least MIN_TLS_RESIDUES residues
// (TlsSegments) - each cut a hypothesis where it differs from the one before; and a U per atom. (Cut into 2, the
// chains changed the choice on 6 of 364 refinements of the battery, none by more than 0.001 in R-free.)
std::vector<AdpHypothesis> AdpHypotheses(const Context &ctx, const RefinementTopology &topo,
const std::vector<TlsBody> &input) {
constexpr size_t MIN_TLS_ATOMS = 50;
constexpr int MIN_TLS_RESIDUES = 10;
std::vector<AdpHypothesis> out;
if (!input.empty()) {
AdpHypothesis h;
h.kind = AdpHypothesis::Kind::Tls;
h.name = fmt::format("TLS, the model's {} groups", input.size());
h.bodies = input;
h.tls_b.assign(ctx.atoms.size(), 0.0);
for (const TlsBody &body : h.bodies)
for (int i : body.atoms)
h.tls_b[i] = gemmi::u_to_b() * body.U(ctx.atoms[i]->pos).trace() / 3.0;
out.push_back(std::move(h));
}
// per chain its residues' refinable atoms but hydrogens
std::vector<std::vector<std::vector<int>>> chains;
int i = 0;
for (const gemmi::Chain &ch : ctx.st.models[0].chains) {
std::vector<std::vector<int>> residues;
size_t atoms = 0;
for (const gemmi::Residue &r : ch.residues) {
std::vector<int> ra;
for (size_t k = 0; k < r.atoms.size(); k++, i++)
if (!r.is_water() && topo.refinable[i] && !r.atoms[k].is_hydrogen())
ra.push_back(i);
if (!ra.empty()) {
atoms += ra.size();
residues.push_back(std::move(ra));
}
}
if (atoms >= MIN_TLS_ATOMS)
chains.push_back(std::move(residues));
}
if (!chains.empty()) {
const std::vector<gemmi::Position> pos = ctx.Positions();
const std::vector<double> b = ctx.B();
size_t last = 0;
for (int k : {1, 4, 8, 16}) {
std::vector<std::vector<int>> groups;
for (const std::vector<std::vector<int>> &residues : chains) {
std::vector<int> starts = TlsSegments(residues, pos, b, k, MIN_TLS_RESIDUES);
starts.push_back(static_cast<int>(residues.size()));
for (size_t s = 0; s + 1 < starts.size(); s++) {
std::vector<int> g;
for (int r = starts[s]; r < starts[s + 1]; r++)
g.insert(g.end(), residues[r].begin(), residues[r].end());
groups.push_back(std::move(g));
}
}
if (groups.size() == last)
continue;
last = groups.size();
std::string name = k == 1 ? fmt::format("TLS, {} groups (one per chain)", groups.size())
: fmt::format("TLS, {} groups (chains in up to {} segments)", groups.size(), k);
out.push_back(TlsHypothesis(ctx, std::move(name), std::move(groups)));
out.back().segments = k;
}
}
AdpHypothesis aniso;
aniso.kind = AdpHypothesis::Kind::Anisotropic;
aniso.name = "anisotropic";
for (const RestraintRecord &rec : topo.adp.records)
if (rec.kind == static_cast<int>(RestraintKind::BSimilarity))
aniso.pairs.emplace_back(rec.atom[0], rec.atom[1]);
out.push_back(std::move(aniso));
return out;
}
// The U X-ray weight that balances the gradients, as BalancedADPWeight() does for B: the U restraints' gradient with
// every U given an anisotropy of 20% of its B at random, over the X-ray target's, for the atoms that refine a U.
double BalancedUWeight(Context &ctx, const URestraints &restraints, const std::vector<int> &u_atoms) {
std::vector<std::array<double, 4>> gx;
ctx.engine->Evaluate(ctx.Model(), &gx);
const std::vector<std::array<double, 6>> gu = FoldedUGradient(ctx, gx);
std::vector<std::array<double, 6>> u = ctx.UOfAtoms();
std::mt19937 rng(3);
for (int i : u_atoms) {
const double t = (u[i][0] + u[i][1] + u[i][2]) / 3.0;
for (int c = 0; c < 6; c++)
u[i][c] += t * PortableNormal(rng, 0.0, 0.2);
}
std::vector<std::array<double, 6>> gr(u.size(), {0, 0, 0, 0, 0, 0});
EvaluateURestraints(restraints, ctx.Positions(), u, &gr);
double nx = 0, ng = 0;
for (int i : u_atoms)
for (int c = 0; c < 6; c++) {
nx += gu[i][c] * gu[i][c];
ng += gr[i][c] * gr[i][c];
}
return nx > 0 ? std::sqrt(ng / nx) : 1.0;
}
// The displacements of the model as given, read where it has TLS groups whose atoms bring no U of their own: their U
// from the groups, its B column taken as the total B (the U's equivalent isotropic B, as phenix.refine writes it) or as
// the residual to the groups (as REFMAC writes it), and the isotropic B as it stands - a displacement model, chosen on
// the free set (FreeSetPrefers), in that order. `tls`: the groups, their atoms those of ctx.atoms but the hydrogens
// (which ride on their parents' B). Returns the R-factors of the reading kept.
// Following Winn, Isupov & Murshudov (2001) Acta Cryst. D57, 122-133 (the B column total or residual to the TLS)
RFactors ReadInputDisplacements(Context &ctx, const std::vector<TlsBody> &tls, Logger &logger, RefinementResult &result) {
int own = 0;
for (const AtomAniso &a : ctx.aniso)
own += a.on;
result.adp_input = own > 0 ? "ANISOU" : "isotropic";
result.adp_input_aniso_atoms = own;
std::vector<std::pair<int, gemmi::SMat33<double>>> tls_u; // atom, U_TLS
for (const TlsBody &body : tls)
for (int i : body.atoms)
if (ctx.aniso.empty() || !ctx.aniso[i].on)
tls_u.emplace_back(i, body.U(ctx.atoms[i]->pos));
RFactors r = ctx.Refit();
if (tls_u.empty())
return r;
const std::vector<double> b0 = ctx.B();
const std::vector<AtomAniso> aniso0 = ctx.aniso;
struct Reading {
const char *name;
std::vector<double> b;
std::vector<AtomAniso> aniso;
};
std::vector<Reading> readings;
for (int residual = 0; residual < 2; residual++) {
Reading read{residual ? "TLS, B residual" : "TLS, B total", b0,
aniso0.empty() ? std::vector<AtomAniso>(ctx.atoms.size()) : aniso0};
for (const auto &[i, u] : tls_u) {
// the atom's B (with U_TLS's trace added for the residual reading) and U_TLS's anisotropy, inside the bounds
const double t_u = u.trace() / 3.0;
const double b_i = residual ? b0[i] + gemmi::u_to_b() * t_u : b0[i];
const gemmi::SMat33<double> b = BoundedB(u.added_kI(-t_u).scaled(gemmi::u_to_b()).added_kI(b_i),
ctx.b_floor, ctx.b_ceiling);
const double t = b.trace() / 3.0;
read.b[i] = t;
read.aniso[i].on = true;
read.aniso[i].dev = {static_cast<float>((b.u11 - t) / gemmi::u_to_b()), static_cast<float>((b.u22 - t) / gemmi::u_to_b()),
static_cast<float>((b.u33 - t) / gemmi::u_to_b()), static_cast<float>(b.u12 / gemmi::u_to_b()),
static_cast<float>(b.u13 / gemmi::u_to_b()), static_cast<float>(b.u23 / gemmi::u_to_b())};
}
readings.push_back(std::move(read));
}
readings.push_back({result.adp_input == "ANISOU" ? "ANISOU" : "isotropic", b0, aniso0});
int best = -1;
std::vector<RFactors> rf;
std::vector<double> nll_free;
std::vector<std::vector<float>> r_terms;
for (size_t k = 0; k < readings.size(); k++) {
ctx.SetB(readings[k].b);
ctx.SetAniso(readings[k].aniso);
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (!ctx.rider_parent.empty() && ctx.rider_parent[i] >= 0)
ctx.atoms[i]->b_iso = ctx.atoms[ctx.rider_parent[i]]->b_iso;
ctx.ForgetStarts();
rf.push_back(ctx.Refit());
nll_free.push_back(ctx.Score().nll_free);
r_terms.push_back(FreeRTerms(ctx));
logger.Info("Refinement: the model's {} TLS groups ({} atoms) read as {}: R-work {:.4f}, R-free {:.4f}, -LL free "
"{:.1f}", tls.size(), tls_u.size(), readings[k].name, rf[k].r_work, rf[k].r_free, nll_free[k]);
if (best < 0 || FreeSetPrefers(r_terms[k], nll_free[k], r_terms[best], nll_free[best]))
best = static_cast<int>(k);
}
ctx.SetB(readings[best].b);
ctx.SetAniso(readings[best].aniso);
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (!ctx.rider_parent.empty() && ctx.rider_parent[i] >= 0)
ctx.atoms[i]->b_iso = ctx.atoms[ctx.rider_parent[i]]->b_iso;
ctx.ForgetStarts();
r = ctx.Refit();
if (best < 2) {
result.adp_input = readings[best].name;
result.adp_input_aniso_atoms = own + static_cast<int>(tls_u.size());
}
return r;
}
// 1/d^2 of the finest amplitude of `merged`, 0 without any.
double MaxInvD2(const std::vector<MergedReflection> &merged) {
double max_inv_d2 = 0;
for (const MergedReflection &r : merged)
if (std::isfinite(r.F) && r.d > 0)
max_inv_d2 = std::max(max_inv_d2, 1.0 / (double(r.d) * r.d));
return max_inv_d2;
}
// The d_min a refinement composes to: just past the finest reflection, so that prepare_asu_data()'s strict limit
// keeps it.
double RefinementDMin(double max_inv_d2) {
return 0.999 / std::sqrt(max_inv_d2);
}
// The Wilson B of the data against the model's atoms: the slope of ln(<|F|^2> / sum_j f_j^2) in s^2 / 2 over shells
// from 4 A to the finest amplitude; 0 where the data do not reach 4 A. The B factors refine around it whatever the
// model's say, so the ceiling of B is set from it as well as from them (BCeiling): a model may come with B factors
// that say nothing of where they will refine - a flat B, a prediction's pLDDT.
// Following Wilson (1949) Acta Cryst. 2, 318-321
double WilsonB(const std::vector<MergedReflection> &merged, const std::vector<gemmi::Atom *> &atoms) {
constexpr int SHELLS = 10;
constexpr double S2_MIN = 1.0 / 16.0; // 4 A
const double s2_max = MaxInvD2(merged);
if (s2_max <= S2_MIN)
return 0;
std::map<std::pair<int, signed char>, int> composition;
for (const gemmi::Atom *a : atoms)
if (!a->is_hydrogen())
composition[{static_cast<int>(a->element.elem), a->charge}]++;
std::vector<double> f2_sum(SHELLS, 0.0), s2_sum(SHELLS, 0.0);
std::vector<int> n(SHELLS, 0);
for (const MergedReflection &r : merged) {
if (!(std::isfinite(r.F) && r.d > 0))
continue;
const double s2 = 1.0 / (double(r.d) * r.d);
if (s2 < S2_MIN)
continue;
const int k = std::min(SHELLS - 1, static_cast<int>((s2 - S2_MIN) / (s2_max - S2_MIN) * SHELLS));
f2_sum[k] += double(r.F) * r.F;
s2_sum[k] += s2;
n[k]++;
}
// the line y = ln k - B x through the shells (x = s^2 / 2), each weighted by its reflections
double sw = 0, sx = 0, sy = 0, sxx = 0, sxy = 0;
for (int k = 0; k < SHELLS; k++) {
if (n[k] < 20)
continue;
const double s2 = s2_sum[k] / n[k];
double f2 = 0;
for (const auto &[ion, count] : composition)
f2 += count * std::pow(IonFormFactor<double>(static_cast<gemmi::El>(ion.first), ion.second).calculate_sf(s2 / 4.0), 2);
const double x = s2 / 2.0, y = std::log(f2_sum[k] / n[k] / f2), w = n[k];
sw += w;
sx += w * x;
sy += w * y;
sxx += w * x * x;
sxy += w * x * y;
}
const double det = sw * sxx - sx * sx;
return det > 0 ? std::max(0.0, -(sw * sxy - sx * sy) / det) : 0.0;
}
// One refinement of a model, with the solvent build in it where settings.solvent asks for it.
// `twin_laws`: against the data as twinned by them (one domain each beside the model's own), empty for untwinned;
// `null_first`: the first of them the null (TwinNull) replaces the mates of.
// What a refinement holds of the card that holds the most of it (OwnSlotPlaces), said once (RefineModel).
struct CardHeld {
std::promise<size_t> bytes;
std::once_flag said;
void Say(size_t b) {
std::call_once(said, [&] { bytes.set_value(b); });
}
};
RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<MergedReflection> &merged,
const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double wavelength_A,
Logger &logger, const RefinementSettings &settings,
const std::vector<gemmi::Op> &twin_laws = {}, size_t null_first = 0,
CardHeld *held = nullptr) {
const auto start = std::chrono::steady_clock::now();
RefinementResult result;
Context ctx;
ctx.st = PrepareModel(placed, cell, sg);
ctx.cell = cell;
ctx.sg = &sg;
ctx.nthreads = settings.nthreads;
ctx.ml_threads = settings.nthreads;
ctx.cancel = settings.cancel;
ctx.atoms = AtomList(ctx.Model());
result.atoms = static_cast<int>(ctx.atoms.size());
double b_max = 0;
for (const gemmi::Atom *a : ctx.atoms)
b_max = std::max<double>(b_max, a->aniso.nonzero() ? AnisoRadiusB(a->aniso.scaled(gemmi::u_to_b())) : a->b_iso);
// The model's TLS groups, where it has them; read as the residual to them, its B may go up by their trace.
const std::vector<TlsBody> tls_bodies =
ctx.st.meta.get_tls_groups() != nullptr ? TlsBodiesFromModel(ctx.Model(), *ctx.st.meta.get_tls_groups())
: std::vector<TlsBody>();
for (const TlsBody &body : tls_bodies)
for (int i : body.atoms)
b_max = std::max(b_max, ctx.atoms[i]->b_iso + gemmi::u_to_b() * body.U(ctx.atoms[i]->pos).trace() / 3.0);
if (ctx.atoms.empty()) {
result.failure_reason = "the model has no atoms";
return result;
}
result.wilson_b = WilsonB(merged, ctx.atoms);
ctx.b_ceiling = BCeiling(std::max(b_max, B_CEILING_WILSON * result.wilson_b));
const double max_inv_d2 = MaxInvD2(merged);
if (max_inv_d2 <= 0) {
result.failure_reason = "no amplitudes";
return result;
}
ctx.d_min = RefinementDMin(max_inv_d2);
result.d_min = 1.0 / std::sqrt(max_inv_d2);
ctx.refl = PrepareRefinementReflections(merged, cell, sg, ctx.d_min);
if (ctx.refl.Size() < 100) {
result.failure_reason = "too few reflections";
return result;
}
if (!twin_laws.empty()) {
result.twin_split_pairs = SetTwinLaws(ctx.refl, cell, sg, ctx.d_min, twin_laws);
result.twinned = true;
for (const gemmi::Op &law : twin_laws)
result.twin_law += (result.twin_law.empty() ? "" : ";") + law.as_hkl().triplet('h');
}
ctx.mask = std::make_unique<BulkSolventMask>(cell, sg, ctx.d_min, settings.nthreads);
// The solvent build's placeholders, before the slots copy the model.
const bool build = settings.solvent && settings.restrained && !settings.score_only &&
result.d_min <= SOLVENT_BUILD_D_MIN;
SolventPool pool;
if (build) {
pool.n = PlaceholderCount(ctx.Model());
for (const gemmi::Chain &ch : ctx.Model().chains)
for (const gemmi::Residue &r : ch.residues)
if (r.seqid.num != gemmi::SeqId::OptionalNum::None)
pool.last[ch.name] = std::max(pool.last[ch.name], static_cast<int>(r.seqid.num));
AddPlaceholders(ctx.st, pool.n);
pool.chain.assign(pool.n, "");
pool.seqid.assign(pool.n, gemmi::SeqId());
}
// The dictionary's view of the model, and its hydrogens riding on their parents from the start, so that the
// model is scored as the restrained stage refines it.
const bool restrained = settings.restrained || settings.score_only;
// A side chain placed through another copy or a symmetry mate is turned clear first (ClashRelief.h): the
// repulsion would otherwise push it out along the line of the overlap, into a rotamer of its own choosing.
if (restrained && !settings.score_only) {
auto chis = [](const std::vector<double> &v) {
std::string s;
for (double x : v)
s += fmt::format("{}{:.0f}", s.empty() ? "" : "/", x);
return s;
};
for (const SideChainTurn &t : RelieveSideChainClashes(ctx.st)) {
result.side_chains_turned++;
logger.Info("Refinement: {} turned out of a {:.1f} A overlap with another residue, chi {} -> {}, {:.1f} A left",
t.name, t.overlap_before, chis(t.chi_before), chis(t.chi_after), t.overlap_after);
}
}
// The weight trials' slots start from a copy of the model as it is before the topology is made.
const gemmi::Structure before_topology = ctx.st;
RefinementTopology topo;
if (restrained) {
std::optional<RefinementStage> topo_stage("topology");
topo = BuildRefinementTopology(ctx.st, settings.riding_hydrogens, settings.dictionary);
topo_stage.reset();
ctx.atoms = AtomList(ctx.Model());
ctx.SetRiderParents(topo.rider_parent);
PlaceRiders(ctx, topo);
}
pool.first = static_cast<int>(ctx.atoms.size()) - pool.n; // the placeholders are the last atoms (no hydrogens)
// The copies of a molecule, restrained to each other in trials beside those that refine them independently.
NcsCopies ncs;
if (settings.restrained && settings.ncs)
ncs = FindNcsCopies(ctx.Model(), topo);
const NcsCopies *ncs_p = ncs.Empty() ? nullptr : &ncs;
ctx.TakeAniso();
// The engine's restraint sets as the model has them now - the geometry with its repulsions and the NCS restraints,
// and the B restraints - with a quarter more for the contacts a trial or the solvent build adds.
size_t restraint_records = 0, restraint_slots = 0;
if (restrained) {
for (const RestraintSet &set : {RestraintsNow(ctx, topo, ncs_p, 1.0), BRestraintsNow(ctx, topo)}) {
restraint_records += set.records.size() + set.plane_atoms.size();
restraint_slots += set.n_slots;
}
restraint_records += restraint_records / 4;
restraint_slots += restraint_slots / 4;
}
// The positions' normal matrix a trial holds on the engine's card (MinimiseAtoms): the geometry as above, the
// parameter atoms in their residues' blocks.
size_t normal_bytes = 0;
#ifdef JFJOCH_USE_CUDA
if (restrained) {
std::vector<int> block_atoms;
size_t i = 0;
for (const gemmi::Chain &ch : ctx.Model().chains)
for (const gemmi::Residue &r : ch.residues) {
int a = 0;
for (size_t k = 0; k < r.atoms.size(); k++, i++)
if (ctx.rider_parent.empty() || ctx.rider_parent[i] < 0)
a++;
block_atoms.push_back(a);
}
normal_bytes = RestraintNormalMatrixGPU::DeviceBytes(RestraintsNow(ctx, topo, ncs_p, 1.0), ctx.atoms.size(),
block_atoms, NORMAL_CG_ITERATIONS);
logger.Debug("Refinement: the positions' normal matrix {:.2f} GB per trial on a GPU", normal_bytes / 1e9);
}
#endif
MakeEngine(ctx, settings.gpu, settings, logger, restraint_records, restraint_slots, normal_bytes);
result.on_gpu = ctx.engine->OnGPU();
std::vector<TlsBody> input_tls;
if (ctx.st.meta.get_tls_groups() != nullptr) {
input_tls = TlsBodiesFromModel(ctx.Model(), *ctx.st.meta.get_tls_groups());
for (TlsBody &body : input_tls)
std::erase_if(body.atoms, [&](int i) { return ctx.atoms[i]->is_hydrogen(); });
}
RFactors r = ReadInputDisplacements(ctx, input_tls, logger, result);
result.n_work = r.n_work;
result.n_free = r.n_free;
result.r_work_start = r.r_work;
result.r_free_start = r.r_free;
result.score_start = ctx.Score();
const std::vector<float> start_r_terms = FreeRTerms(ctx);
logger.Info("Refinement{}: {} atoms ({} riding hydrogens), {} reflections to {:.2f} A ({} free), on the {}; as "
"given R-work {:.4f}, R-free {:.4f}; k_sol {:.3f}, b_sol {:.1f}, overall B {:.1f}, Wilson B {:.0f}, B "
"ceiling {:.0f}, mask grid {:.2f} A",
ctx.refl.Twinned() ? fmt::format(" against the data twinned by {} (fraction {:.3f}, {} mates not "
"observed)", result.twin_law, FractionSum(ctx.rows),
ctx.refl.mate_hkl.size())
: std::string(),
ctx.atoms.size(), result.score_start.hydrogens, ctx.refl.Size(), result.d_min, r.n_free,
result.on_gpu ? "GPU" : "CPU", r.r_work, r.r_free, ctx.k_sol, ctx.b_sol, result.score_start.b_iso,
result.wilson_b, ctx.b_ceiling, result.score_start.mask_spacing);
// The model as given, scaled: a candidate of the final choice beside the trials (the restrained stage, 5.), with
// its atoms and residues as they are, which the solvent build edits in place.
TrialOutcome given;
given.trial.r_work = r.r_work;
given.trial.r_free = r.r_free;
given.trial.nll_free = result.score_start.nll_free;
given.trial.free_r_terms = start_r_terms;
given.pos = ctx.Positions();
given.b = ctx.B();
given.aniso = ctx.aniso;
given.fit = ctx.Fitted();
given.ran = true;
std::vector<gemmi::Atom> given_atoms;
std::vector<std::pair<std::string, gemmi::EntityType>> given_residues;
const SolventPool given_pool = pool;
if (settings.restrained && !settings.score_only)
for (const gemmi::Chain &ch : ctx.Model().chains)
for (const gemmi::Residue &res : ch.residues) {
given_residues.emplace_back(res.name, res.entity_type);
given_atoms.insert(given_atoms.end(), res.atoms.begin(), res.atoms.end());
}
// The waters and ions the build relabelled and kept (SolventEdit::site their old name), for the metal records the
// refined model is written with.
std::vector<SolventEdit> relabels;
// The maps, the blobs and the model of where the refinement ended.
auto finish = [&]() {
MakeMaps(ctx, merged, wavelength_A, result);
result.score = ctx.Score(); // at the scale and solvent MakeMaps refitted
if (ctx.refl.Twinned()) {
result.twin_fraction = FractionSum(ctx.rows);
result.twin_multi_domains = DomainText(ctx.refl, ctx.rows);
result.twin_domain_fractions.assign(ctx.rows.twin_fractions.begin(), ctx.rows.twin_fractions.end());
TwinNull(ctx, result, null_first);
}
// the anisotropic atoms' U into the model written out (ANISOU), and the TLS groups refined (the model's own
// stay where the U of their atoms was held, and go where every atom refined a U of its own)
ApplyAniso(ctx.Model(), ctx.aniso);
for (const AtomAniso &a : ctx.aniso)
result.aniso_atoms += a.on;
if (!ctx.tls.empty()) {
if (ctx.st.meta.refinement.empty())
ctx.st.meta.refinement.emplace_back();
ctx.st.meta.refinement[0].tls_groups = TlsBodiesToGroups(ctx.Model(), ctx.tls);
for (size_t k = 1; k < ctx.st.meta.refinement.size(); k++)
ctx.st.meta.refinement[k].tls_groups.clear();
} else if (result.adp_model.starts_with("anisotropic"))
for (gemmi::RefinementInfo &ref : ctx.st.meta.refinement)
ref.tls_groups.clear();
if (const std::vector<gemmi::TlsGroup> *groups = ctx.st.meta.get_tls_groups())
result.tls_groups = static_cast<int>(groups->size());
// the model as the build left it: the waters put in in their chains, those taken out out, and each metal it
// relabelled with the records of its inner sphere in place of its old ones
gemmi::Structure model = build ? SolventView(ctx.st, pool) : ctx.st;
int record = 0;
for (const SolventEdit &e : relabels) {
const std::string from = e.site.substr(0, e.site.find(' '));
std::erase_if(model.connections, [&](const gemmi::Connection &c) {
auto is_site = [&](const gemmi::AtomAddress &a) {
return a.chain_name == e.chain && a.res_id.seqid == e.seqid && a.res_id.name == from;
};
return is_site(c.partner1) || is_site(c.partner2);
});
for (const gemmi::Chain &ch : model.models[0].chains)
for (const gemmi::Residue &r : ch.residues)
if (ch.name == e.chain && r.seqid == e.seqid && r.name == e.species)
AddMetalRecords(model, ch, r, record);
}
{
const RefinementStage stage("blobs");
result.blobs = FindDensityBlobs(*result.difference_map, result.difference_map_rms, model, result.d_min,
result.anomalous_map.get(), result.anomalous_map_rms);
}
if (settings.solvent) {
const RefinementStage stage("solvent");
const SolventMaps maps{result.map.get(), result.map_rms, result.difference_map.get(),
result.difference_map_rms, result.anomalous_map.get(), result.anomalous_map_rms,
wavelength_A};
result.solvent = AnalyseSolventSites(model, result.d_min, maps, ctx.nthreads);
}
result.b_ceiling = ctx.b_ceiling;
result.k_sol = ctx.k_sol;
result.b_sol = ctx.b_sol;
if (build)
for (gemmi::Chain &ch : model.models[0].chains)
for (gemmi::Residue &r : ch.residues)
for (gemmi::Atom &a : r.atoms)
a.serial = 0; // SolventView's indices
result.model = std::make_shared<gemmi::Structure>(std::move(model));
result.ok = true;
result.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
};
if (settings.score_only) {
result.score_start.hkl = ctx.refl.hkl;
result.score_start.fcalc = ctx.fcalc;
result.score_start.fmask = ctx.rows.fmask;
result.score_start.k_total = ctx.rows.k_total;
result.score_start.d = ctx.rows.d;
result.score_start.var = ctx.rows.var;
result.r_work_placement = result.r_work_start;
result.r_free_placement = result.r_free_start;
result.restrained_residues = topo.restrained_residues;
result.fixed_residues = topo.fixed_residues;
result.fixed_names = topo.fixed_names;
result.geometry_start = MeasureGeometry(topo.geometry, BuildRepulsions(topo, ctx.Model(), cell, sg, 0.0),
ctx.XyzB(), &topo.metal_bonds);
finish();
const GeometryStatistics &g = result.geometry_start;
logger.Info("Refinement: the model scored only; geometry rmsd bonds {:.4f} A (rmsZ {:.2f}, {} bonds), metal "
"bonds {:.4f} A (rmsZ {:.2f}, {}; {} links made), both {:.4f} A; angles {:.2f} deg (rmsZ {:.2f})",
g.rmsd_bonds, g.rmsz_bonds, g.bonds, g.rmsd_metal, g.rmsz_metal, g.metal_bonds, topo.metal_links,
g.RmsdAllBonds(), g.rmsd_angles, g.rmsz_angles);
return result;
}
// The trials run side by side, as many at a time as the longest set of them has and the cards take engines:
// on the main model and engine, and on copies with engines of their own; each with a share of the threads. The
// copies are made while the placement runs: they start from the model as it was before the topology was made
// and take nothing from the main context that the placement changes.
int wanted = static_cast<int>(std::max({settings.weight_ladder.size() * settings.adp_ladder.size() *
(ncs_p != nullptr ? 2 : 1),
settings.final_ladder.size(), size_t{3}}));
if (settings.max_trials_at_once > 0)
wanted = std::min(wanted, static_cast<int>(settings.max_trials_at_once));
size_t busiest = 0;
const std::vector<SlotPlace> devices = OwnSlotPlaces(ctx, settings.restrained ? wanted - 1 : 0, settings, busiest);
if (held != nullptr)
held->Say(busiest);
const size_t slot_threads = std::max<size_t>(1, settings.nthreads / (devices.size() + 1));
ScalePools pools;
#ifdef JFJOCH_USE_CUDA
if (ctx.scale_gpu)
pools.pool[ctx.device] = ctx.scale_gpu;
#endif
std::vector<std::unique_ptr<OwnSlot>> own(devices.size());
std::vector<std::thread> slot_builders;
// joined before the slots are taken, or here where the placement throws
struct JoinAll {
std::vector<std::thread> &threads;
~JoinAll() {
for (std::thread &t : threads)
if (t.joinable())
t.join();
}
} join_slot_builders{slot_builders};
for (size_t k = 0; k < devices.size(); k++)
slot_builders.emplace_back([&, k]() {
#ifdef JFJOCH_USE_CUDA
const ScopedGpuPin pin(devices[k].device >= 0 ? devices[k].device : get_gpu());
#endif
// a slot that cannot be made is left out, as where the card cannot take its engine
try {
own[k] = MakeOwnSlot(before_topology, ctx, devices[k].device, devices[k].own_lane, slot_threads, settings,
pools);
} catch (const std::exception &e) {
logger.Warning("Refinement: a trial slot could not be made ({}), the trials share fewer", e.what());
}
});
RefinePlacement(ctx, settings.group_b, result);
if (restrained)
PlaceRiders(ctx, topo);
r = ctx.Refit();
result.r_work_placement = r.r_work;
result.r_free_placement = r.r_free;
// The overall B of the scale into the atoms (MoveOverallB, as every trial does), so that the weights are balanced
// and the trials started at the level the B factors refine at, not the one they came at: the B similarity's
// gradient goes as 1 / B, and from a prediction's B (6 A^2 under an overall B of 45) the balanced weight was a
// hundred times too stiff - what the second round of the weights (4b.) was mostly correcting.
if (settings.restrained)
ctx.MoveOverallB();
logger.Info("Refinement, placement ({} rigid bodies kept, B per residue {}): R-work {:.4f}, R-free {:.4f}",
result.rigid_bodies, result.group_b ? "kept" : "not kept", r.r_work, r.r_free);
if (settings.restrained) {
ctx.Refit();
result.restrained_residues = topo.restrained_residues - pool.n; // not the placeholders
result.fixed_residues = topo.fixed_residues;
result.fixed_names = topo.fixed_names;
std::vector<int> refinable;
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (topo.refinable[i])
refinable.push_back(static_cast<int>(i));
result.refined_atoms = static_cast<int>(refinable.size());
logger.Info("Refinement: {} residues restrained, {} held fixed{}; {} bonds, {} metal bonds ({} links made, {} records "
"beyond the inner shell left as contacts), {} sugar links made, {} "
"angles, {} torsions, {} chiral volumes, {} planes",
result.restrained_residues, topo.fixed_residues,
topo.fixed_names.empty() ? std::string() : " (" + Joined(topo.fixed_names) + ")",
topo.bonds, topo.metal_bonds.records.size(), topo.metal_links, topo.metal_links_refused, topo.sugar_links, topo.angles, topo.torsions, topo.chirals, topo.planes);
if (ncs_p != nullptr) {
result.ncs_groups = ncs.groups;
result.ncs_copies = ncs.copies;
result.ncs_restraints = static_cast<int>(ncs.distances.records.size());
result.ncs_chains = ncs.description;
result.ncs_rmsd_start = NcsRmsd(ncs, ctx.XyzB());
logger.Info("Refinement: NCS, {} molecule{} in {} copies ({}), {} local distance restraints; copies {:.3f} A "
"apart (C-alpha/P rmsd after superposition)", ncs.groups, ncs.groups == 1 ? "" : "s",
ncs.copies, ncs.description, result.ncs_restraints, result.ncs_rmsd_start);
}
ctx.engine->SetRestraints(1, topo.adp);
{
const RestraintSet set = RestraintsNow(ctx, topo);
ctx.engine->SetRestraints(0, set);
RestraintSet repulsions = BuildRepulsions(topo, ctx.Model(), cell, sg, 0.0, ctx.nthreads);
result.geometry_start = MeasureGeometry(topo.geometry, repulsions, ctx.XyzB(), &topo.metal_bonds);
}
// (the placement moves rigid bodies, which leaves the bonds and angles as given)
given.trial.geometry = result.geometry_start;
given.trial.within_bounds = given.trial.geometry.rmsd_bonds <= MAX_RMSD_BONDS &&
given.trial.geometry.rmsd_angles <= MAX_RMSD_ANGLES;
given.trial.adp = result.adp_input;
double w0 = 0, adp_w0 = 0;
{
const RefinementStage stage("balanced_weight");
w0 = BalancedWeight(ctx, refinable);
adp_w0 = BalancedADPWeight(ctx, BRestraintsNow(ctx, topo), refinable);
}
logger.Info("Refinement: geometry as given rmsd bonds {:.4f} A, angles {:.2f} deg, {} clashes; balanced X-ray "
"weights {:.3g} (positions), {:.3g} (B)", result.geometry_start.rmsd_bonds,
result.geometry_start.rmsd_angles, result.geometry_start.clashes, w0, adp_w0);
const std::vector<gemmi::Position> pos0 = ctx.Positions();
const std::vector<double> b0 = ctx.B();
std::optional<RefinementStage> slot_stage("slots");
for (std::thread &t : slot_builders)
t.join();
const size_t main_threads = ctx.nthreads;
ctx.SetThreads(slot_threads);
std::vector<TrialSlot> trial_slots{{&ctx, &topo}};
for (const std::unique_ptr<OwnSlot> &o : own)
if (o)
trial_slots.push_back({&o->ctx, &o->topo});
slot_stage.reset();
logger.Debug("Refinement: {} weight trials at a time", trial_slots.size());
// Trials whose need is decided by others are run with them where the slots take them all at once - but not
// where the slots share one card, whose time they would take from the trials that are needed (a model of 43k
// atoms: the ladder 10 s ahead, 5 s not), nor on the CPU, where the slots share the threads and a trial run
// ahead takes them from those that are needed - and left out where it is decided they were not needed, with
// their evaluations, which do not count: the outcome and the count are the same whatever the slots.
std::set<int> cards;
for (const TrialSlot &s : trial_slots)
cards.insert(s.ctx->device);
const bool one_card = cards.size() == 1 && *cards.begin() >= 0;
const bool on_gpu = ctx.engine->OnGPU();
auto ahead = [&](size_t trials) { return on_gpu && !one_card && trial_slots.size() >= trials; };
int ahead_evaluations = 0;
auto drop_ahead = [&](TrialOutcome &o) {
ahead_evaluations += o.evaluations;
o = TrialOutcome();
};
// A trial at other weights (or the model as given) displaces another only where its geometry is within
// bounds and the free set prefers it (FreeSetPrefers).
auto displaces = [](const RefinementWeightTrial &t, const RefinementWeightTrial &b) {
return t.within_bounds &&
(!b.within_bounds || FreeSetPrefers(t.free_r_terms, t.nll_free, b.free_r_terms, b.nll_free));
};
// A trial taken on from `s`: at the weights of s, where the work-set target (WorkTarget) is lower than that of s
// by more than TARGET_TOLERANCE, and the geometry within bounds where that of s is. A trial at other weights
// (X-ray weights or NCS restraints) is not compared on a target that changed with them - the positions and the
// B factors minimise targets of their own, whose sum a change of weight can raise as it lowers each: the
// weights are the free set's to decide (displaces).
auto advances = [&displaces](const RefinementWeightTrial &t, const RefinementWeightTrial &s) {
if (t.weight != s.weight || t.adp_weight != s.adp_weight || t.ncs_sigma != s.ncs_sigma)
return displaces(t, s);
if (t.within_bounds < s.within_bounds)
return false;
const double before = WorkTarget(s, s.weight, s.adp_weight);
return WorkTarget(t, s.weight, s.adp_weight) < before - TARGET_TOLERANCE * std::abs(before);
};
// Of a set of trials, the one kept: they come in order of trust, the first kept unless a later one displaces
// it. -1: none ran.
auto best_of = [&](const std::vector<TrialOutcome> &outcomes) {
int best = -1;
for (size_t k = 0; k < outcomes.size(); k++)
if (outcomes[k].ran && (best < 0 || displaces(outcomes[k].trial, outcomes[best].trial)))
best = static_cast<int>(k);
return best;
};
auto choose = [&](std::vector<TrialOutcome> &outcomes, const char *stage) {
for (const TrialOutcome &o : outcomes)
if (o.ran) {
logger.Info("Refinement, {}: {}", stage, o.line);
result.trials.push_back(o.trial);
}
return best_of(outcomes);
};
const double w_max = w0 * settings.max_weight;
// NCS restraints kept where R-free keeps them, the copies' B factors refined independently: following
// Kleywegt (1996) Acta Cryst. D52, 842-857, Uson et al. (1999) Acta Cryst. D55, 1158-1167, and Headd et al.
// (2014) Acta Cryst. D70, 1346-1356
// 1. The weights from the placed model: the ladder's rungs, tighter geometry first, each with the B factors
// held and refined at their own balanced weight; where there are copies, each rung without NCS restraints
// and then with them - those with them at the B factors' weight the rungs without chose, which NCS does
// not change (on 39 refinements of 21 structures with copies it never did; the X-ray weight it does), and
// at the rung they chose and the next looser one (on 198 refinements of the battery with copies, NCS was
// never kept at any other).
// The B factors are refined first at the middle rung only, and at the others unless holding them there
// displaces refining them (on 364 refinements of the battery B held was kept by 281). Their X-ray weight is tried at the balanced
// one and at a quarter and a sixteenth of it: a model whose B factors arrive far from where they refine
// (a prediction's pLDDT) balances a weight that is not the one its B wants, and the lower rungs took
// 0.002-0.010 off R-free on five such starts, nothing off eight deposited models.
const std::vector<AtomAniso> aniso0 = ctx.aniso;
std::vector<TrialOutcome> outcomes;
// The ladder from `from` (positions, B, anisotropy; `fit` what was fitted there, null to fit it), at the
// balanced weights as they stand: the trial kept, or one that did not run where none did.
auto ladder_choice = [&](const std::vector<gemmi::Position> &from_pos, const std::vector<double> &from_b,
const std::vector<AtomAniso> &from_aniso, const Context::Fit *fit) {
std::vector<TrialWeights> ladder;
for (double rung : settings.weight_ladder)
for (double adp : settings.adp_ladder) {
ladder.push_back({w0 * rung, adp_w0 * adp, 0});
if (ncs_p != nullptr)
ladder.push_back({w0 * rung, adp_w0 * adp, settings.ncs_sigma});
}
outcomes.assign(ladder.size(), TrialOutcome());
auto run_ladder = [&](const std::function<bool(const TrialWeights &)> &these) {
std::vector<TrialWeights> t;
for (const TrialWeights &l : ladder)
if (these(l))
t.push_back(l);
std::vector<TrialOutcome> ran = RunTrials(trial_slots, t, refinable, from_pos, from_b, from_aniso, fit,
settings.macro_cycles, settings, ncs_p);
for (size_t i = 0, m = 0; i < ladder.size(); i++)
if (these(ladder[i]))
outcomes[i] = std::move(ran[m++]);
};
const double mid = w0 * settings.weight_ladder[settings.weight_ladder.size() / 2];
auto first = [&](const TrialWeights &l) { return l.ncs == 0 && (l.adp == 0 || l.w == mid); };
const bool all_ahead = ahead(ladder.size());
run_ladder([&](const TrialWeights &l) { return first(l) || all_ahead; });
// B refined against B held at the middle rung (where both ran)
const TrialOutcome *held = nullptr, *refined = nullptr;
for (size_t i = 0; i < ladder.size(); i++)
if (ladder[i].w == mid && ladder[i].ncs == 0)
(ladder[i].adp == 0 ? held : refined) = &outcomes[i];
const bool refine_b = held == nullptr || refined == nullptr ||
(refined->ran && refined->trial.within_bounds && !displaces(held->trial, refined->trial));
auto rest_b = [&](const TrialWeights &l) { return l.ncs == 0 && !first(l); };
if (refine_b && !all_ahead)
run_ladder(rest_b);
// the best of those without NCS restraints, those with them set aside
std::vector<TrialOutcome> with_ncs(ladder.size());
for (size_t i = 0; i < ladder.size(); i++)
if (ladder[i].ncs > 0 || (!refine_b && rest_b(ladder[i])))
std::swap(with_ncs[i], outcomes[i]);
if (const int b = best_of(outcomes); ncs_p != nullptr && b >= 0) {
const double adp = outcomes[b].trial.adp_weight, w = outcomes[b].trial.weight;
double looser = w;
for (double rung : settings.weight_ladder)
if (w0 * rung > w && (looser == w || w0 * rung < looser))
looser = w0 * rung;
auto these = [&](const TrialWeights &l) { return l.ncs > 0 && l.adp == adp && (l.w == w || l.w == looser); };
for (size_t i = 0; i < ladder.size(); i++)
if (these(ladder[i]))
std::swap(with_ncs[i], outcomes[i]);
if (!all_ahead)
run_ladder(these);
}
for (TrialOutcome &o : with_ncs)
drop_ahead(o);
const int k = choose(outcomes, "ladder");
return k >= 0 ? std::move(outcomes[k]) : TrialOutcome();
};
TrialOutcome state = ladder_choice(pos0, b0, aniso0, nullptr);
// 1a. A twinned refinement (DecideTwin) stops where the ladder leaves its law far from being taken, hardly
// ahead of its null (TwinNull, TWIN_STOP_Z) - its threads go to the untwinned refinement beside it, and
// the law is decided where it stopped.
bool stopped = false;
if (settings.twin_arm_stops && state.ran && !ctx.Cancelled()) {
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
ctx.SetAniso(state.aniso);
ctx.Restore(state.fit);
ctx.FModel();
RefinementResult null;
TwinNull(ctx, null, null_first);
stopped = null.twin_null_z < TWIN_STOP_Z;
logger.Info("Refinement, twin law: after the ladder fraction {:.3f}, ahead of its null by {:.1f} standard "
"errors{}", FractionSum(ctx.rows), null.twin_null_z, stopped ? " - not taken further" : "");
}
// 2. The trajectory: steps of a few macro-cycles from where the last one ended, each trying the weights as they
// are and, in turn, the positions' X-ray weight halved and doubled (the ladder's rungs are a factor of eight
// apart, and a trajectory that goes on from a looser weight than the data want can tighten it only so) and
// the B factors' divided and multiplied by four; where there are copies, also the NCS restraints changed:
// switched on where they are off, and in turn tightened and loosened (to off beyond NCS_SIGMA_MAX) where
// they are on.
// The weights of a step are chosen on the free set among its trials (displaces, the weights as they are
// first); the step is taken while that trial advances from where the last one ended (advances), and the
// trajectory ends where the last step taken ended.
// The trials of the step that went from where the trajectory ended: the final ladder goes from there too, and
// a trial of the same weights from the same model is the same computation.
std::vector<TrialOutcome> from_best;
// `fresh`: the first step fits the scale, the solvent and the likelihood anew (the model changed since).
// `label`: the steps' name in the log.
auto trajectory = [&](TrialOutcome state, bool fresh, std::vector<TrialOutcome> &from_best, const char *label) {
from_best.clear();
for (int step = 0; step < settings.max_steps && state.ran && !ctx.Cancelled(); step++) {
const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma;
std::vector<TrialWeights> trials = {{w, adp, ncs_s}};
if (step % 2 == 1 && adp > 0) {
trials.push_back({w, adp / 4, ncs_s});
trials.push_back({w, 4 * adp, ncs_s});
} else {
trials.push_back({w / 2, adp, ncs_s});
if (2 * w <= w_max)
trials.push_back({2 * w, adp, ncs_s});
}
if (ncs_p != nullptr) {
const double changed = ncs_s == 0 ? settings.ncs_sigma : step % 2 == 0 ? ncs_s / 2 : 2 * ncs_s;
trials.push_back({w, adp, changed <= NCS_SIGMA_MAX ? changed : 0});
}
std::vector<TrialOutcome> outcomes = RunTrials(trial_slots, trials, refinable, state.pos, state.b, state.aniso,
fresh && step == 0 ? nullptr : &state.fit,
settings.macro_cycles, settings, ncs_p);
from_best = outcomes;
const int k = choose(outcomes, fmt::format("{} {}", label, step + 1).c_str());
if (k < 0)
break;
// after a change of the model the first step is taken whatever it gains: it is where the change is
// judged from
if (!((fresh && step == 0) || advances(outcomes[k].trial, state.trial)))
break;
state = std::move(outcomes[k]);
from_best.clear();
}
return state;
};
if (!stopped)
state = trajectory(std::move(state), false, from_best, "step");
// 2a. The population: from where the trajectory ended, members that refine the positions on the data blurred
// along a schedule of their own (TrialWeights::blur: at 60 A^2 the data reach 4 A, at 20 A^2 2.2 A, then
// the resolution as it is), from the positions as they are and kicked, all in one wave - where one
// descent sees one basin, the members see the basins around it. Then their crossover: on one 2mFo-DFc
// map averaged over the members with the free reflections left out (PopulationMap), each residue is taken
// from the member whose atoms sit in the most density (following Nwachukwu et al. (2013) Structure 21,
// 1923-1930); the child's junctions are mended by a macro-cycle
// at the ladder's tightest rung and the whole refined by one trial at the state's weights. The child is taken where it
// advances on the work-set target, as a step is (the free set does not see the members).
// The members are at most as many as the trial slots, one wave: on a large model, where the card holds
// few, more members would cost a wave each.
const size_t population = std::min<size_t>(std::max(settings.population, 0), trial_slots.size());
if (population > 1 && !stopped && state.ran && !ctx.Cancelled()) {
const RefinementStage pop_stage("population");
const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma;
// the members, the most different first: the positions as they are and kicked, on the data as they are
// and blurred to 4 A then 2.2 A, to 5 A then 2.8 A, to 2.8 A then 2 A, each a macro-cycle longer than a
// step so that every member settles on the data as they are before the crossover (members a step long
// left the basin a step longer reached on a placed prediction)
const std::vector<std::pair<std::vector<double>, double>> variants = {
{{}, 0.0}, {{60, 20}, 0.0}, {{60, 20}, 0.3}, {{100, 30}, 0.2}, {{30, 10}, 0.3}, {{}, 0.2},
{{100, 30}, 0.0}, {{30, 10}, 0.0}, {{60, 20}, 0.2}, {{100, 30}, 0.3}, {{30, 10}, 0.2}, {{}, 0.3}};
std::vector<TrialWeights> members;
for (const auto &[blur, kick] : variants)
if (members.size() < population)
members.push_back({w, adp, ncs_s, blur, kick, static_cast<unsigned>(members.size() + 1)});
std::vector<TrialOutcome> outcomes = RunTrials(trial_slots, members, refinable, state.pos, state.b, state.aniso,
&state.fit, settings.macro_cycles + 1, settings, ncs_p);
int best = -1;
for (size_t k = 0; k < outcomes.size(); k++) {
if (!outcomes[k].ran)
continue;
std::string blur;
for (double b : members[k].blur)
blur += fmt::format("{}{:.0f}", blur.empty() ? "" : "/", b);
logger.Info("Refinement, population: blur {}, kick {:.1f} A: {}", blur.empty() ? "none" : blur,
members[k].kick, outcomes[k].line);
if (outcomes[k].trial.within_bounds &&
(best < 0 || WorkTarget(outcomes[k].trial, w, adp) < WorkTarget(outcomes[best].trial, w, adp)))
best = static_cast<int>(k);
}
if (best >= 0) {
const gemmi::Grid<float> map = PopulationMap(ctx, outcomes);
const std::vector<int> residue = ResidueOfAtom(ctx.Model());
const int n_res = residue.back() + 1;
std::vector<std::vector<double>> density(outcomes.size(), std::vector<double>(n_res, 0.0));
for (size_t k = 0; k < outcomes.size(); k++)
if (outcomes[k].ran)
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (!ctx.atoms[i]->is_hydrogen() && ctx.atoms[i]->occ > 0)
density[k][residue[i]] += map.interpolate_value(outcomes[k].pos[i]);
std::vector<int> pick(n_res, -1);
for (int r = 0; r < n_res; r++)
for (size_t k = 0; k < outcomes.size(); k++)
if (outcomes[k].ran && (pick[r] < 0 || density[k][r] > density[pick[r]][r]))
pick[r] = static_cast<int>(k);
std::vector<gemmi::Position> child_pos = state.pos;
std::vector<double> child_b = state.b;
std::set<int> used;
for (size_t i = 0; i < ctx.atoms.size(); i++) {
child_pos[i] = outcomes[pick[residue[i]]].pos[i];
child_b[i] = outcomes[pick[residue[i]]].b[i];
used.insert(pick[residue[i]]);
}
// the junctions between residues of different members mended by one macro-cycle at the ladder's
// tightest rung, B held (at the state's weight the chimera ends out of the geometry bounds), then the
// step at the state's weights
std::vector<TrialOutcome> child = RunTrials(trial_slots, {{w0 * settings.weight_ladder.front(), 0, ncs_s}},
refinable, child_pos, child_b, state.aniso, nullptr, 1, settings,
ncs_p);
child = RunTrials(trial_slots, {{w, adp, ncs_s}}, refinable, child[0].pos, child[0].b, state.aniso,
&child[0].fit, settings.macro_cycles, settings, ncs_p);
const RefinementWeightTrial &c = child[0].trial;
const bool kept = child[0].ran && advances(c, state.trial);
logger.Info("Refinement, population: {} members, the best on the work-set target {:.6g}; crossover from "
"{} members: {}; {} (state target {:.6g})", outcomes.size(),
WorkTarget(outcomes[best].trial, w, adp), used.size(), child[0].line, kept ? "kept" : "not kept",
WorkTarget(state.trial, w, adp));
if (kept) {
state = std::move(child[0]);
from_best.clear();
}
}
}
// 2b. The solvent build (SolventBuild.h), from where the trajectory ended: waters put in at the difference
// map's peaks, taken out where the maps no longer hold them, and waters and ions relabelled, on the maps
// and the solvent audit of that model; then the solvent alone refined, the rest held, at the weights the
// trajectory ended at. A round is kept where that lowers the work-set target at those weights, and undone otherwise (the build ends
// there); at most SOLVENT_BUILD_ROUNDS of them. What they kept is then refined with the rest by the final
// ladder (the trajectory run again from there gained nothing on twelve sets). Each round thus costs a map,
// an audit and a refinement of the solvent, where a refinement of its own cost the whole weight search
// (the build was 2-4 times the refinement).
if (build && !stopped && state.ran && !ctx.Cancelled()) {
const auto build_start = std::chrono::steady_clock::now();
const RefinementStage build_stage("solvent_build");
SolventBuildResult &b = result.solvent_build;
b.ran = true;
std::set<std::string> added; // the waters put in and still there, by name
std::set<std::string> relabelled; // the sites relabelled, by their new names
std::vector<SolventEdit> not_solvent;
// per slot, the residue of each atom (the atoms stay where they are in memory)
std::vector<std::vector<gemmi::Residue *>> residue_of(trial_slots.size());
for (size_t sl = 0; sl < trial_slots.size(); sl++)
for (gemmi::Chain &ch : trial_slots[sl].ctx->Model().chains)
for (gemmi::Residue &r : ch.residues)
for (size_t a = 0; a < r.atoms.size(); a++)
residue_of[sl].push_back(&r);
for (int round = 1; !ctx.Cancelled(); round++) {
if (round > SOLVENT_BUILD_ROUNDS) {
b.stop = fmt::format("{} rounds", SOLVENT_BUILD_ROUNDS);
break;
}
// the maps and the audit of where the trajectory ended, on all of the threads (the trials left the
// main slot its share of them)
ctx.SetThreads(main_threads);
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
PlaceRiders(ctx, topo);
// the maps' Refit() from the trajectory's own fit, whichever slot ran it last
ctx.Restore(state.fit);
RefinementResult scratch;
MakeMaps(ctx, merged, wavelength_A, scratch);
std::optional<RefinementStage> pick_stage("build.view");
const gemmi::Structure view = SolventView(ctx.st, pool);
const SolventMaps maps{scratch.map.get(), scratch.map_rms, scratch.difference_map.get(),
scratch.difference_map_rms, scratch.anomalous_map.get(),
scratch.anomalous_map_rms, wavelength_A};
pick_stage.emplace("build.audit");
const std::vector<SolventSite> sites = AnalyseSolventSites(view, result.d_min, maps, settings.nthreads);
pick_stage.emplace("build.pick");
SolventRound picked = PickSolvent(view, maps, sites, added, relabelled, ctx.b_ceiling, round);
pick_stage.reset();
not_solvent.clear();
for (const SolventEdit &e : picked.edits)
if (e.kind == SolventEdit::Kind::NotSolvent)
not_solvent.push_back(e);
b.not_solvent = picked.not_solvent;
std::string refused;
for (const auto &[why, n] : picked.refused)
refused += fmt::format("{}{} {}", refused.empty() ? "" : ", ", n, why);
logger.Info("Solvent build, round {}: {} waters to add, {} to remove, {} sites to relabel, {} peaks "
"not solvent{}", round, picked.added, picked.removed, picked.relabelled,
picked.not_solvent, refused.empty() ? std::string() : "; peaks refused: " + refused);
if (picked.added + picked.removed + picked.relabelled == 0) {
b.stop = "nothing to change";
break;
}
// the atoms the round edits (an index in the model of every slot), each with what it was
std::map<std::string, int> by_name; // "HOH A401" (and "|altloc" where it has one) -> atom index
for (const gemmi::Chain &ch : view.models[0].chains)
for (const gemmi::Residue &r : ch.residues)
for (const gemmi::Atom &a : r.atoms)
if (!a.is_hydrogen())
by_name[r.name + " " + ch.name + r.seqid.str() +
(a.altloc ? std::string("|") + a.altloc : std::string())] = a.serial;
struct Undo {
int index;
gemmi::Atom atom;
std::string resname;
gemmi::EntityType entity;
};
std::vector<std::vector<Undo>> undo(trial_slots.size());
std::vector<RefinementTopology> topo_before(trial_slots.size());
const SolventPool pool_before = pool;
// the edits, in order, made below in every slot - each slot on a thread of its own
using Edit = std::function<void(gemmi::Atom &, gemmi::Residue &, size_t)>;
std::vector<std::pair<int, Edit>> edits;
std::vector<char> retyped(ctx.atoms.size(), 0); // the atoms whose B similarities go
auto edit_atom = [&](int index, Edit f) { edits.emplace_back(index, std::move(f)); };
std::vector<SolventEdit> kept;
for (SolventEdit &e : picked.edits) {
if (e.kind == SolventEdit::Kind::Added) {
const int k = pool.Free();
if (k < 0)
continue; // no placeholder left: the weaker peaks wait
pool.chain[k] = e.chain;
pool.seqid[k] = gemmi::SeqId(++pool.last[e.chain], ' ');
e.site = "HOH " + e.chain + pool.seqid[k].str();
// on a special position (PickSolvent put it exactly there), held on it
SpecialPosition site = SiteSymmetry(view.cell, e.pos);
site.atom = pool.first + k;
const int index = pool.first + k;
edit_atom(index, [&trial_slots, e, index, site](gemmi::Atom &a, gemmi::Residue &, size_t sl) {
a.pos = e.pos;
a.occ = static_cast<float>(e.occupancy);
a.b_iso = static_cast<float>(e.b_iso);
RefinementTopology &t = *trial_slots[sl].topo;
t.refinable[index] = 1;
if (!site.ops.empty()) {
t.special_index[index] = static_cast<int>(t.special.size());
t.special.push_back(site);
}
});
} else if (e.kind == SolventEdit::Kind::Removed) {
const auto it = by_name.find(e.site + (e.altloc ? std::string("|") + e.altloc : std::string()));
if (it == by_name.end())
continue;
edit_atom(it->second, [&trial_slots, index = it->second](gemmi::Atom &a, gemmi::Residue &, size_t sl) {
a.occ = 0;
trial_slots[sl].topo->refinable[index] = 0;
});
} else if (e.kind == SolventEdit::Kind::Relabelled) {
const auto it = by_name.find(e.site);
if (it == by_name.end())
continue;
const int i = it->second;
// its inner sphere, for a metal's bonds, read in the view and numbered as the slots number it
std::vector<int> donors;
if (gemmi::Element(SolventSpeciesElement(e.species)).is_metal())
for (const gemmi::Chain &ch : view.models[0].chains)
for (const gemmi::Residue &r : ch.residues)
if (ch.name == e.chain && r.seqid == e.seqid && r.atoms.size() == 1) {
gemmi::Residue as_metal = r;
as_metal.atoms[0].element = gemmi::Element(SolventSpeciesElement(e.species));
for (const InnerSphereDonor &d : InnerSphereDonors(view.models[0], r, as_metal.atoms[0]))
donors.push_back(view.models[0].chains[d.chain].residues[d.residue].atoms[d.atom].serial);
}
retyped[i] = 1;
edit_atom(i, [&trial_slots, e, i, donors](gemmi::Atom &a, gemmi::Residue &r, size_t sl) {
const bool water = e.species == "HOH";
r.name = e.species;
r.entity_type = water ? gemmi::EntityType::Water : gemmi::EntityType::NonPolymer;
a.element = gemmi::Element(SolventSpeciesElement(e.species));
a.name = water ? "O" : e.species;
a.charge = ScatteringCharge(r, a);
RetypeAtom(*trial_slots[sl].topo, r, a, i, donors);
});
} else
continue;
kept.push_back(e);
}
std::optional<RefinementStage> edit_stage("build.edit");
std::vector<std::thread> editors;
for (size_t sl = 0; sl < trial_slots.size(); sl++)
editors.emplace_back([&, sl]() {
topo_before[sl] = *trial_slots[sl].topo;
for (const auto &[index, f] : edits) {
gemmi::Residue &r = *residue_of[sl][index];
gemmi::Atom &a = *trial_slots[sl].ctx->atoms[index];
undo[sl].push_back({index, a, r.name, r.entity_type});
f(a, r, sl);
}
if (std::find(retyped.begin(), retyped.end(), 1) != retyped.end())
DropBRestraints(*trial_slots[sl].topo, retyped);
});
for (std::thread &t : editors)
t.join();
edit_stage.reset();
refinable.clear();
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (topo.refinable[i])
refinable.push_back(static_cast<int>(i));
// the solvent alone refined, the rest held, at the weights the trajectory ended at: what the round's
// changes are judged by
std::vector<int> solvent;
for (int i : refinable) {
const gemmi::Residue &r = *residue_of[0][i];
if (r.is_water() || (r.atoms.size() == 1 && (r.atoms[0].element.is_metal() || r.atoms[0].element == gemmi::El::Cl)))
solvent.push_back(i);
}
const std::vector<gemmi::Position> changed_pos = ctx.Positions();
const std::vector<double> changed_b = ctx.B();
// the round as built, nothing refined: where refining the solvent alone loses what the sites gained,
// it is judged as built
ctx.Restore(state.fit);
const RFactors built = ctx.Refit();
const Context::Fit built_fit = ctx.Fitted();
RefinementWeightTrial built_trial = state.trial;
built_trial.r_work = built.r_work;
built_trial.r_free = built.r_free;
MeasureWorkTarget(ctx, topo, ncs_p, state.trial.ncs_sigma, built_trial);
// the round judged on the work-set target at the weights the trajectory ended at
const double target_before = WorkTarget(state.trial, state.trial.weight, state.trial.adp_weight);
auto lowers = [&](const RefinementWeightTrial &t) {
return WorkTarget(t, state.trial.weight, state.trial.adp_weight) <
target_before - TARGET_TOLERANCE * std::abs(target_before);
};
// (B refined, at its balanced weight where the trajectory held B: a new water's starting B is a
// guess; fewer iterations, the waters' few parameters converging fast)
RefinementSettings solvent_settings = settings;
solvent_settings.xyz_iterations = SOLVENT_ITERATIONS;
solvent_settings.b_iterations = SOLVENT_ITERATIONS;
std::vector<TrialOutcome> local =
RunTrials(trial_slots, {{state.trial.weight, state.trial.adp_weight > 0 ? state.trial.adp_weight : adp_w0,
state.trial.ncs_sigma}},
solvent, changed_pos, changed_b, state.aniso, nullptr, settings.macro_cycles, solvent_settings,
ncs_p);
logger.Info("Solvent build, round {}: the solvent refined alone: {}", round, local[0].line);
TrialOutcome after = std::move(local[0]);
if (after.ran && !lowers(after.trial) && lowers(built_trial)) {
logger.Info("Solvent build, round {}: kept as built, R-work {:.4f}, R-free {:.4f}, target {:.6g}",
round, built.r_work, built.r_free,
WorkTarget(built_trial, state.trial.weight, state.trial.adp_weight));
after.pos = changed_pos;
after.b = changed_b;
after.aniso = state.aniso;
after.fit = built_fit;
const double adp_weight = after.trial.adp_weight;
after.trial = built_trial;
after.trial.adp_weight = adp_weight;
}
if (!after.ran || !lowers(after.trial)) {
logger.Info("Solvent build, round {}: not kept (target {:.6g}, before {:.6g})", round,
WorkTarget(after.trial, state.trial.weight, state.trial.adp_weight), target_before);
b.stop = fmt::format("round {} did not lower the target", round);
// undone, in every slot
for (size_t sl = 0; sl < trial_slots.size(); sl++) {
for (auto u = undo[sl].rbegin(); u != undo[sl].rend(); ++u) {
*trial_slots[sl].ctx->atoms[u->index] = u->atom;
residue_of[sl][u->index]->name = u->resname;
residue_of[sl][u->index]->entity_type = u->entity;
}
*trial_slots[sl].topo = topo_before[sl];
}
pool = pool_before;
refinable.clear();
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (topo.refinable[i])
refinable.push_back(static_cast<int>(i));
break;
}
for (const SolventEdit &e : kept) {
if (e.kind == SolventEdit::Kind::Added)
added.insert(e.site);
else if (e.kind == SolventEdit::Kind::Removed) {
if (added.erase(e.site) == 0)
b.removed++;
} else {
const std::string now = e.species + e.site.substr(e.site.find(' '));
if (added.erase(e.site) > 0)
added.insert(now);
relabelled.insert(now);
relabels.push_back(e);
b.relabelled++;
}
b.edits.push_back(e);
}
b.rounds = round;
// the solvent's B refined for the judge only: the rest goes on with the B factors as the trajectory
// had them
after.trial.adp_weight = state.trial.adp_weight;
state = std::move(after);
from_best.clear();
}
b.added = static_cast<int>(added.size());
b.edits.insert(b.edits.end(), not_solvent.begin(), not_solvent.end());
b.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - build_start).count();
logger.Info("Solvent build: {} waters added, {} removed, {} sites relabelled in {} round{} kept ({}), {} "
"peaks not solvent; R-free {:.4f}; {:.1f} s", b.added, b.removed, b.relabelled, b.rounds,
b.rounds == 1 ? "" : "s", b.stop, b.not_solvent, state.trial.r_free, b.seconds);
}
// 2c. The side-chain field (SideChainField.h), from where the trajectory (and the solvent build) ended: every
// side chain the field can search put in the rotamer of least energy - the -ln L of the working
// reflections at the fit as it stands, plus the refinement's own repulsion term at its X-ray weight -
// given the others, or pruned to CB where no rotamer fits; then one step at the trajectory's weights from
// the field, judged against the same step from the model as it was: kept where it advances on the
// work-set target from that step (advances), undone otherwise. Neither the field nor its test sees the
// free reflections. A model whose side chains the data already hold moves nothing and
// costs the sweep alone. The atoms it prunes are left at occupancy 0, as the solvent build's placeholders
// are, and held.
if (settings.side_chain_field && !stopped && state.ran && !ctx.Cancelled() && !ctx.refl.Twinned() &&
std::none_of(state.aniso.begin(), state.aniso.end(), [](const AtomAniso &a) { return a.on; })) {
const RefinementStage field_stage("side_chain_field");
const auto field_start = std::chrono::steady_clock::now();
ctx.SetThreads(main_threads);
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
ctx.SetAniso(state.aniso);
PlaceRiders(ctx, topo);
ctx.Restore(state.fit);
ctx.engine->Fcalc(ctx.Model(), ctx.fcalc);
FieldReflections fr;
for (size_t m = 0; m < ctx.refl.Size(); m++) {
const gemmi::Miller &h = ctx.refl.hkl[m];
fr.hkl.push_back({static_cast<float>(h[0]), static_cast<float>(h[1]), static_cast<float>(h[2])});
fr.stol2.push_back(static_cast<float>(ctx.refl.inv_d2[m] / 4.0));
fr.fo.push_back(ctx.refl.fo[m]);
fr.d.push_back(ctx.rows.d[m]);
fr.var.push_back(ctx.rows.var[m]);
fr.centric.push_back(ctx.refl.centric[m]);
fr.work.push_back(!ctx.refl.free[m]);
fr.k_total.push_back(ctx.rows.k_total[m]);
fr.fmask.push_back(ctx.rows.fmask[m]);
}
// the form factors of the model's elements, each once
std::vector<FormFactor> elements;
std::vector<gemmi::El> element_list;
std::vector<int> element_of(ctx.atoms.size());
for (size_t i = 0; i < ctx.atoms.size(); i++) {
const gemmi::El el = ctx.atoms[i]->element.elem;
auto it = std::find(element_list.begin(), element_list.end(), el);
if (it == element_list.end()) {
element_list.push_back(el);
elements.push_back(gemmi::IT92<float>::get(el, 0).coefs);
it = element_list.end() - 1;
}
element_of[i] = static_cast<int>(it - element_list.begin());
}
std::unique_ptr<SideChainFieldEvaluator> evaluator;
#ifdef JFJOCH_USE_CUDA
// on every card, each with a part of the reflections, where a part's states fit beside the engines -
// decided up front on the card's total, as the engines are; on the CPU otherwise, rather than failing at
// an allocation inside a sweep
if (ctx.engine->OnGPU()) {
const size_t ops = FieldOperators(sg).size();
const int cards = get_gpu_count();
const size_t wanted = SideChainFieldDeviceBytes(fr.Size() / cards + 1, ops),
card = get_gpu_total_memory(ctx.device);
std::vector<int> devices;
for (int k = 0; k < cards; k++)
devices.push_back((ctx.device + k) % cards);
if (wanted < card / 2)
evaluator = MakeSideChainFieldEvaluatorGPUs(devices, fr, FieldOperators(sg), elements);
else
logger.Info("Side-chain field on the CPU: {} MiB wanted on the card of {} MiB", wanted >> 20,
card >> 20);
}
#endif
if (!evaluator)
evaluator = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(sg), elements, ctx.nthreads, settings.gpu);
std::vector<std::complex<float>> f_total(ctx.fcalc.begin(), ctx.fcalc.begin() + ctx.refl.Size());
evaluator->SetTotal(f_total);
// the copies of each residue, from the NCS anchors (corresponding C-alpha atoms): per residue, numbered
// over the model in order, the lowest ordinal among its copies as the group
std::vector<int> ncs_group;
if (ncs_p != nullptr) {
std::vector<int> residue_of;
int ordinal = 0;
for (const gemmi::Chain &ch : ctx.Model().chains)
for (const gemmi::Residue &r : ch.residues) {
residue_of.insert(residue_of.end(), r.atoms.size(), ordinal);
ordinal++;
}
ncs_group.assign(ordinal, -1);
for (const std::vector<std::array<int, 2>> &pairs : ncs.anchors)
for (const std::array<int, 2> &pair : pairs) {
const int a = residue_of[pair[0]], b = residue_of[pair[1]];
const int ga = ncs_group[a] < 0 ? a : ncs_group[a], gb = ncs_group[b] < 0 ? b : ncs_group[b];
const int g = std::min(ga, gb);
for (int &x : ncs_group)
if (x == ga || x == gb)
x = g;
ncs_group[a] = ncs_group[b] = g;
}
}
const SideChainFieldResult field = OptimiseSideChainField(ctx.st, element_of, topo.refinable, ncs_group,
state.trial.weight, fr.Size(), *evaluator);
evaluator.reset();
ctx.SetThreads(slot_threads);
result.side_chain_field_candidates = field.candidates;
auto chis = [](const std::vector<double> &v) {
std::string s;
for (double x : v)
s += fmt::format("{}{:.0f}", s.empty() ? "" : "/", x);
return s.empty() ? "absent" : s;
};
std::string names;
int pruned = 0;
for (const SideChainFieldMove &m : field.moves) {
names += fmt::format("{}{} {} -> {} ({:.0f})", names.empty() ? "" : ", ", m.name, chis(m.chi_before),
chis(m.chi_after), m.energy_gain);
pruned += m.absent;
}
for (const SideChainFieldAlternative &a : field.alternatives)
if (a.gain > 1)
logger.Debug("Side-chain field: {} as given {}, best alternative {} would gain {:.1f}", a.name,
chis(a.chi_given), chis(a.chi_best), a.gain);
logger.Info("Side-chain field: {} residues searched{}, {} moved ({} pruned) in {} sweep{}, energy {:.1f} -> "
"{:.1f}, {:.2f} s{}{}", field.candidates, field.copies_together ? " (copies together)" : "",
field.moves.size(), pruned, field.sweeps,
field.sweeps == 1 ? "" : "s", field.energy_before, field.energy_after,
std::chrono::duration<double>(std::chrono::steady_clock::now() - field_start).count(),
names.empty() ? "" : ": ", names);
if (!field.moves.empty()) {
// one step at the trajectory's weights, from the field and from the model as it is
auto one_step = [&](const std::vector<gemmi::Position> &pos) {
std::vector<TrialOutcome> o =
RunTrials(trial_slots, {{state.trial.weight, state.trial.adp_weight, state.trial.ncs_sigma}},
refinable, pos, state.b, state.aniso, nullptr, settings.macro_cycles, settings, ncs_p);
return std::move(o[0]);
};
std::vector<gemmi::Position> moved = state.pos;
std::vector<int> pruned_atoms;
for (const SideChainFieldMove &m : field.moves) {
for (const auto &[a, p] : m.atoms)
moved[a] = p;
pruned_atoms.insert(pruned_atoms.end(), m.pruned.begin(), m.pruned.end());
}
const TrialOutcome unchanged = one_step(state.pos);
// the pruned atoms absent in every slot: no density, no mask, no contact, not refined
std::vector<RefinementTopology> topo_before(trial_slots.size());
for (size_t sl = 0; sl < trial_slots.size(); sl++) {
topo_before[sl] = *trial_slots[sl].topo;
for (int i : pruned_atoms) {
trial_slots[sl].ctx->atoms[i]->occ = 0;
trial_slots[sl].topo->refinable[i] = 0;
}
}
const std::vector<int> refinable_before = refinable;
refinable.clear();
for (size_t i = 0; i < ctx.atoms.size(); i++)
if (topo.refinable[i])
refinable.push_back(static_cast<int>(i));
TrialOutcome after = one_step(moved);
const bool kept = after.ran && unchanged.ran && advances(after.trial, unchanged.trial);
logger.Info("Side-chain field: refined from the field R-free {:.4f}, -LL free {:.1f}, target {:.6g}; from "
"the model as it was R-free {:.4f}, -LL free {:.1f}, target {:.6g}: {}", after.trial.r_free,
after.trial.nll_free, WorkTarget(after.trial, after.trial.weight, after.trial.adp_weight),
unchanged.trial.r_free, unchanged.trial.nll_free,
WorkTarget(unchanged.trial, after.trial.weight, after.trial.adp_weight), kept ? "kept" : "not kept");
if (kept) {
result.side_chain_field_kept = true;
result.side_chain_field_moved = static_cast<int>(field.moves.size()) - pruned;
result.side_chain_field_pruned = pruned;
state = std::move(after);
from_best.clear();
} else {
for (size_t sl = 0; sl < trial_slots.size(); sl++) {
for (int i : pruned_atoms)
trial_slots[sl].ctx->atoms[i]->occ = 1;
*trial_slots[sl].topo = topo_before[sl];
}
refinable = refinable_before;
}
}
}
// 3. The final weights: a short ladder from where the trajectory ended, the weight as it is first, its choice
// on the free set taken where it advances from there (advances). A rung the step from there tried already is taken from that step.
if (!stopped && state.ran && !ctx.Cancelled()) {
const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma;
std::vector<TrialWeights> trials;
for (double f : settings.final_ladder)
if (w * f <= w_max)
trials.push_back({w * f, adp, ncs_s});
outcomes.assign(trials.size(), TrialOutcome());
std::vector<TrialWeights> missing;
for (size_t i = 0; i < trials.size(); i++) {
for (const TrialOutcome &o : from_best)
if (o.ran && o.trial.weight == trials[i].w && o.trial.adp_weight == trials[i].adp &&
o.trial.ncs_sigma == trials[i].ncs)
outcomes[i] = o;
if (!outcomes[i].ran)
missing.push_back(trials[i]);
}
std::vector<TrialOutcome> ran = RunTrials(trial_slots, missing, refinable, state.pos, state.b, state.aniso,
&state.fit, settings.macro_cycles, settings, ncs_p);
for (size_t i = 0, m = 0; i < trials.size(); i++)
if (!outcomes[i].ran)
outcomes[i] = std::move(ran[m++]);
const int k = choose(outcomes, "final");
if (k >= 0 && advances(outcomes[k].trial, state.trial))
state = std::move(outcomes[k]);
}
// 4. The displacement model, decided late, on the free set: from where the weights ended, TLS bodies (the
// model's own groups, one per chain, the chains in TLSMD segments) with a residual B, and a U per atom at
// its balanced weight and a quarter of it - each trial as long as a step, on the positions where the weights
// ended, held, so that the trials differ in the displacement model alone; the model chosen then takes a
// step with the positions refined, kept where it advances on the work-set target from its trial (against trials a macro-cycle longer
// that refined the positions too, on 50 refinements of the battery R-free 0.001 lower on average and none
// higher by more than 0.0015, at 78% of the evaluations). Where the geometry is out of bounds there, which
// held positions cannot mend, each trial refines them too, a macro-cycle longer than a step, and no step
// follows. One displaces the one before it (the first
// where the weights ended, the B factors as they are) only where its geometry is within bounds and the free
// set prefers it (FreeSetPrefers: its -ln L of the free reflections lower, its R-free not higher beyond its
// standard error): the free set is the test of the parameters it adds.
// Following Merritt (2012) Acta Cryst. D68, 468-477 (the displacement model the data support, decided by test)
// The trials that would not change the choice are left out: the model's own TLS groups are tried only where
// one body per chain or the finest cut lowers R-free, the cuts between those only where also the finest
// fits the free set better, the U at its balanced weight only where the U at a quarter of it lowers R-free
// or the -ln L of the free reflections (run ahead where the slots take them), and the U at four times its
// balanced weight not at all (on 49 refinements of 33 structures, no choice taken changed).
// `own_tls`: the TLS groups the model has, tried as a hypothesis of their own.
auto displacements = [&](const std::vector<TlsBody> &own_tls) {
const RefinementStage adp_stage("adp_hypotheses");
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
ctx.SetAniso(state.aniso);
ctx.Restore(state.fit);
PlaceRiders(ctx, topo);
state.trial.nll_free = ctx.Score().nll_free;
logger.Info("Refinement, displacements: from where the weights ended, R-free {:.4f}, -LL free {:.1f}",
state.trial.r_free, state.trial.nll_free);
std::vector<AdpHypothesis> hyps = AdpHypotheses(ctx, topo, own_tls);
const double w = state.trial.weight, adp = state.trial.adp_weight > 0 ? state.trial.adp_weight : adp_w0;
// a U per atom only where it is affordable: where the working reflections outnumber the parameters of the
// model with it, the positions and the six U of every atom, counted unrestrained (Merritt (2012) counts
// the positions with them) - the end of the range of restraint weights where Hamilton's test can be taken
// at all
int atoms_b = 0;
for (int i : refinable)
atoms_b += !ctx.atoms[i]->is_hydrogen();
const bool u_tried = result.n_work > 9 * atoms_b;
// the U weight balanced on the model with every refinable atom given a U
double u_w0 = adp;
if (u_tried) {
const std::vector<AtomAniso> keep = ctx.aniso;
std::vector<AtomAniso> all = keep.empty() ? std::vector<AtomAniso>(ctx.atoms.size()) : keep;
std::vector<char> refines_u(ctx.atoms.size(), 0);
std::vector<int> u_atoms;
for (int i : refinable)
if (!ctx.atoms[i]->is_hydrogen()) {
all[i].on = true;
refines_u[i] = 1;
u_atoms.push_back(i);
}
ctx.SetAniso(all);
u_w0 = BalancedUWeight(ctx, MakeURestraints(hyps.back().pairs, refines_u), u_atoms);
ctx.SetAniso(keep);
}
const double ncs_s = state.trial.ncs_sigma;
// every trial in order of trust; the model's own TLS groups and the cuts between one TLS body per chain
// and the finest, and the U at its balanced weight, run after the others where those ask for them
std::vector<TrialWeights> trials;
std::vector<const AdpHypothesis *> which;
int own = -1, coarsest = -1, finest = -1, u_quarter = -1;
for (const AdpHypothesis &h : hyps) {
if (h.kind == AdpHypothesis::Kind::Anisotropic) {
if (!u_tried)
continue;
u_quarter = static_cast<int>(trials.size());
for (double f : {0.25, 1.0}) {
trials.push_back({w, u_w0 * f, ncs_s});
which.push_back(&h);
}
} else {
if (h.segments == 0)
own = static_cast<int>(trials.size());
else {
if (coarsest < 0)
coarsest = static_cast<int>(trials.size());
finest = static_cast<int>(trials.size());
}
trials.push_back({w, adp, ncs_s});
which.push_back(&h);
}
}
std::vector<char> later(trials.size(), 0);
for (int k = coarsest + 1; k < finest; k++)
later[k] = 1;
if (own >= 0 && coarsest >= 0)
later[own] = 1;
if (u_quarter >= 0)
later[u_quarter + 1] = 1;
outcomes.assign(trials.size(), TrialOutcome());
// the positions held only where the geometry is within its bounds where the weights ended: a trial that
// holds them cannot bring it there, one that refines them can (and is then the whole trial, no step after)
const bool hold = state.trial.within_bounds;
auto run = [&](const std::vector<char> &these) {
std::vector<TrialWeights> t;
std::vector<const AdpHypothesis *> h;
for (size_t k = 0; k < trials.size(); k++)
if (these[k]) {
t.push_back(trials[k]);
h.push_back(which[k]);
}
if (t.empty())
return;
std::vector<TrialOutcome> ran = RunTrials(trial_slots, t, refinable, state.pos, state.b, state.aniso,
&state.fit, settings.macro_cycles + (hold ? 0 : 1), settings,
ncs_p, h, hold);
for (size_t k = 0, m = 0; k < trials.size(); k++)
if (these[k])
outcomes[k] = std::move(ran[m++]);
};
const bool later_ahead = ahead(trials.size());
std::vector<char> now(trials.size());
for (size_t k = 0; k < trials.size(); k++)
now[k] = !later[k] || later_ahead;
run(now);
// the model's own groups where one body per chain or the finest cut lowers R-free from where the weights
// ended, the cuts between where also the finest fits the free set better than one body per chain, the U
// at its balanced weight where the U at a quarter of it lowers R-free or -ln L free (R-free alone left out
// the trial that would have been kept where the quarter lowered -ln L free but not R-free)
auto lowers = [&](int k) {
return k >= 0 && outcomes[k].ran && outcomes[k].trial.within_bounds &&
outcomes[k].trial.r_free < state.trial.r_free;
};
// the U at a quarter fitting the free reflections better than where the weights ended asks for the
// balanced weight as well: the choice below is made on -ln L free as much as on R-free
auto fits_better = [&](int k) {
return k >= 0 && outcomes[k].ran && outcomes[k].trial.within_bounds &&
outcomes[k].trial.nll_free < state.trial.nll_free;
};
const bool tls = lowers(coarsest) || lowers(finest), u_full = lowers(u_quarter) || fits_better(u_quarter);
const bool cuts = tls && finest > coarsest && outcomes[finest].ran && outcomes[coarsest].ran &&
outcomes[finest].trial.nll_free < outcomes[coarsest].trial.nll_free;
std::vector<char> needed(trials.size(), 0);
for (int k = 0; k < static_cast<int>(trials.size()); k++)
needed[k] = later[k] && (k == u_quarter + 1 ? u_full : k == own ? tls : cuts);
if (!later_ahead)
run(needed);
for (size_t k = 0; k < trials.size(); k++)
if (later[k] && !needed[k])
drop_ahead(outcomes[k]);
int best = -1; // where the weights ended
auto trial = [&](int k) -> const RefinementWeightTrial & { return k < 0 ? state.trial : outcomes[k].trial; };
for (size_t k = 0; k < outcomes.size(); k++) {
const RefinementWeightTrial &t = outcomes[k].trial;
if (!outcomes[k].ran)
continue;
logger.Info("Refinement, displacements: {}", outcomes[k].line);
result.adp_trials.push_back(t);
// the larger model where Hamilton's tests take it, the smaller where they refuse it, the free set's
// choice where they are inconclusive (or the two have as many parameters) - a larger model then with
// its R-free lower, not within the tie: the burden of the evidence is on the parameters it adds
const RefinementWeightTrial &b = trial(best);
const bool larger = t.adp_parameters > b.adp_parameters;
const double pass = larger ? HamiltonPassFraction(t, b, result.n_work)
: HamiltonPassFraction(b, t, result.n_work);
bool takes = displaces(t, b) && (!larger || t.r_free < b.r_free);
if (t.adp_parameters != b.adp_parameters && (pass > HAMILTON_LARGER || pass < HAMILTON_SMALLER))
takes = t.within_bounds && (!b.within_bounds || (pass > HAMILTON_LARGER) == larger);
logger.Info("Refinement, displacements: {} against {}: the larger passes {:.0f}% of Hamilton's tests{}",
t.adp.empty() ? "the B factors as they are" : t.adp,
b.adp.empty() ? "the B factors as they are" : b.adp, 100 * pass, takes ? ", taken" : "");
if (takes)
best = static_cast<int>(k);
}
// (the model where the weights ended keeps the displacement model a round before chose for it, its U held by
// the weight trials since)
if (result.adp_model.empty())
result.adp_model = result.adp_input == "isotropic" ? "isotropic" : "the model's U held";
else if (state.trial.adp.empty() && result.adp_model != "isotropic" && !result.adp_model.ends_with("held"))
result.adp_model += ", U held";
if (best >= 0) {
result.adp_model = outcomes[best].trial.adp;
state = std::move(outcomes[best]);
}
if (best >= 0 && hold) {
// the model chosen refined on with the positions, a step from where its trial ended, kept where it
// advances on the work-set target from there
AdpHypothesis chosen = *which[best];
chosen.bodies = state.tls;
chosen.tls_b = state.tls_b;
std::vector<TrialOutcome> on = RunTrials(trial_slots, {trials[best]}, refinable, state.pos, state.b,
state.aniso, &state.fit, settings.macro_cycles, settings,
ncs_p, {&chosen});
if (on[0].ran) {
logger.Info("Refinement, displacements, positions refined: {}", on[0].line);
if (advances(on[0].trial, state.trial))
state = std::move(on[0]);
}
}
logger.Info("Refinement: displacement model {}", result.adp_model);
};
if (settings.adp_hypotheses && !stopped && state.ran && !ctx.Cancelled())
displacements(input_tls);
// 4b. The weights and the displacement model decided again, from the model as it now is: the balanced
// weights on its B factors, the ladder (1.) from it with its displacements as they are (the U of TLS or
// of the anisotropic model held, B held or refined), the trajectory (2.) from the rung kept, and the
// displacement hypotheses (4.) from where that ends - the TLS bodies refined from the B factors the first
// choice left, which they could not be from the B factors as given. A round is kept where it advances as a step
// does (advances) and undone otherwise, RefinementSettings::rounds in all: from a placed prediction, whose B
// factors are its pLDDT (6 A^2 under an overall B of 45), the first round balanced the B factors' weight
// a hundred times too stiff - the B similarity's gradient goes as 1 / B - and chose the middle rung with
// NCS; the second chose the tightest rung without, and from the B factors the first left, its TLS bodies
// 0.02 lower in R-free. Where the ladder and the trajectory leave the model where it was, the hypotheses
// would run the same trials from the same model again, so the round ends there: from a model refined
// before, a round is its ladder and one step. No round where the model as given is kept over the
// refinement (5.).
// The model as given kept over the refinement: where the free set prefers it, as one weight is preferred to
// another (displaces), and the refinement is not better on both R-work and R-free. The work-set target cannot
// choose here, the refinement having minimised it from the model as given.
auto given_kept = [&]() {
return displaces(given.trial, state.trial) &&
!(state.trial.r_free < given.trial.r_free && state.trial.r_work < given.trial.r_work);
};
for (int round = 2; round <= settings.rounds && settings.adp_hypotheses && !stopped && state.ran &&
!ctx.Cancelled() && (!settings.as_given_competes || !given_kept());
round++) {
const TrialOutcome before = state;
const std::vector<TlsBody> own_tls = state.tls;
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
ctx.SetAniso(state.aniso);
ctx.Restore(state.fit);
PlaceRiders(ctx, topo);
// the ceiling of B from the model as it now is, as from the model as given: a placed prediction comes
// with B factors of 6 A^2 under an overall B of 45 (its ceiling 100) and refines at B near 50-70, where
// the TLS bodies and the U per atom were held at the ceiling; where it rises the engines are made again
// (the gather is reserved for it)
double b_max = 0;
for (size_t i = 0; i < ctx.atoms.size(); i++)
b_max = std::max<double>(b_max, ctx.atoms[i]->b_iso);
const double ceiling = BCeiling(b_max);
if (ceiling > ctx.b_ceiling) {
const RefinementStage stage("engine");
for (std::unique_ptr<OwnSlot> &o : own)
if (o)
o->ctx.engine.reset();
ctx.b_ceiling = ceiling;
MakeEngine(ctx, settings.gpu, settings, logger, restraint_records, restraint_slots, normal_bytes);
ctx.engine->SetRestraints(0, RestraintsNow(ctx, topo));
ctx.engine->SetRestraints(1, topo.adp);
for (size_t k = 0; k < own.size(); k++)
if (own[k]) {
Context &o = own[k]->ctx;
o.b_ceiling = ceiling;
#ifdef JFJOCH_USE_CUDA
if (ctx.engine->OnGPU())
o.engine = MakeRefinementEngineGPU(*ctx.engine, devices[k].device, o.nthreads, devices[k].own_lane);
#endif
if (!o.engine)
o.engine = MakeRefinementEngineCPU(o.cell, *o.sg, o.d_min, o.refl, o.nthreads);
o.engine->SetAniso(&o.aniso);
o.engine->SetRestraints(1, own[k]->topo.adp);
}
ctx.Restore(state.fit);
}
{
const RefinementStage stage("balanced_weight");
w0 = BalancedWeight(ctx, refinable);
adp_w0 = BalancedADPWeight(ctx, BRestraintsNow(ctx, topo), refinable);
}
logger.Info("Refinement, round {}: balanced X-ray weights {:.3g} (positions), {:.3g} (B), B ceiling {:.0f}", round,
w0, adp_w0, ctx.b_ceiling);
TrialOutcome rung = ladder_choice(state.pos, state.b, state.aniso, &state.fit);
if (rung.ran && displaces(rung.trial, state.trial))
state = std::move(rung);
std::vector<TrialOutcome> unused;
const std::string step = fmt::format("round {} step", round);
state = trajectory(std::move(state), false, unused, step.c_str());
if (state.trial.r_work == before.trial.r_work && state.trial.r_free == before.trial.r_free) {
logger.Info("Refinement, round {}: the weights leave the model where it was", round);
break;
}
const std::string adp_before = result.adp_model;
displacements(own_tls.empty() ? input_tls : own_tls);
const bool kept = advances(state.trial, before.trial);
logger.Info("Refinement, round {}: R-free {:.4f} -> {:.4f}, target {:.6g} -> {:.6g}{}", round,
before.trial.r_free, state.trial.r_free,
WorkTarget(before.trial, state.trial.weight, state.trial.adp_weight),
WorkTarget(state.trial, state.trial.weight, state.trial.adp_weight), kept ? "" : ", not kept");
if (!kept) {
state = before;
result.adp_model = adp_before;
break;
}
}
// 5. The model as given: where it is kept over the refinement (given_kept), it is given back as it came (the
// solvent build's edits undone), so that what comes out is never worse on both free measures than what went
// in - where the free set is free to the model as given (RefinementSettings::as_given_competes).
if (settings.as_given_competes && state.ran && !ctx.Cancelled() && given_kept()) {
logger.Info("Refinement: the model as given kept (R-work {:.4f}, R-free {:.4f}, -LL free {:.1f}), the "
"refinement did not displace it (R-work {:.4f}, R-free {:.4f}, -LL free {:.1f})",
given.trial.r_work, given.trial.r_free, given.trial.nll_free, state.trial.r_work,
state.trial.r_free, state.trial.nll_free);
size_t a = 0, k = 0;
for (gemmi::Chain &ch : ctx.Model().chains)
for (gemmi::Residue &res : ch.residues) {
std::tie(res.name, res.entity_type) = given_residues[k++];
for (gemmi::Atom &at : res.atoms)
at = given_atoms[a++];
}
pool = given_pool;
relabels.clear();
result.solvent_build = SolventBuildResult();
result.adp_model = result.adp_input == "isotropic" ? "isotropic" : "the model's U held";
result.as_given = true;
state = given;
}
for (size_t s = 1; s < trial_slots.size(); s++)
ctx.evaluations += trial_slots[s].ctx->evaluations;
ctx.evaluations -= ahead_evaluations;
if (ahead_evaluations > 0)
logger.Debug("Refinement: {} target evaluations in trials run ahead and not needed", ahead_evaluations);
ctx.SetThreads(main_threads);
if (state.ran) {
ctx.SetPositions(state.pos);
ctx.SetB(state.b);
ctx.SetAniso(state.aniso);
ctx.tls = state.tls;
ctx.tls_b = state.tls_b;
PlaceRiders(ctx, topo);
// the maps' Refit() from the kept trial's own fit (the solvent grid and the likelihood's D it starts
// from), not from whichever trial the main slot ran last: the same whatever the slots
ctx.Restore(state.fit);
result.restrained = true;
result.weight = state.trial.weight;
result.adp_weight = state.trial.adp_weight;
result.ncs_sigma = state.trial.ncs_sigma;
if (ncs_p != nullptr) {
result.ncs_rmsd = NcsRmsd(ncs, ctx.XyzB());
result.ncs_released = NcsReleasedResidues(ncs, ctx.XyzB());
logger.Info("Refinement: NCS {}; copies {:.3f} A apart (as placed {:.3f} A), {} residues released",
result.ncs_sigma > 0 ? fmt::format("restrained at {:.3g} A", result.ncs_sigma)
: std::string("not kept, the copies refined independently"),
result.ncs_rmsd, result.ncs_rmsd_start, result.ncs_released);
}
result.r_work = state.trial.r_work;
result.r_free = state.trial.r_free;
result.geometry = state.trial.geometry;
} else
ctx.Refit(); // no trial ran: the model as placed, its scale and solvent for the maps
}
result.cancelled = ctx.Cancelled();
if (result.cancelled)
logger.Warning("Refinement: cancelled, {} weight trials ran", result.trials.size());
finish();
logger.Info("Refinement: done in {:.1f} s, {} target evaluations{}", result.seconds, ctx.evaluations,
result.restrained ? fmt::format("; R-work {:.4f}, R-free {:.4f}, weight {:.3g}", result.r_work,
result.r_free, result.weight)
: std::string());
RefinementStage::Log(logger);
return result;
}
// The R-free a refinement ended at: the restrained stage's, or the placement's where that did not run.
double FinalRFree(const RefinementResult &r) {
return r.restrained ? r.r_free : r.r_free_placement;
}
double FinalRWork(const RefinementResult &r) {
return r.restrained ? r.r_work : r.r_work_placement;
}
// The twin hypotheses beside the untwinned refinement `result`, as a ladder. First one law: the placed model refined
// again, solvent build and displacement model included, against the data as twinned by a law of the lattice, and the
// twinned refinement taken (into `result`) where the law beats its own null (TwinNull) on the free reflections by
// TWIN_NULL_Z standard errors, or by TWIN_NULL_Z_DATA where the caller says the data read twinned (AdoptTwinLaw) - not
// on its R-free against the untwinned one's, which compares two refinement paths and so carries their noise - so that
// neither the drop in R any twin target buys by averaging, nor a pseudo-symmetry the model already carries (it
// predicts I(Th) ~ I(h), and the law then changes nothing) is read as a twin. A law is tested only where the free set
// keeps its twin pairs together; of several, the one the placed model asks the largest fraction of. Where that law is
// taken and the lattice has more domains (RefinementTwinDomains), the next rung: every domain at once, each with its
// fraction, taken where the domains added beat their own null - their mates unrelated, the law's kept - by TWIN_NULL_Z
// standard errors, each added domain at a fraction of TWIN_DOMAIN_MIN at least.
void DecideTwin(const gemmi::Structure &placed, const std::vector<MergedReflection> &merged,
const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double wavelength_A, Logger &logger,
const RefinementSettings &settings, RefinementResult &result, const std::shared_future<void> &untwinned,
const std::function<void()> &arm_done) {
const RefinementStage stage("twin");
const std::vector<gemmi::Op> laws = RefinementTwinLaws(cell, sg);
const std::vector<gemmi::Op> domains = RefinementTwinDomains(cell, sg);
// The twinned arm reads only the placed model and the data, so it runs beside the untwinned refinement
// (RefineModel): `result` is read and written only once `untwinned` says that refinement is in it, and
// nothing is decided where it failed or was cut short.
std::string split_law;
int split_pairs = 0;
auto take_untwinned = [&]() {
untwinned.wait();
if (!result.ok || result.cancelled)
return false;
result.untwinned_r_work = FinalRWork(result);
result.untwinned_r_free = FinalRFree(result);
result.twin_laws = static_cast<int>(laws.size());
if (!split_law.empty()) {
result.twin_law = split_law;
result.twin_split_pairs = split_pairs;
}
return true;
};
if (laws.empty()) {
if (take_untwinned())
result.twin_decision = "NO_TWIN_LAW";
return;
}
// the laws whose twin pairs the free set keeps together
RefinementReflections refl = PrepareRefinementReflections(merged, cell, sg, RefinementDMin(MaxInvD2(merged)));
std::vector<gemmi::Op> usable;
for (const gemmi::Op &law : laws) {
const int split = SetTwinLaw(refl, cell, sg, RefinementDMin(MaxInvD2(merged)), law);
if (split == 0)
usable.push_back(law);
else {
logger.Info("Refinement, twin law {}: not tested - {} of its twin pairs are split between the working and "
"the free set", law.as_hkl().triplet('h'), split);
split_law = law.as_hkl().triplet('h');
split_pairs = split;
}
}
if (usable.empty()) {
if (take_untwinned())
result.twin_decision = "FREE_SET_SPLIT";
return;
}
split_law.clear();
gemmi::Op law = usable[0];
if (usable.size() > 1) {
RefinementSettings score = settings;
score.score_only = true;
double best = -1;
for (const gemmi::Op &candidate : usable) {
const RefinementResult s = RefineOnce(placed, merged, cell, sg, wavelength_A, logger, score, {candidate});
logger.Info("Refinement, twin law {}: the placed model asks a fraction of {:.3f}",
candidate.as_hkl().triplet('h'), s.score_start.twin_fraction);
if (s.ok && s.score_start.twin_fraction > best) {
best = s.score_start.twin_fraction;
law = candidate;
}
}
}
RefinementSettings arm = settings;
arm.twin_arm_stops = true;
RefinementResult twinned = RefineOnce(placed, merged, cell, sg, wavelength_A, logger, arm, {law});
arm_done();
if (!take_untwinned())
return;
result.twin_split_pairs = 0;
result.twin_law = law.as_hkl().triplet('h');
if (!twinned.ok || twinned.cancelled) {
result.twin_decision = "UNTWINNED";
return;
}
result.twin_fraction = twinned.twin_fraction;
result.twin_r_work = FinalRWork(twinned);
result.twin_r_free = FinalRFree(twinned);
result.twin_r_free_null = twinned.twin_r_free_null;
result.twin_fraction_null = twinned.twin_fraction_null;
result.twin_null_z = twinned.twin_null_z;
const bool adopt = AdoptTwinLaw(twinned.twin_fraction, result.twin_null_z, settings.twin_data_indicated);
result.twin_decision = twinned.twin_fraction == 0 ? "ZERO_FRACTION" : adopt ? "TWINNED" : "UNTWINNED";
if (twinned.twin_fraction > 0)
result.twin_rule = adopt && result.twin_null_z < TWIN_NULL_Z ? "NULL+LTEST" : "NULL";
logger.Info("Refinement, twin law {}: fraction {:.3f}, R-work {:.4f}, R-free {:.4f}; untwinned R-free {:.4f}; null "
"(mates unrelated, fraction {:.3f}) R-free {:.4f}, the law ahead of it by {:.1f} standard errors{} -> {}",
result.twin_law, result.twin_fraction, result.twin_r_work, result.twin_r_free, result.untwinned_r_free,
result.twin_fraction_null, result.twin_r_free_null, result.twin_null_z,
settings.twin_data_indicated ? ", the data read twinned" : "", result.twin_decision);
if (!adopt)
return;
// the twinned refinement is the result, with the evidence that chose it
twinned.twin_decision = result.twin_decision;
twinned.twin_rule = result.twin_rule;
twinned.twin_laws = result.twin_laws;
twinned.twin_free_closed = result.twin_free_closed;
twinned.twin_r_work = result.twin_r_work;
twinned.twin_r_free = result.twin_r_free;
twinned.untwinned_r_work = result.untwinned_r_work;
twinned.untwinned_r_free = result.untwinned_r_free;
result = std::move(twinned);
// the next rung: every domain of the lattice, the adopted law's first
std::vector<gemmi::Op> all{law};
for (const gemmi::Op &op : domains)
if (op.rot != law.rot)
all.push_back(op);
if (all.size() < 2)
return;
// Refined, the added domains under TWIN_DOMAIN_MIN dropped and the rest refined again, until every one left
// has it; none left: the single law stands.
RefinementResult multi;
while (true) {
multi = RefineOnce(placed, merged, cell, sg, wavelength_A, logger, settings, all, 1);
if (!multi.ok || multi.cancelled)
return;
std::vector<gemmi::Op> kept{law};
for (size_t d = 1; d < all.size(); d++)
if (d < multi.twin_domain_fractions.size() && multi.twin_domain_fractions[d] >= TWIN_DOMAIN_MIN)
kept.push_back(all[d]);
if (kept.size() == all.size())
break;
logger.Info("Refinement, {} twin domains ({}): {} below a fraction of {:.2f}{}", all.size() + 1,
multi.twin_multi_domains, all.size() - kept.size(), TWIN_DOMAIN_MIN,
kept.size() > 1 ? ", refined again without" : " - the single law stands");
if (kept.size() == 1) {
result.twin_multi_decision = "MULTI_UNTWINNED";
result.twin_multi_domains = multi.twin_multi_domains;
return;
}
all = kept;
}
result.twin_multi_domains = multi.twin_multi_domains;
result.twin_multi_r_work = FinalRWork(multi);
result.twin_multi_r_free = FinalRFree(multi);
result.twin_multi_r_free_null = multi.twin_r_free_null;
result.twin_multi_null_z = multi.twin_null_z;
const bool adopt_multi = multi.twin_null_z >= TWIN_NULL_Z;
result.twin_multi_decision = adopt_multi ? "MULTI_TWINNED" : "MULTI_UNTWINNED";
logger.Info("Refinement, {} twin domains ({}): R-work {:.4f}, R-free {:.4f}; one law R-free {:.4f}; null (the added "
"domains' mates unrelated) R-free {:.4f}, the domains ahead of it by {:.1f} standard errors -> {}",
all.size() + 1, multi.twin_multi_domains, result.twin_multi_r_work, result.twin_multi_r_free,
result.twin_r_free, result.twin_multi_r_free_null, result.twin_multi_null_z, result.twin_multi_decision);
if (!adopt_multi)
return;
multi.twin_decision = result.twin_decision;
multi.twin_rule = result.twin_rule;
multi.twin_laws = result.twin_laws;
multi.twin_free_closed = result.twin_free_closed;
multi.twin_fraction_null = result.twin_fraction_null;
multi.twin_r_free_null = result.twin_r_free_null;
multi.twin_null_z = result.twin_null_z;
multi.twin_r_work = result.twin_r_work;
multi.twin_r_free = result.twin_r_free;
multi.untwinned_r_work = result.untwinned_r_work;
multi.untwinned_r_free = result.untwinned_r_free;
multi.twin_multi_decision = result.twin_multi_decision;
multi.twin_multi_r_work = result.twin_multi_r_work;
multi.twin_multi_r_free = result.twin_multi_r_free;
multi.twin_multi_r_free_null = result.twin_multi_r_free_null;
multi.twin_multi_null_z = result.twin_multi_null_z;
result = std::move(multi);
}
} // namespace
int CloseFreeSet(std::vector<MergedReflection> &merged, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg) {
const std::vector<gemmi::Op> laws = RefinementTwinDomains(cell, sg);
if (laws.empty())
return 0;
const gemmi::GroupOps gops = sg.operations();
const gemmi::ReciprocalAsu asu(&sg);
auto key = [&](const gemmi::Miller &h) { return asu.to_asu(h, gops).first; };
// each reflection's orbit, by the ASU index of every reflection the laws reach from it; named by its lowest index
std::map<gemmi::Miller, int> orbit_of;
std::vector<gemmi::Miller> name;
std::vector<int> orbit(merged.size());
for (size_t i = 0; i < merged.size(); i++) {
const gemmi::Miller h = key(gemmi::Miller{{merged[i].h, merged[i].k, merged[i].l}});
if (const auto it = orbit_of.find(h); it != orbit_of.end()) {
orbit[i] = it->second;
continue;
}
const int o = static_cast<int>(name.size());
name.push_back(h);
std::vector<gemmi::Miller> todo{h};
orbit_of[h] = o;
while (!todo.empty()) {
const gemmi::Miller x = todo.back();
todo.pop_back();
name[o] = std::min(name[o], x);
for (const gemmi::Op &law : laws)
if (const gemmi::Miller y = key(law.apply_to_hkl(x)); orbit_of.emplace(y, o).second)
todo.push_back(y);
}
orbit[i] = o;
}
std::vector<int> free(name.size(), 0), all(name.size(), 0);
for (size_t i = 0; i < merged.size(); i++) {
free[orbit[i]] += merged[i].rfree_flag;
all[orbit[i]]++;
}
// splitmix64 of the orbit's name -> uniform in [0, 1), salted: unsalted it is AssignRfreeFlags' own draw of a
// reflection the name often is, and an orbit would go free whenever that member was
auto uniform = [](const gemmi::Miller &h) {
uint64_t z = (static_cast<uint64_t>(h[0] + (1 << 20)) << 42 | static_cast<uint64_t>(h[1] + (1 << 20)) << 21
| static_cast<uint64_t>(h[2] + (1 << 20))) + 0x9e3779b97f4a7c15ULL * 2;
z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL;
z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL;
return static_cast<double>((z ^ (z >> 31)) >> 11) * (1.0 / 9007199254740992.0);
};
int moved = 0;
for (size_t i = 0; i < merged.size(); i++) {
const int o = orbit[i];
const bool flag = free[o] == all[o] || (free[o] > 0 && uniform(name[o]) * all[o] < free[o]);
moved += merged[i].rfree_flag != flag;
merged[i].rfree_flag = flag;
}
return moved;
}
bool AdoptTwinLaw(double twin_fraction, double null_z, bool data_indicated) {
return twin_fraction > 0 && (null_z >= TWIN_NULL_Z || (data_indicated && null_z >= TWIN_NULL_Z_DATA));
}
bool ProperSubgroup(const gemmi::GroupOps &sub, const gemmi::GroupOps &super) {
if (sub.order() >= super.order())
return false;
for (const gemmi::Op &op : sub.sym_ops)
if (super.find_by_rotation(op.rot) == nullptr)
return false;
for (const gemmi::Op::Tran &cen : sub.cen_ops)
if (std::find(super.cen_ops.begin(), super.cen_ops.end(), cen) == super.cen_ops.end())
return false;
// The translations too, on some origin of sub: moved by p, its operator (R, t) reads (R, t + (R - 1) p). The
// rotations alone also pass a 2 where super has a 2(1) - P 1 2 1 is a subgroup of P 21 21 2 on no origin, and a
// model refined under it would overlay copies where the data have none.
constexpr int D = gemmi::Op::DEN;
for (int px = 0; px < D; ++px)
for (int py = 0; py < D; ++py)
for (int pz = 0; pz < D; ++pz) {
const int p[3] = {px, py, pz};
bool all = true;
for (const gemmi::Op &op : sub.sym_ops) {
gemmi::Op moved = op;
for (int i = 0; i < 3; ++i)
moved.tran[i] += (op.rot[i][0] * px + op.rot[i][1] * py + op.rot[i][2] * pz) / D - p[i];
moved.wrap();
const gemmi::Op &s = *super.find_by_rotation(op.rot);
if (std::none_of(super.cen_ops.begin(), super.cen_ops.end(),
[&](const gemmi::Op::Tran &cen) { return s.add_centering(cen) == moved; })) {
all = false;
break;
}
}
if (all)
return true;
}
return false;
}
std::vector<MergedReflection> ExpandToSubgroup(const std::vector<MergedReflection> &merged, const gemmi::SpaceGroup &from,
const gemmi::SpaceGroup &to) {
const gemmi::GroupOps from_ops = from.operations(), to_ops = to.operations();
const gemmi::ReciprocalAsu asu(&to);
std::set<gemmi::Miller> seen;
std::vector<MergedReflection> out;
for (const MergedReflection &r : merged)
for (const gemmi::Op &op : from_ops.sym_ops) { // the centring changes no index
const auto [h, isym] = asu.to_asu(op.apply_to_hkl({{r.h, r.k, r.l}}), to_ops);
if (!seen.insert(h).second)
continue;
MergedReflection e = r;
e.h = h[0];
e.k = h[1];
e.l = h[2];
if (isym % 2 == 0) { // the Friedel opposite
std::swap(e.I_plus, e.I_minus);
std::swap(e.sigma_plus, e.sigma_minus);
std::swap(e.F_plus, e.F_minus);
std::swap(e.sigmaF_plus, e.sigmaF_minus);
}
out.push_back(e);
}
return out;
}
RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector<MergedReflection> &merged_in,
const gemmi::UnitCell &cell, const gemmi::SpaceGroup &data_sg, double wavelength_A,
Logger &logger, const RefinementSettings &settings_in) {
// In the model's own group where that is a proper subgroup of the data's: a model validated in P2(1), its copies
// pseudo-symmetric, against a merge in P4(1)2(1)2 the data could not tell from it. Its copies refine under their
// own symmetry, against the merge expanded to the subgroup - in the data's group the operators the model does not
// have would overlay its copies, and R would be the null's. The data carry the larger group's symmetry exactly,
// so a twin law of the subgroup's lattice has nothing to be tested on.
const gemmi::SpaceGroup *own = placed.find_spacegroup();
const bool subgroup = own != nullptr && ProperSubgroup(own->operations(), data_sg.operations());
const gemmi::SpaceGroup &sg = subgroup ? *own : data_sg;
RefinementSettings settings = settings_in;
std::vector<MergedReflection> expanded;
if (subgroup) {
expanded = ExpandToSubgroup(merged_in, data_sg, sg);
settings.twin = false;
logger.Info("Refinement: in the model's {}, a subgroup of the data's {} - the copies that group would overlay "
"refined under their own symmetry, against the {} reflections expanded to {} in the subgroup; no "
"twin law tested, the data having the larger group's symmetry exactly",
sg.xhm(), data_sg.xhm(), merged_in.size(), expanded.size());
}
const std::vector<MergedReflection> &merged = subgroup ? expanded : merged_in;
// The free set closed under the lattice's twin domains once, before either arm, so that both share it.
std::vector<MergedReflection> closed;
int moved = 0;
if (settings.twin) {
closed = merged;
moved = CloseFreeSet(closed, cell, sg);
if (moved > 0)
logger.Info("Refinement: the free set closed under the lattice's twin domains, each group of twin-related "
"reflections in one set - {} reflections changed set", moved);
}
const std::vector<MergedReflection> &data = moved > 0 ? closed : merged;
#ifdef JFJOCH_USE_CUDA
const bool on_cpu = !settings.gpu || get_gpu_count() == 0;
#else
const bool on_cpu = true;
#endif
// Each arm, untwinned and twinned, is a whole refinement - the solvent built and the displacement model chosen
// against its own target - and the decision is taken on where they ended. A model only scored is scored both
// ways and decided by the same rule, so the R-free it reports is not an untwinned one by default.
// The arms run side by side, the twinned one on the next card where there is more than one, once the untwinned
// one has placed its engines and slots - and where twice what it holds of its busiest card does not fit the
// budget, after it, so that the twinned arm's engines are made as they would be alone. Its log is kept and
// follows the untwinned one's, as when they ran one after the other.
// On the CPU, where nothing in an arm depends on how many threads it has once there is no population (whose
// members are as many as the trial slots), the arms run side by side, each on half of the threads.
RefinementResult result;
if (!settings.twin)
result = RefineOnce(placed, data, cell, sg, wavelength_A, logger, settings);
else {
// on the CPU each arm on half of the threads - where the lattice admits a twin law, without which the twinned
// arm ends at once and its half would sit idle
const bool halves = on_cpu && settings.population <= 1 && settings.nthreads > 1 &&
!RefinementTwinLaws(cell, sg).empty();
RefinementSettings untwinned_settings = settings, twin_settings = settings;
// and each arm, done, gives its threads to the other
std::atomic<size_t> spare_for_untwinned{0}, spare_for_twin{0};
if (halves) {
twin_settings.nthreads = settings.nthreads / 2;
untwinned_settings.nthreads = settings.nthreads - twin_settings.nthreads;
untwinned_settings.spare_arm_threads = &spare_for_untwinned;
twin_settings.spare_arm_threads = &spare_for_twin;
}
auto twin_done = [&]() {
if (halves)
spare_for_untwinned = twin_settings.nthreads;
};
Logger twin_logger = Logger::Buffered();
CardHeld held;
std::future<size_t> held_bytes = held.bytes.get_future();
std::promise<void> untwinned_done;
const std::shared_future<void> untwinned = untwinned_done.get_future().share();
size_t budget = 0;
#ifdef JFJOCH_USE_CUDA
const int twin_card = get_gpu_count() > 1 ? (get_gpu() + 1) % get_gpu_count() : get_gpu();
if (get_gpu_count() > 0) {
const size_t usable = get_gpu_total_memory(twin_card) / 5 * 4;
budget = usable > settings.gpu_resident_bytes ? usable - settings.gpu_resident_bytes : 0;
}
#endif
std::future<void> twin_arm = std::async(std::launch::async, [&]() {
#ifdef JFJOCH_USE_CUDA
const ScopedGpuPin pin(twin_card);
#endif
// Nothing held on a card: the untwinned arm is on the CPU, where a second arm only shares its threads -
// unless each arm has half of them (halves).
const size_t held_on_card = held_bytes.get();
if (!halves && (held_on_card == 0 || 2 * held_on_card > budget))
untwinned.wait();
// A failure here (the card's memory, shared with the untwinned arm) leaves the untwinned result standing.
try {
DecideTwin(placed, data, cell, sg, wavelength_A, twin_logger, twin_settings, result, untwinned,
twin_done);
} catch (const std::exception &e) {
twin_logger.Warning("Refinement: twin hypothesis not tested - {}", e.what());
}
twin_done(); // where it ended before its refinement
});
try {
result = RefineOnce(placed, data, cell, sg, wavelength_A, logger, untwinned_settings, {}, 0, &held);
} catch (...) {
result = RefinementResult(); // not ok: the twinned arm decides nothing
held.Say(0);
untwinned_done.set_value();
twin_arm.wait();
throw;
}
held.Say(0); // where it stopped before placing its slots
if (halves)
spare_for_twin = untwinned_settings.nthreads;
untwinned_done.set_value();
twin_arm.get();
twin_logger.ReplayInto(logger);
}
result.twin_free_closed = moved;
result.space_group = sg.xhm();
result.map.reset();
return result;
}