The field's search is iterated conditional modes over the residues, one after another, and each step evaluates -ln L over every working reflection for each combination of states. On a large model it ran on the main engine's card alone while the others idled: a 242k-atom model with 45 NCS copies and 663k reflections spent 1743 s of a 2392 s refinement in it on one of four T4s. -ln L is a sum over the reflections, so MakeSideChainFieldEvaluatorGPUs gives each card a contiguous part of them and adds the parts' sums in card order. The search, its order and its decisions are unchanged; only the last bits of each sum move, where the parts meet. One card keeps the evaluator as it was. On that model, 4 x T4: the field 1743 -> 456 s, the refinement 2392 -> 1119 s, the refined model bit-identical (same R-free, same md5). Two small models refine bit-identically, in the same time. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
211 lines
8.9 KiB
C++
211 lines
8.9 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
|
|
#include <cmath>
|
|
#include <complex>
|
|
#include <vector>
|
|
|
|
#include "gemmi/it92.hpp"
|
|
#include "gemmi/sfcalc.hpp"
|
|
#include "gemmi/symmetry.hpp"
|
|
#include "../image_analysis/structure_refinement/SideChainField.h"
|
|
#ifdef JFJOCH_USE_CUDA
|
|
#include <cuda_runtime.h>
|
|
#endif
|
|
|
|
namespace {
|
|
|
|
constexpr double EDGE = 24.0; // A, a P 21 21 21 box: the sum runs over four operators
|
|
constexpr double D_MIN = 2.0;
|
|
constexpr double DEG = 3.14159265358979323846 / 180.0;
|
|
|
|
// The atom bonded to c at `bond`, at `angle` b-c-x and the torsion a-b-c-x (degrees).
|
|
gemmi::Position place(const gemmi::Position &a, const gemmi::Position &b, const gemmi::Position &c, double bond,
|
|
double angle, double torsion) {
|
|
const gemmi::Vec3 bc = (c - b).normalized();
|
|
const gemmi::Vec3 n = (b - a).cross(bc).normalized();
|
|
const gemmi::Vec3 m = n.cross(bc);
|
|
const double x = -bond * std::cos(angle * DEG);
|
|
const double y = bond * std::sin(angle * DEG) * std::cos(torsion * DEG);
|
|
const double z = bond * std::sin(angle * DEG) * std::sin(torsion * DEG);
|
|
return gemmi::Position(gemmi::Vec3(c) + bc * x + m * y + n * z);
|
|
}
|
|
|
|
// A lysine at these chi angles, its main chain near the middle of the box.
|
|
gemmi::Residue lysine(double chi1, double chi2, double chi3, double chi4) {
|
|
gemmi::Residue r;
|
|
r.name = "LYS";
|
|
r.seqid = gemmi::SeqId(1, ' ');
|
|
const gemmi::Position n(10, 10, 10), ca(11.46, 10, 10);
|
|
const gemmi::Position c = place(gemmi::Position(10, 11, 10), n, ca, 1.52, 111, 180);
|
|
const gemmi::Position o = place(n, ca, c, 1.23, 121, 0);
|
|
const gemmi::Position cb = place(c, n, ca, 1.53, 110.5, 122.5);
|
|
const gemmi::Position cg = place(n, ca, cb, 1.52, 114, chi1);
|
|
const gemmi::Position cd = place(ca, cb, cg, 1.52, 111, chi2);
|
|
const gemmi::Position ce = place(cb, cg, cd, 1.52, 111, chi3);
|
|
const gemmi::Position nz = place(cg, cd, ce, 1.49, 112, chi4);
|
|
const std::pair<const char *, gemmi::Position> atoms[] = {{"N", n}, {"CA", ca}, {"C", c}, {"O", o}, {"CB", cb},
|
|
{"CG", cg}, {"CD", cd}, {"CE", ce}, {"NZ", nz}};
|
|
for (const auto &[name, pos] : atoms) {
|
|
gemmi::Atom a;
|
|
a.name = name;
|
|
a.element = gemmi::Element(name[0] == 'N' ? gemmi::El::N : name[0] == 'O' ? gemmi::El::O : gemmi::El::C);
|
|
a.pos = pos;
|
|
a.occ = 1.0f;
|
|
a.b_iso = 20.0f;
|
|
r.atoms.push_back(a);
|
|
}
|
|
return r;
|
|
}
|
|
|
|
gemmi::Structure box(const gemmi::Residue &residue) {
|
|
gemmi::Structure st;
|
|
st.cell = gemmi::UnitCell(EDGE, EDGE, EDGE, 90, 90, 90);
|
|
st.spacegroup_hm = "P 21 21 21";
|
|
st.setup_cell_images();
|
|
st.models.emplace_back(1);
|
|
st.models[0].chains.emplace_back("A");
|
|
st.models[0].chains[0].residues.push_back(residue);
|
|
return st;
|
|
}
|
|
|
|
// The reflections of the box to D_MIN, one per orbit, every one a working reflection, at D = 1 and a small variance.
|
|
FieldReflections reflections(const gemmi::Structure &st) {
|
|
const gemmi::SpaceGroup &sg = *st.find_spacegroup();
|
|
const gemmi::GroupOps gops = sg.operations();
|
|
const gemmi::ReciprocalAsu asu(&sg);
|
|
FieldReflections fr;
|
|
const int hmax = static_cast<int>(EDGE / D_MIN);
|
|
for (int h = -hmax; h <= hmax; h++)
|
|
for (int k = -hmax; k <= hmax; k++)
|
|
for (int l = -hmax; l <= hmax; l++) {
|
|
const gemmi::Miller hkl{h, k, l};
|
|
const double inv_d2 = st.cell.calculate_1_d2(hkl);
|
|
if ((h == 0 && k == 0 && l == 0) || inv_d2 > 1.0 / (D_MIN * D_MIN) || !asu.is_in(hkl))
|
|
continue;
|
|
fr.hkl.push_back({static_cast<float>(h), static_cast<float>(k), static_cast<float>(l)});
|
|
fr.stol2.push_back(static_cast<float>(inv_d2 / 4.0));
|
|
fr.fo.push_back(0);
|
|
fr.d.push_back(1.0f);
|
|
fr.var.push_back(4.0f);
|
|
fr.centric.push_back(gops.is_reflection_centric(hkl));
|
|
fr.work.push_back(1);
|
|
fr.k_total.push_back(1.0f);
|
|
fr.fmask.push_back(0);
|
|
}
|
|
return fr;
|
|
}
|
|
|
|
std::vector<FormFactor> elements() {
|
|
return {gemmi::IT92<float>::get(gemmi::El::C, 0).coefs, gemmi::IT92<float>::get(gemmi::El::N, 0).coefs,
|
|
gemmi::IT92<float>::get(gemmi::El::O, 0).coefs};
|
|
}
|
|
int element_index(gemmi::El el) { return el == gemmi::El::C ? 0 : el == gemmi::El::N ? 1 : 2; }
|
|
|
|
// Every atom of the residue as a state.
|
|
std::vector<FieldAtom> all_atoms(const gemmi::Structure &st) {
|
|
std::vector<FieldAtom> out;
|
|
for (const gemmi::Atom &a : st.models[0].chains[0].residues[0].atoms) {
|
|
const gemmi::Fractional f = st.cell.fractionalize(a.pos);
|
|
out.push_back({static_cast<float>(f.x), static_cast<float>(f.y), static_cast<float>(f.z), a.b_iso, a.occ,
|
|
element_index(a.element.elem)});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
bool gpu_present() {
|
|
#ifdef JFJOCH_USE_CUDA
|
|
int n = 0;
|
|
return cudaGetDeviceCount(&n) == cudaSuccess && n > 0;
|
|
#else
|
|
return false;
|
|
#endif
|
|
}
|
|
|
|
} // namespace
|
|
|
|
TEST_CASE("SideChainField_direct_sum_matches_gemmi", "[Refinement]") {
|
|
const gemmi::Structure st = box(lysine(-60, 180, 180, 60));
|
|
const FieldReflections fr = reflections(st);
|
|
REQUIRE(fr.Size() > 500);
|
|
FieldStates states;
|
|
states.Add(all_atoms(st));
|
|
auto cpu = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(*st.find_spacegroup()), elements(), 2);
|
|
const std::vector<std::complex<float>> f = cpu->Contribution(states, 0);
|
|
gemmi::StructureFactorCalculator<gemmi::IT92<float>> calc(st.cell);
|
|
double num = 0, den = 0;
|
|
for (size_t m = 0; m < fr.Size(); m++) {
|
|
const gemmi::Miller hkl{static_cast<int>(fr.hkl[m][0]), static_cast<int>(fr.hkl[m][1]), static_cast<int>(fr.hkl[m][2])};
|
|
const std::complex<double> ref = calc.calculate_sf_from_model(st.models[0], hkl);
|
|
num += std::norm(std::complex<double>(f[m]) - ref);
|
|
den += std::norm(ref);
|
|
}
|
|
CHECK(std::sqrt(num / den) < 1e-4);
|
|
if (gpu_present()) {
|
|
#ifdef JFJOCH_USE_CUDA
|
|
auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, FieldOperators(*st.find_spacegroup()), elements());
|
|
const std::vector<std::complex<float>> g = gpu->Contribution(states, 0);
|
|
double gnum = 0;
|
|
for (size_t m = 0; m < fr.Size(); m++)
|
|
gnum += std::norm(std::complex<double>(f[m]) - std::complex<double>(g[m]));
|
|
CHECK(std::sqrt(gnum / den) < 1e-4);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") {
|
|
// the data: |F| of the lysine at one rotamer; the model starts at another
|
|
const gemmi::Structure truth = box(lysine(180, 180, 180, 180));
|
|
FieldReflections fr = reflections(truth);
|
|
FieldStates truth_state;
|
|
truth_state.Add(all_atoms(truth));
|
|
const std::vector<FormFactor> ff = elements();
|
|
const std::vector<std::array<float, 12>> ops = FieldOperators(*truth.find_spacegroup());
|
|
{
|
|
auto e = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2);
|
|
const std::vector<std::complex<float>> f = e->Contribution(truth_state, 0);
|
|
for (size_t m = 0; m < fr.Size(); m++)
|
|
fr.fo[m] = std::abs(f[m]);
|
|
}
|
|
const gemmi::Structure start = box(lysine(-60, 180, 60, 180));
|
|
const std::vector<int> element_of = {1, 0, 0, 2, 0, 0, 0, 0, 1};
|
|
const std::vector<char> refinable(9, 1);
|
|
auto run = [&](SideChainFieldEvaluator &e) {
|
|
FieldStates s;
|
|
s.Add(all_atoms(start));
|
|
e.SetTotal(e.Contribution(s, 0));
|
|
return OptimiseSideChainField(start, element_of, refinable, {}, 1.0, fr.Size(), e);
|
|
};
|
|
auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2);
|
|
const SideChainFieldResult r = run(*cpu);
|
|
REQUIRE(r.candidates == 1);
|
|
REQUIRE(r.moves.size() == 1);
|
|
REQUIRE(!r.moves[0].absent);
|
|
REQUIRE(r.moves[0].chi_after.size() == 4);
|
|
for (double chi : r.moves[0].chi_after)
|
|
CHECK(std::abs(std::remainder(chi - 180.0, 360.0)) < 1.0);
|
|
CHECK(r.energy_after < r.energy_before);
|
|
if (gpu_present()) {
|
|
#ifdef JFJOCH_USE_CUDA
|
|
auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, ops, ff);
|
|
const SideChainFieldResult g = run(*gpu);
|
|
REQUIRE(g.moves.size() == 1);
|
|
CHECK(g.moves[0].chi_after == r.moves[0].chi_after);
|
|
CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4));
|
|
// on every card, the reflections split among them: the same move, the energy summed in another order
|
|
int n = 0;
|
|
cudaGetDeviceCount(&n);
|
|
std::vector<int> cards;
|
|
for (int k = 0; k < n; k++)
|
|
cards.push_back(k);
|
|
auto split = MakeSideChainFieldEvaluatorGPUs(cards, fr, ops, ff);
|
|
const SideChainFieldResult s = run(*split);
|
|
REQUIRE(s.moves.size() == 1);
|
|
CHECK(s.moves[0].chi_after == g.moves[0].chi_after);
|
|
CHECK(s.energy_after == Catch::Approx(g.energy_after).epsilon(1e-9));
|
|
#endif
|
|
}
|
|
}
|