Files
Jungfraujoch/tests/SideChainFieldTest.cpp
jungfrauandClaude Opus 5.5 f07493c9bf Side-chain field: the reflections split across every card
The field's search is iterated conditional modes over the residues, one after another, and
each step evaluates -ln L over every working reflection for each combination of states. On a
large model it ran on the main engine's card alone while the others idled: a 242k-atom model
with 45 NCS copies and 663k reflections spent 1743 s of a 2392 s refinement in it on one of
four T4s.

-ln L is a sum over the reflections, so MakeSideChainFieldEvaluatorGPUs gives each card a
contiguous part of them and adds the parts' sums in card order. The search, its order and its
decisions are unchanged; only the last bits of each sum move, where the parts meet. One card
keeps the evaluator as it was.

On that model, 4 x T4: the field 1743 -> 456 s, the refinement 2392 -> 1119 s, the refined
model bit-identical (same R-free, same md5). Two small models refine bit-identically, in the
same time.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
2026-11-17 11:27:24 +01:00

211 lines
8.9 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <cmath>
#include <complex>
#include <vector>
#include "gemmi/it92.hpp"
#include "gemmi/sfcalc.hpp"
#include "gemmi/symmetry.hpp"
#include "../image_analysis/structure_refinement/SideChainField.h"
#ifdef JFJOCH_USE_CUDA
#include <cuda_runtime.h>
#endif
namespace {
constexpr double EDGE = 24.0; // A, a P 21 21 21 box: the sum runs over four operators
constexpr double D_MIN = 2.0;
constexpr double DEG = 3.14159265358979323846 / 180.0;
// The atom bonded to c at `bond`, at `angle` b-c-x and the torsion a-b-c-x (degrees).
gemmi::Position place(const gemmi::Position &a, const gemmi::Position &b, const gemmi::Position &c, double bond,
double angle, double torsion) {
const gemmi::Vec3 bc = (c - b).normalized();
const gemmi::Vec3 n = (b - a).cross(bc).normalized();
const gemmi::Vec3 m = n.cross(bc);
const double x = -bond * std::cos(angle * DEG);
const double y = bond * std::sin(angle * DEG) * std::cos(torsion * DEG);
const double z = bond * std::sin(angle * DEG) * std::sin(torsion * DEG);
return gemmi::Position(gemmi::Vec3(c) + bc * x + m * y + n * z);
}
// A lysine at these chi angles, its main chain near the middle of the box.
gemmi::Residue lysine(double chi1, double chi2, double chi3, double chi4) {
gemmi::Residue r;
r.name = "LYS";
r.seqid = gemmi::SeqId(1, ' ');
const gemmi::Position n(10, 10, 10), ca(11.46, 10, 10);
const gemmi::Position c = place(gemmi::Position(10, 11, 10), n, ca, 1.52, 111, 180);
const gemmi::Position o = place(n, ca, c, 1.23, 121, 0);
const gemmi::Position cb = place(c, n, ca, 1.53, 110.5, 122.5);
const gemmi::Position cg = place(n, ca, cb, 1.52, 114, chi1);
const gemmi::Position cd = place(ca, cb, cg, 1.52, 111, chi2);
const gemmi::Position ce = place(cb, cg, cd, 1.52, 111, chi3);
const gemmi::Position nz = place(cg, cd, ce, 1.49, 112, chi4);
const std::pair<const char *, gemmi::Position> atoms[] = {{"N", n}, {"CA", ca}, {"C", c}, {"O", o}, {"CB", cb},
{"CG", cg}, {"CD", cd}, {"CE", ce}, {"NZ", nz}};
for (const auto &[name, pos] : atoms) {
gemmi::Atom a;
a.name = name;
a.element = gemmi::Element(name[0] == 'N' ? gemmi::El::N : name[0] == 'O' ? gemmi::El::O : gemmi::El::C);
a.pos = pos;
a.occ = 1.0f;
a.b_iso = 20.0f;
r.atoms.push_back(a);
}
return r;
}
gemmi::Structure box(const gemmi::Residue &residue) {
gemmi::Structure st;
st.cell = gemmi::UnitCell(EDGE, EDGE, EDGE, 90, 90, 90);
st.spacegroup_hm = "P 21 21 21";
st.setup_cell_images();
st.models.emplace_back(1);
st.models[0].chains.emplace_back("A");
st.models[0].chains[0].residues.push_back(residue);
return st;
}
// The reflections of the box to D_MIN, one per orbit, every one a working reflection, at D = 1 and a small variance.
FieldReflections reflections(const gemmi::Structure &st) {
const gemmi::SpaceGroup &sg = *st.find_spacegroup();
const gemmi::GroupOps gops = sg.operations();
const gemmi::ReciprocalAsu asu(&sg);
FieldReflections fr;
const int hmax = static_cast<int>(EDGE / D_MIN);
for (int h = -hmax; h <= hmax; h++)
for (int k = -hmax; k <= hmax; k++)
for (int l = -hmax; l <= hmax; l++) {
const gemmi::Miller hkl{h, k, l};
const double inv_d2 = st.cell.calculate_1_d2(hkl);
if ((h == 0 && k == 0 && l == 0) || inv_d2 > 1.0 / (D_MIN * D_MIN) || !asu.is_in(hkl))
continue;
fr.hkl.push_back({static_cast<float>(h), static_cast<float>(k), static_cast<float>(l)});
fr.stol2.push_back(static_cast<float>(inv_d2 / 4.0));
fr.fo.push_back(0);
fr.d.push_back(1.0f);
fr.var.push_back(4.0f);
fr.centric.push_back(gops.is_reflection_centric(hkl));
fr.work.push_back(1);
fr.k_total.push_back(1.0f);
fr.fmask.push_back(0);
}
return fr;
}
std::vector<FormFactor> elements() {
return {gemmi::IT92<float>::get(gemmi::El::C, 0).coefs, gemmi::IT92<float>::get(gemmi::El::N, 0).coefs,
gemmi::IT92<float>::get(gemmi::El::O, 0).coefs};
}
int element_index(gemmi::El el) { return el == gemmi::El::C ? 0 : el == gemmi::El::N ? 1 : 2; }
// Every atom of the residue as a state.
std::vector<FieldAtom> all_atoms(const gemmi::Structure &st) {
std::vector<FieldAtom> out;
for (const gemmi::Atom &a : st.models[0].chains[0].residues[0].atoms) {
const gemmi::Fractional f = st.cell.fractionalize(a.pos);
out.push_back({static_cast<float>(f.x), static_cast<float>(f.y), static_cast<float>(f.z), a.b_iso, a.occ,
element_index(a.element.elem)});
}
return out;
}
bool gpu_present() {
#ifdef JFJOCH_USE_CUDA
int n = 0;
return cudaGetDeviceCount(&n) == cudaSuccess && n > 0;
#else
return false;
#endif
}
} // namespace
TEST_CASE("SideChainField_direct_sum_matches_gemmi", "[Refinement]") {
const gemmi::Structure st = box(lysine(-60, 180, 180, 60));
const FieldReflections fr = reflections(st);
REQUIRE(fr.Size() > 500);
FieldStates states;
states.Add(all_atoms(st));
auto cpu = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(*st.find_spacegroup()), elements(), 2);
const std::vector<std::complex<float>> f = cpu->Contribution(states, 0);
gemmi::StructureFactorCalculator<gemmi::IT92<float>> calc(st.cell);
double num = 0, den = 0;
for (size_t m = 0; m < fr.Size(); m++) {
const gemmi::Miller hkl{static_cast<int>(fr.hkl[m][0]), static_cast<int>(fr.hkl[m][1]), static_cast<int>(fr.hkl[m][2])};
const std::complex<double> ref = calc.calculate_sf_from_model(st.models[0], hkl);
num += std::norm(std::complex<double>(f[m]) - ref);
den += std::norm(ref);
}
CHECK(std::sqrt(num / den) < 1e-4);
if (gpu_present()) {
#ifdef JFJOCH_USE_CUDA
auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, FieldOperators(*st.find_spacegroup()), elements());
const std::vector<std::complex<float>> g = gpu->Contribution(states, 0);
double gnum = 0;
for (size_t m = 0; m < fr.Size(); m++)
gnum += std::norm(std::complex<double>(f[m]) - std::complex<double>(g[m]));
CHECK(std::sqrt(gnum / den) < 1e-4);
#endif
}
}
TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") {
// the data: |F| of the lysine at one rotamer; the model starts at another
const gemmi::Structure truth = box(lysine(180, 180, 180, 180));
FieldReflections fr = reflections(truth);
FieldStates truth_state;
truth_state.Add(all_atoms(truth));
const std::vector<FormFactor> ff = elements();
const std::vector<std::array<float, 12>> ops = FieldOperators(*truth.find_spacegroup());
{
auto e = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2);
const std::vector<std::complex<float>> f = e->Contribution(truth_state, 0);
for (size_t m = 0; m < fr.Size(); m++)
fr.fo[m] = std::abs(f[m]);
}
const gemmi::Structure start = box(lysine(-60, 180, 60, 180));
const std::vector<int> element_of = {1, 0, 0, 2, 0, 0, 0, 0, 1};
const std::vector<char> refinable(9, 1);
auto run = [&](SideChainFieldEvaluator &e) {
FieldStates s;
s.Add(all_atoms(start));
e.SetTotal(e.Contribution(s, 0));
return OptimiseSideChainField(start, element_of, refinable, {}, 1.0, fr.Size(), e);
};
auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2);
const SideChainFieldResult r = run(*cpu);
REQUIRE(r.candidates == 1);
REQUIRE(r.moves.size() == 1);
REQUIRE(!r.moves[0].absent);
REQUIRE(r.moves[0].chi_after.size() == 4);
for (double chi : r.moves[0].chi_after)
CHECK(std::abs(std::remainder(chi - 180.0, 360.0)) < 1.0);
CHECK(r.energy_after < r.energy_before);
if (gpu_present()) {
#ifdef JFJOCH_USE_CUDA
auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, ops, ff);
const SideChainFieldResult g = run(*gpu);
REQUIRE(g.moves.size() == 1);
CHECK(g.moves[0].chi_after == r.moves[0].chi_after);
CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4));
// on every card, the reflections split among them: the same move, the energy summed in another order
int n = 0;
cudaGetDeviceCount(&n);
std::vector<int> cards;
for (int k = 0; k < n; k++)
cards.push_back(k);
auto split = MakeSideChainFieldEvaluatorGPUs(cards, fr, ops, ff);
const SideChainFieldResult s = run(*split);
REQUIRE(s.moves.size() == 1);
CHECK(s.moves[0].chi_after == g.moves[0].chi_after);
CHECK(s.energy_after == Catch::Approx(g.energy_after).epsilon(1e-9));
#endif
}
}