Merge rc187-cytc-tail: tNCS phase classes from exp tables, live MR carried across the indexing ambiguity, tNCS sort keys

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-14 07:28:28 +01:00
commit dc984cd7e8
3 files changed
+173 -18

No files matched your search

+66 -12
View File
@@ -161,6 +161,52 @@ namespace {
return best;
}
// TncsClass - cos(2 pi h.v) averaged over `vectors` - for every h up to the order m_max, as the real
// part of exp(2 pi i h v_x) exp(2 pi i k v_y) exp(2 pi i l v_z), each factor read from a table over
// the orders: the searches below ask for it thousands of times on up to a few hundred thousand
// reflections, and a libm cosine per reflection per vector made it most of the CPU time of a whole
// run on a merge in P1.
class ClassTable {
public:
ClassTable(const std::vector<std::array<double, 3>> &vectors, int m_max)
: m_max_(m_max), width_(2 * m_max + 1), count_(vectors.size()),
re_(count_ * 3 * width_), im_(count_ * 3 * width_) {
for (size_t v = 0; v < count_; ++v)
for (int axis = 0; axis < 3; ++axis)
for (int m = -m_max; m <= m_max; ++m) {
const double phase = 2.0 * std::numbers::pi * m * vectors[v][axis];
re_[(v * 3 + axis) * width_ + (m_max + m)] = std::cos(phase);
im_[(v * 3 + axis) * width_ + (m_max + m)] = std::sin(phase);
}
}
double operator()(const std::array<int, 3> &h) const {
double c = 0.0;
for (size_t v = 0; v < count_; ++v) {
const size_t a = v * 3 * width_ + (m_max_ + h[0]);
const size_t b = (v * 3 + 1) * width_ + (m_max_ + h[1]);
const size_t d = (v * 3 + 2) * width_ + (m_max_ + h[2]);
const double ab_re = re_[a] * re_[b] - im_[a] * im_[b];
const double ab_im = re_[a] * im_[b] + im_[a] * re_[b];
c += ab_re * re_[d] - ab_im * im_[d];
}
return c / static_cast<double>(count_);
}
private:
int m_max_;
size_t width_, count_;
std::vector<double> re_, im_;
};
// The highest |index| among the first `n` reflections: the order a ClassTable for them reaches.
int MaxOrder(const std::vector<std::array<int, 3>> &hkl, size_t n) {
int m_max = 0;
for (size_t i = 0; i < n; ++i)
m_max = std::max({m_max, std::abs(hkl[i][0]), std::abs(hkl[i][1]), std::abs(hkl[i][2])});
return m_max;
}
// How well a vector explains the intensities of the first `n` reflections under the physical
// model <E^2> = A (1 + f cos(2 pi h.u)): the correlation between E^2 and cos(2 pi h.u).
//
@@ -177,10 +223,10 @@ namespace {
// With `rots` the cosine is the class averaged over them (see `modulation_averaged`).
double CosineCorrelation(const std::vector<std::array<int, 3>> &hkl, const std::vector<double> &e2,
const std::array<double, 3> &u, size_t n, const std::vector<Rot3> &rots) {
const auto vectors = Images(u, rots);
const ClassTable cls(Images(u, rots), MaxOrder(hkl, n));
double sc = 0.0, scc = 0.0, sy = 0.0, syy = 0.0, scy = 0.0;
for (size_t i = 0; i < n; ++i) {
const double c = TncsClass(hkl[i][0], hkl[i][1], hkl[i][2], vectors);
const double c = cls(hkl[i]);
sc += c;
scc += c * c;
sy += e2[i];
@@ -489,8 +535,12 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
if (std::isfinite(r.I) && std::isfinite(r.d) && acentric(r))
mod_idx.push_back(static_cast<int>(i));
}
std::sort(mod_idx.begin(), mod_idx.end(),
[&](int a, int b) { return merged[a].d != merged[b].d ? merged[a].d > merged[b].d : a < b; });
// the comparisons read d from an array of its own rather than each from a whole MergedReflection:
// this sort, of every acentric reflection, was most of the analysis' serial time on a large merge
std::vector<float> d(merged.size());
for (int i : mod_idx)
d[i] = merged[i].d;
std::sort(mod_idx.begin(), mod_idx.end(), [&](int a, int b) { return d[a] != d[b] ? d[a] > d[b] : a < b; });
const int n_beyond = static_cast<int>(std::count_if(mod_idx.begin(), mod_idx.end(),
[&](int i) { return merged[i].d >= kModulationDMin; }));
mod_idx.resize(std::min(kMaxModulationBand, std::max(n_beyond, std::min<int>(kMinModulationBand,
@@ -574,9 +624,10 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
// `rots` empty bins by the phase frac(h.u) itself; otherwise by the class averaged over the
// rotations, arccos(<cos(2 pi h.Ru)>)/pi, which for the identity alone is the same phase folded
// about 1/2 - the bins then centred on the strongest and the weakest class.
const int mod_max_order = MaxOrder(mod_hkl, mod_hkl.size());
auto contrast_over = [&](const std::vector<double> &e2v, const std::array<double, 3> &u,
std::vector<double> *profile, const std::vector<Rot3> &rots) {
const auto vectors = Images(u, rots);
const ClassTable cls(Images(u, rots), mod_max_order);
std::array<double, kPhaseBins> sum{}, cnt{};
sum.fill(0.0);
cnt.fill(0.0);
@@ -586,8 +637,7 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
const double t = mod_hkl[i][0] * u[0] + mod_hkl[i][1] * u[1] + mod_hkl[i][2] * u[2];
f = t - std::floor(t);
} else
f = std::acos(std::clamp(TncsClass(mod_hkl[i][0], mod_hkl[i][1], mod_hkl[i][2], vectors),
-1.0, 1.0)) / std::numbers::pi;
f = std::acos(std::clamp(cls(mod_hkl[i]), -1.0, 1.0)) / std::numbers::pi;
const int b = std::min(static_cast<int>(f * kPhaseBins), kPhaseBins - 1);
sum[b] += e2v[i];
cnt[b] += 1.0;
@@ -818,12 +868,16 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
idx.push_back(static_cast<int>(i));
// Equal-count shells sorted by |h|, so that every stage of the ladder reads a population
// that is already normalised and can be taken as a prefix.
std::sort(idx.begin(), idx.end(), [&](int a, int b) {
return std::max({std::abs(merged[a].h), std::abs(merged[a].k), std::abs(merged[a].l)})
< std::max({std::abs(merged[b].h), std::abs(merged[b].k), std::abs(merged[b].l)});
});
// the keys in arrays of their own, as the modulation band's sort above
std::vector<int> order(merged.size());
std::vector<float> d(merged.size());
for (int i : idx) {
order[i] = std::max({std::abs(merged[i].h), std::abs(merged[i].k), std::abs(merged[i].l)});
d[i] = merged[i].d;
}
std::sort(idx.begin(), idx.end(), [&](int a, int b) { return order[a] < order[b]; });
std::vector<int> by_d = idx;
std::sort(by_d.begin(), by_d.end(), [&](int a, int b) { return merged[a].d > merged[b].d; });
std::sort(by_d.begin(), by_d.end(), [&](int a, int b) { return d[a] > d[b]; });
std::vector<double> e2_of(merged.size(), std::numeric_limits<double>::quiet_NaN());
const size_t n_shells = std::max<size_t>(1, by_d.size() / 500);
for (size_t s = 0; s < n_shells; ++s) {
@@ -26,6 +26,8 @@
#include <gemmi/dencalc.hpp> // DensityCalculator
#include <gemmi/it92.hpp> // IT92 x-ray form factors
#include <gemmi/scaling.hpp> // Scaling (bulk solvent + anisotropic B)
#include <gemmi/twin.hpp> // find_twin_laws
#include <gemmi/asudata.hpp> // ReciprocalAsu
#include "../../common/JFJochMath.h" // PI
#include "../../common/Logger.h"
@@ -4092,6 +4094,46 @@ MolecularReplacementResult RunMolecularReplacement(const RugnuxModel &model, con
return result;
}
// The operator of the lattice's indexing ambiguity (the identity among them) that takes the reflections of `searched`
// onto those of `merge`, by the correlation of their intensities: the two passes of a run index the sweep each on
// its own, and a lattice with an ambiguity can come out in either hand of it.
static gemmi::Op AmbiguityBetween(const MRMerge &searched, const MRMerge &merge, const gemmi::SpaceGroup &sg) {
const gemmi::GroupOps gops = sg.operations();
const gemmi::ReciprocalAsu asu(&sg);
std::unordered_map<int64_t, float> intensity;
const auto key = [&](const gemmi::Miller &h) {
const gemmi::Miller a = asu.to_asu(h, gops).first;
return (static_cast<int64_t>(a[0] + 512) << 20) | (static_cast<int64_t>(a[1] + 512) << 10) | (a[2] + 512);
};
for (const MergedReflection &r : *merge.merged)
if (std::isfinite(r.I))
intensity[key({{r.h, r.k, r.l}})] = r.I;
gemmi::Op best = gemmi::Op::identity();
double best_cc = -2.0;
std::vector<gemmi::Op> ops{gemmi::Op::identity()};
for (const gemmi::Op &op : gemmi::find_twin_laws(gemmi::UnitCell(merge.cell), &sg, 3.0, false))
ops.push_back(op);
for (const gemmi::Op &op : ops) {
double n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
for (const MergedReflection &r : *searched.merged) {
if (!std::isfinite(r.I))
continue;
const auto it = intensity.find(key(op.apply_to_hkl({{r.h, r.k, r.l}})));
if (it == intensity.end())
continue;
n += 1;
sx += r.I; sy += it->second;
sxx += r.I * r.I; syy += it->second * it->second; sxy += r.I * it->second;
}
const double cc = n > 1 ? (sxy - sx * sy / n) / std::sqrt((sxx - sx * sx / n) * (syy - sy * sy / n)) : -1.0;
if (cc > best_cc) {
best_cc = cc;
best = op;
}
}
return best;
}
std::optional<MolecularReplacementResult> CarryMolecularReplacement(const MolecularReplacementResult &early,
const MRMerge &searched, const MRMerge &merge,
Logger &logger, size_t nthreads,
@@ -4117,12 +4159,18 @@ std::optional<MolecularReplacementResult> CarryMolecularReplacement(const Molecu
if (r.d > 0)
data_d_min = std::min<double>(data_d_min, r.d);
const double d_min = std::max(SCORE_D_MIN, data_d_min);
// Every copy at its fractional position in this cell.
// Every copy at its fractional position in this cell, turned by the indexing ambiguity between the two merges:
// a reflection h of `searched` is op(h) here, so a position x there is op^-1 x here.
const gemmi::Op op = AmbiguityBetween(searched, merge, sg);
const gemmi::Op to_here = op.inverse();
std::vector<gemmi::Structure> body = SplitCopies(*c.placed, c.copies);
for (gemmi::Structure &b : body) {
std::vector<gemmi::Position> pos = ModelPositions(b.models[0]);
for (gemmi::Position &x : pos)
x = cell.orthogonalize(b.cell.fractionalize(x));
for (gemmi::Position &x : pos) {
const gemmi::Fractional f = b.cell.fractionalize(x);
const std::array<double, 3> t = to_here.apply_to_xyz({f.x, f.y, f.z});
x = cell.orthogonalize(gemmi::Fractional(t[0], t[1], t[2]));
}
SetModelPositions(b.models[0], pos);
b.cell = cell;
b.spacegroup_hm = sg.xhm();
@@ -4165,9 +4213,11 @@ std::optional<MolecularReplacementResult> CarryMolecularReplacement(const Molecu
auto placed = std::make_shared<gemmi::Structure>(AssembleCopies(body));
const PlacementScore score = score_placement(*placed, sg, *merge.merged, d_min, nthreads, &scorers);
const bool holds = score.r_free <= c.r_free + R_FREE_MARGIN;
logger.Info("Molecular replacement: the search on the {} ({}): its placement in {} ({} copies, R-free {:.3f} there) refined against "
"this merge: R-work {:.3f} R-free {:.3f} ({:.2f} s) - {}", searched.name,
early.decision, sg.xhm(), c.copies, c.r_free, score.r_work, score.r_free,
logger.Info("Molecular replacement: the search on the {} ({}): its placement in {} ({} copies, R-free {:.3f} there){} "
"refined against this merge: R-work {:.3f} R-free {:.3f} ({:.2f} s) - {}", searched.name,
early.decision, sg.xhm(), c.copies, c.r_free,
op == gemmi::Op::identity() ? std::string() : ", indexed there as " + op.as_hkl().triplet('h') + " here,",
score.r_work, score.r_free,
seconds_since(t0), holds ? "it holds, not searched again" : "it does not hold, searched again");
if (!holds)
return std::nullopt;
+51
View File
@@ -13,6 +13,7 @@
#include <gemmi/dencalc.hpp>
#include <gemmi/it92.hpp>
#include <gemmi/asudata.hpp>
#include "../image_analysis/structure_refinement/ModelFFT.h"
#include "../image_analysis/structure_refinement/MolecularReplacement.h"
@@ -817,6 +818,56 @@ TEST_CASE("MolecularReplacement_CarryToAnotherMerge", "[MolecularReplacement]")
CHECK_FALSE(CarryMolecularReplacement(early, searched, {"MERGE", &merged, uc, other}, logger, 4));
}
// The two merges of one crystal in the two hands of its indexing ambiguity (P 41: k,h,-l), as two passes that index
// the sweep each on its own can give: the placement found on the one is turned onto the other (here up to a symmetry operator of the group) and
// holds there.
TEST_CASE("MolecularReplacement_CarryAcrossIndexingAmbiguity", "[MolecularReplacement]") {
const SearchModel model = synthetic_model();
const gemmi::SpaceGroup &sg = gemmi::get_spacegroup_by_name("P 41");
const gemmi::UnitCell cell(60, 60, 70, 90, 90, 90);
const gemmi::Structure truth = PlaceSearchModel(model, axis_angle({1, 2, 3}, 50), gemmi::Fractional(0.17, 0.31, 0.09),
cell, sg);
auto merged = observed(truth);
for (size_t i = 0; i < merged.size(); i++)
merged[i].rfree_flag = i % 10 == 0;
// The other hand: every reflection h of this merge is op(h) there, and the model placed there is op^-1 x.
const gemmi::Op op = gemmi::parse_triplet("y,x,-z");
const gemmi::GroupOps gops = sg.operations();
const gemmi::ReciprocalAsu asu(&sg);
std::vector<MergedReflection> other_hand = merged;
for (MergedReflection &r : other_hand) {
const gemmi::Miller h = asu.to_asu(op.apply_to_hkl({{r.h, r.k, r.l}}), gops).first;
r.h = h[0]; r.k = h[1]; r.l = h[2];
}
gemmi::Structure there = truth;
std::vector<gemmi::Position> pos = ModelPositions(there.models[0]);
for (gemmi::Position &x : pos) {
const gemmi::Fractional f = cell.fractionalize(x);
const std::array<double, 3> t = op.inverse().apply_to_xyz({f.x, f.y, f.z});
x = cell.orthogonalize(gemmi::Fractional(t[0], t[1], t[2]));
}
SetModelPositions(there.models[0], pos);
MolecularReplacementResult early;
early.ok = true;
early.chosen = 0;
early.decision = "AGREES";
MRCandidate c;
c.space_group = sg;
c.copies = 1;
c.r_free = 0.30;
c.placed = std::make_shared<gemmi::Structure>(there);
early.candidates.push_back(c);
const UnitCell uc{60, 60, 70, 90, 90, 90};
Logger logger("MolecularReplacement_CarryAcrossIndexingAmbiguity");
const auto carried = CarryMolecularReplacement(early, {"first merge", &other_hand, uc, sg},
{"MERGE", &merged, uc, sg}, logger, 4);
// the placement as it came, untouched by the ambiguity, fits this merge at R-free 0.5 and more
REQUIRE(carried);
CHECK(carried->candidates[0].r_free < 0.2);
}
#ifdef JFJOCH_USE_CUDA
TEST_CASE("MolecularReplacement_GPUMatchesCPU", "[MolecularReplacement]") {
if (get_gpu_count() == 0)