The reference of a joint multi-sweep merge was the best sweep on its own (resolved ISa, finest d_min, then ISa). On a set of three non-merohedrally twinned crystals the first look made the reference a sweep that had integrated the crystal in a wrong C2 setting: it reached 1.65 A and resolved an ISa, but its CC1/2 was 0.02 and no other sweep correlated with it (CC -0.02..0.04). Every other sweep then had "no axes" and was processed again given its cell, and the final merge was one sweep at 37.5% completeness - while two of the others correlated with each other at 0.92 and 0.58. Every sweep is now put on the axes of every other (ChooseReferenceSetting, all ordered pairs, which also serves the star relation that follows), and among the sweeps with a resolved ISa those on whose axes the most others reach MIN_REFERENCE_CC are the candidates; the existing rule (finest d_min within 1%, then ISa) picks among them. Where every sweep agrees with every other - the passing multi-sweep sets measured - the reference is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
259 lines
12 KiB
C++
259 lines
12 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
|
|
#include <cmath>
|
|
#include <algorithm>
|
|
#include <map>
|
|
#include <random>
|
|
|
|
#include "../rugnux/RugnuxJointSweeps.h"
|
|
|
|
namespace {
|
|
|
|
// A tetragonal P4 crystal: Laue class 4/m on a 4/mmm lattice, so (k, h, -l) indexes it a second way.
|
|
const gemmi::SpaceGroup &P4() { return *gemmi::find_spacegroup_by_name("P 4"); }
|
|
constexpr float A = 60.0f, C = 40.0f;
|
|
|
|
std::array<int, 3> P4Key(int h, int k, int l) {
|
|
// the 4/m ASU representative, enough to give equivalents one intensity
|
|
std::array<int, 3> best{h, k, l};
|
|
const std::array<std::array<int, 3>, 8> eq{{{h, k, l}, {-k, h, l}, {-h, -k, l}, {k, -h, l},
|
|
{-h, -k, -l}, {k, -h, -l}, {h, k, -l}, {-k, h, -l}}};
|
|
for (const auto &e: eq) best = std::max(best, e);
|
|
return best;
|
|
}
|
|
|
|
// Four observations of every reflection, with counting noise of `sigma`.
|
|
JointMemberSweep MakeSweep(const std::map<std::array<int, 3>, double> &truth, const gemmi::Op &indexed_as,
|
|
int n_frames, uint32_t seed, double sigma = 10.0) {
|
|
std::mt19937 rng(seed);
|
|
std::normal_distribution<double> noise(0.0, sigma);
|
|
JointMemberSweep s;
|
|
s.n_frames = n_frames;
|
|
s.cell = UnitCell{A, A, C, 90, 90, 90};
|
|
s.space_group = P4();
|
|
s.isa = 1000.0 / sigma; // what its error model would resolve: the noise level against the intensities
|
|
for (const auto &[hkl, I]: truth) {
|
|
const auto m = indexed_as.apply_to_hkl({{hkl[0], hkl[1], hkl[2]}});
|
|
const auto d = static_cast<float>(1.0 / std::sqrt((hkl[0] * hkl[0] + hkl[1] * hkl[1]) / (A * A)
|
|
+ hkl[2] * hkl[2] / (C * C)));
|
|
double sum = 0;
|
|
for (int i = 0; i < 4; ++i) {
|
|
ScaledFull f;
|
|
f.h = m[0]; f.k = m[1]; f.l = m[2];
|
|
f.I = static_cast<float>(I + noise(rng));
|
|
f.sigma = static_cast<float>(sigma);
|
|
f.image_number = static_cast<float>(rng() % n_frames);
|
|
f.var_bkg = 4.0f;
|
|
f.var_per_I = 0.5f;
|
|
f.capture = 0.0f;
|
|
f.d = d;
|
|
s.fulls.push_back(f);
|
|
sum += f.I;
|
|
}
|
|
MergedReflection r;
|
|
r.h = m[0]; r.k = m[1]; r.l = m[2];
|
|
r.I = static_cast<float>(sum / 4);
|
|
r.sigma = static_cast<float>(sigma / 2);
|
|
r.I_half[0] = (s.fulls[s.fulls.size() - 4].I + s.fulls[s.fulls.size() - 2].I) / 2;
|
|
r.I_half[1] = (s.fulls[s.fulls.size() - 3].I + s.fulls[s.fulls.size() - 1].I) / 2;
|
|
r.d = d;
|
|
s.merged.push_back(r);
|
|
}
|
|
s.d_min = s.merged.empty() ? NAN : std::min_element(s.merged.begin(), s.merged.end(), [](const auto &a, const auto &b) {
|
|
return a.d < b.d;
|
|
})->d;
|
|
return s;
|
|
}
|
|
|
|
// The reflections of `truth` finer than `d` dropped: a sweep that stops there.
|
|
std::map<std::array<int, 3>, double> CoarserThan(const std::map<std::array<int, 3>, double> &truth, double d) {
|
|
std::map<std::array<int, 3>, double> t;
|
|
for (const auto &[hkl, I]: truth)
|
|
if (1.0 / std::sqrt((hkl[0] * hkl[0] + hkl[1] * hkl[1]) / (A * A) + hkl[2] * hkl[2] / (C * C)) >= d)
|
|
t[hkl] = I;
|
|
return t;
|
|
}
|
|
|
|
std::map<std::array<int, 3>, double> Truth(uint32_t seed) {
|
|
std::mt19937 rng(seed);
|
|
std::exponential_distribution<double> wilson(1.0);
|
|
std::map<std::array<int, 3>, double> t;
|
|
for (int h = -12; h <= 12; ++h)
|
|
for (int k = -12; k <= 12; ++k)
|
|
for (int l = -8; l <= 8; ++l) {
|
|
if (!(h || k || l)) continue;
|
|
const auto key = P4Key(h, k, l);
|
|
if (!t.contains(key)) t[key] = 1000.0 * wilson(rng);
|
|
}
|
|
return t;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
TEST_CASE("RelateJointSweeps_AmbiguityRecoveredAndAStrangerLeftOut", "[Rugnux]") {
|
|
const auto truth = Truth(1);
|
|
const gemmi::Op twin = gemmi::parse_triplet("y,x,-z"); // h,k,l -> k,h,-l
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 2, 10.0)); // the reference: the best measured
|
|
sweeps.push_back(MakeSweep(truth, twin, 80, 3, 20.0));
|
|
sweeps.push_back(MakeSweep(Truth(99), gemmi::Op::identity(), 50, 4, 20.0)); // another crystal
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
REQUIRE(relation.reference == 0);
|
|
CHECK(relation.sweeps[0].included);
|
|
REQUIRE(relation.sweeps[1].included);
|
|
CHECK(relation.sweeps[1].cc > 0.99);
|
|
CHECK(relation.sweeps[1].identity_cc < 0.5);
|
|
// the operator found takes sweep 2's indices back onto the reference's (up to the Laue group)
|
|
const auto back = relation.sweeps[1].cob.apply_to_hkl(twin.apply_to_hkl({{3, 1, 2}}));
|
|
CHECK(P4Key(back[0], back[1], back[2]) == P4Key(3, 1, 2));
|
|
CHECK(relation.sweeps[1].decision == "isomorphous");
|
|
// another crystal's intensities correlate with the reference's on no axes
|
|
CHECK_FALSE(relation.sweeps[2].included);
|
|
CHECK(relation.sweeps[2].decision == "no axes");
|
|
|
|
const auto input = BuildJointMergeInput(sweeps, relation);
|
|
REQUIRE(input.segments.size() == 2);
|
|
CHECK(input.segments[0].first_frame == 0);
|
|
CHECK(input.segments[0].n_frames == 100);
|
|
CHECK(input.segments[1].first_frame == 100);
|
|
CHECK(input.segments[1].n_frames == 80);
|
|
CHECK(input.segment_sweep == std::vector<int>{0, 1});
|
|
REQUIRE(input.outcomes.size() == 180);
|
|
// a full as the combine takes it: count I/c at correction c = var_per_I, variance var_bkg / c^2
|
|
const Reflection *r = nullptr;
|
|
for (const auto &o: input.outcomes)
|
|
if (!o.reflections.empty()) { r = &o.reflections.front(); break; }
|
|
REQUIRE(r != nullptr);
|
|
CHECK(r->image_scale_corr == Catch::Approx(0.5));
|
|
CHECK(r->var_bkg == Catch::Approx(16.0));
|
|
CHECK(r->partiality == 1.0f);
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_DoubtfulSweepDecidedOnTheMerge", "[Rugnux]") {
|
|
// Sweep 1: the same crystal measured noisily, its ratio uncertain; sweep 2: a crystal whose
|
|
// intensities differ by a few percent from the reference's, measured well enough to show it. Both fall in the doubtful band, and
|
|
// the held-out residual keeps the first and leaves out the second.
|
|
const auto truth = Truth(1);
|
|
auto shifted = truth;
|
|
std::mt19937 rng(7);
|
|
std::normal_distribution<double> change(0.0, 0.14);
|
|
for (auto &[hkl, I]: shifted)
|
|
I *= std::exp(change(rng));
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 2, 50.0));
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 3, 1500.0));
|
|
sweeps.push_back(MakeSweep(shifted, gemmi::Op::identity(), 100, 4, 100.0));
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
REQUIRE(relation.reference == 0);
|
|
INFO("ratios " << relation.sweeps[1].iso_ratio << " +- " << relation.sweeps[1].iso_se << ", "
|
|
<< relation.sweeps[2].iso_ratio << " +- " << relation.sweeps[2].iso_se);
|
|
CHECK(relation.sweeps[1].decision == "doubtful: kept");
|
|
CHECK(relation.sweeps[1].included);
|
|
CHECK(relation.sweeps[1].held_out_change < 0.0);
|
|
CHECK(relation.sweeps[2].decision == "doubtful: left out");
|
|
CHECK_FALSE(relation.sweeps[2].included);
|
|
CHECK(relation.sweeps[2].held_out_change > 0.0);
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_ReferenceIsTheBestMeasuredSweepNotTheFinestResolution", "[Rugnux]") {
|
|
// Sweep 1: the crystal measured well to 5 A. Sweep 2: the same reflections, and some finer ones, all of
|
|
// them noise - what a lattice that integrated nothing but background to the edge of the detector merges,
|
|
// with no ISa its error model could resolve. Its own merge reaches the finest resolution; the sweep
|
|
// with a measured accuracy is the reference.
|
|
const auto truth = Truth(1);
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(CoarserThan(truth, 5.0), gemmi::Op::identity(), 100, 2, 10.0));
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 3, 5000.0));
|
|
sweeps[1].isa = NAN;
|
|
REQUIRE(sweeps[1].d_min < sweeps[0].d_min);
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
INFO("sweep 2: cc " << relation.sweeps[1].cc << ", ratio " << relation.sweeps[1].iso_ratio << " +- "
|
|
<< relation.sweeps[1].iso_se << ", held-out " << relation.sweeps[1].held_out_change << " +- "
|
|
<< relation.sweeps[1].held_out_se << ", " << relation.sweeps[1].decision);
|
|
CHECK(relation.reference == 0);
|
|
CHECK(relation.sweeps[0].decision == "reference");
|
|
CHECK_FALSE(relation.sweeps[1].included);
|
|
CHECK(relation.sweeps[1].decision == "no axes");
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_SweepsAtTheSameEdgeAreTiedOnResolution", "[Rugnux]") {
|
|
// Both sweeps stop at the same detector edge; the second's d_min is finer in the third decimal only, by
|
|
// the reflections it happened to measure there. That is a tie, and the higher ISa (the first) decides it.
|
|
const auto truth = Truth(1);
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 2, 10.0));
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 3, 10.0));
|
|
sweeps[0].isa = 40.0;
|
|
sweeps[1].isa = 20.0;
|
|
sweeps[1].d_min = sweeps[0].d_min * 0.995;
|
|
|
|
CHECK(RelateJointSweeps(sweeps).reference == 0);
|
|
// ... but a resolution finer beyond that is a better sweep.
|
|
sweeps[1].d_min = sweeps[0].d_min * 0.95;
|
|
CHECK(RelateJointSweeps(sweeps).reference == 1);
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_ReferenceIsASweepTheOthersAgreeWith", "[Rugnux]") {
|
|
// Two sweeps of the crystal to 3 A, and a third with a resolved ISa reaching the finest resolution of
|
|
// all whose intensities are another crystal's - what the crystal integrated in a wrong setting merges.
|
|
// On its own it is the best sweep, but no other correlates with it: the reference is one of the two.
|
|
const auto truth = Truth(1);
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(Truth(99), gemmi::Op::identity(), 100, 2, 10.0));
|
|
sweeps.push_back(MakeSweep(CoarserThan(truth, 3.0), gemmi::Op::identity(), 100, 3, 10.0));
|
|
sweeps.push_back(MakeSweep(CoarserThan(truth, 3.0), gemmi::Op::identity(), 100, 4, 20.0));
|
|
REQUIRE(sweeps[0].d_min < 0.99 * sweeps[1].d_min);
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
CHECK(relation.reference == 1);
|
|
CHECK(relation.sweeps[2].included);
|
|
CHECK_FALSE(relation.sweeps[0].included);
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_ASweepThatKeptTheTwinOverlapIsLeftOut", "[Rugnux]") {
|
|
// The reference's integration left out the reflections the other twin domain overlaps (index 2); the
|
|
// third sweep's kept them. Its intensities agree elsewhere, so only the integration tells them apart.
|
|
const auto truth = Truth(1);
|
|
std::vector<JointMemberSweep> sweeps;
|
|
for (uint32_t seed = 2; seed <= 4; ++seed)
|
|
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, seed, 10.0));
|
|
sweeps[0].isa = 200.0;
|
|
sweeps[0].twin_overlap_index = sweeps[1].twin_overlap_index = 2;
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
REQUIRE(relation.reference == 0);
|
|
CHECK(relation.sweeps[1].included);
|
|
CHECK_FALSE(relation.sweeps[2].included);
|
|
CHECK(relation.sweeps[2].decision == "twin overlap");
|
|
}
|
|
|
|
TEST_CASE("RelateJointSweeps_ASweepTheMergeCannotMeasureJoins", "[Rugnux]") {
|
|
// A cell of a few dozen unique reflections: too few for the isomorphism ratio, and the held-out
|
|
// residual of the merge moves by less than its own noise. Nothing measured refuses the sweep, so it
|
|
// joins - the second measurement of the same reflections is what a small-molecule merge needs.
|
|
std::map<std::array<int, 3>, double> small;
|
|
int n = 0;
|
|
for (const auto &[hkl, I]: Truth(1))
|
|
if (n++ % 50 == 0)
|
|
small[hkl] = I;
|
|
REQUIRE(small.size() < 200);
|
|
std::vector<JointMemberSweep> sweeps;
|
|
sweeps.push_back(MakeSweep(small, gemmi::Op::identity(), 100, 2, 200.0));
|
|
sweeps.push_back(MakeSweep(small, gemmi::Op::identity(), 100, 3, 300.0));
|
|
|
|
const auto relation = RelateJointSweeps(sweeps);
|
|
REQUIRE(relation.reference == 0);
|
|
INFO("held-out change " << relation.sweeps[1].held_out_change << " +- " << relation.sweeps[1].held_out_se
|
|
<< " on " << relation.sweeps[1].held_out_n << " reflections");
|
|
CHECK(std::isnan(relation.sweeps[1].iso_ratio));
|
|
CHECK(std::isnan(relation.sweeps[1].held_out_change));
|
|
CHECK(relation.sweeps[1].decision == "doubtful: kept");
|
|
CHECK(relation.sweeps[1].included);
|
|
}
|