Serial stills: the indexing ambiguity resolved from the data when no reference or model is given

Where the space group is given (-S) and its lattice admits alternative indexings, a stills run without
-z or --model used to merge the crystals' random indexings together (and warned). The crystals are now
made consistent with each other before anything is scaled (ReferenceFreeReindex): each takes the
indexing in which it agrees best with all the others (Brehm & Diederichs 2014), agreement being the
sum over shared reflections of the products of shell-normalised, per-crystal standardised intensities,
weighted by 1/(1+(sigma/sd)^2). The sum over all pairs is (|M|^2 - const)/2 for M the running sum, so a
pass over the crystals is one pass over the reflections rather than over all pairs: coordinate ascent,
four seeded starts, stop when <= 1 crystal in 1000 moves, the overall relabel that moves the fewest
crystals. Deterministic (fixed summation order, sequential passes).

A twin law of a pseudo-merohedral cell whose axes differ by a few per cent is decided by the crystals'
own refined cells, which the indexer has already used: where >= 90% of the crystals' cells fit better as
indexed, those crystals are anchors and only the others are decided by the intensities. Without this,
two public sets got worse (pal90 R-free 0.183 -> 0.207, lcls242_t 0.296 -> 0.350).

With a reference or model the per-image choice stands; the same resolution runs as a check, reported as
AMBIGUITY_DATA_DISAGREES. New report keys (section 2): INDEXING_AMBIGUITY_RESOLVED, AMBIGUITY_CRYSTALS,
AMBIGUITY_REINDEXED, AMBIGUITY_SEPARATION_SIGMA, AMBIGUITY_DECIDED_FRACTION (+ AMBIGUITY_TIME_S, dev).
--mode scale is not changed.

Still battery, no --model (R-free, REFMAC 0-cycle against the deposited model; before -> after; with
--model for comparison): euxfel83 0.321 -> 0.207 (0.204), euxfel152 0.346 -> 0.257 (0.245), alvra213
0.209 -> 0.201 (0.207), alvra213_int16 0.210 -> 0.202 (0.208), pal90 0.183 -> 0.183 (0.188), lcls242_t
0.296 -> 0.294 (0.285); controls sacla237 / lcls222 unchanged (no ambiguity, no-op). The step takes
0.2-1.6 s on 2.7k-54k crystals; synthetic 64k crystals x 200 reflections 0.5 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
(cherry picked from commit 80d378a1fe92364fd5b99a17f5c9ee8498412d36)

tests/ReindexAmbiguityTest.cpp

@@ -3,7 +3,10 @@
 
 #include <catch2/catch_all.hpp>
 
+#include <algorithm>
+#include <array>
 #include <cmath>
+#include <random>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -11,6 +14,7 @@
 #include "../image_analysis/scale_merge/CrystalSetting.h"
 #include "../image_analysis/scale_merge/HKLKey.h"
 #include "../image_analysis/scale_merge/ReindexAmbiguity.h"
+#include "../image_analysis/scale_merge/ReferenceFreeReindex.h"
 #include "../image_analysis/scale_merge/SublatticeRelabel.h"
 
 namespace {
@@ -306,3 +310,158 @@ TEST_CASE("Sublattice: a frame on the given cell in another setting is described
     // An unrelated cell is not put on it.
     CHECK_FALSE(CellOnReference(UnitCell{55.0f, 66.0f, 77.0f, 90.0f, 90.0f, 90.0f}, given, settings).has_value());
 }
+
+TEST_CASE("Reindex: a stills set indexed in random hands is made consistent from the data alone", "[reindex]") {
+    // P4 on a tetragonal lattice: two indexings, related by a twin law. Every crystal records a random
+    // part of the same synthetic structure with its own scale and noise, and half of them, at random,
+    // carry the other indexing.
+    const UnitCell cell = Tetragonal();
+    DiffractionExperiment x;
+    x.SetUnitCell(cell).SpaceGroupNumber(75);
+    const auto laws = ReindexAmbiguityOperators(cell, x.GetSpaceGroupOrP1());
+    REQUIRE(laws.size() == 1);
+
+    const HKLKeyGenerator key(true, x.GetSpaceGroupOrP1());
+    std::mt19937 rng(7);
+    std::exponential_distribution<double> wilson(1.0);
+    std::normal_distribution<double> noise(0.0, 0.3);
+    std::unordered_map<uint64_t, double> structure;
+    std::vector<std::array<int, 3>> hkls;
+    for (int h = -12; h <= 12; ++h)
+        for (int k = -12; k <= 12; ++k)
+            for (int l = -6; l <= 6; ++l) {
+                if (h == 0 && k == 0 && l == 0)
+                    continue;
+                hkls.push_back({h, k, l});
+                structure.try_emplace(key(h, k, l).pack(), wilson(rng));
+            }
+
+    const int n_crystals = 300;
+    std::vector<IntegrationOutcome> outcomes(n_crystals);
+    std::vector<bool> flipped(n_crystals);
+    for (int c = 0; c < n_crystals; ++c) {
+        flipped[c] = rng() % 2;
+        const double scale = 0.5 + (rng() % 1000) / 1000.0;
+        outcomes[c].latt = CrystalLattice(cell);
+        for (int i = 0; i < 150; ++i) {
+            const auto &t = hkls[rng() % hkls.size()];
+            const auto m = flipped[c] ? laws[0].apply_to_hkl({{t[0], t[1], t[2]}}) : gemmi::Op::Miller{{t[0], t[1], t[2]}};
+            Reflection r{};
+            r.h = m[0]; r.k = m[1]; r.l = m[2];
+            r.d = 1.0f / std::sqrt((t[0] * t[0] + t[1] * t[1]) / (cell.a * cell.a) + t[2] * t[2] / (cell.c * cell.c));
+            r.I = static_cast<float>(scale * structure.at(key(t[0], t[1], t[2]).pack()) * (1.0 + noise(rng)));
+            r.sigma = 1.0f;
+            r.partiality = 1.0f;
+            r.prescaling_corr = 1.0f;
+            outcomes[c].reflections.push_back(r);
+        }
+    }
+
+    const auto result = ResolveReindexReferenceFree(outcomes, x, cell, laws, 4, /*apply=*/true);
+    CHECK(result.crystals == n_crystals);
+    // The overall hand is free: either exactly the flipped crystals are reindexed, or exactly the others.
+    int agree = 0;
+    for (int c = 0; c < n_crystals; ++c)
+        agree += (result.choice[c] != 0) == flipped[c];
+    CHECK((agree == n_crystals || agree == 0));
+    const size_t n_flipped = std::count(flipped.begin(), flipped.end(), true);
+    CHECK(result.reindexed == std::min(n_flipped, n_crystals - n_flipped));
+    CHECK(result.reindexed <= n_crystals / 2);   // the overall hand that moves the fewest crystals
+    CHECK(result.decided_fraction > 0.9);
+
+    // Done again on the consistent set, nothing moves.
+    const auto again = ResolveReindexReferenceFree(outcomes, x, cell, laws, 4, /*apply=*/false);
+    CHECK(again.reindexed == 0);
+}
+
+TEST_CASE("Reindex: a holohedral crystal gives the stills resolution nothing to do", "[reindex]") {
+    DiffractionExperiment x;
+    x.SetUnitCell(Tetragonal()).SpaceGroupNumber(89);   // P422
+    std::vector<IntegrationOutcome> outcomes(3);
+    const auto laws = ReindexAmbiguityOperators(Tetragonal(), x.GetSpaceGroupOrP1());
+    const auto result = ResolveReindexReferenceFree(outcomes, x, Tetragonal(), laws, 1, true);
+    CHECK(laws.empty());
+    CHECK(result.reindexed == 0);
+    CHECK(result.crystals == 0);
+}
+
+TEST_CASE("Reindex: an operator the crystals' own cells decide is left to them", "[reindex]") {
+    // Orthorhombic with a and b 3% apart: a twin law of the metric to within the obliquity tolerance,
+    // but every crystal's refined cell says which axis is which - nothing for the data to decide.
+    const UnitCell cell{78.0f, 80.4f, 37.0f, 90.0f, 90.0f, 90.0f};
+    DiffractionExperiment x;
+    x.SetUnitCell(cell).SpaceGroupNumber(16);   // P222
+    const auto laws = ReindexAmbiguityOperators(cell, x.GetSpaceGroupOrP1());
+    REQUIRE_FALSE(laws.empty());
+
+    std::vector<IntegrationOutcome> outcomes(50);
+    std::mt19937 rng(3);
+    for (auto &o : outcomes) {
+        o.latt = CrystalLattice(cell);
+        for (int i = 0; i < 100; ++i) {
+            Reflection r{};
+            r.h = static_cast<int>(rng() % 15) - 7; r.k = static_cast<int>(rng() % 15) - 7; r.l = static_cast<int>(rng() % 7) + 1;
+            r.d = 3.0f; r.I = static_cast<float>(rng() % 1000); r.sigma = 1.0f; r.partiality = 1.0f; r.prescaling_corr = 1.0f;
+            o.reflections.push_back(r);
+        }
+    }
+    const auto result = ResolveReindexReferenceFree(outcomes, x, cell, laws, 1, true);
+    REQUIRE(result.metric_preference.size() == laws.size());
+    for (double p : result.metric_preference)
+        CHECK(p == 1.0);
+    CHECK(result.crystals == 0);
+    CHECK(result.reindexed == 0);
+}
+
+TEST_CASE("Reindex: where the cells decide, the crystals whose cell disagrees are decided by the intensities",
+          "[reindex]") {
+    // As above, a and b 3% apart, but a few crystals were indexed on swapped axes: their own cells say so,
+    // and the intensities, read against the crystals the cells anchor, put them back.
+    const UnitCell cell{78.0f, 80.4f, 37.0f, 90.0f, 90.0f, 90.0f};
+    DiffractionExperiment x;
+    x.SetUnitCell(cell).SpaceGroupNumber(16);   // P222
+    const auto laws = ReindexAmbiguityOperators(cell, x.GetSpaceGroupOrP1());
+    REQUIRE(laws.size() == 1);
+
+    const HKLKeyGenerator key(true, x.GetSpaceGroupOrP1());
+    std::mt19937 rng(11);
+    std::exponential_distribution<double> wilson(1.0);
+    std::normal_distribution<double> noise(0.0, 0.3);
+    std::unordered_map<uint64_t, double> structure;
+    std::vector<std::array<int, 3>> hkls;
+    for (int h = -12; h <= 12; ++h)
+        for (int k = -12; k <= 12; ++k)
+            for (int l = -6; l <= 6; ++l)
+                if (h != 0 || k != 0 || l != 0) {
+                    hkls.push_back({h, k, l});
+                    structure.try_emplace(key(h, k, l).pack(), wilson(rng));
+                }
+
+    const int n_crystals = 300;
+    std::vector<IntegrationOutcome> outcomes(n_crystals);
+    std::vector<bool> swapped(n_crystals, false);
+    for (int c = 0; c < n_crystals; ++c) {
+        swapped[c] = c % 12 == 5;   // 25 of 300: the cells still decide the law
+        outcomes[c].latt = swapped[c] ? ReindexLattice(CrystalLattice(cell), laws[0]) : CrystalLattice(cell);
+        for (int i = 0; i < 150; ++i) {
+            const auto &t = hkls[rng() % hkls.size()];
+            const auto m = swapped[c] ? laws[0].apply_to_hkl({{t[0], t[1], t[2]}}) : gemmi::Op::Miller{{t[0], t[1], t[2]}};
+            Reflection r{};
+            r.h = m[0]; r.k = m[1]; r.l = m[2];
+            r.d = 1.0f / std::sqrt(t[0] * t[0] / (cell.a * cell.a) + t[1] * t[1] / (cell.b * cell.b)
+                                   + t[2] * t[2] / (cell.c * cell.c));
+            r.I = static_cast<float>(structure.at(key(t[0], t[1], t[2]).pack()) * (1.0 + noise(rng)));
+            r.sigma = 0.1f;
+            r.partiality = 1.0f;
+            r.prescaling_corr = 1.0f;
+            outcomes[c].reflections.push_back(r);
+        }
+    }
+
+    const auto result = ResolveReindexReferenceFree(outcomes, x, cell, laws, 4, /*apply=*/true);
+    REQUIRE(result.metric_preference.size() == 1);
+    CHECK(result.metric_preference[0] >= METRIC_DECIDED);
+    CHECK(result.crystals == 25);
+    for (int c = 0; c < n_crystals; ++c)
+        CHECK((result.choice[c] != 0) == swapped[c]);
+}