Jungfraujoch/tests/SearchSpaceGroupTest.cpp

#include <catch2/catch_all.hpp>

#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/scale_merge/CentreOfSymmetry.h"
#include "gemmi/symmetry.hpp"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <random>
#include <string>
#include <tuple>
#include <unordered_set>
#include <vector>

namespace {
    struct HKL {
        int h = 0;
        int k = 0;
        int l = 0;

        bool operator==(const HKL& o) const noexcept {
            return h == o.h && k == o.k && l == o.l;
        }
    };

    struct HKLHash {
        size_t operator()(const HKL& x) const noexcept {
            auto mix = [](uint64_t v) {
                v ^= v >> 33;
                v *= 0xff51afd7ed558ccdULL;
                v ^= v >> 33;
                v *= 0xc4ceb9fe1a85ec53ULL;
                v ^= v >> 33;
                return v;
            };
            return static_cast<size_t>(
                mix(static_cast<uint64_t>(x.h)) ^
                (mix(static_cast<uint64_t>(x.k)) << 1) ^
                (mix(static_cast<uint64_t>(x.l)) << 2));
        }
    };

    double CalcSyntheticD(int h, int k, int l) {
        const double q2 = static_cast<double>(h * h + k * k + l * l);
        return 40.0 / std::sqrt(q2 + 1.0);
    }

    double SyntheticIntensityFromAsu(const gemmi::Op::Miller& asu) {
        uint64_t x = static_cast<uint64_t>((asu[0] + 31) * 73856093u) ^
                     static_cast<uint64_t>((asu[1] + 37) * 19349663u) ^
                     static_cast<uint64_t>((asu[2] + 41) * 83492791u);
        x ^= x >> 13;
        x *= 0x9e3779b97f4a7c15ULL;
        x ^= x >> 17;
        return 100.0 + static_cast<double>(x % 500);
    }

    std::vector<MergedReflection> GenerateMergedReflectionsForSpaceGroup(
        const gemmi::SpaceGroup& sg,
        int hmax = 8) {

        std::vector<MergedReflection> merged;
        std::unordered_set<HKL, HKLHash> added;

        const gemmi::GroupOps gops = sg.operations();
        const gemmi::ReciprocalAsu rasu(&sg);

        for (int h = -hmax; h <= hmax; ++h) {
            for (int k = -hmax; k <= hmax; ++k) {
                for (int l = -hmax; l <= hmax; ++l) {
                    if (h == 0 && k == 0 && l == 0)
                        continue;

                    bool absent = false;
                    gemmi::Op::Miller hkl{{h, k, l}};
                    if (gops.is_systematically_absent(hkl))
                        absent = true;

                    const auto [asu, sign_plus] = rasu.to_asu_sign(hkl, gops);
                    if (!sign_plus)
                        continue;

                    const HKL key{h, k, l};
                    if (added.find(key) != added.end())
                        continue;
                    added.insert(key);

                    merged.push_back(MergedReflection{
                        .h = h,
                        .k = k,
                        .l = l,
                        .I = static_cast<float>(absent ? 0.0 : SyntheticIntensityFromAsu(asu)),
                        .sigma = 1.0,
                        .d = static_cast<float>(CalcSyntheticD(h, k, l))
                    });
                }
            }
        }

        return merged;
    }
}

TEST_CASE("SearchSpaceGroup detects synthetic space groups") {
    struct Case {
        std::string input_name;
        std::string expected_short_name;
    };

    const std::vector<Case> cases = {
        {"P 1", "P1"},
        {"P 1 2 1", "P2"},
        {"P 3 2 1", "P321"},
        {"P 4 2 2", "P422"},
        {"P 4 3 2", "P432"},
        {"P 43 21 2", "P43212"},
        {"P 6 2 2", "P622"},
        {"C 1 2 1", "C2"},
        {"C 2 2 2", "C222"},
        {"I 4 3 2", "I432"},
        {"I 21 21 21", "I212121"},
        {"I 2 1 3", "I213"},
    };

    for (const auto& tc : cases) {
        DYNAMIC_SECTION(tc.expected_short_name) {
            const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name(tc.input_name);
            const auto merged = GenerateMergedReflectionsForSpaceGroup(sg);

            SearchSpaceGroupOptions opt;
            opt.merge_friedel = true;

            const auto result = SearchSpaceGroup(merged, opt);

            // Several inputs cannot be told apart from intensities alone: enantiomorphic partners
            // (P4_3 vs P4_1) and origin-ambiguous pairs (I2_12_12_1 vs I222, I2_13 vs I2_3) share
            // the same systematic absences. The search reports those as alternatives, so the
            // expected group must appear among the best group and its alternatives.
            std::vector<std::string> accepted;
            if (result.best_space_group.has_value())
                accepted.push_back(result.best_space_group->short_name());
            for (const auto& alt : result.alternatives)
                accepted.push_back(alt.short_name());

            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            CHECK(std::find(accepted.begin(), accepted.end(), tc.expected_short_name) != accepted.end());
        }
    }
}

// Regression: a real screw axis whose systematically-absent reflections carry a genuinely weak
// intensity but an UNDER-estimated sigma (so their I/sigma clears the "present" cut) must still be
// found. Reproduces a monoclinic 2_1 miss on weakly-diffracting monoclinic data, where the merged sigmas on
// the 0k0-odd reflections were ~2x too small and faked screw-axis violations. The E^2 intensity gate
// (present_e_squared) is what keeps those reflections classified absent.
TEST_CASE("SearchSpaceGroup finds a screw axis despite under-estimated sigmas on absent reflections") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 21 1");
    auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 18);

    // Every systematically-absent (0k0, k odd) reflection: small-but-nonzero intensity (~2% of a
    // normal reflection) with a far-too-small sigma, so I/sigma ~ 27 fakes a "present" reflection.
    const gemmi::GroupOps gops = sg.operations();
    int absent_count = 0;
    for (auto& r : merged) {
        const gemmi::Op::Miller hkl{{r.h, r.k, r.l}};
        if (gops.is_systematically_absent(hkl)) {
            r.I = 8.0f;
            r.sigma = 0.3f;
            ++absent_count;
        }
    }
    REQUIRE(absent_count >= 8);   // enough predicted-absent reflections to be trusted

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    SECTION("intensity gate on (default): screw recovered") {
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->short_name() == "P21");
    }

    SECTION("intensity gate off (I/sigma only): the deferral recovers the screw anyway") {
        // This section used to document the failure the E^2 gate fixes - with I/sigma alone the
        // too-small sigmas fake violations and the search fell back to the symmorphic group. There
        // are now TWO independent defences and the second one holds here without the first: the
        // fabricated violations are still counted, but the absent class sits at 2% of its own row,
        // so the zone is dead per reflection and carries no measured pseudo-translation, which is
        // what licenses the absence evidence to override the count.
        //
        // Kept rather than deleted, because it pins the two apart: if a future change makes this
        // read P2 again, the deferral has stopped licensing a zone that is genuinely extinct.
        opt.present_e_squared = 0.0;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->short_name() == "P21");
    }
}

// Regression: the E^2 gate above compares a reflection to the mean of its RESOLUTION SHELL, which
// falls off with resolution, while a systematically-absent reflection keeps a small non-decaying
// residual (background / profile leakage). On a crystal whose axial rows are much stronger than an
// average reflection, that turns the high-resolution residuals into screw-axis violations and the
// screw is lost, although the reflections beside them in the same row are tens of times stronger.
// A tetragonal 42_12 case failed exactly this way (18 of 47 absent 00l over the cut, all beyond
// 3.7 A, at 1-2% of the l=4n reflections next to them). The threshold is therefore taken relative to
// the axial row the screw constrains, not to the shell.
TEST_CASE("SearchSpaceGroup finds a screw axis whose absent class is weak only within its own row") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 43 21 2");
    auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 12);

    // Axial rows 40x stronger than a general reflection, and an absent class carrying ~2% of its own
    // row - but half of a general reflection, so a threshold set against the shell calls every one of
    // them a violation while a threshold set against the row calls none.
    const gemmi::GroupOps gops = sg.operations();
    int absent_on_axis = 0;
    for (auto& r : merged) {
        const gemmi::Op::Miller hkl{{r.h, r.k, r.l}};
        if (gops.epsilon_factor_without_centering(hkl) <= 1)
            continue;
        if (gops.is_systematically_absent(hkl)) {
            r.I = 300.0f;
            r.sigma = 1.0f;
            ++absent_on_axis;
        } else {
            r.I *= 40.0f;
        }
    }
    REQUIRE(absent_on_axis >= 8);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    const auto result = SearchSpaceGroup(merged, opt);
    INFO(SearchSpaceGroupResultToText(result));
    REQUIRE(result.best_space_group.has_value());
    // P4_1 2_1 2 and P4_3 2_1 2 are enantiomorphs and indistinguishable from intensities.
    std::vector<std::string> accepted{result.best_space_group->short_name()};
    for (const auto& alt : result.alternatives)
        accepted.push_back(alt.short_name());
    CHECK(std::find(accepted.begin(), accepted.end(), "P43212") != accepted.end());
}
// Regression: a screw's predicted-absent class is one row of reciprocal space, and that row is often
// the one a rotation sweep records least - it lies near the spindle, where the blind cusp maps onto
// itself and symmetry cannot fill it in. Counting the class therefore measures the geometry of the
// sweep, not the strength of the evidence, and a count gate refused a monoclinic crystal its 2_1 for
// having six 0k0-odd reflections rather than eight, every one of them measured at a thousandth of the
// row beside them. The class is judged by ScrewZoneEvidence instead, which reads the contrast
// against the row - so few-but-decisive is accepted and many-but-marginal is not.
TEST_CASE("SearchSpaceGroup weighs a screw's absences by evidence, not by how many were recorded") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 21 1");
    const gemmi::GroupOps gops = sg.operations();

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    SECTION("five decisive absences, below min_absent_observed: the screw is still found") {
        auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 18);
        // Keep five of the 0k0-odd reflections, at a thousandth of their row, and drop the rest - as a
        // sweep along the 2-fold does, leaving too few to satisfy a count but plenty to decide.
        int kept = 0;
        std::erase_if(merged, [&](MergedReflection& r) {
            if (!gops.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}}))
                return false;
            if (kept >= 5)
                return true;
            ++kept;
            r.I = 0.5;
            return false;
        });
        REQUIRE(kept == 5);
        REQUIRE(kept < opt.min_absent_observed);

        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->short_name() == "P21");
    }

    // A short monoclinic axis on weak data: four 0k0-odd reflections in range, each at 2% of its row
    // rather than at the floor. That zone reads ~11 nats - refused under the old bound of 20 on five
    // crystals that are P2_1 - and must be claimed; the same four at a fifth of their row must not.
    SECTION("four absences at a few percent of a short row: the screw is found, at a fifth it is not") {
        for (const auto& [fraction, expected] : {std::pair{0.02, "P21"}, std::pair{0.2, "P2"}}) {
            auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 18);
            int kept = 0;
            std::erase_if(merged, [&](MergedReflection& r) {
                if (!gops.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}}))
                    return false;
                if (kept >= 4)
                    return true;
                ++kept;
                r.I = static_cast<float>(fraction * 350.0);
                return false;
            });
            REQUIRE(kept == 4);

            const auto result = SearchSpaceGroup(merged, opt);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            CHECK(result.best_space_group->short_name() == expected);
        }
    }

    SECTION("a uniformly weak axial row decides nothing, however many absences it holds") {
        // The whole 0k0 row badly measured: the predicted-absent reflections are weak, but so is the
        // rest of their row, so there is no contrast and no screw to claim. A violation count cannot
        // see this - nothing on the row clears an absolute cut, so it reads zero violations and, with
        // enough reflections to satisfy the count, would claim the 2_1 from no evidence at all.
        auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 18);
        int absent_on_row = 0;
        for (auto& r : merged) {
            if (r.h != 0 || r.l != 0)
                continue;
            const bool absent = gops.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}});
            r.I = absent ? 4.0 : 5.0;
            absent_on_row += absent ? 1 : 0;
        }
        REQUIRE(absent_on_row >= opt.min_absent_observed);

        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->short_name() == "P2");
    }
}

// The operator correlation is on resolution-normalised E^2, not on raw I (see SearchSpaceGroup.cpp).
// Both members of a symmetry pair sit at the same |s|, so on raw intensities the resolution fall-off
// is variance shared perfectly between the two arms of every pair and reads as a correlation for ANY
// pairing at all. These two cases pin that down from both sides.
TEST_CASE("SearchSpaceGroup operator correlation reads symmetry, not the resolution fall-off",
          "[SearchSpaceGroup]") {
    // Intensities that are a smooth function of resolution times an INDEPENDENT per-reflection
    // factor: a Wilson-like fall-off with no symmetry in it whatsoever.
    auto radial_only = [](int hmax) {
        std::vector<MergedReflection> merged;
        for (int h = -hmax; h <= hmax; ++h)
            for (int k = -hmax; k <= hmax; ++k)
                for (int l = -hmax; l <= hmax; ++l) {
                    if ((h == 0 && k == 0 && l == 0) || std::make_tuple(-h, -k, -l) < std::make_tuple(h, k, l))
                        continue;
                    const double d = CalcSyntheticD(h, k, l);
                    const double falloff = std::exp(-30.0 / (d * d));
                    // Deterministic, independent of any symmetry mate: reuse the hash on the raw index.
                    const double jitter = SyntheticIntensityFromAsu(gemmi::Op::Miller{{h, k, l}}) / 350.0;
                    const double I = 1.0e5 * falloff * jitter;
                    merged.push_back(MergedReflection{
                        .h = h, .k = k, .l = l, .I = static_cast<float>(I),
                        .sigma = static_cast<float>(I / 20.0), .d = static_cast<float>(d)});
                }
        return merged;
    };

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    SECTION("a fall-off with no symmetry in it confirms no operator") {
        const auto result = SearchSpaceGroup(radial_only(8), opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.operator_scores.size() > 1);
        for (const auto& s : result.operator_scores) {
            INFO("operator " << s.op_triplet_hkl);
            CHECK(s.n_pairs >= opt.min_pairs_per_operator);
            CHECK(s.cc < opt.min_operator_cc);
            CHECK_FALSE(s.present);
        }
        CHECK(result.point_group_hm == "1");
    }

    SECTION("a real operator under the same fall-off is confirmed, and does not move with the cut") {
        // Same fall-off, but the intensities now carry a genuine monoclinic 2-fold.
        const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 2 1");
        const gemmi::ReciprocalAsu rasu(&sg);
        const gemmi::GroupOps gops = sg.operations();
        auto merged = radial_only(8);
        for (auto& r : merged) {
            const auto [asu, plus] = rasu.to_asu_sign(gemmi::Op::Miller{{r.h, r.k, r.l}}, gops);
            const double falloff = std::exp(-30.0 / (r.d * r.d));
            r.I = 1.0e5 * falloff * SyntheticIntensityFromAsu(asu) / 350.0;
            r.sigma = r.I / 20.0;
        }
        auto two_fold_cc = [&](double d_min) {
            SearchSpaceGroupOptions o = opt;
            o.d_min_limit_A = d_min;
            const auto result = SearchSpaceGroup(merged, o);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.point_group_hm == "2");
            double cc = -2.0;
            for (const auto& s : result.operator_scores)
                if (s.present)
                    cc = s.cc;
            REQUIRE(cc > opt.min_operator_cc);
            return cc;
        };
        // The whole point of normalising: how much of the fall-off is inside the merge no longer
        // moves the operator's score, so the search resolution cut cannot decide the symmetry.
        CHECK(std::fabs(two_fold_cc(0.0) - two_fold_cc(6.0)) < 0.05);
    }
}

// The enumeration reaches the settings gemmi does not call the reference one, and only when the cell
// has the axes they name. Both halves are pinned here: `P 1 1 2_1` puts its 2-fold and its screw on
// c, which no reference setting can express (Stage A never offers the rotation and Stage B never
// offers the group), so without the two options the answer is P1; with them, and with a cell whose
// unique axis IS c, it is named; and with a cell whose unique axis is b the same candidate is
// refused rather than adopted on axes the crystal does not have.
TEST_CASE("SearchSpaceGroup names a non-reference setting only on a cell that hosts it") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 1 21");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 12);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    SECTION("narrow enumeration cannot name it") {
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->number == 1);
    }

    SECTION("widened enumeration names it on a c-unique cell") {
        opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.0, 90.0, 100.0);
        opt.enumerate_all_settings = true;
        opt.enumerate_all_rotation_sets = true;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->xhm() == "P 1 1 21");
    }

    SECTION("a b-unique cell refuses it") {
        opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.0, 100.0, 90.0);
        opt.enumerate_all_settings = true;
        opt.enumerate_all_rotation_sets = true;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->number == 1);
    }
}

// The screw axes of an orthorhombic crystal can lie on any pair of axes, and only one of the three
// namings of #18 is a reference setting. With the narrow enumeration the group that predicts a
// SUBSET of the real absences and nothing else wins on no evidence at all, so the reported group is
// wrong rather than low - the widening is what makes the correct one available.
TEST_CASE("SearchSpaceGroup names an orthorhombic screw pair on the axes it lies on") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 2 21 21");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 14);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;
    opt.lattice_system = gemmi::CrystalSystem::Orthorhombic;

    SECTION("narrow enumeration reports the wrong group") {
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->number != 18);
    }

    SECTION("widened enumeration reports it") {
        opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.0, 90.0, 90.0);
        opt.enumerate_all_settings = true;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->xhm() == "P 2 21 21");
    }

    // A cell refined free after integration is a few tenths of a degree off 90. The reference
    // candidates hold the same rotations and are offered on it, so the other settings must be too.
    SECTION("a free cell a few tenths off 90 still offers it") {
        opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.32, 90.19, 90.19);
        opt.enumerate_all_settings = true;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->xhm() == "P 2 21 21");
    }
}

// A screw on a row the sweep never recorded is not a group the data refused, it is a question
// nobody asked: the run writes the member claiming no screw because a reflection file must carry
// one group, and without this the only trace of the ambiguity is a list of names that does not say
// which axis is open. Modelled on a real hexagonal set whose 00l row lies in the spindle's blind
// cone and which is reported as P 6 against a deposited P 63.
TEST_CASE("SearchSpaceGroup says which axis a missing row left the screw open on",
          "[SearchSpaceGroup]") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 21 21 21");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 14);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;
    opt.lattice_system = gemmi::CrystalSystem::Orthorhombic;

    SECTION("every row measured - nothing is open") {
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->xhm() == "P 21 21 21");
        CHECK(result.undetermined_screws.empty());
    }

    SECTION("the 00l row removed - the c screw is undetermined") {
        std::vector<MergedReflection> without_00l;
        for (const auto& r : merged)
            if (r.h != 0 || r.k != 0)
                without_00l.push_back(r);
        const auto result = SearchSpaceGroup(without_00l, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        REQUIRE(result.undetermined_screws.size() == 1);
        CHECK(result.undetermined_screws[0].axis == 'c');
        CHECK(result.undetermined_screws[0].row_label == "00l");
        CHECK(result.undetermined_screws[0].n_observed == 0);
        // The a and b screws were measured and are unaffected: what the missing row costs is the
        // third condition, not the two the data still carry.
        CHECK(result.best_space_group->operations().is_systematically_absent({{1, 0, 0}}));
        CHECK(result.best_space_group->operations().is_systematically_absent({{0, 1, 0}}));
        // ...and the group the c row would have decided between is offered, not silently dropped.
        bool offers_a_group_without_the_c_screw = false;
        for (const auto& alt : result.alternatives)
            if (!alt.operations().is_systematically_absent({{0, 0, 1}}))
                offers_a_group_without_the_c_screw = true;
        CHECK((offers_a_group_without_the_c_screw
               || !result.best_space_group->operations().is_systematically_absent({{0, 0, 1}})));
    }
}

// The centring half of the same widening. A, B and C centring on one orthorhombic cell are three
// different lattices, and only C is a reference setting, so an A-centred crystal used to have its
// centring refused (its absent class is not the one C predicts) and came out primitive. The
// candidate is now offered, and it has to be adopted from its own absences rather than from the
// metric, which cannot tell A from C at all.
TEST_CASE("SearchSpaceGroup names an A-centred orthorhombic lattice") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("A 2 2 2");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 12);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;
    opt.lattice_system = gemmi::CrystalSystem::Orthorhombic;

    SECTION("narrow enumeration cannot name it") {
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->centring_type() != 'A');
    }

    SECTION("widened enumeration names it") {
        opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.0, 90.0, 90.0);
        opt.enumerate_all_settings = true;
        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->xhm() == "A 2 2 2");
    }
}

// The null the widening has to survive. Stage A's second pass offers the a- and c-unique 2-folds on
// any metric that could host them, which is every orthorhombic one - so a genuinely triclinic
// crystal sitting on a pseudo-orthorhombic cell is now offered three promotions where it used to be
// offered one. It must still be refused all three: the added candidates go through the same operator
// correlation as every other, and a rotation the intensities do not have scores nothing.
TEST_CASE("SearchSpaceGroup does not promote triclinic data on a pseudo-orthorhombic cell") {
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 10);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;
    opt.lattice_system = gemmi::CrystalSystem::Orthorhombic;
    opt.cell = gemmi::UnitCell(40.0, 50.0, 60.0, 90.0, 90.0, 90.0);
    opt.enumerate_all_settings = true;
    opt.enumerate_all_rotation_sets = true;

    const auto result = SearchSpaceGroup(merged, opt);
    INFO(SearchSpaceGroupResultToText(result));
    REQUIRE(result.best_space_group.has_value());
    CHECK(result.best_space_group->number == 1);
    CHECK(result.point_group_order == 1);
}

// A screw zone is a handful of axial reflections and its evidence is a SUM over them, so it is
// decided by its largest member. Measured on a monoclinic crystal whose eight absent 0k0 are dead in
// every run: between two scaling passes that differed only in which weak frames were rejected, ONE
// of the eight moved from 14 +- 9 to 99 +- 10 (its two half-set merges reading 198 and 2.5, so it
// was never measured to the precision its sigma claimed) while the other seven did not move at all -
// and the zone went from 30.1 nats to 17.1 and lost the 2(1) under a bound of 20. Trimming the
// largest member and rescaling for the trim makes the two passes agree.
TEST_CASE("A screw zone's evidence does not hang on its largest absence", "[SearchSpaceGroup]") {
    // Seven reflections at a hundredth of their row, and one that moved between the two passes.
    const double dead_seven = 7 * 0.01;
    const double before = ScrewZoneEvidence(TrimmedZoneSum(dead_seven + 0.03, 0.03, 8), 8);
    const double after = ScrewZoneEvidence(TrimmedZoneSum(dead_seven + 0.39, 0.39, 8), 8);

    CHECK(before == Catch::Approx(after).margin(0.01));   // the same seven reflections, the same verdict
    CHECK(after > 20.0);                                  // and the screw survives the move
    // Untrimmed, that one reflection is the whole difference and it crosses the bound.
    CHECK(ScrewZoneEvidence(dead_seven + 0.03, 8) > 20.0);
    CHECK(ScrewZoneEvidence(dead_seven + 0.39, 8) < 20.0);

    // Only ONE member is trimmed, whatever the zone holds: a zone with two strong absences is not a
    // zone with a bad reflection in it, it is a zone that is not extinct.
    CHECK(ScrewZoneEvidence(TrimmedZoneSum(dead_seven + 0.39 + 0.39, 0.39, 8), 8) < 20.0);

    // On a uniform zone the rescale under-states rather than over-states - the safe direction.
    CHECK(ScrewZoneEvidence(TrimmedZoneSum(8 * 0.01, 0.01, 8), 8) <
          ScrewZoneEvidence(8 * 0.01, 8));
    // A class that merged non-positive throughout is unchanged: sum and max are both zero, so the
    // floor in ScrewZoneEvidence is what answers, exactly as before.
    CHECK(TrimmedZoneSum(0.0, 0.0, 8) == 0.0);
}

// Screw ORDERS on one axial row are nested: 6_1 extinguishes l != 6n and 6_2/6_4 extinguish
// l != 3n, so 6_1's absent class is 6_2's plus the l = 3n that are not 6n, and the whole of the
// evidence between the two lies in that difference. Trimming the zone's largest member defends
// against one badly-measured reflection, but where the difference class holds the strongest
// reflection on the row it trimmed away the only datum that refutes 6_1 - which then read the row
// as dead, won on its two extra absences, and used the inflated evidence to excuse the very
// violation it had discarded. A member that is both flagged PRESENT and standing at its row's own
// mean is not an outlier, so it is not trimmed. Measured on a hexagonal crystal: nine absences at
// 50.6 nats with one violation beat seven at 43.3 with none, and read 6.8 once the violation - at
// 1.94 of its row - stayed in. The two sections here are the two sides of that bound.
TEST_CASE("SearchSpaceGroup does not trim away the reflection that refutes a screw order",
          "[SearchSpaceGroup]") {
    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;

    SECTION("an absence at its row's own strength decides against the order that claims it") {
        const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 64");
        auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 12);

        // The 00l row as such a crystal records it: l = 3n present, everything else dead, and one
        // l = 3n that is NOT 6n - the class 6_1 has to call absent and 6_4 does not - by far the
        // strongest reflection on the row.
        int on_row = 0;
        for (auto& r : merged) {
            if (r.h != 0 || r.k != 0)
                continue;
            const int l = std::abs(r.l);
            ++on_row;
            if (l % 3 != 0)      r.I = 0.0f;    // extinguished by the 3n condition, in both candidates
            else if (l % 6 == 0) r.I = 300.0f;  // the control class 6_1 keeps for itself
            else                 r.I = (l == 9) ? 4000.0f : 5.0f;
        }
        REQUIRE(on_row >= 8);

        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        std::vector<std::string> accepted{result.best_space_group->short_name()};
        for (const auto& alt : result.alternatives)
            accepted.push_back(alt.short_name());
        // P6_2 and P6_4 are enantiomorphs and indistinguishable from intensities; P6_1/P6_5 are a
        // different claim and must not be what comes out.
        CHECK(std::find(accepted.begin(), accepted.end(), "P64") != accepted.end());
        CHECK(std::find(accepted.begin(), accepted.end(), "P61") == accepted.end());
        CHECK(std::find(accepted.begin(), accepted.end(), "P65") == accepted.end());
    }

    SECTION("one weak absence that moved is still an outlier, and the screw survives it") {
        // The other side of the bound, and the case the trim was built for: a genuine 2_1 whose
        // 0k0-odd class is dead but for one reflection at a third of its row, measured with a sigma
        // that makes it read present. Trimmable as before - it is nowhere near the row's strength -
        // and losing that would cost a real screw.
        const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 21 1");
        auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 18);

        const gemmi::GroupOps gops = sg.operations();
        int absent = 0;
        float row_strength = 0.0f;
        for (const auto& r : merged)
            if (r.h == 0 && r.l == 0 && !gops.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}}))
                row_strength = std::max(row_strength, r.I);
        for (auto& r : merged) {
            if (r.h != 0 || r.l != 0)
                continue;
            if (!gops.is_systematically_absent(gemmi::Op::Miller{{r.h, r.k, r.l}}))
                continue;
            ++absent;
            r.I = absent == 1 ? 0.3f * row_strength : 0.0f;
            r.sigma = 0.3f;
        }
        REQUIRE(absent >= 6);

        const auto result = SearchSpaceGroup(merged, opt);
        INFO(SearchSpaceGroupResultToText(result));
        REQUIRE(result.best_space_group.has_value());
        CHECK(result.best_space_group->short_name() == "P21");
    }
}

// A zone whose predicted absences were never measurable must not outscore a zone that is genuinely
// dead. sum_u is a sum of max(0, E^2)/row_mean, so it is EXACTLY zero when every absent reflection in
// the zone merged non-positive - and the Beta tail then diverges, worth ~690 nats per reflection. That
// was harmless while the number only had to clear a bound; it is now summed across zones and ranks the
// candidates, so it made a candidate claiming a screw on an UNMEASURED row beat one whose rows are
// actually dead. The evidence is scored through the same entry point for both kinds of absence.
TEST_CASE("AbsenceEvidence does not reward a zone that was never measurable", "[SearchSpaceGroup]") {
    // 2 absences that all merged non-positive, against a control of 8...
    const double unmeasurable = AbsenceEvidence(0.0, 2, 8);
    // ...against a genuinely dead zone: 6 absences at 1% of their row's mean, same control.
    const double genuine = AbsenceEvidence(0.06, 6, 8);

    CHECK(std::isfinite(unmeasurable));
    CHECK(unmeasurable < genuine);                 // the ordering that was inverted
    CHECK(unmeasurable < 20.0);                    // and it does not clear min_screw_absence_evidence

    // The floor is far below any real measurement, so a genuine zone is untouched by it.
    CHECK(genuine == Catch::Approx(22.0).margin(0.2));
    CHECK(AbsenceEvidence(0.22, 22, 8) == Catch::Approx(65.4).margin(0.3));

    // More dead reflections still means more evidence, which is the property the sum relies on.
    CHECK(AbsenceEvidence(0.0, 6, 8) > AbsenceEvidence(0.0, 2, 8));
}

// ---------------------------------------------------------------------------------------------
// Glide planes (small-molecule space groups).
//
// A glide extinguishes a two-dimensional ZONE where a screw extinguishes a row, so it is the same
// absence test on a plane. What these cases pin is not that the test works - it is the two places
// it must NOT act: on a Sohncke group (a chiral crystal has no glide, and the corpus measurement
// that licensed this feature is a zero false-positive rate on 140 protein datasets), and on an
// inversion centre (Friedel's law makes it unmeasurable, so it must never be claimed).
// ---------------------------------------------------------------------------------------------
TEST_CASE("SearchSpaceGroup names a glide plane") {
    struct Case {
        std::string input_name;
        std::string expected_xhm;
        gemmi::UnitCell cell;
    };
    const gemmi::UnitCell monoclinic(11.0, 13.0, 17.0, 90.0, 101.0, 90.0);
    const gemmi::UnitCell orthorhombic(11.0, 13.0, 17.0, 90.0, 90.0, 90.0);

    const std::vector<Case> cases = {
        // The reference setting, and the non-reference one the same group takes when the data are
        // indexed with the glide on a: a setting names the mirror by AXIS, so both have to be
        // reachable or a crystal indexed the other way round is named wrongly or not at all.
        {"P 1 21/c 1", "P 1 21/c 1", monoclinic},
        {"P 1 21/a 1", "P 1 21/a 1", monoclinic},
        {"C 1 2/c 1",  "C 1 2/c 1",  monoclinic},
        // Three glide planes at once: every zone must be dead, not just the best one.
        {"P b c a",    "P b c a",    orthorhombic},
    };

    for (const auto& tc : cases) {
        DYNAMIC_SECTION(tc.expected_xhm) {
            const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name(tc.input_name);
            const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 10);

            SearchSpaceGroupOptions opt;
            opt.merge_friedel = true;
            opt.cell = tc.cell;
            opt.enumerate_all_settings = true;

            const auto result = SearchSpaceGroup(merged, opt);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            CHECK(result.best_space_group->xhm() == tc.expected_xhm);
            // The Sohncke answer is reported alongside on the same run, so a reader who knows the
            // sample is chiral never has to process the images again to see it.
            CHECK(result.sohncke_space_group.has_value());
            CHECK(result.sohncke_space_group->is_sohncke());
            REQUIRE_FALSE(result.glide_zones.empty());
            for (const auto& z : result.glide_zones)
                CHECK(z.evidence_per_reflection >= opt.min_glide_evidence_per_reflection);
        }
    }
}

// The inert direction, which is the one that matters: a chiral crystal has no glide plane, so on
// Sohncke data the glide machinery must add nothing at all - not a different group, not a zone.
TEST_CASE("SearchSpaceGroup claims no glide on Sohncke data") {
    const gemmi::UnitCell monoclinic(11.0, 13.0, 17.0, 90.0, 101.0, 90.0);
    for (const std::string name : {"P 1 21 1", "P 1 2 1", "C 1 2 1"}) {
        DYNAMIC_SECTION(name) {
            const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name(name);
            const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 10);

            SearchSpaceGroupOptions opt;
            opt.merge_friedel = true;
            opt.cell = monoclinic;
            opt.enumerate_all_settings = true;

            const auto result = SearchSpaceGroup(merged, opt);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            CHECK(result.best_space_group->is_sohncke());
            CHECK_FALSE(result.glide_space_group.has_value());
            CHECK(result.glide_zones.empty());
        }
    }
}

// The centre of symmetry is NOT determinable and must never be claimed: Friedel's law makes the
// diffraction pattern centrosymmetric whether or not the crystal is, so P 1 2/m 1 predicts exactly
// what P 1 2 1 predicts. Data generated in the centrosymmetric group must still come out Sohncke -
// which is the enumeration refusing any non-Sohncke group whose absences a Sohncke one already has.
TEST_CASE("SearchSpaceGroup never claims an inversion centre") {
    const gemmi::UnitCell monoclinic(11.0, 13.0, 17.0, 90.0, 101.0, 90.0);
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 2/m 1");
    const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 10);

    SearchSpaceGroupOptions opt;
    opt.merge_friedel = true;
    opt.cell = monoclinic;
    opt.enumerate_all_settings = true;

    const auto result = SearchSpaceGroup(merged, opt);
    INFO(SearchSpaceGroupResultToText(result));
    REQUIRE(result.best_space_group.has_value());
    CHECK(result.best_space_group->is_sohncke());
    CHECK_FALSE(result.glide_space_group.has_value());
}

// A NON-centrosymmetric glide group holds only half the rotations of its Laue class, so it has to be
// enumerated by Laue class to be a candidate at all. Where its absences have no centrosymmetric
// carrier (I-42d, Fdd2, P-42_1c, I-43d) it is the only group naming the glide; I-42d and I4_1md
// predict the same absences, so the one not written is reported beside the one that is.
TEST_CASE("SearchSpaceGroup names a glide with no centrosymmetric carrier") {
    struct Case {
        std::string input_name;
        std::vector<std::string> accepted;   // the written group and its same-absence twins
        gemmi::UnitCell cell;
    };
    const std::vector<Case> cases = {
        {"I -4 2 d",  {"I 41 m d", "I -4 2 d"}, gemmi::UnitCell(7.4, 7.4, 6.9, 90, 90, 90)},
        {"P -4 21 c", {"P -4 21 c"},            gemmi::UnitCell(7.4, 7.4, 6.9, 90, 90, 90)},
        {"F d d 2",   {"F d d 2"},              gemmi::UnitCell(11.0, 13.0, 17.0, 90, 90, 90)},
        {"I -4 3 d",  {"I -4 3 d"},             gemmi::UnitCell(9.0, 9.0, 9.0, 90, 90, 90)},
    };
    for (const auto& tc : cases) {
        DYNAMIC_SECTION(tc.input_name) {
            const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name(tc.input_name);
            const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 12);

            SearchSpaceGroupOptions opt;
            opt.merge_friedel = true;
            opt.cell = tc.cell;
            opt.enumerate_all_settings = true;

            const auto result = SearchSpaceGroup(merged, opt);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            std::vector<std::string> named{result.best_space_group->xhm()};
            for (const auto& alt : result.alternatives)
                named.push_back(alt.xhm());
            CHECK_THAT(named, Catch::Matchers::UnorderedEquals(tc.accepted));
            CHECK(result.glide_space_group.has_value());
            REQUIRE(result.sohncke_space_group.has_value());
            CHECK(result.sohncke_space_group->is_sohncke());
        }
    }
}

// Pc and P2/c (Pna2_1 and Pnma) predict the same absences and differ by an inversion centre, which
// Friedel's law hides. By convention the CENTROSYMMETRIC one is written, whichever the data came
// from, and the other is reported as an alternative - never dropped, so the run does not claim a
// centre nothing measured.
TEST_CASE("SearchSpaceGroup writes the centrosymmetric group and reports its non-centrosymmetric twin") {
    struct Case {
        std::string input_name;
        std::string written;
        std::string alternative;
        gemmi::UnitCell cell;
    };
    const gemmi::UnitCell monoclinic(11.0, 13.0, 17.0, 90.0, 101.0, 90.0);
    const gemmi::UnitCell orthorhombic(11.0, 13.0, 17.0, 90.0, 90.0, 90.0);
    const std::vector<Case> cases = {
        {"P 1 c 1",    "P 1 2/c 1", "P 1 c 1",   monoclinic},
        {"P 1 2/c 1",  "P 1 2/c 1", "P 1 c 1",   monoclinic},
        {"C 1 c 1",    "C 1 2/c 1", "C 1 c 1",   monoclinic},
        {"P n a 21",   "P n a m",   "P n a 21",  orthorhombic},
    };
    for (const auto& tc : cases) {
        DYNAMIC_SECTION(tc.input_name) {
            const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name(tc.input_name);
            const auto merged = GenerateMergedReflectionsForSpaceGroup(sg, 10);

            SearchSpaceGroupOptions opt;
            opt.merge_friedel = true;
            opt.cell = tc.cell;
            opt.enumerate_all_settings = true;

            const auto result = SearchSpaceGroup(merged, opt);
            INFO(SearchSpaceGroupResultToText(result));
            REQUIRE(result.best_space_group.has_value());
            CHECK(result.best_space_group->xhm() == tc.written);
            CHECK(std::any_of(result.alternatives.begin(), result.alternatives.end(),
                              [&](const gemmi::SpaceGroup& a) { return a.xhm() == tc.alternative; }));
        }
    }
}

// The centre-of-symmetry statistics on synthetic Wilson data: acentric and centric intensities on the
// general reflections of a 2/m Laue class, with the reflections a 2-fold maps to their Friedel mate
// (h0l) centric in both. The calibrated readings must land at their ends, and only the acentric set
// may read ACENTRIC.
TEST_CASE("AnalyzeCentreOfSymmetry separates acentric from centric Wilson data") {
    const gemmi::UnitCell cell(11.0, 13.0, 17.0, 90.0, 101.0, 90.0);
    const gemmi::SpaceGroup& sg = gemmi::get_spacegroup_by_name("P 1 2/c 1");
    for (const bool centric : {false, true}) {
        DYNAMIC_SECTION((centric ? "centric" : "acentric")) {
            std::mt19937 rng(7);
            std::exponential_distribution<double> expo(1.0);
            std::normal_distribution<double> gauss(0.0, 1.0);
            std::vector<MergedReflection> merged;
            const gemmi::GroupOps gops = sg.operations();
            const gemmi::ReciprocalAsu asu(&sg);
            for (int h = -10; h <= 10; ++h)
                for (int k = 0; k <= 10; ++k)
                    for (int l = -12; l <= 12; ++l) {
                        const gemmi::Op::Miller hkl{{h, k, l}};
                        if ((h == 0 && k == 0 && l == 0) || !asu.is_in(hkl) || gops.is_systematically_absent(hkl))
                            continue;
                        const double d = cell.calculate_d(hkl);
                        if (d < 0.8)
                            continue;
                        // h0l is centric under the 2-fold whatever the crystal is.
                        const bool c = centric || k == 0;
                        const double z = gauss(rng);
                        const double I = 1000.0 * std::exp(-2.0 / (d * d)) * (c ? z * z : expo(rng));
                        merged.push_back(MergedReflection{.h = h, .k = k, .l = l,
                                                          .I = static_cast<float>(I), .sigma = 1.0f,
                                                          .d = static_cast<float>(d)});
                    }
            const auto r = AnalyzeCentreOfSymmetry(merged, sg);
            INFO("n " << r.n_general << " pairs " << r.l_pairs << " L f " << r.mean_abs_l.f << " E f "
                 << r.mean_abs_e2_minus_1.f << " N f " << r.n_z_01.f << " control f "
                 << r.control_mean_abs_e2_minus_1.f);
            REQUIRE(r.n_general > 1000);
            CHECK(r.applicable);
            CHECK(r.mean_abs_l.f == Catch::Approx(centric ? 1.0 : 0.0).margin(0.2));
            CHECK(r.mean_abs_e2_minus_1.f == Catch::Approx(centric ? 1.0 : 0.0).margin(0.2));
            CHECK(r.reads_acentric == !centric);
        }
    }
}