Jungfraujoch/image_analysis/scale_merge/SearchSpaceGroup.h

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#pragma once

#include <optional>
#include <string>
#include <vector>

#include "../../common/Reflection.h"
#include "gemmi/symmetry.hpp"

// Determine the likely space group of a dataset from its P1-merged intensities, in the spirit
// of POINTLESS (Evans 2006):
//
//   Stage A - point group (Laue) symmetry. Every candidate rotation operator is scored once by
//             the correlation of I(h) with I(Rh). The chosen point group is the largest one all
//             of whose operators are confirmed (high CC). A wrong operator scores ~0, so this is
//             self-pruning - the unit cell is not needed.
//
//   Stage B - space group within that point group. Each Sohncke space group of the point group
//             predicts a set of systematically absent reflections (centering + screw axes). The
//             chosen space group is the one that explains the MOST absences while every reflection
//             it predicts absent is in fact weak. The symmorphic group (no absences) is the
//             fallback when no screw/centering is supported. Enantiomorphic pairs (e.g. P4_1 vs
//             P4_3) are indistinguishable from intensities and are reported as a pair.

struct SpaceGroupOperatorScore {
    std::string op_triplet_hkl;   // reciprocal-space triplet of the rotation, e.g. "-h,-k,l"
    double cc = 0.0;              // correlation of I(h) with I(Rh)
    int n_pairs = 0;             // independent reflection pairs the CC was computed from
    bool present = false;        // operator confirmed as a real symmetry of the intensities
    // MEDIAN |I1-I2|/(I1+I2) over this operator's pairs - the disagreement the operator implies, with no
    // sigma in it. Unlike chi^2 and the systematic-b, which are ratios to a merge error model that
    // drifts with multiplicity, this is a property of the intensities alone; compared against an
    // operator already confirmed on the same reflections it is what separates a real symmetry from a
    // twin law (see max_operator_h_ratio). The median rather than the mean because a merohedral twin
    // perturbs EVERY pair while a badly integrated minority perturbs only the tail.
    double h_stat = 0.0;
};

struct SpaceGroupCandidateScore {
    gemmi::SpaceGroup space_group;
    int absent_observed = 0;          // observed reflections this SG predicts systematically absent
    int absent_violations = 0;        // of those, how many are nonetheless strongly present
    double absent_mean_i_over_sigma = 0.0;
    double present_mean_i_over_sigma = 0.0;
    // Screw evidence, as median E^2 = I/<I>(shell): the reflections a screw axis extinguishes against
    // the rest of that same axial row. This is the comparison the screw test makes, and the only one
    // that means anything for it - the I/sigma means above are dominated by the merged-sigma floor
    // (sigma >= b|I| pins I/sigma at ISa), so they read the same for absent and present alike.
    double screw_absent_median_e_squared = 0.0;
    double screw_row_median_e_squared = 0.0;
    bool consistent = false;          // absent class confirmed weak (few violations)
    bool selected = false;            // chosen result (or its enantiomorph)
};

struct SearchSpaceGroupOptions {
    // Lattice (metric) symmetry from LatticeSearch. When set, the point-group search is limited to
    // the Sohncke subgroups of this system's holohedry - the metric is an upper bound on the
    // intensity symmetry, so e.g. a tetragonal metric tests {1,2,222,4,422} and never 3/6/23. All
    // subgroups are still tested (down to P1), so a pseudo-symmetric metric never forces a higher
    // symmetry than the intensities support. Unset = search every Sohncke system.
    std::optional<gemmi::CrystalSystem> lattice_system;

    // Centering is NOT taken from the lattice metric: an indexer that returns the conventional cell
    // (e.g. cubic 'P'-looking axes for a body-centered lattice) hides the centering, which lives only
    // in the systematic absences. Stage B therefore tests every centering allowed by the point group
    // and confirms it from the data (h+k+l etc. absent), rather than trusting a geometric hint.

    // Friedel mates are treated as equivalent when matching HKLs (i.e. anomalous signal ignored).
    bool merge_friedel = true;

    // Ignore reflections beyond this resolution (smaller d = higher resolution). 0 disables.
    double d_min_limit_A = 0.0;

    // Drop reflections weaker than this from the correlation stage only (the absence stage must
    // keep weak reflections - that is where the screw-axis signal lives).
    double min_i_over_sigma = 0.0;

    // Drop reflections STRONGER than this resolution-normalised E^2 = I/<I>(shell) from the correlation
    // stage only. A second lattice deposits intensity on one reciprocal position but not its symmetry
    // mate, so an overlap-contaminated reflection is a one-sided E^2 outlier that poisons an operator's
    // I(h)/I(Rh) correlation (it flipped a pseudo-merohedral P2_1 case to P1 under -A: excluding the E^2>9 tail, ~0.4%
    // of reflections, lifted the 2-fold CC 0.33->0.53 back over the gate). Clean Wilson-distributed data
    // almost never reaches E^2=9 (P(E^2>9) ~ 0.01-0.3%), so this removes essentially nothing there and
    // only trims the overlap tail. 0 disables. Absences are unaffected (they need the weak tail).
    double max_e_squared_for_cc = 9.0;

    // --- Stage A: point group ---
    // A rotation is accepted as a real symmetry when its I(h)/I(Rh) correlation reaches this over
    // at least min_pairs_per_operator independent pairs.
    double min_operator_cc = 0.5;
    int min_pairs_per_operator = 20;

    // Per-operator CC alone cannot tell a real weak operator from a false strong one (a noisy crystal's
    // genuine 2-fold can score below a pseudo-symmetric crystal's near-perfect false one). So a point
    // group is also required to be SELF-CONSISTENT: merging the intensities under it must not inflate the
    // reduced chi^2 (within-orbit scatter / sigma^2) beyond this factor times the most-consistent
    // candidate. A false operator forces non-equivalent reflections together so they disagree by many
    // sigma and chi^2 blows up; a real one leaves it ~flat even when the operator CC is only moderate.
    // Calibrated on the rotation-test battery: every correct point group stays within ~1.7x the best
    // subgroup even on weak / badly-integrated data (worst real case a P41212 at 1.71), while a twin
    // law or pseudo-symmetry lands clearly higher (a merohedral R3->R32 twin 2-fold at 2.01). 1.85 sits
    // between the two, so a partial merohedral twin is kept in its true lower symmetry (R3), not
    // over-promoted to the holohedral R32.
    double max_merge_chi2_ratio = 1.85;

    // A genuine high-symmetry merge can drift just past max_merge_chi2_ratio when its data are only
    // imperfectly scaled: each real symmetry step then adds a little systematic scatter, so a weakly-
    // scaled cubic case lands at ratio ~2.0 - right where a merohedral twin (an R3->R32 case at ~1.95)
    // also lands, so the chi^2 ratio alone cannot separate them. A candidate whose chi^2 is
    // only this far past the best subgroup is therefore rescued if the SYSTEMATIC part of the extra
    // scatter stayed small (max_systematic_b_ratio): merging under a genuine operator gains multiplicity
    // without intensity-proportional disagreement, so the merge error model's b barely moves, whereas a
    // twin forces non-equivalent reflections together and b balloons (a genuine cubic step b x1.6 vs a
    // merohedral twin x2.2, measured against the largest confirmed subgroup). Both bounds sit in the gap; the rescue only ever
    // promotes, and only in this narrow chi^2 band.
    double max_merge_chi2_rescue = 2.30;
    double max_systematic_b_ratio = 1.90;

    // Veto bound. The systematic-b test above is otherwise only a rescue - it can promote a chi^2-borderline
    // genuine step but never demote a chi^2-passing one. A merohedral twin whose within-orbit scatter looks
    // self-consistent (chi^2 ratio below max_merge_chi2_ratio) therefore slips through on chi^2 alone even
    // though merging its non-equivalent reflections balloons b. So a chi^2-passing high-symmetry promotion
    // whose b, relative to the largest confirmed subgroup, exceeds this bound is vetoed and kept in its true
    // lower symmetry. Calibrated on the rotation-test battery, where the largest genuine step (a P422
    // tetragonal) sits at b-ratio ~1.8 and a merohedral R3->R32 twin at ~2.6 - an empty gap - so 2.0 keeps
    // every genuine high-symmetry merge (they are never demoted) while catching the twin. The veto only ever
    // keeps a clearly-ballooned promotion down, never promotes.
    double max_systematic_b_veto = 2.0;

    // Floor on the parent's systematic-b when forming the veto ratio. On excellent data a genuine merge's
    // b is near zero (ISa well above 20), so even a small, harmless absolute increase gives a huge b-ratio
    // and the veto would wrongly reject real high symmetry (a true tetragonal 422 whose parent 222 sits at
    // b=0.008 and 422 at b=0.049 - both tiny - reads as a 6x balloon). A real twin, by contrast, drives b
    // to a large ABSOLUTE value (~0.19). Flooring the parent b here makes the ratio meaningful only once b
    // is a non-negligible fraction of I; below the floor the increase is treated as noise, not a twin.
    double min_systematic_b_for_veto = 0.05;

    // Promotion gate on the operator disagreement H = median|I1-I2|/(I1+I2), taken as the ratio of the
    // operators the promotion ADDS to the operators of the parent group already confirmed on the same
    // reflections. A real symmetry operator relates equal intensities, so its H matches the parent's
    // (ratio ~1); a merohedral twin law relates DIFFERENT reflections mixed in proportion alpha, so its
    // H is systematically larger. Unlike the chi^2 and systematic-b ratios (genuine 1.00-3.47 /
    // 1.09-3.89 vs twin 1.35-3.32 / 1.77-4.76, fully interleaved) it does not drift with multiplicity,
    // because there is no sigma in it and the parent normalisation cancels data quality.
    //
    // A MEDIAN, not a mean. A merohedral twin mixes every reflection with its twin mate, so it shifts the
    // whole distribution of |I1-I2|/(I1+I2); a minority of badly measured reflections shifts only the
    // tail. Measured on the same real crystals, moving from the mean to the median leaves genuine
    // promotions where they are (1.016 -> 1.013, 1.051 -> 1.067, 1.238 -> 1.231) and pushes every real
    // twin UP (1.280 -> 1.622, 1.427 -> 2.010, 1.272 -> 1.447, 1.441 -> 1.522), widening the margin
    // around this bound from 2.7% to 17.5%. Battery-neutral: 33 crystals, no point group changed.
    //
    // The parent normalisation is what makes this work, and it is not optional. Symmetry-related
    // reflections never agree exactly on real data - absorption, illumination and partiality differ
    // between them - and that systematic floor varies by crystal AND by operator (a cubic 3-fold permutes
    // axes, relating far-apart parts of reciprocal space, so it disagrees more than the 2-folds of its
    // parent even when the symmetry is perfectly real). Measuring an added operator against the parent's
    // own operators, on the same reflections, is what divides that floor out. An ABSOLUTE bound on any
    // per-operator agreement statistic cannot: measured absolute values for genuine symmetry span the
    // whole range from 0.99 on strong data down to 0.71 on weak, straddling every twin.
    double max_operator_h_ratio = 1.25;

    // The H test needs at least this many pairs on both sides to mean anything.
    int min_pairs_for_h = 200;

    // Above this reduced chi^2 for the best subgroup (chi2_ref), the merged error model is treated as
    // badly miscalibrated (weak, low-resolution data whose sigmas are far too small): the fixed-sigma
    // chi^2 ratio then grows with point-group order for genuine high symmetry too and can no longer
    // arbitrate, so a promotion is confirmed on the systematic-b test alone (which re-fits its own error
    // and stays valid). A well-calibrated merge sits near 1; 3.0 (sigmas ~1.7x too small) marks the point
    // where the ratio stops being trustworthy. The balloon veto still guards against a twin.
    double chi2_ref_reliable = 3.0;

    // Adopt this space group's point group instead of the one Stage A would choose, and go straight to
    // Stage B with it. Used when one merge decides the point group and a different merge has to decide
    // the absences: a caller cannot pass the point group by name, because gemmi reports both P321 and
    // P312 as "32", so the group itself is passed and its rotations are taken from it. Stage A still
    // runs (its operator scores and its refusal report are still wanted); only the choice is overridden.
    std::optional<gemmi::SpaceGroup> fixed_point_group;

    // --- Stage B: space group (screw axes / centering) ---
    bool determine_space_group = true;     // false: stop at the symmorphic representative

    // Signed I/sigma above which a reflection counts as genuinely present. Signed, so negative
    // noise is never mistaken for a real reflection. Used both to spot reflections that violate a
    // wrongly-assumed absence, and to keep the correlation stage from pairing near-zero reflections.
    double present_i_over_sigma = 3.0;

    // Extra intensity gate for the systematic-absence test: a reflection also has to reach this
    // resolution-normalised intensity E^2 = I / <I>(shell) to count as violating a predicted absence.
    // The error model can under-estimate sigma on weak axial reflections and fake a high I/sigma, so a
    // reflection at a few percent of the shell-mean intensity is judged absent regardless of its sigma
    // (e.g. the monoclinic 2_1: 0k0-odd at ~1% of 0k0-even). 0 disables the gate (I/sigma only).
    //
    // For a SCREW the threshold is this fraction of the median E^2 of the axial row the screw
    // constrains (floored at the plain value), not of an average reflection at that resolution - an
    // axial row is often far stronger than the shell mean, and only the row is a fair comparison.
    double present_e_squared = 0.3;

    // A candidate's SCREW/glide absence conditions are accepted when at most this fraction of the
    // reflections it predicts absent are in fact strongly present.
    double max_absent_violation_fraction = 0.10;

    // A CENTERING is accepted when its systematically-absent class is this much weaker than the present
    // class - mean signed I/sigma of the centering-absent reflections <= this fraction of the present
    // mean. A real centering cancels structure factors so its absent class sits near zero (ratio ~0-0.3
    // across the test battery) even on noisy or obverse/reverse-twinned data; a false centering leaves it
    // as strong as the present class (ratio ~1.0). 0.5 separates the two with wide margin. This strength
    // test replaces a per-reflection violation-count gate for centering, which was brittle when noise
    // pushed genuinely-absent reflections over I/sigma>3 (a true R3 was lost at 13.5% violations).
    double max_absent_present_ratio = 0.5;

    // Need at least this many observed reflections in the predicted-absent class before a
    // screw/centering is claimed (guards against deciding from a handful of reflections).
    int min_absent_observed = 8;
};

struct SearchSpaceGroupResult {
    std::optional<gemmi::SpaceGroup> best_space_group;
    // Other space groups that fit the data equally well (same systematic absences): enantiomorphic
    // partners (P4_1 vs P4_3), origin-ambiguous pairs (I222 vs I2_12_12_1), or groups left
    // undetermined by incomplete data. The data cannot choose between best_space_group and these.
    std::vector<gemmi::SpaceGroup> alternatives;

    std::string point_group_hm;                            // chosen point group, e.g. "422"
    // The symmorphic space group representing that point group. This, not the name, identifies it:
    // gemmi reports both P321 and P312 as "32", so two searches that disagree about which 2-folds are
    // real look identical by name. Also what a caller passes back as fixed_point_group.
    std::optional<gemmi::SpaceGroup> point_group_representative;
    // Order of that point group (its number of rotations). Reported separately because Stage B can
    // leave best_space_group unset - no candidate had enough absences to be eligible - while Stage A
    // has confirmed the point group perfectly well, and a caller comparing two searches has to see
    // the symmetry that was found either way. 0 only when no point group was chosen at all.
    int point_group_order = 0;
    std::vector<SpaceGroupOperatorScore> operator_scores;  // Stage A, all distinct operators tested
    std::vector<SpaceGroupCandidateScore> candidates;      // Stage B, ranked

    // A HIGHER point group whose operators the intensities confirmed (Stage A) but whose promotion the
    // consistency tests refused, with the reason. Processing continues in the lower group, which is the
    // safe direction: merging a twinned crystal in the twin's holohedry averages non-equivalent
    // reflections into each other and is unrecoverable from the output (and makes the run report that no
    // twin law exists), whereas keeping the subgroup costs only redundancy and can be promoted later.
    // Empty when nothing was refused. Surfaced to the user - a silent demotion is how a twin gets missed.
    std::string refused_point_group_hm;
    std::string refused_reason;
};

SearchSpaceGroupResult SearchSpaceGroup(
    const std::vector<MergedReflection>& merged,
    const SearchSpaceGroupOptions& opt = {});

std::string SearchSpaceGroupResultToText(
    const SearchSpaceGroupResult& result,
    size_t max_candidates_to_print = 20);