Jungfraujoch/image_analysis/scale_merge/SearchSpaceGroup.h

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#pragma once

#include <optional>
#include <string>
#include <vector>

#include "../../common/Reflection.h"
#include "gemmi/symmetry.hpp"

// Determine the likely space group of a dataset from its P1-merged intensities, in the spirit
// of POINTLESS (Evans 2006):
//
//   Stage A - point group (Laue) symmetry. Every candidate rotation operator is scored once by
//             the correlation of I(h) with I(Rh). The chosen point group is the largest one all
//             of whose operators are confirmed (high CC). A wrong operator scores ~0, so this is
//             self-pruning - the unit cell is not needed.
//
//   Stage B - space group within that point group. Each Sohncke space group of the point group
//             predicts a set of systematically absent reflections (centering + screw axes). The
//             chosen space group is the one that explains the MOST absences while every reflection
//             it predicts absent is in fact weak. The symmorphic group (no absences) is the
//             fallback when no screw/centering is supported. Enantiomorphic pairs (e.g. P4_1 vs
//             P4_3) are indistinguishable from intensities and are reported as a pair.

struct SpaceGroupOperatorScore {
    std::string op_triplet_hkl;   // reciprocal-space triplet of the rotation, e.g. "-h,-k,l"
    double cc = 0.0;              // correlation of I(h) with I(Rh)
    int n_pairs = 0;             // independent reflection pairs the CC was computed from
    bool present = false;        // operator confirmed as a real symmetry of the intensities
    // MEDIAN |I1-I2|/(I1+I2) over this operator's pairs - the disagreement the operator implies, with no
    // sigma in it. Unlike chi^2 and the systematic-b, which are ratios to a merge error model that
    // drifts with multiplicity, this is a property of the intensities alone; compared against an
    // operator already confirmed on the same reflections it is what separates a real symmetry from a
    // twin law (see max_operator_h_ratio). The median rather than the mean because a merohedral twin
    // perturbs EVERY pair while a badly integrated minority perturbs only the tail.
    double h_stat = 0.0;
};

struct SpaceGroupCandidateScore {
    gemmi::SpaceGroup space_group;
    int absent_observed = 0;          // observed reflections this SG predicts systematically absent
    int absent_violations = 0;        // of those, how many are nonetheless strongly present
    int centering_absent = 0;         // of absent_observed, the ones extinguished by the CENTERING
    int screw_absent = 0;             // ... and the ones extinguished by a SCREW (i.e. on axial rows)
    // How unlikely the screw-absent class would be if the screw did not exist, in nats - the quantity
    // that decides whether a screw may be claimed at all (see min_screw_absence_evidence). 0 when there
    // are no screw absences, or when no axial row carried a control class to judge them against.
    double screw_absence_evidence = 0.0;
    double absent_mean_i_over_sigma = 0.0;
    double present_mean_i_over_sigma = 0.0;
    bool consistent = false;          // absent class confirmed weak (few violations)
    bool selected = false;            // chosen result (or its enantiomorph)
};

struct SearchSpaceGroupOptions {
    // Lattice (metric) symmetry from LatticeSearch. When set, the point-group search is limited to
    // the Sohncke subgroups of this system's holohedry - the metric is an upper bound on the
    // intensity symmetry, so e.g. a tetragonal metric tests {1,2,222,4,422} and never 3/6/23. All
    // subgroups are still tested (down to P1), so a pseudo-symmetric metric never forces a higher
    // symmetry than the intensities support. Unset = search every Sohncke system.
    std::optional<gemmi::CrystalSystem> lattice_system;

    // Centering is NOT taken from the lattice metric: an indexer that returns the conventional cell
    // (e.g. cubic 'P'-looking axes for a body-centered lattice) hides the centering, which lives only
    // in the systematic absences. Stage B therefore tests every centering allowed by the point group
    // and confirms it from the data (h+k+l etc. absent), rather than trusting a geometric hint.

    // Friedel mates are treated as equivalent when matching HKLs (i.e. anomalous signal ignored).
    bool merge_friedel = true;

    // Ignore reflections beyond this resolution (smaller d = higher resolution). 0 disables.
    double d_min_limit_A = 0.0;

    // Drop reflections weaker than this from the correlation stage only (the absence stage must
    // keep weak reflections - that is where the screw-axis signal lives).
    double min_i_over_sigma = 0.0;

    // Drop reflections STRONGER than this resolution-normalised E^2 = I/<I>(shell) from the correlation
    // stage only. A second lattice deposits intensity on one reciprocal position but not its symmetry
    // mate, so an overlap-contaminated reflection is a one-sided E^2 outlier that poisons an operator's
    // I(h)/I(Rh) correlation (it flipped a pseudo-merohedral P2_1 case to P1 under -A: excluding the E^2>9 tail, ~0.4%
    // of reflections, lifted the 2-fold CC 0.33->0.53 back over the gate). Clean Wilson-distributed data
    // almost never reaches E^2=9 (P(E^2>9) ~ 0.01-0.3%), so this removes essentially nothing there and
    // only trims the overlap tail. 0 disables. Absences are unaffected (they need the weak tail).
    double max_e_squared_for_cc = 9.0;

    // --- Stage A: point group ---
    // A rotation is accepted as a real symmetry when its I(h)/I(Rh) correlation reaches this over
    // at least min_pairs_per_operator independent pairs.
    double min_operator_cc = 0.5;
    int min_pairs_per_operator = 20;

    // Per-operator CC alone cannot tell a real weak operator from a false strong one (a noisy crystal's
    // genuine 2-fold can score below a pseudo-symmetric crystal's near-perfect false one). So a point
    // group is also required to be SELF-CONSISTENT: merging the intensities under it must not inflate the
    // reduced chi^2 (within-orbit scatter / sigma^2) beyond this factor times the most-consistent
    // candidate. A false operator forces non-equivalent reflections together so they disagree by many
    // sigma and chi^2 blows up; a real one leaves it ~flat even when the operator CC is only moderate.
    // Calibrated on the rotation-test battery: every correct point group stays within ~1.7x the best
    // subgroup even on weak / badly-integrated data (worst real case a P41212 at 1.71), while a twin
    // law or pseudo-symmetry lands clearly higher (a merohedral R3->R32 twin 2-fold at 2.01). 1.85 sits
    // between the two, so a partial merohedral twin is kept in its true lower symmetry (R3), not
    // over-promoted to the holohedral R32.
    double max_merge_chi2_ratio = 1.85;

    // A genuine high-symmetry merge can drift just past max_merge_chi2_ratio when its data are only
    // imperfectly scaled: each real symmetry step then adds a little systematic scatter, so a weakly-
    // scaled cubic case lands at ratio ~2.0 - right where a merohedral twin (an R3->R32 case at ~1.95)
    // also lands, so the chi^2 ratio alone cannot separate them. A candidate whose chi^2 is
    // only this far past the best subgroup is therefore rescued if the SYSTEMATIC part of the extra
    // scatter stayed small (max_systematic_b_ratio): merging under a genuine operator gains multiplicity
    // without intensity-proportional disagreement, so the merge error model's b barely moves, whereas a
    // twin forces non-equivalent reflections together and b balloons (a genuine cubic step b x1.6 vs a
    // merohedral twin x2.2, measured against the largest confirmed subgroup). Both bounds sit in the gap; the rescue only ever
    // promotes, and only in this narrow chi^2 band.
    double max_merge_chi2_rescue = 2.30;
    double max_systematic_b_ratio = 1.90;

    // Veto bound. The systematic-b test above is otherwise only a rescue - it can promote a chi^2-borderline
    // genuine step but never demote a chi^2-passing one. A merohedral twin whose within-orbit scatter looks
    // self-consistent (chi^2 ratio below max_merge_chi2_ratio) therefore slips through on chi^2 alone even
    // though merging its non-equivalent reflections balloons b. So a chi^2-passing high-symmetry promotion
    // whose b, relative to the largest confirmed subgroup, exceeds this bound is vetoed and kept in its true
    // lower symmetry. Calibrated on the rotation-test battery against a merohedral R3->R32 twin at b-ratio
    // ~2.6, with the largest genuine step then measured at ~1.8.
    //
    // That gap is far narrower than it looked: a genuine cubic 222 -> 23 step on weak low-energy data sits
    // at 1.94. Masking the beam-stop shadow moved it to 2.07 and demoted the crystal - and it did so while
    // IMPROVING both merges, because b is fitted against the merged sigmas and the subgroup's b improved
    // more (0.164 -> 0.148) than the cubic merge's (0.319 -> 0.306). A ratio with 3% of headroom, which the
    // measurement can move in either direction, cannot decide a symmetry on its own. So the veto is applied
    // only where the H test has NOT confirmed the promotion: H compares intensities with intensities, has no
    // sigma in it, and is the statistic that was actually measured to separate a twin (see
    // max_operator_h_ratio). Where both fire the promotion is still refused, and every twin on the battery
    // and in the synthetic harness fires both. The veto only ever keeps a clearly-ballooned promotion down,
    // never promotes.
    double max_systematic_b_veto = 2.0;

    // Floor on the parent's systematic-b when forming the veto ratio. On excellent data a genuine merge's
    // b is near zero (ISa well above 20), so even a small, harmless absolute increase gives a huge b-ratio
    // and the veto would wrongly reject real high symmetry (a true tetragonal 422 whose parent 222 sits at
    // b=0.008 and 422 at b=0.049 - both tiny - reads as a 6x balloon). A real twin, by contrast, drives b
    // to a large ABSOLUTE value (~0.19). Flooring the parent b here makes the ratio meaningful only once b
    // is a non-negligible fraction of I; below the floor the increase is treated as noise, not a twin.
    double min_systematic_b_for_veto = 0.05;

    // Promotion gate on the operator disagreement H = median|I1-I2|/(I1+I2), taken as the ratio of the
    // operators the promotion ADDS to the operators of the parent group already confirmed on the same
    // reflections. A real symmetry operator relates equal intensities, so its H matches the parent's
    // (ratio ~1); a merohedral twin law relates DIFFERENT reflections mixed in proportion alpha, so its
    // H is systematically larger. Unlike the chi^2 and systematic-b ratios (genuine 1.00-3.47 /
    // 1.09-3.89 vs twin 1.35-3.32 / 1.77-4.76, fully interleaved) it does not drift with multiplicity,
    // because there is no sigma in it and the parent normalisation cancels data quality.
    //
    // A MEDIAN, not a mean. A merohedral twin mixes every reflection with its twin mate, so it shifts the
    // whole distribution of |I1-I2|/(I1+I2); a minority of badly measured reflections shifts only the
    // tail. Measured on the same real crystals, moving from the mean to the median leaves genuine
    // promotions where they are (1.016 -> 1.013, 1.051 -> 1.067, 1.238 -> 1.231) and pushes every real
    // twin UP (1.280 -> 1.622, 1.427 -> 2.010, 1.272 -> 1.447, 1.441 -> 1.522), widening the margin
    // around this bound from 2.7% to 17.5%. Battery-neutral: 33 crystals, no point group changed.
    //
    // The parent normalisation is what makes this work, and it is not optional. Symmetry-related
    // reflections never agree exactly on real data - absorption, illumination and partiality differ
    // between them - and that systematic floor varies by crystal AND by operator (a cubic 3-fold permutes
    // axes, relating far-apart parts of reciprocal space, so it disagrees more than the 2-folds of its
    // parent even when the symmetry is perfectly real). Measuring an added operator against the parent's
    // own operators, on the same reflections, is what divides that floor out. An ABSOLUTE bound on any
    // per-operator agreement statistic cannot: measured absolute values for genuine symmetry span the
    // whole range from 0.99 on strong data down to 0.71 on weak, straddling every twin.
    //
    // Its known limit is angular coverage, not data quality. What the parent normalisation cannot divide
    // out is the part of the systematic floor that differs BETWEEN the added operators and the parent's,
    // and that part grows as the measurement gets less uniform: on a full sweep a real tetragonal crystal
    // reads the same H on all seven operators of 422 (0.051-0.057, ratio 1.03), but on a quarter of the
    // frames of the same crystal the seven spread over 0.082-0.168 and the ratio reaches 1.45, the two
    // diagonal 2-folds - left with the fewest surviving pairs - carrying the whole excess. So a refusal
    // from a merge built on a small or lopsided part of the sweep says as much about the coverage as about
    // the symmetry; the fix for that is upstream, in getting the sweep indexed.
    //
    // The bound is set from the gap between the two populations, measured over the rotation battery:
    // genuine promotions read 0.85-1.57 (and 2.48 on a genuine orthorhombic step in a pair-starved arm),
    // real merohedral twins 1.82 and 4.01. 1.25 sat INSIDE the genuine range - four genuine promotions
    // already exceeded it and survived only because the two-arm rule happened to cover them, and one
    // cubic case with no such cover was refused outright by a margin of 0.4%.
    //
    // Re-measured over ~780 reports (three batteries and four de-novo integration-radius arms), H is the
    // only Stage-A guard that ever fires: 14 refusals on three crystals, no chi-squared-ratio refusal and
    // no b veto anywhere. Two of the three are the genuine trigonal twins, reading 1.90/1.91 on every arm
    // and 3.61-4.80; the third is a weak tetragonal crystal reading 1.73-1.80 whose 422 is confirmed by
    // every operator correlation and which three independent tests - our own L test on the merge it was
    // demoted into, xtriage on the same merge ("most likely due to an NCS axis parallel to the twin
    // axis"), and the arithmetic that a twin at the implied fraction caps pair correlation at 0.55 while
    // the added operators score 0.84-0.90 - agree is not twinned. Its excess is the coverage limit named
    // above: the added 422 operators exchange h and k and carry a systematic floor the parent's
    // sign-flipping 2-folds do not. So the populations collide at 1.80 and 1.90 rather than at 1.70, and
    // the bound sits at the midpoint. Degrading the data drives the ratio towards 1, never up, and the
    // estimator is precise (bootstrap standard error 0.2-3.7% of the ratio), so the collision is a real
    // overlap and not sampling noise - which is also why scaling the bound by that error cannot work.
    double max_operator_h_ratio = 1.85;

    // The H test needs at least this many pairs on both sides to mean anything.
    int min_pairs_for_h = 200;

    // Above this reduced chi^2 for the best subgroup (chi2_ref), the merged error model is treated as
    // badly miscalibrated (weak, low-resolution data whose sigmas are far too small): the fixed-sigma
    // chi^2 ratio then grows with point-group order for genuine high symmetry too and can no longer
    // arbitrate, so a promotion is confirmed on the systematic-b test alone (which re-fits its own error
    // and stays valid). A well-calibrated merge sits near 1; 3.0 (sigmas ~1.7x too small) marks the point
    // where the ratio stops being trustworthy. The balloon veto still guards against a twin.
    double chi2_ref_reliable = 3.0;

    // Adopt this space group's point group instead of the one Stage A would choose, and go straight to
    // Stage B with it. Used when one merge decides the point group and a different merge has to decide
    // the absences: a caller cannot pass the point group by name, because gemmi reports both P321 and
    // P312 as "32", so the group itself is passed and its rotations are taken from it. Stage A still
    // runs (its operator scores and its refusal report are still wanted); only the choice is overridden.
    std::optional<gemmi::SpaceGroup> fixed_point_group;

    // --- Stage B: space group (screw axes / centering) ---
    bool determine_space_group = true;     // false: stop at the symmorphic representative

    // Signed I/sigma above which a reflection counts as genuinely present. Signed, so negative
    // noise is never mistaken for a real reflection. Used both to spot reflections that violate a
    // wrongly-assumed absence, and to keep the correlation stage from pairing near-zero reflections.
    // For the CORRELATION stage this is an upper bound on the cut rather than the cut itself: merged
    // sigma is floored at b|I|, so no reflection can read above ISa = 1/b, and a merge whose ISa fell
    // below this value would otherwise contribute nothing at all (see SearchSpaceGroup.cpp).
    double present_i_over_sigma = 3.0;

    // Extra intensity gate for the systematic-absence test: a reflection also has to reach this
    // resolution-normalised intensity E^2 = I / <I>(shell) to count as violating a predicted absence.
    // The error model can under-estimate sigma on weak axial reflections and fake a high I/sigma, so a
    // reflection at a few percent of the shell-mean intensity is judged absent regardless of its sigma
    // (e.g. the monoclinic 2_1: 0k0-odd at ~1% of 0k0-even). 0 disables the gate (I/sigma only).
    //
    // For a SCREW the threshold is this fraction of the median E^2 of the axial row the screw
    // constrains (floored at the plain value), not of an average reflection at that resolution - an
    // axial row is often far stronger than the shell mean, and only the row is a fair comparison.
    double present_e_squared = 0.3;

    // A candidate's SCREW/glide absence conditions are accepted when at most this fraction of the
    // reflections it predicts absent are in fact strongly present.
    double max_absent_violation_fraction = 0.10;

    // A CENTERING is accepted when its systematically-absent class is this much weaker than the present
    // class - mean signed I/sigma of the centering-absent reflections <= this fraction of the present
    // mean. A real centering cancels structure factors so its absent class sits near zero (ratio ~0-0.3
    // across the test battery) even on noisy or obverse/reverse-twinned data; a false centering leaves it
    // as strong as the present class (ratio ~1.0). 0.5 separates the two with wide margin. This strength
    // test replaces a per-reflection violation-count gate for centering, which was brittle when noise
    // pushed genuinely-absent reflections over I/sigma>3 (a true R3 was lost at 13.5% violations).
    double max_absent_present_ratio = 0.5;

    // Need at least this many observed reflections in the CENTERING-absent class before a centering is
    // claimed (guards against deciding from a handful of reflections). A real centering extinguishes a
    // third to a half of every reflection in the data set, so this is never the binding constraint on a
    // centering that exists - it only refuses one the data barely sampled.
    //
    // It does NOT apply to screws. A screw's absent class is one row of reciprocal space by
    // construction, and that row is often the one a rotation sweep records least: it lies near the
    // spindle, where the blind cusp maps onto itself and symmetry cannot fill it in. Counting it
    // measures the geometry of the sweep, not the strength of the evidence. Applied to screws this
    // bound cost a monoclinic crystal its 2_1 - six 0k0-odd reflections, every one of them measured at
    // |E^2| <= 0.013 with zero violations, against a 0k0 row averaging 1.44x the shell mean, refused
    // for being six rather than eight. Screws answer to min_screw_absence_evidence below instead.
    int min_absent_observed = 8;

    // Evidence, in nats, that a screw's predicted-absent class really is absent, required before the
    // screw may be claimed. In the spirit of the POINTLESS zone test (Evans, Acta Cryst D67, 282-292
    // (2011), Appendix A3), which likewise scores an absence against the rest of its own axial row
    // rather than against a global mean or a fixed cut, and likewise lets the confidence fall away with
    // the number of axial reflections instead of refusing outright below a count.
    //
    // The statistic (see ScrewAbsenceEvidence) is the -log Beta tail of the absent class against its
    // row's control class. Two properties are what a violation count lacks: the row's own strength
    // cancels, so a uniformly weak axial row decides nothing rather than reading as "absent"; and the
    // scale is set by the number of reflections, so few-but-decisive and many-but-marginal are told
    // apart. It is sigma-free by design - the merged sigma carries the error model's
    // intensity-proportional term and so shrinks with I, reading much the same on an absent reflection
    // as on a present one (the <I/s> columns of the candidate table show this directly).
    //
    // Measured over the probe crystals, genuine screw conditions read 34-800 nats and the false ones
    // (the 4_1/4_3 conditions of a cubic crystal that has no screw at all, whose predicted-absent class
    // is STRONGER than its control row) read -7 to -8.5. The gap is ~40 nats wide and this bound sits
    // in it, at p <= 2e-9 - loose enough that three well-measured dead axial reflections clear it,
    // tight enough that two do not.
    double min_screw_absence_evidence = 20.0;

    // Workers for the operator-correlation stage, which is the bulk of the search: one pass over the
    // whole merge per candidate rotation, and the rotations are independent of each other. 1 = serial.
    size_t nthreads = 1;
};

struct SearchSpaceGroupResult {
    std::optional<gemmi::SpaceGroup> best_space_group;
    // Other space groups that fit the data equally well (same systematic absences): enantiomorphic
    // partners (P4_1 vs P4_3), origin-ambiguous pairs (I222 vs I2_12_12_1), or groups left
    // undetermined by incomplete data. The data cannot choose between best_space_group and these.
    std::vector<gemmi::SpaceGroup> alternatives;

    std::string point_group_hm;                            // chosen point group, e.g. "422"
    // The symmorphic space group representing that point group. This, not the name, identifies it:
    // gemmi reports both P321 and P312 as "32", so two searches that disagree about which 2-folds are
    // real look identical by name. Also what a caller passes back as fixed_point_group.
    std::optional<gemmi::SpaceGroup> point_group_representative;
    // Order of that point group (its number of rotations). Reported separately because Stage B can
    // leave best_space_group unset - no candidate had enough absences to be eligible - while Stage A
    // has confirmed the point group perfectly well, and a caller comparing two searches has to see
    // the symmetry that was found either way. 0 only when no point group was chosen at all.
    int point_group_order = 0;
    std::vector<SpaceGroupOperatorScore> operator_scores;  // Stage A, all distinct operators tested
    std::vector<SpaceGroupCandidateScore> candidates;      // Stage B, ranked

    // A HIGHER point group whose operators the intensities confirmed (Stage A) but whose promotion the
    // consistency tests refused, with the reason. Processing continues in the lower group, which is the
    // safe direction: merging a twinned crystal in the twin's holohedry averages non-equivalent
    // reflections into each other and is unrecoverable from the output (and makes the run report that no
    // twin law exists), whereas keeping the subgroup costs only redundancy and can be promoted later.
    // Empty when nothing was refused. Surfaced to the user - a silent demotion is how a twin gets missed.
    std::string refused_point_group_hm;
    std::string refused_reason;
};

SearchSpaceGroupResult SearchSpaceGroup(
    const std::vector<MergedReflection>& merged,
    const SearchSpaceGroupOptions& opt = {});

std::string SearchSpaceGroupResultToText(
    const SearchSpaceGroupResult& result,
    size_t max_candidates_to_print = 20);