Jungfraujoch/tools/battery/test_score.py

"""Targeted tests of score.py, the part a wrong answer would be laundered through.

Plain stdlib unittest, so it runs with `python3 tools/battery/test_score.py` (or
`python3 -m unittest`) from this directory, without pytest and without a battery run.
"""
import os
import tempfile
import unittest

import score


def report(sg, sgno, cell, **extra):
    """The three report keys the scorer needs, plus whatever a case adds."""
    return dict({"SPACE_GROUP_NAME": sg, "SPACE_GROUP_NUMBER": str(sgno),
                 "UNIT_CELL_CONSTANTS": " ".join(f"{x:.3f}" for x in cell)}, **extra)


# A deposited trigonal reference and the higher group our reduction reads on it: the shape of
# the accepted-alternative rows, with a synthetic cell, not a real one.
TRIGONAL = [80.0, 80.0, 150.0, 90.0, 90.0, 120.0]
DEPOSITED = {"sg": "P 32", "sgno": 145, "cell": TRIGONAL, "dmin": 2.0}


def entry(**extra):
    return dict({"id": "test_set", "arm": "open", "ref": DEPOSITED}, **extra)


class AcceptedAlternatives(unittest.TestCase):
    def test_deposition_still_decides_on_its_own(self):
        """Without alternatives the higher group is a symmetry failure, as before."""
        r = score.judge(entry(), report("P 31 2 1", 152, TRIGONAL), "")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_over"))

    def test_either_answer_passes(self):
        """Both the deposited group and the alternative pass, and only the second is marked."""
        e = entry(ref_alternatives=[{"sg": "P 31 2 1", "sgno": 152, "why": "a stated reason"}])
        dep = score.judge(e, report("P 32", 145, TRIGONAL), "")
        self.assertEqual((dep["verdict"], dep["cause"], dep["accepted_alt"]), ("pass", None, None))
        alt = score.judge(e, report("P 31 2 1", 152, TRIGONAL), "")
        self.assertEqual((alt["verdict"], alt["cause"]), ("pass", "accepted_alternative"))
        self.assertEqual(alt["accepted_alt"], "P 31 2 1")
        self.assertIn("a stated reason", alt["accepted_alt_why"])
        # the row still describes the answer against the DEPOSITION, not against the alternative
        self.assertEqual(alt["sg_ref"], "P 32")

    def test_the_other_hand_of_an_alternative_passes_too(self):
        """P 31 2 1 and P 32 2 1 differ by hand only, which intensities cannot decide."""
        e = entry(ref_alternatives=[{"sg": "P 31 2 1", "sgno": 152, "why": "a stated reason"}])
        r = score.judge(e, report("P 32 2 1", 154, TRIGONAL), "")
        self.assertEqual((r["verdict"], r["cause"]), ("pass", "accepted_alternative"))

    def test_a_third_answer_still_fails(self):
        """An alternative accepts one other answer, not any other answer."""
        e = entry(ref_alternatives=[{"sg": "P 31 2 1", "sgno": 152, "why": "a stated reason"}])
        r = score.judge(e, report("P 31 1 2", 151, TRIGONAL), "")
        self.assertEqual(r["verdict"], "fail")

    def test_alternative_cell(self):
        """A supercell alternative: the deposited cell is its (0,1/2,1/2) sublattice, so the
        deposition alone scores the answer as a doubled lattice and the alternative accepts it."""
        sub = [35.869, 39.297, 100.916, 98.3, 90.32, 90.09]
        sup = [35.869, 39.297, 199.976, 87.087, 90.341, 90.09]      # c' = b + 2c, centring removed
        e = {"id": "test_cell", "arm": "open", "ref": {"sg": "P 1", "sgno": 1, "cell": sub}}
        r = score.judge(e, report("P 1", 1, sup), "")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "lattice_doubled"))
        e["ref_alternatives"] = [{"cell": sup, "why": "a stated reason"}]
        r = score.judge(e, report("P 1", 1, sup), "")
        self.assertEqual((r["verdict"], r["cause"]), ("pass", "accepted_alternative"))
        self.assertTrue(r["accepted_alt"].startswith("cell "))
        self.assertEqual(r["volume_ratio"], 2.0)      # still reported against the deposition

    def test_a_justification_is_mandatory(self):
        """An accepted alternative with no reason is a schema error, not a silent pass: this is
        the check that keeps the mechanism from becoming a way to launder a failure."""
        for alt in ({"sg": "P 31 2 1", "sgno": 152},
                    {"sg": "P 31 2 1", "sgno": 152, "why": "  "}):
            with self.assertRaises(ValueError) as cm:
                score.judge(entry(ref_alternatives=[alt]), report("P 31 2 1", 152, TRIGONAL), "")
            self.assertIn("why", str(cm.exception))

    def test_an_alternative_must_replace_something(self):
        with self.assertRaises(ValueError):
            score.judge(entry(ref_alternatives=[{"why": "a stated reason"}]),
                        report("P 32", 145, TRIGONAL), "")

    def test_the_manifest_rows_are_valid(self):
        """Every committed row's alternatives pass the schema check."""
        import json
        import os
        for name in ("open.json", "inhouse.json"):
            path = os.path.join(os.path.dirname(os.path.abspath(__file__)), name)
            with open(path) as f:
                for e in json.load(f)["sets"]:
                    score.alternatives(e)


# A screw along an axis the sweep never recorded: an unanswerable question, not a wrong answer.
# The cells are synthetic, and the second is the first cyclically permuted.
ORTHO = [97.0, 108.0, 113.0, 90.0, 90.0, 90.0]
ORTHO_PERMUTED = [113.0, 97.0, 108.0, 90.0, 90.0, 90.0]


class ScrewUndetermined(unittest.TestCase):
    def judge(self, ref_cell, undetermined, alternatives):
        """Our P 21 21 21 on ORTHO against a deposited P 21 21 2 on ref_cell."""
        e = {"id": "test_screw", "arm": "open",
             "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": ref_cell}}
        return score.judge(e, report("P 21 21 21", 19, ORTHO,
                                     SPACE_GROUP_SCREW_UNDETERMINED=undetermined,
                                     SPACE_GROUP_ALTERNATIVES=alternatives), "")

    def test_same_setting(self):
        """The plain case: the reference is offered verbatim and the open axis is the one they
        differ on."""
        r = self.judge(ORTHO, "c", "P 21 21 2")
        self.assertEqual((r["verdict"], r["cause"]), ("unscored", "screw_undetermined"))

    def test_permuted_setting(self):
        """The deposition's axes are a cyclic permutation of ours, so it names the open screw c
        while we name it b. The comparison must be made in OUR setting - against the offered
        group of the reference's number - or the rule misses a row it covers."""
        r = self.judge(ORTHO_PERMUTED, "b", "P 21 2 21")
        self.assertEqual((r["verdict"], r["cause"]), ("unscored", "screw_undetermined"))
        self.assertIn("the screw along b", r["reason"])

    def test_a_wrong_screw_still_fails(self):
        """The open axis is not the axis the two differ on: in our own setting the disagreement
        sits on b while the run says it could not determine a, so the b row WAS measured. That is
        a wrong answer and stays one - the new framing must not turn every permutation into an
        excuse."""
        r = self.judge(ORTHO_PERMUTED, "a", "P 21 2 21")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_screw"))

    def test_nothing_undetermined_still_fails(self):
        """Every axial row measured: no rule to apply."""
        r = self.judge(ORTHO, "NONE", "")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_screw"))

    def test_the_reference_must_be_among_the_offered(self):
        """An open axis alone does not excuse a group the run never offered."""
        r = self.judge(ORTHO, "b", "P 21 21 21")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_screw"))


# Space groups of one Laue class with identical absences: the merged intensities cannot separate
# them. Synthetic cells.
TETRA_I = [7.4, 7.4, 6.9, 90.0, 90.0, 90.0]
MONO = [11.0, 6.5, 11.0, 90.0, 96.0, 90.0]
ORTHO_SMALL = [8.3, 9.6, 27.0, 90.0, 90.0, 90.0]


class IndistinguishableAlternatives(unittest.TestCase):
    def judge(self, arm, ref_sg, ref_sgno, cell, sg, sgno, alternatives=""):
        e = {"id": "test_same_absences", "arm": arm,
             "ref": {"sg": ref_sg, "sgno": ref_sgno, "cell": cell}}
        return score.judge(e, report(sg, sgno, cell, SPACE_GROUP_ALTERNATIVES=alternatives), "")

    def test_same_absences_pass_on_both_arms(self):
        """Acentric vs acentric (I41md / I-42d), acentric vs centric (Pc / P2/c, Pna21 / Pnma,
        P2 / P2/m): a pass with its own cause, whichever arm, whichever way round."""
        cases = [("I -4 2 d", 122, TETRA_I, "I 41 m d", 109),
                 ("P 1 2/c 1", 13, MONO, "P 1 c 1", 7),
                 ("P 1 c 1", 7, MONO, "P 1 2/c 1", 13),
                 ("P n m a", 62, ORTHO_SMALL, "P n a m", 62),       # same number: plain match
                 ("P n a 21", 33, ORTHO_SMALL, "P n a m", 62),
                 ("P 1 2 1", 3, MONO, "P 1 2/m 1", 10)]
        for arm in ("open", "inhouse"):
            for ref_sg, ref_sgno, cell, sg, sgno in cases:
                r = self.judge(arm, ref_sg, ref_sgno, cell, sg, sgno)
                want = (("pass", None) if sgno == ref_sgno
                        else ("pass", "indistinguishable_alternative"))
                self.assertEqual((r["verdict"], r["cause"]), want, (arm, ref_sg, sg))

    def test_the_reason_says_whether_rugnux_listed_it(self):
        r = self.judge("inhouse", "I -4 2 d", 122, TETRA_I, "I 41 m d", 109, "I -4 2 d")
        self.assertIn("listed in SPACE_GROUP_ALTERNATIVES", r["reason"])
        self.assertEqual(r["sg_alternatives"], "I -4 2 d")
        r = self.judge("inhouse", "I -4 2 d", 122, TETRA_I, "I 41 m d", 109)
        self.assertIn("NOT listed", r["reason"])

    def test_different_absences_still_fail(self):
        """Same Laue class, different absences: a real difference, listed or not."""
        r = self.judge("open", "P 1 21/c 1", 14, MONO, "P 1 21 1", 4, "P 1 21/c 1")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_under"))
        r = self.judge("open", "I -4 2 d", 122, TETRA_I, "I 41 2 2", 98, "I -4 2 d")
        self.assertEqual(r["verdict"], "fail")
        r = self.judge("inhouse", "I -4 2 d", 122, TETRA_I, "I 4 m m", 107, "I -4 2 d")
        self.assertEqual(r["verdict"], "fail")

    def test_different_laue_class_still_fails(self):
        r = self.judge("open", "P 4 2 2", 89, [50.0, 50.0, 70.0, 90, 90, 90], "P 4", 75, "P 4 2 2")
        self.assertEqual((r["verdict"], r["cause"]), ("fail", "sym_under"))


class Crashes(unittest.TestCase):
    def log(self, text, exit_code):
        with tempfile.TemporaryDirectory() as wd:
            open(os.path.join(wd, "p_log.txt"), "w").write(text)
            return score.crash_note(exit_code, wd)

    def test_a_crash_fails_even_a_correct_answer(self):
        """A crash is decided first: a report that would pass does not save the set."""
        r = score.judge(entry(), report("P 32", 145, TRIGONAL), "crash: exit 1: Processing failed: x")
        self.assertEqual((r["verdict"], r["cause"]), ("crash", "crash"))

    def test_a_crash_fails_a_no_crystal_control(self):
        r = score.judge(entry(expect="no_lattice"), {}, "crash: killed by signal 11")
        self.assertEqual(r["verdict"], "crash")

    def test_what_is_a_crash(self):
        self.assertIn("unspecified launch failure",
                      self.log("[t] [rugnux] [error] Processing failed: CUDA (GPU) error "
                               "(RotationScaleMergeGPU: upload: unspecified launch failure)\n", 1))
        # a device fault rugnux handled still counts, exit 0 or not; a planned capacity fallback does not
        self.assertTrue(self.log("[t] [error] Indexer thread 1 failed: CUDA (GPU) error "
                                 "(an illegal memory access was encountered)\n", 0))
        self.assertEqual(self.log("[t] [Rugnux] [warning] Model validation: the rigid body failed on the "
                                  "GPU (CUDA (GPU) error (rigid body: zone over the engine's reserve)); "
                                  "validating again on the CPU\n", 0), "")
        self.assertTrue(self.log("", -11))
        # rugnux's own refusal, a CUDA build's banner and a timeout are not crashes
        self.assertEqual(self.log("[t] [rugnux] [info] rugnux Version 1 - CUDA build\n"
                                  "[t] [rugnux] [error] No crystal lattice found: few spots\n", 1), "")
        self.assertEqual(self.log("", None), "")


if __name__ == "__main__":
    unittest.main()