diff --git a/image_analysis/lattice_search/LatticeSearch.cpp b/image_analysis/lattice_search/LatticeSearch.cpp index fbeb22a6..d8e5e85a 100644 --- a/image_analysis/lattice_search/LatticeSearch.cpp +++ b/image_analysis/lattice_search/LatticeSearch.cpp @@ -8,6 +8,15 @@ #include #include +// How close the reduced beta has to be to 90 degrees for the two Niggli types to be genuinely +// interchangeable (see the retry at the end of LatticeSearch). Not the angle tolerance: that is how +// far a metric may sit from an ideal one and still be called it, which is far too generous here - a +// cell 2 degrees off the boundary is a real type-1 cell, and presenting it in the obtuse setting +// promotes a general triclinic lattice to C-centred monoclinic on residuals of ~2 degrees. Measured: +// the crystal this was found on sits 0.07 degrees from the boundary and matches on 0.006 to 0.135; +// the triclinic cell that must not be promoted sits 2.0 degrees from it. +constexpr double NIGGLI_TYPE_BOUNDARY_DEG = 0.5; + struct NiggliClass { int number; int type; @@ -352,10 +361,12 @@ LatticeSearchResult LatticeSearch(const CrystalLattice &L, double dist_tolerance return std::nullopt; }; - if (auto found = match(L_niggli, D, E, F)) + // Character 44 fits any cell, so a match is always found - "nothing fits" is reported as triclinic. + auto found = match(L_niggli, D, E, F); + if (found && found->system != gemmi::CrystalSystem::Triclinic) return *found; - // A reduced cell whose beta is 90 to within the angle tolerance sits ON the boundary between the two + // A reduced cell whose beta is 90 to within NIGGLI_TYPE_BOUNDARY_DEG sits ON the boundary between the two // Niggli types: the same lattice reduces to an all-acute cell or an all-obtuse one according to the // last digits of whatever refinement produced it. The type-1 characters are skipped for such a cell // (just above) and the type-2 ones are stated for the obtuse setting, so an acute cell can match none @@ -363,12 +374,16 @@ LatticeSearchResult LatticeSearch(const CrystalLattice &L, double dist_tolerance // and c keeps the lattice and beta and turns alpha and gamma into their supplements. Measured on a // C-centred monoclinic crystal whose reduced beta sits 0.07 deg from 90: its centring was read or // missed according to the sign of that 0.07 deg, and with it the space group of the whole run. - if (L_niggli.GetUnitCell().beta >= 90 - angle_tolerance && D > 0 && E > 0 && F > 0) { + if (L_niggli.GetUnitCell().beta >= 90 - NIGGLI_TYPE_BOUNDARY_DEG && D > 0 && E > 0 && F > 0) { const gemmi::Mat33 obtuse(-1, 0, 0, 0, 1, 0, 0, 0, -1); - if (auto found = match(L_niggli.Multiply(obtuse), -D, E, -F)) - return *found; + const auto flipped = match(L_niggli.Multiply(obtuse), -D, E, -F); + if (flipped && flipped->system != gemmi::CrystalSystem::Triclinic) + return *flipped; } + if (found) + return *found; + return LatticeSearchResult{ .niggli_class = 44, .primitive_reduced = L_niggli,