Indexing: a coplanar cell leaves the frame unindexed instead of throwing

The online broker cancelled a collection with "Crystal lattice is coplanar and has no reciprocal
cell". The candidate filter in PostIndexingRefinement (used by FFT, FFTW and ffbidx) tests lengths,
angles and spot count only; three rows 120 deg apart in one plane pass all of them, and spots along
the plane normal index as (0,0,0). Such a candidate - from ffbidx, or one the least-squares
refinement walked flat - went on to LatticeSearch and XtalOptimizer, which refuses it and leaves it
in place, and then reached Astar() in AnalyzeIndexing/prediction, which throws; the receiver turns
that into a cancelled acquisition.

- Refine() rejects a candidate below MIN_BASIS_VOLUME_FRACTION, like any other bad candidate.
- XtalOptimizerInternal builds the refined lattice first and counts a coplanar result as a failed
  refinement, writing nothing back (the residual's volume clamp lets a solve end there and report
  success).

Test: PostIndexingRefinement_CoplanarCandidateIsRejected threw the exact message before the fix.
rugnux p.mtz unchanged on the three in-house reference sets at --prepass-fraction 1.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
This commit is contained in:
2026-10-07 13:55:44 +02:00
co-authored by Claude Opus 5.5
parent 280ff0a4a0
commit e6bbfea4c4
3 changed files with 72 additions and 18 deletions
@@ -457,6 +457,31 @@ bool XtalOptimizerInternal(XtalOptimizerData &data,
if (!summary.IsSolutionUsable())
return false;
// A solve can walk the basis flat and still report success - the volume clamp in XtalResidual
// keeps every step finite on the way. A coplanar cell has no reciprocal basis, so that is a
// failed refinement: refuse it like any other, before anything is written back.
CrystalLattice latt;
if (data.crystal_system == gemmi::CrystalSystem::Orthorhombic)
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
else if (data.crystal_system == gemmi::CrystalSystem::Tetragonal) {
latt_vec1[1] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Cubic) {
latt_vec1[1] = latt_vec1[0];
latt_vec1[2] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Hexagonal) {
latt_vec1[1] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1,PI / 2.0, PI / 2.0, 2.0 * PI / 3.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Monoclinic) {
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, latt_vec2[0], PI / 2.0);
} else {
// Triclinic via the same generic builder
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, latt_vec2[0], latt_vec2[1], latt_vec2[2]);
}
if (latt.VolumeFraction() < MIN_BASIS_VOLUME_FRACTION)
return false;
if (data.refine_beam_center) {
data.beam_corr_x = data.geom.GetBeamX_pxl() - beam[0];
data.beam_corr_y = data.geom.GetBeamY_pxl() - beam[1];
@@ -470,24 +495,7 @@ bool XtalOptimizerInternal(XtalOptimizerData &data,
if (data.axis && data.refine_rotation_axis)
data.axis.value().Axis(Coord(rot_vec[0], rot_vec[1], rot_vec[2]));
if (data.crystal_system == gemmi::CrystalSystem::Orthorhombic)
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
else if (data.crystal_system == gemmi::CrystalSystem::Tetragonal) {
latt_vec1[1] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Cubic) {
latt_vec1[1] = latt_vec1[0];
latt_vec1[2] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Hexagonal) {
latt_vec1[1] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1,PI / 2.0, PI / 2.0, 2.0 * PI / 3.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Monoclinic) {
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, latt_vec2[0], PI / 2.0);
} else {
// Triclinic via the same generic builder
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, latt_vec2[0], latt_vec2[1], latt_vec2[2]);
}
data.latt = latt;
return true;
} catch (...) {
// Convergence problems, likely not updated