Indexing: a coplanar cell leaves the frame unindexed instead of throwing

The online broker cancelled a collection with "Crystal lattice is coplanar and has no reciprocal
cell". The candidate filter in PostIndexingRefinement (used by FFT, FFTW and ffbidx) tests lengths,
angles and spot count only; three rows 120 deg apart in one plane pass all of them, and spots along
the plane normal index as (0,0,0). Such a candidate - from ffbidx, or one the least-squares
refinement walked flat - went on to LatticeSearch and XtalOptimizer, which refuses it and leaves it
in place, and then reached Astar() in AnalyzeIndexing/prediction, which throws; the receiver turns
that into a cancelled acquisition.

- Refine() rejects a candidate below MIN_BASIS_VOLUME_FRACTION, like any other bad candidate.
- XtalOptimizerInternal builds the refined lattice first and counts a coplanar result as a failed
  refinement, writing nothing back (the residual's volume clamp lets a solve end there and report
  success).

Test: PostIndexingRefinement_CoplanarCandidateIsRejected threw the exact message before the fix.
rugnux p.mtz unchanged on the three in-house reference sets at --prepass-fraction 1.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-07 13:55:44 +02:00
1 parent 280ff0a4a0
commit e6bbfea4c4
3 files changed
+72 -18

No files matched your search

@@ -457,6 +457,31 @@ bool XtalOptimizerInternal(XtalOptimizerData &data,
if (!summary.IsSolutionUsable())
return false;
// A solve can walk the basis flat and still report success - the volume clamp in XtalResidual
// keeps every step finite on the way. A coplanar cell has no reciprocal basis, so that is a
// failed refinement: refuse it like any other, before anything is written back.
CrystalLattice latt;
if (data.crystal_system == gemmi::CrystalSystem::Orthorhombic)
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
else if (data.crystal_system == gemmi::CrystalSystem::Tetragonal) {
latt_vec1[1] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Cubic) {
latt_vec1[1] = latt_vec1[0];
latt_vec1[2] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Hexagonal) {
latt_vec1[1] = latt_vec1[0];
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1,PI / 2.0, PI / 2.0, 2.0 * PI / 3.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Monoclinic) {
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, latt_vec2[0], PI / 2.0);
} else {
// Triclinic via the same generic builder
latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, latt_vec2[0], latt_vec2[1], latt_vec2[2]);
}
if (latt.VolumeFraction() < MIN_BASIS_VOLUME_FRACTION)
return false;
if (data.refine_beam_center) {
data.beam_corr_x = data.geom.GetBeamX_pxl() - beam[0];
data.beam_corr_y = data.geom.GetBeamY_pxl() - beam[1];
@@ -470,24 +495,7 @@ bool XtalOptimizerInternal(XtalOptimizerData &data,
if (data.axis && data.refine_rotation_axis)
data.axis.value().Axis(Coord(rot_vec[0], rot_vec[1], rot_vec[2]));
if (data.crystal_system == gemmi::CrystalSystem::Orthorhombic)
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
else if (data.crystal_system == gemmi::CrystalSystem::Tetragonal) {
latt_vec1[1] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Cubic) {
latt_vec1[1] = latt_vec1[0];
latt_vec1[2] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, PI / 2.0, PI / 2.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Hexagonal) {
latt_vec1[1] = latt_vec1[0];
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1,PI / 2.0, PI / 2.0, 2.0 * PI / 3.0);
} else if (data.crystal_system == gemmi::CrystalSystem::Monoclinic) {
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, PI / 2.0, latt_vec2[0], PI / 2.0);
} else {
// Triclinic via the same generic builder
data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, latt_vec2[0], latt_vec2[1], latt_vec2[2]);
}
data.latt = latt;
return true;
} catch (...) {
// Convergence problems, likely not updated
@@ -270,6 +270,12 @@ std::vector<CrystalLattice> Refine(const std::vector<Coord> &in_spots,
cos_gamma > cos_min_angle || cos_gamma < cos_max_angle)
continue;
// Three coplanar rows pass both tests above - 120 deg apart in one plane is inside any angle
// window - and the refinement can walk a candidate there. Such a cell has no reciprocal basis,
// so it is not a lattice: the same volume test as CrystalLattice::VolumeFraction().
if (std::abs(cell_rows.determinant()) < MIN_BASIS_VOLUME_FRACTION * row_norms.prod())
continue;
int64_t indexed_spot_count = 0;
auto indexed_mask = ComputeIndexedMask(spots.topRows(nspots), cell_cols, p.indexing_tolerance, indexed_spot_count);