From 439ca10db15ed781d7d9ba167c7ad50039f0c106 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 13:15:13 +0200 Subject: [PATCH] RESULT-CHANGING (owner approved): stills crystals indexed in a lattice containing the run's are put on the run's cell A still indexed on few spots often gets a lattice that explains them as well as the true one: an axis multiple, whose reciprocal lattice holds every true node plus the nodes between, or the true lattice in another setting. Its cell is then not the run's, the merge leaves it out, and its reflections at the true nodes - as good as any - are lost. On two 25k-frame XFEL runs of a tetragonal crystal these were 26-30% of the indexed frames; for those holding the true cell as a sublattice, the reflections at true nodes had the I/sigma of accepted frames and those between them none. RelabelOnSublattice (new, scale_merge/SublatticeRelabel) compares lattices, not cell parameters: it looks for integer combinations of the frame's reciprocal axes whose full metric is the run's cell within the merge's own tolerances, volume ratio 1-4 (another setting, or a 2x-4x supercell). Where several such bases exist - a nearly symmetric supercell fits along more than one axis - the one with the strongest mean I/sigma at its nodes is taken, since only the true lattice has signal there. The frame's indices are re-derived on that basis (h = h' M^-1), reflections that are not nodes are dropped, as are those finer than any frame already on the run's cell reaches (the wrong lattice was also predicted to a wrong resolution), and the frame gets the run's basis in its own orientation. It runs before the first stills merge, and in --mode scale before the resolutions are recomputed from the cell. --mode scale, both runs (indicative; the mx pairs are queued): 1060 and 920 crystals put back, observations +8%, I/sigma 10.3 -> 10.6 and 10.7 -> 11.0, CC1/2 0.966 -> 0.969 and 0.970 -> 0.966. Tests: SublatticeRelabel_DoubledAxisGoesBackToTheTrueCell, _OtherSettingAndUnrelatedLattice, _SignalPicksTheAxisThatWasDoubled. Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/scale_merge/CMakeLists.txt | 4 +- .../scale_merge/SublatticeRelabel.cpp | 170 ++++++++++++++++++ .../scale_merge/SublatticeRelabel.h | 41 +++++ rugnux/RugnuxScaleMerge.cpp | 10 ++ rugnux/rugnux_cli.cpp | 10 ++ tests/MergeScaleTest.cpp | 117 ++++++++++++ 6 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 image_analysis/scale_merge/SublatticeRelabel.cpp create mode 100644 image_analysis/scale_merge/SublatticeRelabel.h diff --git a/image_analysis/scale_merge/CMakeLists.txt b/image_analysis/scale_merge/CMakeLists.txt index 9e128e029..ae658c62f 100644 --- a/image_analysis/scale_merge/CMakeLists.txt +++ b/image_analysis/scale_merge/CMakeLists.txt @@ -36,7 +36,9 @@ ADD_LIBRARY(JFJochScaleMerge WilsonOutliers.cpp WilsonOutliers.h ErrorModel.cpp - ErrorModel.h) + ErrorModel.h + SublatticeRelabel.cpp + SublatticeRelabel.h) TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon fftw3f) IF (JFJOCH_CUDA_AVAILABLE) diff --git a/image_analysis/scale_merge/SublatticeRelabel.cpp b/image_analysis/scale_merge/SublatticeRelabel.cpp new file mode 100644 index 000000000..ed9b3e0e3 --- /dev/null +++ b/image_analysis/scale_merge/SublatticeRelabel.cpp @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "SublatticeRelabel.h" + +#include +#include +#include + +namespace { + // Largest coefficient tried: a reference axis up to four frame axes long, which covers the 2x, 3x + // and 4x axis multiples a few-spot frame gets indexed in. + constexpr int MAX_COEFFICIENT = 4; + // Largest volume ratio frame / reference accepted: the frame's lattice a 2x, 3x or 4x supercell (or + // the reference itself in another setting). A denser lattice holds near-fits of almost any cell. + constexpr int MAX_INDEX = 4; + + float AngleDeg(const Coord &u, const Coord &v) { + const float c = (u * v) / (u.Length() * v.Length()); + return std::acos(std::clamp(c, -1.0f, 1.0f)) * (180.0f / 3.14159265f); + } + + int Det(const std::array, 3> &m) { + return m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) + - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]); + } +} + +std::vector, 3>> FindSublatticeBases(const CrystalLattice &frame, + const UnitCell &reference, + const IndexingSettings &settings) { + const CrystalLattice ref(reference); + const Coord ref_star[3] = {ref.Astar(), ref.Bstar(), ref.Cstar()}; + const Coord frame_star[3] = {frame.Astar(), frame.Bstar(), frame.Cstar()}; + const float dist_tol = settings.GetUnitCellDistTolerance(); + const float angle_tol = settings.GetUnitCellAngleTolerance_deg(); + + // The frame's reciprocal lattice vectors as long as each reference axis. + struct Candidate { std::array n; Coord v; }; + std::vector axis[3]; + for (int i = -MAX_COEFFICIENT; i <= MAX_COEFFICIENT; ++i) + for (int j = -MAX_COEFFICIENT; j <= MAX_COEFFICIENT; ++j) + for (int k = -MAX_COEFFICIENT; k <= MAX_COEFFICIENT; ++k) { + const Coord v = frame_star[0] * i + frame_star[1] * j + frame_star[2] * k; + for (int a = 0; a < 3; ++a) + if (std::fabs(v.Length() - ref_star[a].Length()) <= dist_tol * ref_star[a].Length()) + axis[a].push_back({{i, j, k}, v}); + } + + const float ref_angle[3] = {AngleDeg(ref_star[1], ref_star[2]), AngleDeg(ref_star[0], ref_star[2]), + AngleDeg(ref_star[0], ref_star[1])}; + std::vector, 3>> bases; + for (const auto &a: axis[0]) + for (const auto &b: axis[1]) { + if (std::fabs(AngleDeg(a.v, b.v) - ref_angle[2]) > angle_tol) + continue; + for (const auto &c: axis[2]) { + if (std::fabs(AngleDeg(b.v, c.v) - ref_angle[0]) > angle_tol + || std::fabs(AngleDeg(a.v, c.v) - ref_angle[1]) > angle_tol) + continue; + const std::array, 3> m = {a.n, b.n, c.n}; + if (Det(m) > 0 && Det(m) <= MAX_INDEX) + bases.push_back(m); + } + } + return bases; +} + +namespace { + // The frame's reflections at nodes of the lattice M picks out, re-indexed on it: a frame index h' + // is h = h' M^-1 = h' adj(M) / det(M) there, a node only where all three are integers. + ReflectionVector NodesOf(const ReflectionVector &reflections, const std::array, 3> &M, + float d_min) { + const int det = Det(M); + int adj[3][3]; + for (int i = 0; i < 3; ++i) + for (int j = 0; j < 3; ++j) { + const int r0 = (j + 1) % 3, r1 = (j + 2) % 3, c0 = (i + 1) % 3, c1 = (i + 2) % 3; + adj[i][j] = M[r0][c0] * M[r1][c1] - M[r0][c1] * M[r1][c0]; + } + ReflectionVector nodes; + for (const auto &r: reflections) { + const int hp[3] = {r.h, r.k, r.l}; + int h[3]; + bool node = true; + for (int j = 0; j < 3; ++j) { + const int s = hp[0] * adj[0][j] + hp[1] * adj[1][j] + hp[2] * adj[2][j]; + node = node && s % det == 0; + h[j] = s / det; + } + if (!node || !(r.d >= d_min)) + continue; + Reflection relabelled = r; + relabelled.h = h[0]; + relabelled.k = h[1]; + relabelled.l = h[2]; + nodes.push_back(relabelled); + } + return nodes; + } + + double MeanISigma(const ReflectionVector &reflections) { + double sum = 0.0; + size_t n = 0; + for (const auto &r: reflections) + if (std::isfinite(r.I) && r.sigma > 0.0f) { + sum += r.I / r.sigma; + ++n; + } + return n > 0 ? sum / static_cast(n) : -std::numeric_limits::infinity(); + } +} + +SublatticeRelabelResult RelabelOnSublattice(std::vector &outcomes, + const UnitCell &reference, + const IndexingSettings &settings) { + const auto on_reference = [&](const IntegrationOutcome &o) { + return o.latt.GetUnitCell().is_close(reference, settings.GetUnitCellDistTolerance(), + settings.GetUnitCellAngleTolerance_deg()); + }; + // A frame indexed in the wrong lattice was also predicted to the wrong resolution - its limit read + // off spots that were mostly noise - so its reflections are kept only as fine as the frames already + // on the reference cell reach. + float d_min = std::numeric_limits::infinity(); + for (const auto &o: outcomes) + if (on_reference(o)) + for (const auto &r: o.reflections) + if (std::isfinite(r.d) && r.d > 0.0f) + d_min = std::min(d_min, r.d); + + SublatticeRelabelResult result; + for (auto &o: outcomes) { + if (o.reflections.empty() || on_reference(o)) + continue; + // A lattice can hold the reference in more than one way - a supercell whose metric is nearly + // symmetric fits it along either of two axes - and only the true one has signal at its nodes: + // the reflections between them are empty. So the bases are told apart by the data. + std::optional, 3>> best; + ReflectionVector best_nodes; + double best_signal = 0.0; + for (const auto &m: FindSublatticeBases(o.latt, reference, settings)) { + auto nodes = NodesOf(o.reflections, m, d_min); + const double signal = MeanISigma(nodes); + if (!best || signal > best_signal) { + best = m; + best_nodes = std::move(nodes); + best_signal = signal; + } + } + if (!best) + continue; + + result.reflections_dropped += o.reflections.size() - best_nodes.size(); + result.reflections_kept += best_nodes.size(); + o.reflections = std::move(best_nodes); + + // The reference basis in this frame's orientation: the reciprocal axes are rows of M times the + // frame's, and the real axes their reciprocal. + const auto &M = *best; + const Coord fs[3] = {o.latt.Astar(), o.latt.Bstar(), o.latt.Cstar()}; + Coord rs[3]; + for (int i = 0; i < 3; ++i) + rs[i] = fs[0] * M[i][0] + fs[1] * M[i][1] + fs[2] * M[i][2]; + const float v_star = rs[0] * (rs[1] % rs[2]); + o.latt = CrystalLattice((rs[1] % rs[2]) / v_star, (rs[2] % rs[0]) / v_star, (rs[0] % rs[1]) / v_star); + ++result.frames; + } + return result; +} diff --git a/image_analysis/scale_merge/SublatticeRelabel.h b/image_analysis/scale_merge/SublatticeRelabel.h new file mode 100644 index 000000000..72b3c7895 --- /dev/null +++ b/image_analysis/scale_merge/SublatticeRelabel.h @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +#include "../../common/CrystalLattice.h" +#include "../../common/IndexingSettings.h" +#include "../../common/UnitCell.h" +#include "../IntegrationOutcome.h" + +// A still indexed on few spots is often given a lattice that explains them just as well as the true +// one: an axis multiple (a supercell, whose reciprocal lattice holds every true node plus nodes in +// between) or the true lattice in another setting. Such a frame's cell is not close to the run's cell, +// so the merge leaves it out, although its reflections at the true nodes are as good as any. +// +// FindSublatticeBases looks for the run's reciprocal lattice inside the frame's: integer combinations +// of the frame's reciprocal axes whose metric (all three lengths and three angles) is the reference +// cell's, within the merge's own cell tolerances. Each is returned as the integer matrix M with +// (a*, b*, c*)_reference = M (a*, b*, c*)_frame, right-handed. +std::vector, 3>> FindSublatticeBases(const CrystalLattice &frame, + const UnitCell &reference, + const IndexingSettings &settings); + +struct SublatticeRelabelResult { + size_t frames = 0; // frames put on the reference cell + size_t reflections_kept = 0; // reflections of those frames at nodes of the reference lattice + size_t reflections_dropped = 0; +}; + +// For every frame whose cell is not close to the reference but whose lattice contains it: re-derive its +// reflection indices on the reference axes - of several ways to place the reference in its lattice, the +// one with the strongest mean I/sigma at its nodes - drop the reflections that are not reference nodes +// or are finer than any frame on the reference cell reaches, and give the frame the reference basis in +// its own orientation. +SublatticeRelabelResult RelabelOnSublattice(std::vector &outcomes, + const UnitCell &reference, + const IndexingSettings &settings); diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index f390ebc3f..6454613d0 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -78,6 +78,7 @@ #include "../image_analysis/scale_merge/HKLKey.h" #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" +#include "../image_analysis/scale_merge/SublatticeRelabel.h" #include "../image_analysis/WriteReflections.h" #include "../image_analysis/bragg_integration/CalcISigma.h" #include "../common/Definitions.h" @@ -1342,6 +1343,15 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { const bool guard_beside = merge_first_part && guard_ahead.valid(); if (guard_beside) rsm->SetWriteBackPerFrameScale(false); + // Stills: a crystal indexed in a lattice that contains the run's - an axis multiple, or the run's in + // another setting - is put on the run's cell before anything is scaled, rather than left out. + if (!rsm && result.consensus_cell) { + const auto relabel = RelabelOnSublattice(indexer->GetIntegrationOutcome(), *result.consensus_cell, + experiment_.GetIndexingSettings()); + logger.Info("Put {} crystal(s) indexed in a lattice containing the run's on the run's cell: {} " + "reflections at its nodes kept, {} between them dropped", + relabel.frames, relabel.reflections_kept, relabel.reflections_dropped); + } const auto initial_sg = experiment_.GetGemmiSpaceGroup(); auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value(), /*measure_cc=*/true); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 6b29580ca..83aeb7401 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -58,6 +58,7 @@ #include "../image_analysis/scale_merge/ReindexAmbiguity.h" // MatchReference #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" +#include "../image_analysis/scale_merge/SublatticeRelabel.h" #include "../image_analysis/scale_merge/RotationScaleMerge.h" #include "../image_analysis/scale_merge/ResolutionCutoff.h" #include "../image_analysis/scale_merge/AnisotropyAnalysis.h" @@ -2003,6 +2004,15 @@ static int RunRugnux(int argc, char **argv) { } } + // Stills: a crystal indexed in a lattice containing the file's cell is put on that cell, as the full + // pipeline does - before the resolution of every reflection is recomputed from the cell below, + // which is right only for reflections indexed on it. + if (!experiment.IsRotationIndexing()) { + const auto relabel = RelabelOnSublattice(reflections, experiment.GetUnitCell().value(), + experiment.GetIndexingSettings()); + logger.Info("Put {} crystal(s) indexed in a lattice containing the file's cell on that cell", + relabel.frames); + } auto refl_stats = UpdateReflectionResolution(experiment.GetUnitCell().value(), reflections); logger.Info("Read {} reflections from {} images", refl_stats.n_reflections, refl_stats.n_images); experiment.ImagesPerTrigger(refl_stats.n_images); diff --git a/tests/MergeScaleTest.cpp b/tests/MergeScaleTest.cpp index d862fd213..6bc6e0ff2 100644 --- a/tests/MergeScaleTest.cpp +++ b/tests/MergeScaleTest.cpp @@ -12,6 +12,7 @@ #include "../image_analysis/scale_merge/ResolutionCutoff.h" #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" +#include "../image_analysis/scale_merge/SublatticeRelabel.h" #include "gemmi/reciproc.hpp" TEST_CASE("HKLKey_NoSG_noMergeFriedel") { @@ -746,3 +747,119 @@ TEST_CASE("StillsPartialityRefine_RecoversTheTilt") { CHECK(mean_tilt < 0.06); CHECK(after < 0.6 * before); // measured 0.54: the prior holds the larger tilts back } + +namespace { + // The basis vectors a, b, c of a cell, turned by a fixed arbitrary rotation, as a frame's lattice. + std::array TurnedAxes(const UnitCell &cell) { + const CrystalLattice l(cell); + const double c = std::cos(0.7), s = std::sin(0.7); // about (1, 2, 3)/sqrt(14) + const double n[3] = {1.0 / std::sqrt(14.0), 2.0 / std::sqrt(14.0), 3.0 / std::sqrt(14.0)}; + auto turn = [&](const Coord &v) { + const double dot = n[0] * v.x + n[1] * v.y + n[2] * v.z; + const double x[3] = {n[1] * v.z - n[2] * v.y, n[2] * v.x - n[0] * v.z, n[0] * v.y - n[1] * v.x}; + return Coord(static_cast(v.x * c + x[0] * s + n[0] * dot * (1 - c)), + static_cast(v.y * c + x[1] * s + n[1] * dot * (1 - c)), + static_cast(v.z * c + x[2] * s + n[2] * dot * (1 - c))); + }; + return {turn(l.Vec0()), turn(l.Vec1()), turn(l.Vec2())}; + } + + Coord ScatteringVector(const CrystalLattice &l, const Reflection &r) { + return l.Astar() * static_cast(r.h) + l.Bstar() * static_cast(r.k) + + l.Cstar() * static_cast(r.l); + } + + IntegrationOutcome FrameWithAllIndices(const CrystalLattice &latt, int hmax) { + IntegrationOutcome o; + o.latt = latt; + for (int h = -hmax; h <= hmax; ++h) + for (int k = -hmax; k <= hmax; ++k) + for (int l = -hmax; l <= hmax; ++l) { + Reflection r{}; + r.h = h; r.k = k; r.l = l; + r.predicted_x = static_cast(o.reflections.size()); // identifies the observation + r.I = 100.0f; + r.sigma = 10.0f; + const float q = ScatteringVector(latt, r).Length(); + r.d = q > 0.0f ? 1.0f / q : 100.0f; + o.reflections.push_back(r); + } + return o; + } +} + +// A frame indexed with its c axis doubled: its reciprocal lattice holds every node of the true one plus +// the planes in between. Relabelling keeps exactly the true nodes, each at the same scattering vector, +// and gives the frame the true cell. +TEST_CASE("SublatticeRelabel_DoubledAxisGoesBackToTheTrueCell") { + const UnitCell reference{60, 60, 25, 90, 90, 90}; // synthetic tetragonal + const auto ax = TurnedAxes(reference); + const CrystalLattice super(ax[0], ax[1], ax[2] * 2.0f); + // A frame on the true cell beside it sets how fine the relabelled reflections may go. + std::vector outcomes{FrameWithAllIndices(super, 4), + FrameWithAllIndices(CrystalLattice(ax[0], ax[1], ax[2]), 4)}; + const auto before = outcomes[0]; + + IndexingSettings settings; + const auto result = RelabelOnSublattice(outcomes, reference, settings); + CHECK(result.frames == 1); + CHECK(result.reflections_kept == 9 * 9 * 5); // l even: -4, -2, 0, 2, 4 + CHECK(result.reflections_dropped == 9 * 9 * 4); + CHECK(outcomes[0].latt.GetUnitCell().is_close(reference, 0.001, 0.01)); + + for (const auto &r: outcomes[0].reflections) { + const auto it = std::find_if(before.reflections.begin(), before.reflections.end(), + [&](const Reflection &b) { return b.predicted_x == r.predicted_x; }); + REQUIRE(it != before.reflections.end()); + CHECK(it->l % 2 == 0); + const Coord q_before = ScatteringVector(before.latt, *it); + const Coord q_after = ScatteringVector(outcomes[0].latt, r); + CHECK((q_after - q_before).Length() < 1e-5f); + } +} + +// The true lattice in another setting (axes permuted) is the same lattice: every reflection is kept and +// only re-indexed. A frame whose lattice does not contain the reference is left alone. +TEST_CASE("SublatticeRelabel_OtherSettingAndUnrelatedLattice") { + const UnitCell reference{40, 50, 60, 90, 90, 90}; // synthetic orthorhombic + const auto ax = TurnedAxes(reference); + std::vector outcomes{FrameWithAllIndices(CrystalLattice(ax[1], ax[2], ax[0]), 3), + FrameWithAllIndices(CrystalLattice(UnitCell{37, 53, 71, 90, 90, 90}), 3), + FrameWithAllIndices(CrystalLattice(ax[0], ax[1], ax[2]), 4)}; + const auto permuted = outcomes[0]; + const auto unrelated = outcomes[1]; + + IndexingSettings settings; + const auto result = RelabelOnSublattice(outcomes, reference, settings); + CHECK(result.frames == 1); + CHECK(result.reflections_dropped == 0); + CHECK(outcomes[0].latt.GetUnitCell().is_close(reference, 0.001, 0.01)); + for (size_t i = 0; i < outcomes[0].reflections.size(); ++i) { + const Coord q_before = ScatteringVector(permuted.latt, permuted.reflections[i]); + const Coord q_after = ScatteringVector(outcomes[0].latt, outcomes[0].reflections[i]); + CHECK((q_after - q_before).Length() < 1e-5f); + } + CHECK(outcomes[1].reflections.size() == unrelated.reflections.size()); + CHECK(outcomes[1].reflections[5].h == unrelated.reflections[5].h); +} + +// A doubled axis that makes the supercell metric cubic fits the reference along any of the three axes; +// only the true one has signal at its nodes, and that is the one taken. +TEST_CASE("SublatticeRelabel_SignalPicksTheAxisThatWasDoubled") { + const UnitCell reference{60, 60, 30, 90, 90, 90}; // synthetic tetragonal; doubled c gives a cube + const auto ax = TurnedAxes(reference); + std::vector outcomes{FrameWithAllIndices(CrystalLattice(ax[0], ax[1], ax[2] * 2.0f), 4), + FrameWithAllIndices(CrystalLattice(ax[0], ax[1], ax[2]), 4)}; + for (auto &r: outcomes[0].reflections) + r.I = r.l % 2 == 0 ? 100.0f : 0.0f; // the nodes between the true ones are empty + const auto before = outcomes[0]; + + IndexingSettings settings; + const auto result = RelabelOnSublattice(outcomes, reference, settings); + REQUIRE(result.frames == 1); + for (const auto &r: outcomes[0].reflections) { + const auto &b = before.reflections[static_cast(r.predicted_x)]; + CHECK(b.l % 2 == 0); + CHECK((ScatteringVector(outcomes[0].latt, r) - ScatteringVector(before.latt, b)).Length() < 1e-5f); + } +}