From 248970018498ced256b02abc35ea983a2e414b45 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 19:17:31 +0100 Subject: [PATCH] Joint sweeps: the shared-cell distance re-run covers the sweeps the given cell rescued, and is adopted only beyond the errors The distance re-run loop read each sweep's decision from the relation made before the -C re-runs, so every sweep those runs rescued ("no merge"/"no axes" on the first look) was skipped and kept its header distance: on a four-sweep set two rescued sweeps stayed at the header distance with cells ~3.5% short while the other two had found a distance ~3% longer. The sweeps are now related again (and the target sweep chosen again) after the -C re-runs, before any cell is compared. A re-run replaced the first run whenever its held-out residual was not higher, however small the difference: runs were taken at differences of 0.01-0.03% (inside the bootstrap error), one with 40% fewer fulls and a cell overshooting the target, one of a second crystal whose cell legitimately differs. A re-run is now taken only where its held-out residual is lower by more than twice the two residuals' errors combined, and where it has no fewer fulls. The log gives both errors and both fulls counts. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/RUGNUX_CLI.md | 10 ++-- rugnux/rugnux_cli.cpp | 132 +++++++++++++++++++++++------------------- 2 files changed, 78 insertions(+), 64 deletions(-) diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md index 7da5065ff..8538e4f8e 100644 --- a/docs/RUGNUX_CLI.md +++ b/docs/RUGNUX_CLI.md @@ -116,10 +116,12 @@ best; where the lattice allows more than one indexing, which one each sweep take of sweeps at once - and each is judged by its isomorphism ratio with it (see [RUGNUX_REPORT](RUGNUX_REPORT.md)): clearly about 1 it joins, clearly below it (another crystal) it is left out, and in between the merge decides - it joins unless it is shown to hurt: adding it raises the held-out residual of the others' merge by more than 1%, -two standard errors down (a sweep left out is named in the log). The crystal has one cell: a joined sweep whose own cell is more than 0.3% off that of -the joined sweep reaching the finest resolution (where the cell is measured rather than the distance) is -processed a second time, from the distance that would make the two cells agree, and that run replaces the -first where its intensities agree better with that sweep's (a lower held-out residual). Their fulls are +two standard errors down (a sweep left out is named in the log). A sweep that found no lattice of the crystal on its own (it merged nothing, or correlates on no axes) +is processed again given the cell of the joined sweep reaching the finest resolution. The crystal has one cell: +any sweep on the reference's axes - the ones just rescued too - whose own cell is more than 0.3% off that sweep's +(where the cell is measured rather than the distance) is processed a second time, from the distance that would +make the two cells agree, and that run replaces the first only where its intensities agree better with the +joined sweeps' (a lower held-out residual by more than twice its error) and it has no fewer fulls. Their fulls are scaled and merged together: a per-frame scale within each sweep, the sweeps' dose order read from their relative B factors (not from the input order or the header times), a decay along that order or one B per sweep, whichever predicts held-out frames better, then the space-group search, the diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 5d1cb6a2a..431d43a85 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -4181,70 +4181,82 @@ int RugnuxCli::RunJointSweeps(const RugnuxSweepPlan &plan) { inputs.push_back(plan.sweeps[k].input); } - // One cell for the crystal (SharedCellTarget): a sweep whose own cell disagrees with it - joined or not - // on the first look - is processed again from the distance that would make the two agree, and that run replaces its first one where it - // agrees better with the target sweep's intensities (lower held-out residual, HeldOutAgainst). + // One cell for the crystal (SharedCellTarget). A sweep that found no lattice of this crystal on its own - + // it merged nothing, or nothing that correlates with the reference on any axes - is processed again given + // the target sweep's cell (-C): de novo indexing of one sweep can miss a large cell that the crystal's other + // sweeps have measured. Its join is then decided on its intensities, as every sweep's is. Only a sweep that + // failed is given the cell: given to every sweep from the start, it moved sweeps that index well de novo + // into another setting or centring. Where -C was given, every sweep already had a cell. JointSweepRelation relation = RelateJointSweeps(members); bool replaced = false; // a sweep's run replaced by its run again: the relation is made again - { - const JointSweepRelation &first = relation; - const int target = SharedCellTarget(members, first); - // A sweep that found no lattice of this crystal on its own - it merged nothing, or nothing that - // correlates with the reference on any axes - is processed again given the target sweep's cell (-C): - // de novo indexing of one sweep can miss a large cell that the crystal's other sweeps have measured. - // Its join is then decided on its intensities, as every sweep's is. Only a sweep that failed is given - // the cell: given to every sweep from the start, it moved sweeps that index well de novo into another - // setting or centring. Where -C was given, every sweep already had a cell. - for (size_t k = 0; target >= 0 && !fixed_reference_unit_cell && k < members.size(); ++k) { - const std::string &decision = first.sweeps[k].decision; - if (decision != "no merge" && decision != "no axes") - continue; - logger.Info("Sweep {}: {}; processed again given sweep {}'s cell", k + 1, first.sweeps[k].why_left_out, - target + 1); - JointMemberSweep again = run_member(k, std::nullopt, members[target].cell, "_known"); - if (!again.merged.empty()) { - members[k] = std::move(again); - replaced = true; - } + int target = SharedCellTarget(members, relation); + for (size_t k = 0; target >= 0 && !fixed_reference_unit_cell && k < members.size(); ++k) { + const std::string &decision = relation.sweeps[k].decision; + if (decision != "no merge" && decision != "no axes") + continue; + logger.Info("Sweep {}: {}; processed again given sweep {}'s cell", k + 1, relation.sweeps[k].why_left_out, + target + 1); + JointMemberSweep again = run_member(k, std::nullopt, members[target].cell, "_known"); + if (!again.merged.empty()) { + members[k] = std::move(again); + replaced = true; } - for (size_t k = 0; target >= 0 && k < members.size(); ++k) { - // Every sweep on the reference's axes, joined or not: a cell off by a few percent is itself a reason - // for intensities to disagree, so the join is decided after this, on the runs it keeps. - const std::string &decision = first.sweeps[k].decision; - if (decision == "no merge" || decision == "no axes" || static_cast(k) == target - || !std::isfinite(members[k].distance_mm)) - continue; - const double scale = VolumeLength(members[target].cell) / VolumeLength(members[k].cell); - if (std::fabs(scale - 1.0) <= SHARED_CELL_AGREEMENT) - continue; - const auto distance = static_cast(members[k].distance_mm * scale); - logger.Info("Sweep {}: its cell is {:+.2f}% off sweep {}'s; processed again from {:.2f} mm", k + 1, - 100.0 * (1.0 / scale - 1.0), target + 1, distance); - JointMemberSweep again = run_member(k, distance, std::nullopt, "_cell"); - if (again.merged.empty()) - continue; - // Against the merge of the other joined sweeps, the target first (the scale). - std::vector base{&members[target]}; - std::vector base_cob{first.sweeps[target].cob}; - for (size_t j = 0; j < members.size(); ++j) - if (first.sweeps[j].included && j != k && static_cast(j) != target) { - base.push_back(&members[j]); - base_cob.push_back(first.sweeps[j].cob); - } - const JointMemberSweep &reference = members[first.reference]; - const std::vector gains = HeldOutAgainst(base, base_cob, reference, {&members[k], &again}); - const HeldOutGain &before = gains[0], &after = gains[1]; - const bool adopt = std::isfinite(after.change) && !(after.change >= before.change); - logger.Info("Sweep {}: against the joined sweeps (scale: sweep {}), held-out residual {:+.2f}% at its own cell ({:.3f} {:.3f} {:.3f}), " - "{:+.2f}% processed again ({:.3f} {:.3f} {:.3f}): {}", k + 1, target + 1, - 100.0 * before.change, members[k].cell.a, members[k].cell.b, members[k].cell.c, - 100.0 * after.change, again.cell.a, again.cell.b, again.cell.c, - adopt ? "the second run is taken" : "the first run stands"); - if (adopt) { - again.reprocessed_from_mm = members[k].distance_mm; - members[k] = std::move(again); - replaced = true; + } + // The sweeps just rescued are related to the others before their cells are compared, as every other sweep's. + if (replaced) { + relation = RelateJointSweeps(members); + target = SharedCellTarget(members, relation); + replaced = false; + } + + // Every sweep on the reference's axes, joined or not - the ones just rescued too - on the target sweep's + // cell: a cell off by a few percent is itself a reason for intensities to disagree, so the join is decided + // after this, on the runs it keeps. A sweep whose cell is off the target's is processed again from the + // distance that would make the two agree, and that run replaces its first one only where it agrees better + // with the joined sweeps' intensities (HeldOutAgainst) by more than the two residuals' errors, and at no loss + // of observations: a distance is not changed on noise, and a run that integrates fewer reflections does not + // measure them better. + for (size_t k = 0; target >= 0 && k < members.size(); ++k) { + const std::string &decision = relation.sweeps[k].decision; + if (decision == "no merge" || decision == "no axes" || static_cast(k) == target + || !std::isfinite(members[k].distance_mm)) + continue; + const double scale = VolumeLength(members[target].cell) / VolumeLength(members[k].cell); + if (std::fabs(scale - 1.0) <= SHARED_CELL_AGREEMENT) + continue; + const auto distance = static_cast(members[k].distance_mm * scale); + logger.Info("Sweep {}: its cell is {:+.2f}% off sweep {}'s; processed again from {:.2f} mm", k + 1, + 100.0 * (1.0 / scale - 1.0), target + 1, distance); + JointMemberSweep again = run_member(k, distance, std::nullopt, "_cell"); + if (again.merged.empty()) + continue; + // Against the merge of the other joined sweeps, the target first (the scale). + std::vector base{&members[target]}; + std::vector base_cob{relation.sweeps[target].cob}; + for (size_t j = 0; j < members.size(); ++j) + if (relation.sweeps[j].included && j != k && static_cast(j) != target) { + base.push_back(&members[j]); + base_cob.push_back(relation.sweeps[j].cob); } + const JointMemberSweep &reference = members[relation.reference]; + const std::vector gains = HeldOutAgainst(base, base_cob, reference, {&members[k], &again}); + const HeldOutGain &before = gains[0], &after = gains[1]; + const double gain = before.change - after.change; + const bool better = std::isfinite(gain) && gain > 2.0 * std::hypot(before.se, after.se); + const bool no_loss = again.fulls.size() >= members[k].fulls.size(); + logger.Info("Sweep {}: against the joined sweeps (scale: sweep {}), held-out residual {:+.2f}% +- {:.2f}% at " + "its own cell ({:.3f} {:.3f} {:.3f}, {} fulls), {:+.2f}% +- {:.2f}% processed again ({:.3f} {:.3f} " + "{:.3f}, {} fulls): {}", k + 1, target + 1, 100.0 * before.change, 100.0 * before.se, + members[k].cell.a, members[k].cell.b, members[k].cell.c, members[k].fulls.size(), + 100.0 * after.change, 100.0 * after.se, again.cell.a, again.cell.b, again.cell.c, + again.fulls.size(), + !better ? "the first run stands - the second agrees no better beyond the errors" + : !no_loss ? "the first run stands - the second has fewer fulls" + : "the second run is taken"); + if (better && no_loss) { + again.reprocessed_from_mm = members[k].distance_mm; + members[k] = std::move(again); + replaced = true; } }