mx/Jungfraujoch
3
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 81 Wiki Activity
Files
07eecb3285f9b025a8ac869f97c8b8ec0657c4e7
Jungfraujoch/rugnux/RugnuxScaleMerge.cpp
T
leonarski_fandClaude Opus 5.5 30c7bee15f rugnux: --af-model becomes --mr, for predictions and experimental structures alike 
- The option is --mr <file>. What kind of model it is is read from the file: a prediction where it
  is ModelCIF (_ma_qa_metric) or has no cell and one B value per residue in 0-100 (AlphaFold PDB/mmCIF);
  an experimental structure otherwise. A prediction is handled as before (pLDDT >= 70 trim, B from
  pLDDT, PAE domains where the AlphaFold DB's PAE file sits beside it). An experimental structure is
  taken as its polymer - no waters, ligands, hydrogens or alternative conformers, occupancy 1, isotropic
  B as refined - with a coordinate error of 0.67 A (a prediction's at pLDDT 90) for sigma_A; a B column
  above 100 no longer stops the search.
- The report section is MOLECULAR REPLACEMENT (EXPERIMENTAL), its keys MR_* (MR, MR_MODEL_FILE,
  MR_MODEL_KIND, MR_DECISION, MR_SPACE_GROUP, MR_COPIES, MR_HAND_CHECK, MR_SECONDS, ...); the <prefix>_mr
  outputs are unchanged.
- The twin hypothesis' merge that is made only for the model to test (a metric re-ask promoted the
  indexed lattice's group) is written and reported only where the model chooses the subgroup.

Raw deposited coordinates of nine control sets, unconverted: every one placed and confirmed against the
deposition (all chains as one body), the groups the same as with the prediction or the prepared model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3
2026-10-12 19:30:22 +02:00

4213 lines
283 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <cstdlib>
#include "Rugnux.h"
#include "../image_analysis/structure_refinement/ModelValidation.h"
#include "../writer/WriteModel.h"
#include "../image_analysis/indexing/SpindleCuspLoss.h"
#include "../image_analysis/bragg_integration/SpotWidth.h"
#include "../image_analysis/bragg_integration/SpotFootprint.h"
#include "../image_analysis/bragg_integration/BraggStencil.h" // BRAGG_FOOTPRINT_NSIGMA
#include "../image_analysis/hot_pixels/HotPixels.h"
#include "../image_analysis/SensorAbsorption.h"
#include "DiagnosticOutput.h"
#include "RugnuxFirstPass.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <deque>
#include <fstream>
#include <cstring>
#include <functional>
#include <future>
#include <mutex>
#include <numeric>
#include <map>
#include <set>
#include <sstream>
#include <thread>
#include "../reader/JFJochHDF5Reader.h"
#include "../common/JFJochMath.h"
#include "../common/ParallelFor.h"
#include "../common/Logger.h"
#include "../common/AzimuthalIntegrationMapping.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include "../common/CUDAWrapper.h"
#include "../common/JFJochException.h"
#include "../common/time_utc.h"
#include "../writer/FileWriter.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
#include "../image_analysis/beam_stop/ShadowFinder.h"
#include "../image_analysis/IndexAndRefine.h"
#include "../image_analysis/indexing/AnalyzeIndexing.h"
#include "../image_analysis/indexing/HarmonicContamination.h"
#include "../image_analysis/geom_refinement/BeamCenterFFT.h"
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
#include "../image_analysis/indexing/IndexerThreadPool.h"
#include "../image_analysis/spot_finding/ImageSpotFinderCPU.h"
#include "../image_analysis/spot_finding/SpotUtils.h"
#include "gemmi/twin.hpp"
#include "../image_analysis/azint/AzIntEngineCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
#ifdef JFJOCH_USE_CUDA
#include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h"
#include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h"
#endif
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/RfreeFlags.h"
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
#include "../image_analysis/scale_merge/ScalingResult.h"
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/geom_refinement/PostRefine.h"
#include "../image_analysis/lattice_search/LatticeSearch.h"
#include "../image_analysis/lattice_search/LePageLattice.h"
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
#include "../image_analysis/scale_merge/FrenchWilson.h"
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
#include "../image_analysis/scale_merge/TranslationalNCS.h"
#include "../image_analysis/scale_merge/HKLKey.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
#include "../image_analysis/WriteReflections.h"
#include "../image_analysis/bragg_integration/CalcISigma.h"
#include "../common/Definitions.h"
#include "../common/CorrelationCoefficient.h"
#include <array>
#include <map>
#include "RugnuxPipeline.h"
using namespace rugnux_internal;
namespace {
// How precisely the measured beam centre has to be placed before the post-refinement takes it as a
// second place the centre is believed to be (the bound on its beam move). The ceiling is the larger
// of this and what the geometry asks of the centre (BeamCenterNeed_pxl).
constexpr float MAX_BEAM_CENTER_SIGMA_PXL = 1.0f;
// The twin-immune zone of the operators `higher` adds over `lower`, read on a P1 merge, as a
// verdict: +1 the added operators are real (the zone is centric with margin), -1 they are a twin
// law or a pseudo-symmetry (acentric with margin), 0 undecided - fewer than two orders apart
// (the immunity argument holds for one coset), no zone reflections, or within the margin.
//
// The margin is the screw-axis convention (SearchSpaceGroupOptions::min_screw_absence_evidence):
// the same log-likelihood-ratio units, and the same bound of 20 nats. Read over three batteries
// the zones of genuine promotions sit at +60 nats and above and the zones behind the known
// twins at -290 nats and below, with genuine trigonal crystals at -46 and -206 nats whose
// promotion no gate refused - twinned by a law other than the 2-folds, which reaches their zone.
// Read at the twin fraction those other laws show (twin_fraction_outside) the -206 one reads
// +209, and the search itself now asks the zone of a promotion on a merge that reads twinned
// (SearchSpaceGroup's twin-immune zone test); here it is read where a gate refused.
//
// Read on the CALIBRATED evidence: the zone's less what the acentric control shows the
// normalisation to add per reflection (TwinImmuneZoneResult::control_excess_per_reflection). The
// raw evidence rescued two refused 622 promotions - a 6/m crystal and a 312 one, H 4.7x and 5.2x
// the parent's, R_meas 0.09 -> 0.34 under the higher group - whose zones read centric only
// because everything did: a 67 A^2 anisotropy and a pseudo-translation had the control at
// +0.10 nats per reflection, the zone at the same, and +124 nats over 947 reflections was the
// normalisation's. Calibrated they read -115 and -86; the genuine promotions measured keep +140
// and above.
int TwinZoneVerdict(const std::vector<MergedReflection> &p1_merged, const gemmi::UnitCell &cell,
const SearchSpaceGroupResult &higher, const SearchSpaceGroupResult &lower,
ProcessResult &result, const std::array<double, 3> *tncs_vector = nullptr) {
constexpr double DECISIVE_NATS = 20.0;
if (!higher.point_group_representative || !lower.point_group_representative)
return 0;
if (higher.point_group_order != 2 * lower.point_group_order)
return 0;
const auto z = AnalyzeTwinImmuneZone(p1_merged, cell, *higher.point_group_representative,
*lower.point_group_representative, tncs_vector, 1,
higher.twin_fraction_outside);
Logger logger("Rugnux");
if (z.zones.empty() || z.zones[0].n == 0) {
logger.Info("Twin-immune zone {} over {}: no zone reflections in the shells with signal "
"({} merged reflections read) - undecided", higher.point_group_hm,
lower.point_group_hm, p1_merged.size());
return 0;
}
const auto &zone = z.zones[0];
const int verdict = zone.calibrated_evidence_nats <= -DECISIVE_NATS ? -1
: zone.calibrated_evidence_nats >= DECISIVE_NATS ? +1 : 0;
const std::string text = fmt::format(
"{} over {}: operators {}; n {} <|E^2-1|> {:.3f} +- {:.3f} (control {:.3f}, {:+.3f} nats per "
"reflection above acentric), centric read twinned at {:.2f}, evidence {:+.1f} nats, calibrated "
"{:+.1f} ({:.2f}-{:.2f} A{}) -> {}",
higher.point_group_hm, lower.point_group_hm, zone.operators, zone.n, zone.mean_abs_e2_minus_1,
zone.standard_error, z.control.mean_abs_e2_minus_1, z.control_excess_per_reflection, z.twin_fraction,
zone.evidence_nats, zone.calibrated_evidence_nats, z.d_max_A, z.d_min_A,
z.tncs_normalised ? ", normalised per pseudo-translation class" : "",
verdict < 0 && z.zones_ambiguous
? "acentric, but the control is compressed beyond a perfect twin, so genuine symmetry would read so too"
: verdict < 0 ? "acentric, the added operators are a twin law or a pseudo-symmetry"
: verdict > 0 ? "centric, the added operators are real" : "undecided");
// Recorded only where the zone DECIDED something. An undecided read decides nothing - the
// rule stays as it was - and recording it had the report announce "a twin gate refused this
// promotion, and the twin-immune zone decided it" over a line that says the opposite.
if (verdict == 0) {
logger.Info("Twin-immune zone {}", text);
return 0;
}
result.twin_zone_decision = text;
// The verdict as a VALUE, for the report to act on. Every caller acts on a decisive read -
// the refusal stands or it is overturned - so recording it here records the decision, where
// matching on the text records only the last comparison the run happened to make.
result.twin_zone_refusal_stood = verdict < 0;
return verdict;
}
// Angle between the crystal's symmetry axes and the spindle. A rotation sweep never records the
// reflections whose reciprocal vector lies close to the spindle - the blind cusp - and normally
// symmetry fills it in from an equivalent elsewhere in reciprocal space. It cannot when a symmetry
// axis IS the spindle: the cusp then maps onto itself, every reflection in it is equivalent only to
// other reflections in it, and the hole stays empty however long the sweep runs. Returns the
// smallest angle in degrees between any proper rotation axis of the space group and the goniometer
// axis, with the order of the axis that achieves it.
std::optional<std::pair<double, int>> ClosestSymmetryAxisToSpindle(const gemmi::SpaceGroup &sg,
const CrystalLattice &lattice,
const Coord &spindle) {
const double spindle_len = spindle.Length();
if (spindle_len < 1e-9)
return std::nullopt;
std::optional<std::pair<double, int>> best;
for (const auto &op : sg.operations().derive_symmorphic().sym_ops) {
if (op.rot == gemmi::Op::identity().rot)
continue;
// Order of the rotation, then project onto its invariant direction by summing its powers:
// (1/n) sum_k W^k annihilates everything except the axis.
const gemmi::Op w{op.rot, {0, 0, 0}, op.notation};
int order = 0;
gemmi::Op cur = gemmi::Op::identity();
std::array<std::array<double, 3>, 3> sum{};
for (int k = 0; k < 6; ++k) {
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
sum[i][j] += static_cast<double>(cur.rot[i][j]) / gemmi::Op::DEN;
cur = cur.combine(w).wrap();
++order;
if (cur.rot == gemmi::Op::identity().rot)
break;
}
if (order < 2)
continue;
// Any non-degenerate column of the projector is the axis in fractional direct coordinates.
for (int c = 0; c < 3; ++c) {
const Coord axis = lattice.Vec0() * static_cast<float>(sum[0][c])
+ lattice.Vec1() * static_cast<float>(sum[1][c])
+ lattice.Vec2() * static_cast<float>(sum[2][c]);
const double len = axis.Length();
if (len < 1e-6 * lattice.Vec0().Length())
continue;
const double cosang = std::abs((axis * spindle) / (len * spindle_len));
const double ang = std::acos(std::min(1.0, cosang)) * 180.0 / PI;
if (!best.has_value() || ang < best->first)
best = std::make_pair(ang, order);
break;
}
}
return best;
}
// The subgroup a twin-law merge is made in (<prefix>_twin_<SG>), and the law it carries, where the
// adopted group may have absorbed a twin law. Candidates are the index-2 subgroups of `group`
// (IndexTwoTwinSubgroups); where a decision of the search named one - the group a refused promotion
// was adopted over, or the indexed lattice's group a metric re-ask promoted from - only that one,
// and otherwise those the search confirmed as point-group finalists (the ledger's eligible rows).
// Among them, the one whose law agrees worst with the intensities (lowest Stage A CC): the
// two-fold most likely to be the twin law. The rest are named for the report.
struct TwinSubgroupChoice {
gemmi::SpaceGroup subgroup;
gemmi::Op law;
std::vector<std::string> others;
};
std::optional<TwinSubgroupChoice> ChooseTwinSubgroup(const gemmi::SpaceGroup &group,
const SearchSpaceGroupResult &search,
const std::optional<gemmi::SpaceGroup> &named) {
const auto rotations = [](const gemmi::SpaceGroup &g) {
std::vector<gemmi::Op::Rot> r;
for (const auto &op : g.operations().sym_ops)
r.push_back(op.rot);
std::sort(r.begin(), r.end());
return r;
};
const auto law_cc = [&](const gemmi::Op &law) {
const std::string t = law.as_hkl().triplet('h');
for (const auto &s : search.operator_scores)
if (s.op_triplet_hkl == t)
return s.cc;
return std::numeric_limits<double>::infinity();
};
std::optional<TwinSubgroupChoice> best;
double best_cc = 0.0;
std::vector<std::string> names;
for (const auto &t : IndexTwoTwinSubgroups(group)) {
if (t.subgroup.operations().sym_ops.size() < 2)
continue; // P1: a twin of it is not what a promotion absorbs
const auto rots = rotations(t.subgroup);
const bool qualifies = named
? rotations(*named) == rots
: std::any_of(search.point_group_ledger.begin(), search.point_group_ledger.end(),
[&](const PointGroupLedgerEntry &e) {
return e.eligible && e.representative
&& rotations(*e.representative) == rots;
});
if (!qualifies)
continue;
names.push_back(t.subgroup.xhm());
const gemmi::Op *law = &t.laws.front();
for (const auto &l : t.laws)
if (law_cc(l) < law_cc(*law))
law = &l;
if (!best || law_cc(*law) < best_cc) {
best = TwinSubgroupChoice{t.subgroup, *law, {}};
best_cc = law_cc(*law);
}
}
if (best)
for (const auto &n : names)
if (n != best->subgroup.xhm())
best->others.push_back(n);
return best;
}
// A twin-law choice on other axes: the group by SpaceGroupInBasis, the law conjugated by the
// Miller-index operator the change of basis moves the reflections by. False where no setting in
// GEMMI's table names the moved group.
bool MoveTwinSubgroupChoice(TwinSubgroupChoice &c, const gemmi::Op &cob) {
const gemmi::SpaceGroup *moved = SpaceGroupInBasis(c.subgroup, cob);
if (moved == nullptr)
return false;
const gemmi::Op m = HklOperator(cob);
c.subgroup = *moved;
c.law = m.inverse().combine(c.law).combine(m); // h' = m h: back by m, the law, forward by m
return true;
}
// `group` carried into the setting a Miller-index reindex q (h' = q h) takes the merge to, with the
// centring that setting describes - the move the metric re-ask makes with the lattice. Empty where
// no setting in GEMMI's table names the result.
std::optional<gemmi::SpaceGroup> GroupInReindexedSetting(const gemmi::SpaceGroup &group,
const gemmi::Mat33 &q, char centring) {
try {
gemmi::Op hkl = gemmi::Op::identity();
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
hkl.rot[i][j] = static_cast<int>(std::lround(q[j][i])) * gemmi::Op::DEN; // apply_to_hkl uses rot^T
gemmi::Op cob = hkl.inverse();
for (int i = 0; i < 12; ++i)
for (int j = 0; j < 12; ++j)
for (int k = 0; k < 12; ++k) {
cob.tran = {i * gemmi::Op::DEN / 12, j * gemmi::Op::DEN / 12, k * gemmi::Op::DEN / 12};
gemmi::GroupOps gops = group.operations();
gops.change_basis_forward(cob);
gops.cen_ops = gemmi::centring_vectors(centring);
if (const gemmi::SpaceGroup *found = gemmi::find_spacegroup_by_ops(gops))
return *found;
}
} catch (const std::exception &) {
}
return std::nullopt;
}
}
// Proper rotations of a crystal system's holohedry - what a lattice of that class carries, and the
// bound SearchSpaceGroup caps its point-group candidates at.
static size_t HolohedryRotations(gemmi::CrystalSystem system) {
switch (system) {
case gemmi::CrystalSystem::Triclinic: return 1;
case gemmi::CrystalSystem::Monoclinic: return 2;
case gemmi::CrystalSystem::Orthorhombic: return 4;
case gemmi::CrystalSystem::Trigonal: return 6;
case gemmi::CrystalSystem::Tetragonal: return 8;
case gemmi::CrystalSystem::Hexagonal: return 12;
case gemmi::CrystalSystem::Cubic: return 24;
}
return 1;
}
// The lattice METRIC can carry more rotational symmetry than the group the intensities supported.
// That is not by itself a mistake - a pseudo-symmetric metric is ordinary, and only the intensities
// can say whether the extra rotations are real - but it is where the corpus's symmetry under-calls
// live, so the run says it out loud rather than leaving the user to notice. It decides nothing: no
// threshold, no promotion, no demotion.
//
// The metric symmetry is Le Page's, and its obliquity is the continuous measure of how nearly a
// two-fold holds; 1 degree is the middle of the band over which the answer is stable.
// Following Le Page (1982) J. Appl. Cryst. 15, 255-259
static void WarnIfMetricHostsMoreSymmetry(const std::optional<UnitCell> &cell,
const gemmi::SpaceGroup &sg) {
if (!cell)
return;
Logger logger("Rugnux");
const gemmi::UnitCell gc(cell->a, cell->b, cell->c, cell->alpha, cell->beta, cell->gamma);
const size_t metric_rotations =
gemmi::find_lattice_symmetry(gc, sg.centring_type(), LATTICE_MAX_OBLIQUITY_DEG).sym_ops.size();
const size_t group_rotations = sg.operations().sym_ops.size();
if (metric_rotations <= group_rotations)
return;
// The proper rotations of each holohedry, which is what find_lattice_symmetry returns.
const char *metric_system = "a higher";
switch (metric_rotations) {
case 2: metric_system = "monoclinic"; break;
case 4: metric_system = "orthorhombic"; break;
case 6: metric_system = "trigonal"; break;
case 8: metric_system = "tetragonal"; break;
case 12: metric_system = "hexagonal"; break;
case 24: metric_system = "cubic"; break;
default: break;
}
logger.Warning("The cell metric is {} - it admits {} rotations where {} has {}. Whether the extra "
"ones were tested against the intensities depends on the lattice class the search "
"was given: where that class is lower than the metric, they were never enumerated, "
"so their absence here is not evidence against them. This is a note, not a "
"correction. Where the higher symmetry is real, merging in it would raise "
"multiplicity and completeness.",
metric_system, metric_rotations, sg.xhm(), group_rotations);
}
bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
auto &logger = p.logger;
auto &result = p.result;
auto &observer = p.observer;
auto &geometry_prepass = p.geometry_prepass;
auto &full = p.full;
auto &images_to_process = p.images_to_process;
auto &prepass_images = p.prepass_images;
auto &write_files = p.write_files;
auto &plots = p.plots;
auto &writer = p.writer;
auto &indexer = p.indexer;
auto &ambiguity_reference = p.ambiguity_reference;
auto &superseded = p.superseded;
auto &census_future = p.census_future;
auto &start_time = p.start_time;
auto &end_msg = p.end_msg;
auto &unmerged_experiment = p.unmerged_experiment;
auto &unmerged_mtz = p.unmerged_mtz;
// Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
// bar reflects what is happening instead of freezing on one label.
auto phase = [&](const std::string &p) {
if (observer) observer->OnPhase(p);
};
phase("Scaling and merging");
Step(geometry_prepass ? "Scaling and post-refinement" : "Scaling and merging");
// Overloads, for the report: the integrated reflections a saturated signal pixel made unmeasurable.
// Counted before anything below drops or flags one. The merge leaves them out (a rotation merge
// drops the whole rocking event) - they are the strongest reflections, mostly at low resolution,
// and what a model's low-resolution terms rest on.
{
for (const auto &io : indexer->GetIntegrationOutcome())
for (const auto &r : io.reflections) {
const bool low_res = r.d >= OVERLOAD_LOW_RES_A;
++result.integrated_reflections;
result.integrated_reflections_low_res += low_res;
if (r.overloaded) {
++result.overloaded_reflections;
result.overloaded_reflections_low_res += low_res;
}
}
if (result.integrated_reflections > 0)
logger.Info("Overloads: {} of {} integrated reflections ({:.3f}%) held a saturated pixel, {:.3f}% of "
"the {} coarser than {:.0f} A", result.overloaded_reflections,
result.integrated_reflections,
100.0 * result.overloaded_reflections / result.integrated_reflections,
result.integrated_reflections_low_res > 0
? 100.0 * result.overloaded_reflections_low_res / result.integrated_reflections_low_res
: 0.0,
result.integrated_reflections_low_res, OVERLOAD_LOW_RES_A);
}
// Ice-ring handling (--detect-ice-rings): flag reflections sitting on a hexagonal-ice powder
// ring. Their integrated intensity is contaminated by the strong, variable ice background, so
// they drag the per-image scale fit. Flag them here so scaling (the per-image G fit and the
// fulls refit) skips them, while the combine, the merge and the statistics keep them - dropping
// them outright guts low/mid-resolution completeness on crystals that merge them fine.
// The eleven hexagonal-ice bands cover 16-26% of the unique reflections at typical resolutions
// whether or not the crystal has any ice, so handling ice unconditionally taxes clean data for
// nothing. Gate the flagging on the run's own measured ice strength (1 = no ice; the strongest
// ring over the smooth radial background, measured on the peak-excluded profile so Bragg peaks
// cannot pose as ice). The score is logged either way, so a run that gates off still reports why.
// Ice is looked for on TWO channels, because it arrives in two forms. Fine-grained ice is a
// smooth powder ring and shows up in the radial profile (ice_ring_score). Ice in large
// crystallites arrives as discrete reflections instead, leaves the profile flat, and is only
// visible as a pile-up of found SPOTS on the ring positions - measured against the same q width
// of ice-free flanks beside them, so the fall-off of spot density with resolution cancels. On
// the rotation battery the two channels barely overlap: the crystals with smooth ice read
// 2.1-2.4 / ~1.0, the ones whose spots are mostly ice read ~1.1 / 3.8-14.6, and a clean crystal
// reads 1.04 on both. Either channel alone is blind to half the cases.
const auto ice_score = plots.GetIceRingScore();
const auto ice_spot_ratio = plots.GetIceRingSpotRatio();
const float ice_min_score = experiment_.GetScalingSettings().GetIceMinScore();
const float ice_min_spot_ratio = experiment_.GetScalingSettings().GetIceMinSpotRatio();
const bool ice_present = (!ice_score.has_value() || *ice_score >= ice_min_score)
|| (ice_min_spot_ratio > 0.0f && ice_spot_ratio.has_value()
&& *ice_spot_ratio >= ice_min_spot_ratio);
result.ice_ring_score = ice_score;
result.ice_spot_ratio = ice_spot_ratio;
if (experiment_.IsDetectIceRings() && !ice_present) {
logger.Info("Ice-ring handling: measured ice score {:.2f} < {:.2f} and spot ratio {:.2f} < "
"{:.2f}, no ice detected - ice-ring handling skipped entirely",
*ice_score, ice_min_score, ice_spot_ratio.value_or(NAN), ice_min_spot_ratio);
} else if (experiment_.IsDetectIceRings()) {
const float ice_width = config_.spot_finding.ice_ring_width_Q_recipA;
size_t total = 0, flagged = 0;
for (auto &outcome : indexer->GetIntegrationOutcome()) {
for (auto &r : outcome.reflections) {
++total;
r.on_ice_ring = IsOnIceRing(r.d, ice_width);
if (r.on_ice_ring)
++flagged;
}
}
result.ice_rings_detected = true;
result.ice_flagged_fraction = total > 0 ? static_cast<double>(flagged) / total : 0.0;
logger.Info("Ice-ring handling: ice score {:.2f} (gate {:.2f}), spot ratio {:.2f} (gate "
"{:.2f}); flagged {} of {} reflections on ice rings (half-width {:.3f} A^-1); "
"excluded from scaling, kept for merging",
ice_score.value_or(NAN), ice_min_score, ice_spot_ratio.value_or(NAN),
ice_min_spot_ratio, flagged, total, ice_width);
}
// Scale the images and merge. Factored so it can run twice: first in P1 to give the
// space-group search a merged dataset, then again in the determined space group so the
// scaling sees symmetry equivalents and the final statistics are in the right symmetry.
// (With a user-fixed space group it simply runs once, already in that symmetry.)
struct ScaleMergeResult {
std::vector<MergedReflection> merged;
MergeStatistics statistics;
// CC1/2 as measured before the correction surfaces (rotation) - the quantity the two-pass
// quality guard in Run() compares the passes on. Equal to the reported CC1/2 wherever no
// surface was fitted, which is every merge on the stills path.
double cc_half_before_corrections = NAN;
// The reference-range table and its ISa (--report-resolution); see ProcessResult.
std::optional<MergeStatistics> reference_statistics;
double reference_isa = 0.0;
// The merge's fulls unmerged, for the .hkl (rotation, written merges only - see below).
std::vector<ScaledFull> scaled_fulls;
};
// The reference path computes each image's G once (per-image scaling against the
// reference); the scaling loop below is skipped, so G is stable across the two passes.
// Smoothing it more than once would compound the correction, so do it only on the first
// pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly.
// Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single
// allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame
// partials, scales the fulls and merges; the whole hot path runs on the GPU when one is present).
// Ingested once here (after ice flagging) and reused across both space-group passes; a fixed
// (forced) mosaicity is handled by the recompute. The decay + absorption correction surfaces run as
// post-scale-fulls stages (on by default, see ScalingSettings::CorrectionSurfaces); external-
// reference scaling, the stills -B (per-image B-factor) and wedge refinement are unsupported here.
// Re-seat the integrated reflections into a different setting of the SAME lattice, for the two
// arms below. The change of basis is read straight off the two lattices: P[i][j] =
// conv_real[i] . indexed_reciprocal[j] is the coefficient of the indexed cell's j-th axis in the
// new cell's i-th axis, and indices transform with the axes, so it is also the matrix that takes
// hkl across. Rounding it to integers - and declining when the residual is large - keeps it exact
// even though the refined cell is not exactly LatticeSearch's Niggli cell.
//
// The per-image reflections and lattices are already in the _process.h5 by now - they go to file
// as each image is processed, long before the group is known - so re-seating them here leaves the
// stored ones in the OLD setting beside a cell in the new one. Accumulate the change of basis
// (composed, in case more than one fires) and hand it to the writer, so the file says which setting
// its per-image data are in and a reader can bring them across.
gemmi::Mat33 stored_to_final; // identity until something below re-seats
const auto record_reindex = [&](const gemmi::Mat33 &m) {
stored_to_final = m.multiply(stored_to_final);
std::array<int32_t, 9> v{};
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
v[3 * i + j] = static_cast<int32_t>(std::lround(stored_to_final.a[i][j]));
end_msg.reindex_matrix = v;
};
const auto reindex_into = [&](const LatticeSearchResult &cand) {
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (reindex_res >= 0.1)
return false;
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(reindex);
for (auto &r : io.reflections) {
const double h = r.h, k = r.k, l = r.l;
r.h = static_cast<int32_t>(std::lround(reindex.a[0][0]*h + reindex.a[0][1]*k + reindex.a[0][2]*l));
r.k = static_cast<int32_t>(std::lround(reindex.a[1][0]*h + reindex.a[1][1]*k + reindex.a[1][2]*l));
r.l = static_cast<int32_t>(std::lround(reindex.a[2][0]*h + reindex.a[2][1]*k + reindex.a[2][2]*l));
}
}
result.consensus_cell = cand.conventional.GetUnitCell();
result.consensus_centering = cand.centering;
end_msg.unit_cell = result.consensus_cell;
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reindex);
end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system };
result.rotation_lattice_type = end_msg.rotation_lattice_type;
record_reindex(reindex);
return true;
};
// The same for a change of basis of the SAME cell - a relabelling of its axes (CrystalSetting.h):
// the integrated reflections, their lattices and the cell go across together.
const auto reseat_integrated = [&](const gemmi::Op &cob) {
const gemmi::Mat33 basis = BasisMatrix(cob);
const gemmi::Op hkl = HklOperator(cob);
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(basis);
for (auto &r : io.reflections) {
const gemmi::Op::Miller h = hkl.apply_to_hkl({{r.h, r.k, r.l}});
r.h = h[0]; r.k = h[1]; r.l = h[2];
}
}
result.consensus_cell = CellInBasis(*result.consensus_cell, cob);
end_msg.unit_cell = result.consensus_cell;
if (end_msg.rotation_lattice.has_value())
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(basis);
record_reindex(basis);
};
// A space group the USER fixed carries its own Bravais lattice, and that is the setting its
// reflections have to be indexed in - but the indexer answers to the metric, not to the group, and
// LatticeSearch hands back the MOST symmetric setting the metric supports. A tetragonal-P lattice
// therefore never comes back C-centred orthorhombic even though the same lattice has that setting,
// and -S C222 on one merges with the C absence rule applied in the wrong frame: measured, half the
// observations thrown away (2.14 M -> 1.08 M) and ISa 28.3 -> 24.6. Ask the character table for the
// group's own class instead and re-seat the reflections into it.
//
// Two ways an indexed lattice can fail to carry the fixed group's Bravais lattice, and one
// re-seating for both. The CENTRING can differ, as above. Or the centring can agree while the
// METRIC does not - the axes permuted, a=37.909 b=78.031 c=77.594 declared P 41 21 2, which
// needs a=b with the 4-fold along c. That one is invisible to a centring test and merges
// anyway, in a frame the reflections are not in: measured, --mode scale -S P41212 on such a
// run reports COMPLETENESS= 195.3.
// A fixed group of lower symmetry than the lattice but with the same centring (-S P21 on an
// orthorhombic-P lattice, say) satisfies both tests where it stands, merges correctly, and is
// left alone. So is a group the RUN determined - it was determined FROM this lattice, and the
// arm above and the check below are its handling; a de-novo run must behave exactly as it did.
if (user_fixed_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_;
const char indexed_centering = end_msg.rotation_lattice_type->centering;
const double violation = end_msg.unit_cell ? MetricViolation(*end_msg.unit_cell, fixed_sg) : 0.0;
if (indexed_centering != fixed_sg.centring_type() || violation > MAX_METRIC_VIOLATION) {
// A trigonal-P group sits on a hexagonal-P lattice; every other system names its own.
const auto want_system = fixed_sg.crystal_system() == gemmi::CrystalSystem::Trigonal
&& fixed_sg.centring_type() == 'P'
? gemmi::CrystalSystem::Hexagonal : fixed_sg.crystal_system();
// Search the PRIMITIVE cell. rotation_lattice is the conventional one and its centring
// is carried beside it, but the character table is reached through a Niggli reduction
// that takes whatever it is handed to be primitive - so a centred conventional cell
// reduces as a lattice with the wrong point set. This arm is entered exactly when the
// centrings differ, which includes every centred indexed lattice, so the distinction is
// not academic: a C-centred orthorhombic conventional cell read as primitive is itself a
// perfectly good oP metric, and -S P212121 on one would match, reindex by the identity
// and merge in a group whose absence rule deletes half the reflections that exist.
// The other lattice arithmetic in this file goes through ToPrimitive for the same reason.
const auto cand = LatticeSearchForClass(
end_msg.rotation_lattice->ToPrimitive(indexed_centering), want_system,
fixed_sg.centring_type());
if (cand && reindex_into(*cand)) {
const auto &uc = *result.consensus_cell;
logger.Info("Reindexed the indexed {}-centred lattice (metric violation {:.3f} against "
"the group's own operators) into the {}-centred setting the fixed space "
"group {} needs: a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}", indexed_centering, violation, cand->centering,
fixed_sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma);
}
}
}
// ...and which of its settings. The centring and the metric are now the group's, but where the
// group's axes are not all alike - P 21 21 2 has its pure two-fold on c - the metric cannot
// say which axis of the cell is which: an orthorhombic cell ordered by length puts the pure
// axis wherever its length puts it, and merging there applies the screw absences to the wrong
// rows and writes a wrong label. The absences say which, so they are asked, on every axial row
// and zone of the integrated reflections (partials summed). A group all of whose settings on
// this cell predict the same absences is left exactly where it is.
if (user_fixed_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()
&& result.consensus_cell.has_value()
&& end_msg.rotation_lattice_type->centering == user_fixed_sg_->centring_type()
&& MetricViolation(*result.consensus_cell, *user_fixed_sg_) <= MAX_METRIC_VIOLATION) {
std::map<std::array<int, 3>, std::pair<double, double>> summed; // I and sigma^2 per index
for (const auto &io : indexer->GetIntegrationOutcome())
for (const auto &r : io.reflections)
if ((r.h == 0 || r.k == 0 || r.l == 0) && std::isfinite(r.I) && std::isfinite(r.sigma)
&& r.sigma > 0.0f && !r.overloaded) {
auto &sum = summed[{r.h, r.k, r.l}];
sum.first += r.I;
sum.second += static_cast<double>(r.sigma) * r.sigma;
}
std::vector<ReflectionZ> axial;
axial.reserve(summed.size());
for (const auto &[hkl, sum] : summed)
axial.push_back({hkl, sum.first / std::sqrt(sum.second)});
const gemmi::Op cob = SeatGroupByAbsences(*result.consensus_cell, *user_fixed_sg_, axial);
if (!(cob == gemmi::Op::identity())) {
reseat_integrated(cob);
const auto &uc = *result.consensus_cell;
logger.Info("The fixed space group {} names its symmetry elements by axis, and the absences "
"on the {} axial and zonal reflections put them on other axes of this cell than "
"the indexing did: reindexed by {} to a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} "
"beta={:.2f} gamma={:.2f}", user_fixed_sg_->xhm(), axial.size(),
IndexTriplet(HklOperator(cob)), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma);
}
}
// The lattice a pass indexed has to carry the Bravais lattice of the group it merges in. Only a
// group the USER fixed can fail that - one the pass determined for itself was determined FROM
// this lattice - and the re-seating above has already had its chance to find a setting that
// does. What comes out otherwise is not a merely suboptimal answer: the absence rule is applied
// across a frame the reflections are not in, and the statistics stop being arithmetic (measured:
// 173.5% complete on a triclinic-P lattice merged in C2, and an undefined R_meas on an F-centred
// cubic one merged in a trigonal-P group).
//
// The two-pass second pass is flagged and not adopted rather than refused: its lattice comes from
// a de-novo re-index at the post-refined geometry, and the first pass - whose lattice did carry
// the group - is still there to go back to. A pass with nothing to fall back on refuses instead;
// the group is the user's assertion, and quietly determining a different one would answer a
// question that was not asked. Name the cell that WAS indexed so the user can act on it.
if (user_fixed_sg_ && end_msg.rotation_lattice_type.has_value() && end_msg.unit_cell.has_value()) {
const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_;
const bool centring_wrong =
end_msg.rotation_lattice_type->centering != fixed_sg.centring_type();
// The same two tests as the re-seating above, asked again now that it has had its chance.
const double violation = MetricViolation(*end_msg.unit_cell, fixed_sg);
if (centring_wrong || violation > MAX_METRIC_VIOLATION) {
const auto &uc = *end_msg.unit_cell;
const std::string why = centring_wrong
? fmt::format("its lattice is {}-centred and this crystal indexes as {}-centred",
fixed_sg.centring_type(), end_msg.rotation_lattice_type->centering)
: fmt::format("this crystal's cell is not one the group's own operators leave "
"invariant (metric violation {:.3f}, tolerance {:.3f}) - the symmetry "
"axes do not run along the axes the group puts them on",
violation, MAX_METRIC_VIOLATION);
if (prepass_result_) {
logger.Warning("Two-pass: the second pass indexed a lattice the fixed space group {} "
"cannot describe - {} - so this pass will not be adopted",
fixed_sg.xhm(), why);
result.lattice_conflicts_with_fixed_sg = true;
} else {
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, fmt::format(
"The space group {} was fixed for this run, but {}: the crystal indexes as a "
"{}-centred {} lattice (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}), and no setting of that lattice carries the group's. Merging in {} "
"would apply its symmetry to reflections that are not in its frame, so nothing it "
"reported would describe this crystal - a completeness above 100% is the usual "
"symptom. Re-run without a fixed space group to have it determined from the data, "
"or fix one whose lattice this crystal has.",
fixed_sg.xhm(), why, end_msg.rotation_lattice_type->centering,
gemmi::crystal_system_str(end_msg.rotation_lattice_type->crystal_system),
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, fixed_sg.xhm()));
}
}
}
const auto &rot_ss = experiment_.GetScalingSettings();
const bool is_rotation = experiment_.IsRotationIndexing(); // rotation indexing -> rotation scaling/merge
std::optional<RotationScaleMerge> rsm;
std::optional<ScaleMergeResult> guard_merge; // what the quality guard reads, where not the search merge
std::optional<PostRefineObservations> prepass_postrefine_obs;
// The rotation geometry post-refinement (see its call sites below for what it is for), in two
// parts. prepare_post_refine reads the pass - the outcomes, the indexer, the scaling engine - and
// returns the fit with everything it needs; fit_post_refine reads only that, so the canonical pass
// can run it beside its merge chain (below).
struct PostRefineJob {
RotationIndexerResult rot;
PostRefineSettings prs;
PostRefineObservations obs;
};
const auto prepare_post_refine = [&](size_t max_images = SIZE_MAX) -> std::optional<PostRefineJob> {
const auto &outcomes = indexer->GetIntegrationOutcome();
if (geometry_prepass) {
prepass_mosaicity_.assign(outcomes.size(), NAN);
std::vector<float> fitted;
for (size_t o = 0; o < outcomes.size(); ++o)
if (outcomes[o].mosaicity_deg) {
prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg;
fitted.push_back(*outcomes[o].mosaicity_deg);
}
// A pre-pass of part of the sweep has fitted nothing for the frames after it, and they are
// predicted at the median of what it did fit, as the merge fills its own frames without a
// fit. Left at the frame's own estimate alone, the second half of a small-molecule sweep
// predicted a fifth fewer reflections and merged at R_meas 7.2 % against 5.2 %.
if (!fitted.empty() && outcomes.size() < static_cast<size_t>(images_to_process)) {
std::nth_element(fitted.begin(), fitted.begin() + fitted.size() / 2, fitted.end());
prepass_mosaicity_.resize(images_to_process, fitted[fitted.size() / 2]);
}
}
if (result.consensus_cell.has_value()) {
const auto rot = indexer->FinalizeRotationIndexing();
indexer->RethrowIndexerError();
if (rot && rot->axis) {
if (geometry_prepass)
prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback
PostRefineSettings prs;
prs.crystal_system = rot->search_result.system;
prs.num_threads = static_cast<int>(config_.nthreads);
prs.refine_geometry = true;
// Only a pass still at the file's distance can hold it as a hypothesis; a pass
// that has walked off it (the geometry walk in RunAllPasses) is past that question.
prs.distance_at_header = !std::isfinite(header_distance_mm_)
|| std::fabs(experiment_.GetDetectorDistance_mm() - header_distance_mm_)
< 1e-3 * header_distance_mm_;
// The centre the pre-scan measured, where the fit was precise enough to be used as
// a centre at all (MAX_BEAM_CENTER_SIGMA_PXL). It does
// not move the geometry - it only tells the bound on step B's beam move where else
// the centre is independently believed to be.
JoinBeamCenterCapture();
if (measured_beam_center_
&& measured_beam_center_->sigma_pxl <= std::max(MAX_BEAM_CENTER_SIGMA_PXL,
BeamCenterNeed_pxl(experiment_)))
prs.measured_beam_px = std::array<double, 2>{
static_cast<double>(measured_beam_center_->beam_x_pxl),
static_cast<double>(measured_beam_center_->beam_y_pxl)};
// The first pass's own geometry, not the header's. The outcomes being post-refined
// were integrated at it, and step B holds the tilt fixed while it fits the beam and
// the distance - so handing it the header tilt fits a beam that has to absorb a tilt
// error the images do not have, and the answer describes neither geometry.
//
// On the pre-pass the observations were gathered right after Ingest (see there) and
// the scaling engine is finished - nothing after this point reads it on that pass
// (the search and the reference re-merge run only on the canonical one) - so it is
// freed BEFORE the fit: the engine holds about twice the partial payload again,
// and with it gone the fit no longer sits on the pass's memory peak.
if (prepass_postrefine_obs)
rsm.reset();
auto pr_obs = prepass_postrefine_obs
? std::move(*prepass_postrefine_obs)
: GatherPostRefineObservations(indexer->GetIntegrationOutcome(),
config_.nthreads,
/*release_reflections=*/false, logger,
max_images);
prepass_postrefine_obs.reset();
return PostRefineJob{*rot, prs, std::move(pr_obs)};
}
}
return std::nullopt;
};
const auto fit_post_refine = [this, &result = p.result, geometry_prepass = p.geometry_prepass]
(PostRefineJob job, Logger &logger, std::vector<std::string> &notes) {
const auto &rot = job.rot;
const auto pr = PostRefineRotationGeometry(std::move(job.obs), *rot.axis,
rot.geom, rot.lattice, job.prs, logger);
result.post_refine = pr;
if (pr.ok && pr.detector_refined) {
// Store, don't apply: this pass continues to write its own (header-geometry) output;
// Run() applies the refined geometry before the second pass.
prepass_detector_geometry_ = std::array<float, 5>{
static_cast<float>(pr.beam_x_after_px),
static_cast<float>(pr.beam_y_after_px),
static_cast<float>(pr.distance_after_mm),
rot.geom.GetPoniRot1_rad(),
rot.geom.GetPoniRot2_rad()};
logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, "
"beam {:.2f},{:.2f} px, tilt {:.4f},{:.4f} deg) - {}",
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px,
rot.geom.GetPoniRot1_rad() * 180.0 / PI,
rot.geom.GetPoniRot2_rad() * 180.0 / PI,
geometry_prepass ? "it will drive the second integration pass"
: "what the fit still asks for at the geometry "
"this pass ran at");
notes.push_back(fmt::format("{}: {:.2f} mm, beam {:.1f}, {:.1f}",
geometry_prepass ? "geometry refined" : "post-refinement asks",
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px));
} else {
logger.Info("Two-pass: geometry post-refine committed no detector change{}",
geometry_prepass ? " - the second pass reproduces the first" : "");
notes.push_back("geometry post-refinement: no detector change committed");
}
// The goniometer rotation scale in pr is only a fit: RunAllPasses walks it to its
// fixed point and decides it on the validation frames of the whole sweep
// (WalkRotationScale). It does not ride on pr.ok - the crystals that have a stage
// fault are exactly the ones whose cell and detector steps do NOT pass, because the
// angle error is what their residual is made of.
};
const auto post_refine_geometry = [&] {
TimingMark mark(logger, "post-refinement");
if (auto job = prepare_post_refine()) {
std::vector<std::string> notes;
fit_post_refine(std::move(*job), logger, notes);
for (const auto &note : notes)
Note(note);
}
};
// On a CANONICAL pass the post-refinement only measures (see below), and it reads nothing the
// scaling engine produces: the reflections' hkl, I, sigma and positions are final once the
// images are integrated, and the engine writes back per-frame fields only. So it is gathered
// here, beside the engine's ingest, and fitted beside the merges that follow.
// The quality guard judges a refined pass against the pre-pass on the search merge, and a
// pre-pass that integrated only the first part of the sweep merged only that part. Against
// it a merge of the whole sweep holds more reflections, more signal and more of the axial
// rows for no reason but the frames, and the guard could never find it worse. So the guard
// reads this pass's own integration of the same frames, merged the way the pre-pass merged
// them. It only reads the outcomes (the per-frame scale is not written back), and so does the
// post-refinement, so it runs beside it; its log lines are held and replayed where it is used.
const bool merge_first_part = is_rotation && quality_guard_pass1_ && prepass_images < images_to_process
&& !experiment_.GetRefineRotationWedgeInScaling() && !rot_ss.GetRotationWedgeForScaling().has_value();
// With two GPUs or more the guard's merge has an engine of its own on the second card, and the
// main engine does not wait for it: its ingest and first merge run beside the guard's merge. The
// guard reads the outcomes only while it ingests - the first merge writes per-frame fields back
// into them (mosaicity among them, which the ingest reads) - so that merge waits for the guard's
// ingest and nothing else. Neither merge reads anything the other writes, and on cards of one
// model a merge gives the same bits on either, so this changes only when the guard runs.
const bool guard_on_second_gpu = get_gpu_count() >= 2;
std::promise<void> guard_ingested;
Logger guard_log = Logger::Buffered();
const auto merge_guard = [&] {
TimingMark mark(guard_log, "merge quality guard (pre-pass frames)");
RotationScaleMerge first_part_rsm(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter), config_.nthreads,
guard_log, "");
first_part_rsm.SetFrameLimit(prepass_images);
first_part_rsm.SetWriteBackPerFrameScale(false);
if (guard_on_second_gpu)
first_part_rsm.SetGpuDevice(1);
{
// Set however the ingest ends: a guard that failed rethrows from its future, later.
struct Ingested { std::promise<void> &p; ~Ingested() { p.set_value(); } } ingested{guard_ingested};
first_part_rsm.Ingest();
}
first_part_rsm.SetExportScaledFulls(false);
first_part_rsm.SetFrenchWilson(false);
auto r = first_part_rsm.Run(!experiment_.GetGemmiSpaceGroup().has_value(),
/*full_stats=*/false, /*measure_cc_before_corrections=*/true);
return ScaleMergeResult{std::move(r.merged), std::move(r.statistics), r.cc_half_before_corrections};
};
std::future<ScaleMergeResult> guard_ahead;
if (merge_first_part && !(postrefine_probe_ && !geometry_prepass && postrefine_probe_only_))
guard_ahead = std::async(std::launch::async, [&] {
if (guard_on_second_gpu)
pin_gpu(1); // the card and its NUMA node, as an image-loop worker has them
return merge_guard();
});
// The scaling engine's ingest reads the outcomes and nothing else either, so its host half runs
// beside the post-refinement too; with one GPU the device half waits for the guard to hand the
// card back.
Logger engine_log = Logger::Buffered();
std::future<void> engine_ahead;
if (is_rotation && postrefine_probe_ && !geometry_prepass && !postrefine_probe_only_
&& !experiment_.GetRefineRotationWedgeInScaling() && !rot_ss.GetRotationWedgeForScaling().has_value())
engine_ahead = std::async(std::launch::async, [&] {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, engine_log, config_.observation_dump_path);
rsm->IngestHost();
});
// The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the
// comparison reads the held-out residual of a post-refinement on the frames the demoted arm
// integrated, so that fit runs here, before the merge, and the pass stops if the demoted arm
// realised the lower residual. The fit only measures and leaves nothing on the run; otherwise the
// pass goes on exactly as the canonical pass it is, measuring its own post-refinement over all
// the frames where it measures one anyway (postrefine_probe_).
const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass
&& is_rotation && !postrefine_probe_only_;
if (lattice_arm_asked) {
LatticeArmCheck &arm = *lattice_arm_check_;
arm.asked = true;
const auto arm_fit = [&](size_t max_images) -> std::optional<PostRefineResult> {
TimingMark mark(logger, "post-refinement (metric-symmetry arm)");
auto job = prepare_post_refine(max_images);
if (!job)
return std::nullopt;
return PostRefineRotationGeometry(std::move(job->obs), *job->rot.axis, job->rot.geom,
job->rot.lattice, job->prs, logger);
};
arm.promoted = arm_fit(arm.prepass_frames ? static_cast<size_t>(prepass_images) : SIZE_MAX);
if (!cancelled_ && arm.promoted && arm.demoted) {
arm.lost = HeldOutResidualFell(*arm.promoted, *arm.demoted);
// Decided on the start of the sweep, a near tie is re-checked on all of it once the run is
// over (RefinedPasses), against this pass's own fit over every frame, taken while its
// reflections are still here. The margin is the residual's distance from the line
// HeldOutResidualFell draws, against the noise that draws it.
const double noise = std::hypot(arm.promoted->held_out_before_se, arm.demoted->held_out_before_se);
const double margin = arm.promoted->held_out_before - arm.demoted->held_out_before - noise;
arm.near_tie = arm.prepass_frames && std::isfinite(noise)
&& std::abs(margin) <= FirstPassRun::NEAR_TIE_SIGMA * noise;
logger.Info("Metric-symmetry check: on the first {} images the held-out residual is {:.3e} +- {:.1e} "
"as the promoted class against {:.3e} +- {:.1e} as the demoted one{}",
arm.prepass_frames ? prepass_images : images_to_process,
arm.promoted->held_out_before, arm.promoted->held_out_before_se,
arm.demoted->held_out_before, arm.demoted->held_out_before_se,
arm.near_tie ? " - a near tie, re-checked on all the frames after the run" : "");
if (arm.near_tie)
arm.promoted_all = arm_fit(SIZE_MAX);
}
if (arm.lost) {
if (census_future.valid())
census_future.wait();
result.processing_time_s = std::chrono::duration<double>(
std::chrono::steady_clock::now() - start_time).count();
logger.Info("Metric-symmetry check: {} images integrated and post-refined in {:.2f} s - the "
"demoted class realises the lower held-out residual, so this pass does not merge",
result.images_processed, result.processing_time_s);
if (!writer)
return true;
}
}
if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_
&& !(lattice_arm_asked && lattice_arm_check_->lost)) {
// Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined
// it), so only the gather runs here: it reads the outcomes, which the chain re-indexes in
// place once the space group is settled. The fit and the speculative probe it may start
// run beside the chain; the probe's copy of the run is taken now, as the chain goes on to
// change the run's members.
TimingMark mark(logger, "post-refinement observations");
if (auto job = prepare_post_refine()) {
std::shared_ptr<Rugnux> probe_run;
if (speculate_geometry_probe_)
probe_run = std::make_shared<Rugnux>(*this);
p.postrefine_ahead = std::async(std::launch::async,
[this, &p, fit_post_refine, job = std::move(*job), probe_run]() mutable {
TimingMark fit_mark(p.postrefine_log, "post-refinement");
fit_post_refine(std::move(job), p.postrefine_log, p.postrefine_notes);
fit_mark.Stop();
if (probe_run && !cancelled_ && prepass_detector_geometry_ && p.result.post_refine
&& p.result.post_refine->detector_refined && !p.result.post_refine->large_move)
StartSpeculativeGeometryProbe(probe_run, *prepass_detector_geometry_);
});
}
}
if (postrefine_probe_ && !geometry_prepass && is_rotation && postrefine_probe_only_) {
post_refine_geometry();
if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine
&& result.post_refine->detector_refined && !result.post_refine->large_move)
StartSpeculativeGeometryProbe(std::make_shared<Rugnux>(*this), *prepass_detector_geometry_);
// A pass the lattice arms run to compare two Bravais classes has now produced the only
// thing the comparison reads - the held-out residual of the fit just measured. Stop here
// rather than scale, search, fit surfaces and merge for a result nothing looks at.
result.processing_time_s = std::chrono::duration<double>(
std::chrono::steady_clock::now() - start_time).count();
logger.Info("Probe pass: {} images integrated and post-refined in {:.2f} s "
"(the comparison reads the post-refinement only, so this pass does not merge)",
result.images_processed, result.processing_time_s);
return true;
}
// A pass RunAllPasses re-runs at the fixed radius for a starved background ring: its merge,
// search and files are all replaced by the re-run's, and what anything reads from it - the
// post-refinement and validation evidence above, and bkg_starved_fraction_ - is in hand. The
// pass after it starts from the group the run was asked for (top of RunPipeline), so the
// space group this pass would have determined is not carried either. Not with a _process.h5
// writer open: that file is written whole by every pass that opens it.
if (rerun_if_starved_ && !writer && !cancelled_ && bragg_adaptive_ && bkg_starved_fraction_
&& *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION) {
if (census_future.valid())
census_future.wait();
result.processing_time_s = std::chrono::duration<double>(
std::chrono::steady_clock::now() - start_time).count();
logger.Info("Starved pass: {} images integrated in {:.2f} s - {:.2f}% of the reflections "
"lost their background ring, so this pass is re-run at the fixed radius and "
"does not merge", result.images_processed, result.processing_time_s,
100.0 * *bkg_starved_fraction_);
return true;
}
// The all-observation arm of the space-group search (below) is the same ingest merged with the
// Lorentz filter off. No merge depends on the merges made before it (RotationScaleMerge::RunArm),
// so it is merged on a copy of the engine, started here and read where the search needs it: beside
// the P1 merge, the re-scaled search merge and the first arm. Started whenever the search will run;
// a search that turns out not to need it (no point group from the first arm) does not read it.
// Declared before the future, which waits for the merge when it goes, so the engine outlives it.
std::unique_ptr<RotationScaleMerge> rsm_all;
Logger all_arm_log = Logger::Buffered();
std::future<RotationScaleMerge::Result> all_arm_ahead;
if (is_rotation) {
if (experiment_.GetRefineRotationWedgeInScaling()
|| rot_ss.GetRotationWedgeForScaling().has_value())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Rotation scaling/merging (RotationScaleMerge) does not support "
"wedge refinement");
if (merge_first_part) {
logger.Info("Two-pass: the quality guard reads this pass's merge of the first {} images, "
"the ones the pre-pass integrated", prepass_images);
if (!guard_ahead.valid() || !guard_on_second_gpu) {
guard_merge = guard_ahead.valid() ? guard_ahead.get() : merge_guard();
guard_log.ReplayInto(logger);
}
}
// A reference MTZ is allowed for rotation: it fixes the space group / cell (on the CLI) and
// resolves the indexing ambiguity (below), but is NOT used to scale - the rotation merge stays
// self-consistent.
if (engine_ahead.valid()) {
engine_ahead.get();
engine_log.ReplayInto(logger);
rsm->SetLogger(logger);
} else {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->IngestHost();
}
// On a card of its own from three cards up (card 0 has this engine, card 1 the quality guard's
// and the speculative passes). Otherwise on the last card, beside the engine already there -
// this one, or the guard's, which merges at most as many observations - and only where two
// engines of this size fit the card's memory, judged on its total, not on what happens to be
// free: the answer must not depend on the moment. What else stays on the card meanwhile - the
// context, the run's indexers - was measured at up to 2.3 GB; a speculative pass running beside
// holds more, but a merge that does not fit beside it waits for it to end. On a 16 GB card that
// is the data sets up to about 50 M observations at high multiplicity; the large ones merge
// their arms one after the other.
const int32_t cards = get_gpu_count();
if (!geometry_prepass && !experiment_.GetGemmiSpaceGroup().has_value() && rsm->GetSearchMinZeta() > 0.0
&& cards > 0) {
const int32_t device = std::min(cards - 1, 2);
constexpr size_t CARD_BYTES_RESERVED = size_t(3) << 30;
if (cards >= 3 || 2 * rsm->DeviceBytes() + CARD_BYTES_RESERVED <= get_gpu_total_memory(device)) {
rsm_all = rsm->CloneIngest(device, all_arm_log);
rsm_all->SetSearchMinZeta(0.0);
rsm_all->SetExportScaledFulls(false); // as scale_and_merge sets them for a search merge
rsm_all->SetFrenchWilson(false);
}
}
rsm->IngestDevice();
if (rsm_all) {
// Its observations are copied from this engine's card arrays, not staged from the
// reflections a second time.
rsm_all->IngestDevice(&*rsm);
all_arm_ahead = std::async(std::launch::async, [&rsm_all, &all_arm_log] {
pin_gpu(rsm_all->GetGpuDevice()); // the card and its NUMA node, as the guard's merge has them
TimingMark mark(all_arm_log, "merge P1, all observations");
return rsm_all->Run(/*for_search=*/true, /*full_stats=*/true,
/*measure_cc_before_corrections=*/false);
});
}
}
// The geometry pre-pass reads the per-image reflections exactly twice more: they were just
// ingested above, and the post-refinement at the end of the pass reads a narrow
// (hkl, frame, I, sigma, observed position) view of them. Nothing else in the pre-pass
// touches them - its reflections are never written, and the scaling engine reads back only
// per-frame fields - so take the post-refine's view NOW and hand each image's vector back as
// it is read: on a fine-sliced long axis the vectors are gigabytes, held to the end of the
// pass for those 32 bytes a partial. A run with a reference MTZ keeps them: resolving the
// indexing ambiguity re-reads them (and re-ingests).
if (geometry_prepass && rsm && config_.reference_data.empty())
prepass_postrefine_obs = GatherPostRefineObservations(
indexer->GetIntegrationOutcome(), config_.nthreads,
/*release_reflections=*/true, logger);
// What a rotation merge leaves in the result, taken in the order the merges are made - which, for
// the all-observation arm merged beside the others, is where the search reads it.
const auto take_merge = [&](const std::string &label, bool for_search, RotationScaleMerge::Result r)
-> ScaleMergeResult {
result.error_model_isa = r.isa;
result.error_model_isa_asymptotic = r.isa_asymptotic;
result.error_model_isa_resolved = r.isa_resolved;
if (r.resolution_fit_A) result.resolution_fit_A = r.resolution_fit_A;
result.error_model_a = r.error_model_a;
result.error_model_b = r.error_model_b;
result.scaling_iterations_partials = r.scaling_iterations_partials;
result.scaling_iterations_fulls = r.scaling_iterations_fulls;
result.scaling_converged = r.scaling_converged;
if (for_search)
result.search_scaling_converged = r.scaling_converged;
Note(fmt::format("merge in {}: {} unique", label, r.merged.size()));
return ScaleMergeResult{std::move(r.merged), std::move(r.statistics),
r.cc_half_before_corrections,
std::move(r.reference_statistics), r.reference_isa,
std::move(r.scaled_fulls)};
};
// measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a
// merge of its own - only the first merge, which the two-pass quality guard reads.
auto scale_and_merge = [&](const std::string &label, bool for_search, bool measure_cc = false)
-> ScaleMergeResult {
TimingMark mark(logger, "merge " + label);
if (rsm) {
phase("Scale/combine/merge (" + label + ")");
// The geometry pre-pass merges only to choose the space group and to give the quality
// guard something to judge the second pass against; its reflections are never written,
// so the parts of the merge that only fill in an output file are skipped there.
// Only a merge that may be written hands its fulls back: they are a copy as long as the
// fulls, and the search merges and the pre-pass never write a .hkl.
rsm->SetExportScaledFulls(!for_search && !geometry_prepass && write_files && config_.write_merged);
// Amplitudes are made once, on the merge that is written, where the anisotropy tensor
// their prior uses is known (see below) - not on every merge the tail makes on its way.
rsm->SetFrenchWilson(false);
return take_merge(label, for_search,
rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
/*measure_cc_before_corrections=*/measure_cc));
}
// Stills (rotation goes through RotationScaleMerge above): self-scale each image against the
// running merge with ScaleOnTheFly (fixed partiality), then merge directly. This runs even
// with an external reference: the reference is used only to break the per-image indexing
// ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor (scaling each
// image against a foreign dataset injects cross-dataset systematics and is a worse reference
// than the data's own merge). One pass: the per-image scale G is the exact one-pass solution,
// so iterating would only rebuild the reference from the freshly-scaled (noisy) data and
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
{
phase("Scaling images (" + label + ")");
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome());
indexer->ScaleAllImages(merge_result);
}
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
// orientation tilt against the running merge and recompute each reflection's partiality + scale
// correction (no re-integration). Never for the P1 search pass.
if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) {
phase("Partiality post-refine (" + label + ")");
StillsPartialityRefine refiner(experiment_);
const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads);
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
// The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally
// when the image has no prior CC - which is exactly the state the guard above leaves. So
// the guard was protecting nothing: measured on a jet dataset, of 367 images it left
// unscaled only 8 were still unscaled in the output, and 53 reached the merge at or below
// a fiftieth of the run median, the worst at a 4525th. Whatever produced the final scale,
// it is the final scale that has to be credible.
ScaleOnTheFly::RejectCollapsedScales(indexer->GetIntegrationOutcome());
}
const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
phase("Merging");
MergeOnTheFly merge_engine(experiment_);
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
// intensities and the error model, so the space-group search sees clean data; the final
// in-symmetry merge keeps them so completeness is not lost.
merge_engine.ExcludeIceRings(for_search);
if (result.consensus_cell.has_value())
merge_engine.ReferenceCell(*result.consensus_cell);
// Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells)
// most indexed crystals do not correlate with the true structure; --min-image-cc drops the
// crystals whose per-image CC to the reference is below the limit, so the merge keeps only
// the coherent (real) ones. Set before the error model so the model, the merge and the
// reported statistics are all fitted over the same set of images. The P1 search pass keeps
// every image: its job is to find the symmetry, not to produce final intensities.
merge_engine.FilterByImageCC(!for_search
&& experiment_.GetScalingSettings().GetMinCCForImage() > 0.0);
merge_engine.RefineErrorModel(merge_input);
if (merge_engine.ErrorModelActive())
{
// XDS convention, so the pair reads directly against a CORRECT.LP; the stills ISa is
// already the whole-range 1/b, there being no asymptote tier on this path.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
}
for (size_t i = 0; i < merge_input.size(); ++i)
merge_engine.AddImage(merge_input[i], static_cast<int64_t>(i));
ScaleMergeResult out;
out.merged = merge_engine.ExportReflections();
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
// range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A
// manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the
// space-group search still sees the full range. (Rotation is cut inside RotationScaleMerge.)
const auto &cut_ss = experiment_.GetScalingSettings();
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A);
phase("Computing statistics");
out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min);
// The same kept reflections binned over the reference range (--report-resolution); the
// stills error model is fitted over the whole range already, so no ISa of its own.
if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search)
out.reference_statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data,
std::nullopt, range);
{ // stills: no asymptote tier, and its ISa already IS the whole-range 1/b
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
result.error_model_isa = em.isa;
result.error_model_a = em.a;
result.error_model_b = em.b;
}
out.cc_half_before_corrections = out.statistics.overall.cc_half; // no surfaces on this path
logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label);
Note(fmt::format("merge in {}: {} unique", label, out.merged.size()));
return out;
};
// First pass: P1 when searching, or directly the user-fixed space group. A guard merging on the
// second GPU (above) is still running; this merge only waits for it to have read the outcomes.
const bool guard_beside = merge_first_part && guard_on_second_gpu && guard_ahead.valid();
if (guard_beside)
guard_ingested.get_future().wait();
const auto initial_sg = experiment_.GetGemmiSpaceGroup();
auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value(),
/*measure_cc=*/true);
if (guard_beside) {
guard_merge = guard_ahead.get();
guard_log.ReplayInto(logger);
}
// The two-pass quality guard judges the second pass against the first on THIS merge, which both
// passes make in the same terms - P1 (or the group the user fixed), full range, no correction
// surfaces - where their final merges need not even be in the same group. See RunAllPasses. Where
// the pre-pass integrated only part of the sweep, it is the same merge of those frames (above).
{
const ScaleMergeResult &counted = guard_merge ? *guard_merge : sm;
const auto &o = counted.statistics.overall;
result.search_merge_completeness_measured = o.possible_unique_reflections > 0;
result.search_merge_completeness = result.search_merge_completeness_measured
? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0;
result.search_merge_cc_half = counted.cc_half_before_corrections;
if (!counted.statistics.shells.empty())
result.search_merge_inner_cc_half = counted.statistics.shells.front().cc_half;
// ... and the reflections it holds on the principal axial rows, which the guard needs
// separately because CC1/2 cannot express them (see RunAllPasses). Only the low-order
// part is counted: a screw is read off the strong start of its row - the evidence gate
// wants four absences in one zone, and on a 70 A axis that is already at 7 A - while the
// high-order axial reflections are as numerous as they are uninformative and would dilute
// exactly the difference this number exists to show.
//
// Both counts are taken over the reflections this pass's own lattice centring allows, and
// that is what makes them the same measurement on the two sides. Each pass merges on its
// OWN conventional cell, so a pass that finds a centring the other did not carries the
// centring-extinguished half (two thirds, R; three quarters, F) of that cell's reflections
// as measured noise: measured, two crystals compare merges holding 2.0x the reflections of
// the other, with different axial rows, for no reason but the setting. What both settings
// describe is the primitive lattice, and these are its reflections.
const auto centring_allows = [c = result.consensus_centering.value_or('P')]
(const MergedReflection &r) {
switch (c) { // rhombohedral is described obverse everywhere in this code
case 'A': return (r.k + r.l) % 2 == 0;
case 'B': return (r.h + r.l) % 2 == 0;
case 'C': return (r.h + r.k) % 2 == 0;
case 'I': return (r.h + r.k + r.l) % 2 == 0;
case 'F': return (r.h + r.k) % 2 == 0 && (r.h + r.l) % 2 == 0;
case 'R': return (-r.h + r.k + r.l) % 3 == 0;
default: return true;
}
};
constexpr float AXIAL_ROW_LOW_ORDER_D_A = 4.0f;
result.search_merge_axial_reflections = std::count_if(
counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) {
return ((r.h == 0) + (r.k == 0) + (r.l == 0)) == 2
&& r.d > AXIAL_ROW_LOW_ORDER_D_A && centring_allows(r);
});
// The signal the pass measured, and the size of the merge it measured it in, which is
// what the guard decides on.
constexpr float STRONG_I_OVER_SIGMA = 2.0f;
result.search_merge_strong_reflections = std::count_if(
counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) {
return r.sigma > 0 && r.I / r.sigma >= STRONG_I_OVER_SIGMA && centring_allows(r);
});
result.search_merge_negative_reflections = std::count_if(
counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) {
return r.sigma > 0 && r.I / r.sigma <= -STRONG_I_OVER_SIGMA && centring_allows(r);
});
result.search_merge_reflections = std::count_if(counted.merged.begin(), counted.merged.end(),
centring_allows);
}
// Rotation two-pass geometry pre-pass: this first scale/merge has now fitted a frame-order-smoothed
// mosaicity and written it back onto the per-frame outcomes. Capture it for the second pass's Bragg
// prediction (so prediction uses the smoothed value, not a fresh per-image estimate) and post-refine +
// apply the detector geometry from this integration. That is all the pre-pass is for: everything
// the second pass consumes from it is fitted here, and the space group is left to the second pass
// (see the search gate below).
// postrefine_probe_ brings the post-refinement into a CANONICAL pass as well, where the
// pre-pass moved the geometry: the fit it committed was made on reflections integrated at the
// geometry BEFORE the move, and whether it has finished walking can only be asked of
// reflections integrated at the geometry after it. The probe measures - it applies nothing and
// leaves the smoothed mosaicity and the pre-pass lattice alone, which stay pass 1's for every
// pass that consumes them; RunAllPasses decides what to do with the answer.
if (geometry_prepass && rsm && experiment_.IsRotationIndexing())
post_refine_geometry();
// For a de-novo search, gate the P1 scaling+merge (the pass that feeds the space-group search) at
// <I/sigma> >= 1 over thin resolution shells: noise-dominated high-res shells otherwise mislead
// the determination, where a single artifact (a zinger the box sum integrates whole, or a
// weak-reflection profile-fit runaway) has an astronomical resolution-normalised E and wrecks the
// intensity-correlation / second-moment statistics. The final in-symmetry merge keeps the full
// range (only this P1 search pass is cut); a manual --scaling-high-resolution, if coarser, wins.
// The geometry pre-pass does not determine the space group. Its integration is at the header
// geometry, which is the thing the second pass exists to improve on, and the symmetry evidence
// moves with the geometry: measured on a rotation crystal, 0.33 mm of distance and 0.65 px of
// beam centre took the screw-axis evidence from -19 to +311 nats, where merging the same
// integration in symmetry instead of P1 moved it 2.5. Deciding on the worse integration also
// fails in one direction only - every statistic the search weighs is a ratio of the candidate
// merge's scatter to the subgroup's, so poorly-integrated data inflate them and REFUSE real
// symmetry (corpus-wide the misses run ten under-calls to one over-call). So the search runs on
// the canonical pass, at the refined geometry, and the pre-pass skips it along with the
// in-symmetry re-merge that only exists to serve it.
const bool search_space_group = !geometry_prepass && !initial_sg.has_value();
double d_min_search = 0.0;
if (search_space_group) {
std::vector<std::pair<float, float>> rs; // (d, I/sigma) over the full P1 merge
rs.reserve(sm.merged.size());
for (const auto &m : sm.merged)
if (std::isfinite(m.I) && std::isfinite(m.sigma) && m.sigma > 0.0f &&
std::isfinite(m.d) && m.d > 0.0f)
rs.emplace_back(m.d, m.I / m.sigma);
if (rs.size() >= 400) {
std::sort(rs.begin(), rs.end(),
[](const auto &a, const auto &b) { return a.first > b.first; }); // low -> high res
const int bins = std::clamp(static_cast<int>(rs.size() / 100), 1, 40);
const size_t per = (rs.size() + bins - 1) / static_cast<size_t>(bins);
// The shell means, then their best NON-INCREASING fit (pool adjacent violators, weighted
// by shell size). The cut is where the FIT drops under 1, not where the first shell
// does: on a merge whose <I/sigma> is flat near 1 - its ISa has collapsed - the first
// single shell to dip under 1 is a coin toss that moved the cut from 0.85 to 1.19 A
// between two compiler flags on one crystal, and the coarser cut left its glide zones
// too few absences to be judged. The fit moves by as much as its input does.
std::vector<double> mean, weight;
for (size_t lo = 0; lo < rs.size(); lo += per) {
const size_t hi = std::min(rs.size(), lo + per);
double sum = 0.0;
for (size_t j = lo; j < hi; ++j) sum += rs[j].second;
mean.push_back(sum / static_cast<double>(hi - lo));
weight.push_back(static_cast<double>(hi - lo));
}
std::vector<double> block_mean, block_weight;
std::vector<size_t> block_size;
for (size_t b = 0; b < mean.size(); ++b) {
block_mean.push_back(mean[b]); block_weight.push_back(weight[b]); block_size.push_back(1);
while (block_mean.size() > 1 && block_mean[block_mean.size() - 2] < block_mean.back()) {
const size_t k = block_mean.size() - 1;
const double w = block_weight[k - 1] + block_weight[k];
block_mean[k - 1] = (block_mean[k - 1] * block_weight[k - 1] + block_mean[k] * block_weight[k]) / w;
block_weight[k - 1] = w;
block_size[k - 1] += block_size[k];
block_mean.pop_back(); block_weight.pop_back(); block_size.pop_back();
}
}
std::vector<double> fit;
for (size_t k = 0; k < block_mean.size(); ++k)
fit.insert(fit.end(), block_size[k], block_mean[k]);
for (size_t b = 0; b < fit.size(); ++b) {
const size_t lo = b * per;
if (fit[b] < 1.0) {
// Cut the noise-dominated high-res shells. When even the lowest-res shell fails,
// keep that shell alone rather than abandoning the cut. The bound is absolute
// while the merged I/sigma it tests saturates at the merge's own ISa (times the
// square root of the multiplicity), so a merge whose ISa has collapsed towards 1
// can fail it in every shell - and that is exactly the merge the search must not
// be handed whole, since the noise which drove ISa down is what the cut exists to
// remove. Abandoning it feeds back: more noise into the search, a weaker operator
// statistic, and a lost symmetry.
if (b > 0)
d_min_search = rs[lo].first;
else if (per >= 400)
d_min_search = rs[per - 1].first;
break;
}
}
}
const double d_manual = experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0);
if (rsm && d_min_search > d_manual) {
logger.Info("Space-group search: re-scaling the P1 pass at {:.2f} A (<I/sigma> >= 1)", d_min_search);
rsm->SetDMinLimit(d_min_search);
sm = scale_and_merge("P1, space-group search", true);
rsm->SetDMinLimit(d_manual > 0.0 ? std::optional<double>(d_manual) : std::nullopt);
}
}
std::ostringstream stats_text;
// The indexing-ambiguity paragraph as it went into stats_text, so that a model which then
// settles the ambiguity can withdraw it; empty where there was none.
std::string ambiguity_warning_text;
// True when the point group below was chosen BY THE SEARCH (rather than given by the user) and is
// above triclinic - i.e. a promotion was made, which is what makes a later "the Laue class is
// holohedral so there is no twin law" conclusion circular.
bool promoted_point_group = false;
// The twin-law merge (<prefix>_twin_<SG>): where the adopted group may have absorbed a twin law,
// the subgroup a second merge is made in and the law it carries, in the scaler's frame, and why.
// The decisions below that name a subgroup leave it here: the group a refused promotion was
// adopted over (a twin gate overruled), and the indexed lattice's group a metric re-ask promoted
// from. Report and an extra file only - nothing the run writes as its answer reads any of it.
std::optional<gemmi::SpaceGroup> twin_overruled_subgroup, twin_metric_subgroup;
std::string twin_overruled_reason;
std::optional<TwinSubgroupChoice> twin_choice;
std::string twin_choice_reason;
// The twin hypothesis' merge made only for --mr to test: written, and reported, only where it chooses it.
bool twin_for_mr_only = false;
int twin_choice_group_number = 0; // the adopted group's, when the choice was made
// A canonical pass the geometry walk is going to replace stops here, where the answer is
// already in (GeometryWalkTakesRound); it is asked again below, before the analyses. Not with a
// _process.h5 open: that file is written whole by every pass that opens it.
if (!writer && GeometryWalkTakesRound(result, p.postrefine_ahead)) {
logger.Info("The geometry walk will re-integrate at the geometry the post-refinement asks for, "
"so this pass stops before its space-group search");
return true;
}
TimingMark search_mark(logger, "space-group search");
if (search_space_group) {
SearchSpaceGroupOptions sg_opts;
sg_opts.nthreads = static_cast<size_t>(std::max(1, config_.nthreads));
sg_opts.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
sg_opts.d_min_limit_A = std::max<double>(
d_min_search, experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0));
// Constrain the search to subgroups of the lattice (metric) symmetry found by rotation
// indexing. Centering is not constrained here - it is determined from the absences.
if (end_msg.rotation_lattice_type.has_value()) {
sg_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
sg_opts.lattice_centring = end_msg.rotation_lattice_type->centering;
}
// The cell the reflections are indexed on, which is what lets the search offer a candidate
// in a setting other than gemmi's reference one: a setting names the symmetry directions by
// axis, so a candidate the cell cannot host is refused before it is scored.
if (result.consensus_cell.has_value())
sg_opts.cell = gemmi::UnitCell(*result.consensus_cell);
sg_opts.enumerate_all_settings = true;
sg_opts.enumerate_all_rotation_sets = true;
// The error model this merge was scaled with, so the search's "genuinely present" cut can be
// read on the counting sigma instead of on the merged one (SearchSpaceGroupOptions::merge_isa).
// Set per merge, not once: the two arms below are scaled separately and their ISa differ by up
// to a factor of six. 0 (b not measurable) leaves the cut as the caller set it.
sg_opts.merge_isa = result.error_model_isa;
// The all-observation merge made beside everything since the ingest (see rsm_all), and its
// search started beside this one's: a search reads its merge and its options and nothing
// else. Whether the all-observation arm is wanted at all is known only from the first arm
// (below), so nothing of it reaches the run before then. A merge that failed beside the
// others is made again where it is wanted, in series, as it would have been without it.
std::optional<RotationScaleMerge::Result> all_merge;
std::future<SearchSpaceGroupResult> all_search_ahead;
if (all_arm_ahead.valid()) {
TimingMark wait_mark(logger, "wait for the all-observation merge");
try {
all_merge = all_arm_ahead.get();
} catch (const std::exception &e) {
logger.Warning("Space-group search: the all-observation merge beside the others failed ({}); "
"it is made again in series", e.what());
}
wait_mark.Stop();
}
if (all_merge) {
SearchSpaceGroupOptions all_opts = sg_opts;
all_opts.merge_isa = all_merge->isa; // as take_merge leaves it below
all_search_ahead = std::async(std::launch::async, [&merged = all_merge->merged, all_opts] {
return SearchSpaceGroup(merged, all_opts);
});
}
TimingMark first_arm_mark(logger, "SearchSpaceGroup (first arm)");
auto sg_search = SearchSpaceGroup(sm.merged, sg_opts);
first_arm_mark.Stop();
// The Lorentz-filtered arm's merge, kept past the point where the all-observation one
// replaces it. The metric re-ask below is a search like any other and answers to the same
// two-arm rule: a promotion is refused by the most damning statistic it can be shown, and
// only one of the two merges has to be starved for that to happen.
std::vector<MergedReflection> merged_filtered;
// ...and that arm's OWN ISa. sg_opts.merge_isa is re-set below for the all-observation
// merge, and the two differ by up to a factor of six, so the re-ask has to keep this one
// or it judges the filtered merge's "genuinely present" cut on the other arm's error model.
double merged_filtered_isa = 0.0;
// Second opinion from a merge that keeps only well-measured observations (see
// RotationScaleMerge::search_min_zeta). Where the two disagree the all-observation merge
// decides. The filter earns its place because a reflection that crosses the Ewald sphere
// near-tangentially is measured worst, and a handful of bad measurements can make a real
// operator look like a twin law; but it discards 40-80% of the observations, which starves
// the operator correlations. Both failure directions are real - the filtered arm has been
// seen to lose an operator the full merge confirms, and, at geometries away from the ones
// this comparison was first checked on, to confirm one the full merge refuses. So the two
// are split rather than ranked: whichever arm finds the LARGER point group decides which
// rotations are real, because a promotion is refused by the most damning statistic it can
// be shown and only one arm has to be starved for that to happen, while the ABSENCES are
// always judged on the all-observation merge, which is the only one holding the weak
// reflections a systematic absence is made of. The vetoes are NOT kept per arm: a promotion
// one arm refused on its twin gates (H ratio, added-operator R) is adopted when the other arm
// confirms it, because the higher order wins. That held on one real twin at three frame
// ranges, where both arms refused, but an H3 partial twin has been seen refused by the
// all-observation arm on both twin gates and promoted to R32 by the filtered one. So the
// losing arm's refusal is logged and carried to the report below; the rule is unchanged.
if (rsm && rsm->GetSearchMinZeta() > 0.0 && !sg_search.point_group_hm.empty()) {
ScaleMergeResult sm_all;
SearchSpaceGroupResult alt;
merged_filtered_isa = sg_opts.merge_isa; // the filtered arm's, before it is replaced
if (all_merge) {
TimingMark all_arm_mark(logger, "SearchSpaceGroup (all-observation arm, wait)");
alt = all_search_ahead.get();
all_arm_mark.Stop();
phase("Scale/combine/merge (P1, all observations)");
all_arm_log.ReplayInto(logger);
rsm_all->WriteBackPerFrameScale();
sm_all = take_merge("P1, all observations", true, std::move(*all_merge));
sg_opts.merge_isa = result.error_model_isa; // this arm's own error model
} else {
const double zeta = rsm->GetSearchMinZeta();
rsm->SetSearchMinZeta(0.0);
sm_all = scale_and_merge("P1, all observations", true);
rsm->SetSearchMinZeta(zeta);
sg_opts.merge_isa = result.error_model_isa; // this arm's own error model
TimingMark all_arm_mark(logger, "SearchSpaceGroup (all-observation arm)");
alt = SearchSpaceGroup(sm_all.merged, sg_opts);
all_arm_mark.Stop();
}
// The order of the point group each arm confirmed. Read from the search, not from the
// space group it went on to pick: Stage B leaves best_space_group unset when no
// candidate has enough absences to be eligible, which is common on the filtered arm
// precisely because the filter removes the weak reflections the absences live in. An
// arm that confirmed 422 would then have scored 0 and lost to a P2 arm - a demotion,
// reported as "taking the higher symmetry".
const auto order_of = [](const SearchSpaceGroupResult &r) { return r.point_group_order; };
logger.Info("Space-group search: all-observation merge -> {} (order {}), "
"Lorentz-filtered -> {} (order {})",
alt.point_group_hm.empty() ? "?" : alt.point_group_hm, order_of(alt),
sg_search.point_group_hm.empty() ? "?" : sg_search.point_group_hm,
order_of(sg_search));
// The screw/centering determination in particular comes from the merge with all the
// observations, because systematic absences are decided by the WEAK reflections and the
// filter throws most of them away. Preferring the filtered arm on a tie cost four
// crystals their screw axes (P2(1) read as P2) for exactly that reason, with the point
// group and every intensity statistic identical.
// Checked while both arms are still intact - below, the all-observation arm is adopted
// by value and the two become the same result.
// Which point group, by its symmorphic representative rather than by name - gemmi calls
// both P321 and P312 "32", so two arms that disagree about WHICH 2-folds are real read
// as agreeing. By the representative's full symbol rather than its NUMBER, for the same
// reason one level down: P 1 2 1 and P 1 1 2 are both number 3 and put their 2-fold on
// different axes. Empty when no point group was chosen.
const auto pg_of = [](const SearchSpaceGroupResult &r) {
return r.point_group_representative.has_value() ? r.point_group_representative->xhm()
: std::string();
};
if (order_of(alt) > 0 && order_of(alt) == order_of(sg_search)
&& pg_of(alt) != pg_of(sg_search)) {
// Same order, different symmetry: the two merges disagree about WHICH operators are
// real, and neither is higher, so there is nothing to prefer. Say so instead of
// picking silently - the two imply different molecular-replacement searches, and
// trying both is cheap next to processing the data again.
const std::string a = alt.best_space_group.has_value()
? alt.best_space_group->short_name() : alt.point_group_hm;
const std::string b = sg_search.best_space_group.has_value()
? sg_search.best_space_group->short_name() : sg_search.point_group_hm;
const std::string msg = fmt::format(
"Space group is AMBIGUOUS between {} and {} (point groups {} and {}, the same "
"order): the merge of all observations and the merge of only the well-measured "
"ones each support one of them, and neither is the higher symmetry, so the data "
"do not decide. Processing continues in {} (the all-observation choice); "
"molecular replacement in both is what settles it, and a re-run with -S {} "
"forces the other one.",
a, b, alt.point_group_hm, sg_search.point_group_hm, a, b);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
result.warnings.push_back({PathologyCode::SYMMETRY_AMBIGUITY, msg});
}
// The arm whose lower answer is about to be overridden, and why it stopped there. A
// refusal by a consistency gate is the case that matters: the higher group is adopted
// anyway, and without this nothing in the output says a gate ever fired.
const auto overridden = [&](const SearchSpaceGroupResult &lower, const char *lower_name,
const char *higher_name) {
std::string msg;
if (!lower.refused_point_group_hm.empty())
msg = fmt::format("the {} merge REFUSED {} ({}); overridden, because the {} merge "
"confirmed the higher point group", lower_name,
lower.refused_point_group_hm, lower.refused_reason, higher_name);
else
msg = fmt::format("the {} merge did not confirm the operators of the higher point "
"group (no gate refused it); the {} merge did", lower_name,
higher_name);
logger.Warning("Space-group search: {}", msg);
result.space_group_arm_override = msg;
};
// A refusal is not simply outvoted by the other arm's confirmation: the gates read
// the operators' own disagreement, which a partial twin at a fraction the other arm's
// coverage happens to hide passes there (an H3 partial twin promoted to R32 by the
// filtered arm; a P31 one to P3121 by the all-observation arm). What decides is the
// twin-immune zone of the added operators, read on the all-observation P1 merge
// (TwinZoneVerdict): acentric with margin and the refusal stands, on all observations;
// centric with margin and the higher group is taken as before; undecided and the rule
// stays as it was.
int zone_verdict = 0;
if (order_of(alt) > order_of(sg_search)) {
logger.Info("Space-group search: all-observation merge supports {} where the "
"Lorentz-filtered one supports {} - taking the higher symmetry",
alt.point_group_hm, sg_search.point_group_hm);
overridden(sg_search, "Lorentz-filtered", "all-observation");
zone_verdict = !sg_search.refused_point_group_hm.empty()
? TwinZoneVerdict(sm_all.merged, gemmi::UnitCell(*result.consensus_cell), alt,
sg_search, result) : 0;
if (zone_verdict < 0) {
logger.Warning("Space-group search: the Lorentz-filtered merge's refusal of {} "
"STANDS - the twin-immune zone of its added operators reads "
"acentric; keeping {}, judged on all observations",
sg_search.refused_point_group_hm, sg_search.point_group_hm);
sg_opts.fixed_point_group = sg_search.point_group_representative;
auto lower = SearchSpaceGroup(sm_all.merged, sg_opts);
sg_opts.fixed_point_group.reset();
lower.refused_point_group_hm = sg_search.refused_point_group_hm;
lower.refused_reason = sg_search.refused_reason + ", and its twin-immune zone reads acentric";
lower.refused_point_group_representative = sg_search.refused_point_group_representative;
lower.refused_by_twin_gate = true;
lower.refused_twin_fraction = sg_search.refused_twin_fraction;
alt = std::move(lower);
}
}
if (order_of(alt) < order_of(sg_search)
&& !alt.refused_point_group_hm.empty()
&& TwinZoneVerdict(sm_all.merged, gemmi::UnitCell(*result.consensus_cell), sg_search, alt,
result) < 0) {
logger.Warning("Space-group search: the all-observation merge's refusal of {} STANDS "
"- the twin-immune zone of its added operators reads acentric; keeping {}",
alt.refused_point_group_hm, alt.point_group_hm);
overridden(alt, "all-observation", "Lorentz-filtered");
result.space_group_arm_override += " - but the refusal stands (see TWIN_ZONE_DECISION)";
alt.refused_reason += ", and its twin-immune zone reads acentric";
sg_search = alt;
} else if (order_of(alt) < order_of(sg_search)) {
// The filtered arm found symmetry the all-observation arm did not. Discarding it
// was costing real point groups: over the corpus the two arms disagree on eight
// crystals and the higher one is right on all eight, because the filter removes
// the near-tangential observations that make a genuine operator look like a twin,
// and a promotion is refused by the most damning statistic it can be shown.
//
// But the two halves of the answer do not come from the same place. The filtered
// merge is deliberately starved, and ABSENCES live in the weak reflections it
// throws away - adopting its result whole recovers the point group and loses the
// screw (a P6_5 crystal comes back P6). So the point group is taken from the arm
// that found it and the absences are re-judged here, on every observation. That
// is what fixed_point_group is for: Stage A still runs and still reports what it
// would have refused, only the choice is overridden.
overridden(alt, "all-observation", "Lorentz-filtered");
sg_opts.fixed_point_group = sg_search.point_group_representative;
auto split = SearchSpaceGroup(sm_all.merged, sg_opts);
sg_opts.fixed_point_group.reset();
logger.Info("Space-group search: Lorentz-filtered merge supports {} where the "
"all-observation one supports {} - taking the higher symmetry, and "
"judging its absences on all observations -> {}",
sg_search.point_group_hm, alt.point_group_hm,
split.best_space_group.has_value() ? split.best_space_group->xhm()
: split.point_group_hm);
sg_search = std::move(split);
} else {
sg_search = alt;
}
// The all-observation merge is the one carried forward: it is what the
// pre-promotion twinning numbers below are measured on, and the filtered merge is the
// second opinion reported above.
merged_filtered = std::move(sm.merged);
sm = std::move(sm_all);
}
// The arm merged beside the search is done with - or was not needed - before anything below
// changes the experiment it reads, and its card memory goes back before the merges below.
if (all_search_ahead.valid())
all_search_ahead.wait();
all_merge.reset();
rsm_all.reset();
// Everything from here to the adopted group's merge: the lattice re-asks and the choices
// between the candidates the arms left.
TimingMark decisions_mark(logger, "space-group decisions");
// Twinning evidence measured BEFORE any promotion, on the P1/subgroup merge the search was
// given. Reported alongside the post-adoption analysis: once a higher Laue class has been
// adopted, its own twinning test can only say "no twin law exists within this class", which
// is circular - the promotion is what a twin would have caused. These numbers are not.
//
// Measured as the post-adoption test is: with the pseudo-translation of this same merge
// declared, so the L-test can be repaired for it, and - the merge being in P1 - without the
// reflections the lattice centring extinguishes, which an R lattice in its hexagonal cell
// still carries (two thirds of them, and every axis step lands on one).
//
// Nothing reads these before the adopted group's merge below, and nothing writes this merge
// until that merge replaces it, so they are measured beside it rather than ahead of it. The
// cell is taken now: a change of basis committed below replaces it.
const char lattice_centring = end_msg.rotation_lattice_type.has_value()
? end_msg.rotation_lattice_type->centering : 'P';
auto pre_promotion = std::async(std::launch::async,
[&merged = sm.merged, cell = result.consensus_cell,
lattice_centring, nthreads = config_.nthreads] {
TranslationalNCSResult tncs;
if (cell)
tncs = AnalyzeTranslationalNCS(merged, gemmi::UnitCell(*cell), nullptr, 30, 0, nthreads);
auto twinning = AnalyzeTwinning(merged, nullptr, 20, TranslationalNCSVectorForLTest(tncs),
lattice_centring);
return std::make_pair(tncs, twinning);
});
TranslationalNCSResult pre_tncs;
const auto join_pre_promotion = [&] {
auto [tncs, twinning] = pre_promotion.get();
pre_tncs = tncs;
result.pre_promotion_tncs_measured = tncs.measurable;
result.pre_promotion_twinning = twinning;
if (twinning.twinning_suspected)
logger.Warning("Twinning indicators BEFORE the space-group decision (subgroup merge): "
"<|L|> = {:.3f}, second moment = {:.3f} (untwinned 0.500 / 2.00) - if the "
"search promoted the point group, treat the promotion with suspicion",
twinning.mean_abs_l, twinning.second_moment);
};
promoted_point_group = !sg_search.point_group_hm.empty() && sg_search.point_group_hm != "1";
if (!sg_search.refused_point_group_hm.empty())
logger.Warning("Higher symmetry {} was confirmed by the operator correlations but refused: "
"{}. Processing in the lower symmetry (the recoverable direction).",
sg_search.refused_point_group_hm, sg_search.refused_reason);
// Miller-index reindex under a change of basis: (h,k,l)_conv = reindex * (h,k,l)_prim.
const auto reindex_hkl = [](auto &r, const gemmi::Mat33 &m) {
const double h = r.h, k = r.k, l = r.l;
r.h = static_cast<int32_t>(std::lround(m.a[0][0] * h + m.a[0][1] * k + m.a[0][2] * l));
r.k = static_cast<int32_t>(std::lround(m.a[1][0] * h + m.a[1][1] * k + m.a[1][2] * l));
r.l = static_cast<int32_t>(std::lround(m.a[2][0] * h + m.a[2][1] * k + m.a[2][2] * l));
};
// Intensity-based centred-lattice test (XDS CORRECT / POINTLESS style). When the indexer
// demoted to a primitive/triclinic cell but the metric is pseudo-symmetric, the point-group
// search above tested no operators (its candidate set is bounded by the indexer's lattice).
// A genuinely centred lattice and a merely pseudo-symmetric triclinic one are indistinguishable
// from spot POSITIONS - the 2-fold is an INTENSITY property - so decide it here, after P1
// integration: reindex the P1-merged data to the metric candidate's conventional setting and
// re-run the search. The I(h) vs I(Rh) correlation confirms a real centred lattice (high CC ->
// commit the centring) and rejects a pseudo-symmetric triclinic (low CC -> stays P1, no reindex).
std::optional<gemmi::Mat33> commit_reindex;
std::optional<UnitCell> commit_cell;
std::optional<CrystalLattice> commit_lattice;
std::optional<LatticeMessage> commit_lattice_type;
// Only a demoted (triclinic) indexing result reaches here with a primitive cell - that is both
// the case the point-group search could not test (a triclinic holohedry offers no operators)
// and the case where LatticeSearch's Niggli reduction is valid (it assumes a primitive input).
const bool searched_trivial = !sg_search.best_space_group || sg_search.best_space_group->number == 1;
const bool demoted_triclinic = end_msg.rotation_lattice_type.has_value()
&& end_msg.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Triclinic;
if (searched_trivial && demoted_triclinic && end_msg.rotation_lattice.has_value()) {
// The metric only generates the candidate; the intensity CC gate below makes the decision
// (a false pseudo-symmetry is rejected by the correlation).
const auto cand = LatticeSearch(*end_msg.rotation_lattice);
// Exact integer reindex from the indexed setting straight to the candidate conventional
// setting: P[i][j] = conv_real[i] . rot_reciprocal[j]. Derived from the two lattices, so it
// holds even if the refined indexed cell is not exactly LatticeSearch's Niggli cell (whose
// primitive_reduced->conventional reindex would otherwise be applied in the wrong setting).
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (cand.system != gemmi::CrystalSystem::Triclinic && reindex_res < 0.1) {
std::vector<MergedReflection> merged_c = sm.merged;
for (auto &m : merged_c)
reindex_hkl(m, reindex);
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand.system;
o2.lattice_centring = cand.centering;
o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell());
const auto s2 = SearchSpaceGroup(merged_c, o2);
if (s2.best_space_group.has_value() && s2.best_space_group->number > 1) {
// The centring cannot be confirmed from absences here (integrated in the primitive
// cell, so the centring-absent reflections do not exist) - it is metric-determined.
// Among the point-group-equivalent candidates the intensities cannot separate
// (e.g. P2 / P21 / C2), pick the one matching the metric candidate's centring.
gemmi::SpaceGroup chosen = *s2.best_space_group;
if (chosen.centring_type() != cand.centering)
for (const auto &alt : s2.alternatives)
if (alt.centring_type() == cand.centering) { chosen = alt; break; }
// Commit only if the adopted centring matches the metric candidate whose conventional
// cell we reindex into - otherwise the space group and the written cell disagree.
if (chosen.centring_type() == cand.centering) {
sg_search = s2;
// Swap the metric-matching candidate into the answer and leave the one it
// displaced among the alternatives, so the reported "or ..." list stays the
// set of groups the data cannot separate, each named once.
if (chosen.number != s2.best_space_group->number) {
sg_search.alternatives.push_back(*s2.best_space_group);
std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) {
return a.number == chosen.number;
});
}
sg_search.best_space_group = chosen;
commit_reindex = reindex;
commit_cell = cand.conventional.GetUnitCell();
commit_lattice = end_msg.rotation_lattice->Multiply(reindex);
commit_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class,
.crystal_system = cand.system };
// Say so at the point of decision. The point group here is confirmed
// from the intensities, but the centering is not and cannot be - the
// reflections it extinguishes were never measured. (The report warns
// about it too, from the candidate's centering_untested flag, which
// survives into the pass that is written.)
// Only for a CENTRED candidate: a primitive one extinguishes nothing, so
// there is no untested absence class to warn about. Before the enumeration
// was widened this branch could only ever adopt a centred group, so the
// condition was implicit.
if (cand.centering != 'P')
logger.Warning("The {} centering of {} comes from the lattice METRIC, "
"not from the intensities: the crystal is indexed on "
"the primitive sub-cell, so the reflections a "
"{}-centred lattice extinguishes were never predicted "
"or integrated. The point group ({}) IS confirmed, by "
"the operator correlations.",
cand.centering, chosen.short_name(), cand.centering,
s2.point_group_hm);
}
}
}
}
// A CLASS DECIDED BY ROUNDING. LatticeSearch names the Bravais class by Niggli-reducing the
// indexed cell and looking the reduced cell up in the 44 lattice characters. For a lattice
// whose Buerger cells straddle the Niggli type-I/type-II boundary that lookup is a coin
// flip - an F-centred cubic lattice has reduced forms on both sides, the two sides carry
// different characters, and which side the reduction lands on is decided by the last digits
// of whatever refinement produced the cell rather than by the data. Measured on such a
// lattice: 43% cubic F, 36% tetragonal I, 21% orthorhombic I, and the split is flat over a
// factor of ten in the noise. The class then caps the point-group search
// (SearchSpaceGroupOptions::lattice_system), so from the I-centred sub-cell the cubic
// three-fold is never enumerated and the run reports that nothing was refused - which is
// accurate, because nothing was asked.
//
// Le Page's two-fold search has no such key: it measures each rotation's obliquity on the
// lattice itself, in a primitive basis, and the conventional cell is then built from the
// axes of the rotations it found. Where it carries more rotations than the adopted class's
// holohedry, offer its conventional cell as a SECOND lattice candidate - reindex the merge
// into it and run the same search there, with every gate live. This only ever ASKS. The
// reindex is committed if and only if the search in the new setting confirms a strictly
// higher point group, and the centring the new cell describes, on these intensities; a
// pseudo-symmetric metric fails that and the answer already in hand stands.
//
// Asked whether or not the centred-lattice test above already committed. What that test
// adopts is a lattice class of its own, and a class it settled from a triclinic indexing is
// exactly the kind that can still be short of the metric's - so the question here is not
// answered by its having run. The two cannot disagree: the ask below only ever replaces the
// answer in hand with a STRICTLY higher point group, and it writes all four commit_ values
// together, so the group, the cell and the lattice type stay one answer.
if (end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
TimingMark metric_mark(logger, "metric lattice re-ask");
// The merge is indexed on the lattice the indexer named, whatever has been committed
// since, so the metric is measured on the primitive form of THAT one.
const CrystalLattice primitive =
end_msg.rotation_lattice->ToPrimitive(end_msg.rotation_lattice_type->centering);
// The lattice class a search has already been given - the indexer's, or the one the
// centred-lattice test above committed. It bounds what any search so far could
// ENUMERATE, which is what says whether the metric's own class is a new question.
const gemmi::CrystalSystem asked_system = commit_lattice_type
? commit_lattice_type->crystal_system
: end_msg.rotation_lattice_type->crystal_system;
const size_t asked_rotations = HolohedryRotations(asked_system);
const auto metric = LePageLattice(primitive);
// Ask the intensities about ONE lattice the metric offers: reindex the merge into its
// conventional setting and run the same search there, with every gate of that search
// live. It commits if and only if the search confirms a strictly higher point group and
// the centring the new cell describes. Called at most twice - the metric's own cell,
// then the sub-lattice below - and only the ask that adopts sets commit_reindex, so the
// second never runs against an answer the first already took.
const auto ask_metric_lattice = [&](const LePageResult &cand, bool sub_lattice) {
// Exact integer reindex from the indexed setting to the candidate's conventional
// one, P[i][j] = conv_real[i] . rot_reciprocal[j], as in the centred-lattice test
// above.
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(),
cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (reindex_res >= 0.1)
return;
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand.system;
o2.lattice_centring = cand.centering;
o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell());
const auto reindexed = [&](const std::vector<MergedReflection> &in) {
std::vector<MergedReflection> out = in;
for (auto &m : out)
reindex_hkl(m, reindex);
return out;
};
const auto merged_all = reindexed(sm.merged);
auto s2 = SearchSpaceGroup(merged_all, o2);
// The same two-arm rule the search above follows: the Lorentz-filtered merge can
// confirm an operator the all-observation one refuses, because the filter removes
// the near-tangential measurements that make a genuine operator look like a twin.
// Where it finds the larger point group, the absences are still judged on all
// observations, with the point group pinned.
// And the same twin-immune zone verdict on a refusal the other arm would outvote:
// a P31 partial twin whose 32 the rule above had refused on the zone was promoted
// by this ask's filtered arm - the same override, one copy later.
if (!merged_filtered.empty()) {
SearchSpaceGroupOptions of = o2;
of.merge_isa = merged_filtered_isa; // this arm's own error model
auto filtered = SearchSpaceGroup(reindexed(merged_filtered), of);
const auto refusal_stands = [&](const SearchSpaceGroupResult &higher,
SearchSpaceGroupResult &lower) {
if (higher.point_group_order <= lower.point_group_order
|| lower.refused_point_group_hm.empty()
|| TwinZoneVerdict(merged_all, *o2.cell, higher, lower, result) >= 0)
return false;
logger.Warning("Space-group search (metric lattice): the refusal of {} STANDS - the "
"twin-immune zone of its added operators reads acentric; keeping {}",
lower.refused_point_group_hm, lower.point_group_hm);
lower.refused_reason += ", and its twin-immune zone reads acentric";
return true;
};
if (filtered.point_group_order > s2.point_group_order && !refusal_stands(filtered, s2)) {
o2.fixed_point_group = filtered.point_group_representative;
s2 = SearchSpaceGroup(merged_all, o2);
o2.fixed_point_group.reset();
} else if (refusal_stands(s2, filtered)) {
o2.fixed_point_group = filtered.point_group_representative;
auto lower = SearchSpaceGroup(merged_all, o2);
o2.fixed_point_group.reset();
lower.refused_point_group_hm = filtered.refused_point_group_hm;
lower.refused_reason = filtered.refused_reason;
lower.refused_point_group_representative = filtered.refused_point_group_representative;
lower.refused_by_twin_gate = true;
lower.refused_twin_fraction = filtered.refused_twin_fraction;
s2 = std::move(lower);
}
}
// The centring cannot be read from the absences in this setting: reindexing
// into the metric candidate's conventional cell leaves its centring-absent class
// holding no data at all, so among the point-group-equivalent candidates the
// intensities cannot separate, the tie-break takes the lowest number, and that is
// the primitive one. Pick the candidate matching the metric's own centring out of
// the alternatives first, exactly as the centred-lattice test above does -
// otherwise a promotion the intensities DID confirm is refused for naming the
// same lattice in its equivalent primitive setting.
std::optional<gemmi::SpaceGroup> chosen = s2.best_space_group;
if (chosen.has_value() && chosen->centring_type() != cand.centering)
for (const auto &alt : s2.alternatives)
if (alt.centring_type() == cand.centering) { chosen = alt; break; }
const bool point_group_higher = s2.best_space_group.has_value()
&& s2.point_group_order > sg_search.point_group_order;
bool higher = point_group_higher
&& chosen->centring_type() == cand.centering;
// A promotion over a refusal is arbitrated by the twin-immune zone of the added
// operators (TwinZoneVerdict), here as where the two arms disagree: where a twin
// gate of the search in hand refused a higher group, this ask reopens the question
// that gate answered, for whichever of the refused group's operators the new
// lattice holds, and its own search cannot answer it - the gates there are relative
// too, and a twin law or a pseudo-symmetry the refused group's other operators
// diluted passes them alone. Measured on a monoclinic crystal on a hexagonal metric,
// a pseudo-merohedral twin: 622 refused on the added-operator R, the C-centred
// orthorhombic sub-lattice then asked, its two in-plane two-folds at R 0.061 against
// 0.045 for the genuine screw, and C 2 2 21 adopted with the zone of those two-folds
// reading acentric. The group in hand is carried into the candidate's setting for
// it: h -> Q h with Q the reindex from the basis it was found in (read where GEMMI's
// table names the result, as every setting the search enumerates is).
std::string zone_refusal;
if (higher && sg_search.refused_by_twin_gate && sg_search.point_group_representative) {
gemmi::Mat33 q = reindex;
if (commit_reindex)
q = reindex.multiply(commit_reindex->inverse());
const gemmi::Mat33 q_inv = q.inverse();
gemmi::GroupOps lower_ops;
lower_ops.cen_ops = gemmi::centring_vectors(cand.centering);
for (const auto &op : sg_search.point_group_representative->operations().sym_ops) {
// apply_to_hkl uses rot^T, so the Miller-index matrix of op is rot^T / DEN.
gemmi::Mat33 m;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
m.a[i][j] = static_cast<double>(op.rot[j][i]) / gemmi::Op::DEN;
const gemmi::Mat33 m_new = q.multiply(m).multiply(q_inv);
gemmi::Op op_new = gemmi::Op::identity();
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
op_new.rot[i][j] = static_cast<int>(std::lround(m_new[j][i])) * gemmi::Op::DEN;
lower_ops.sym_ops.push_back(op_new);
}
if (const gemmi::SpaceGroup *lower_sg = gemmi::find_spacegroup_by_ops(lower_ops)) {
SearchSpaceGroupResult lower = sg_search;
lower.point_group_representative = *lower_sg;
if (TwinZoneVerdict(merged_all, *o2.cell, s2, lower, result) < 0) {
zone_refusal = sg_search.refused_point_group_hm + " was refused (" +
sg_search.refused_reason + "), and the twin-immune zone of the "
"operators this group adds reads acentric";
higher = false;
}
}
}
const auto &uc = cand.conventional.GetUnitCell();
// The two asks are told apart by what they were asked against: the class a search
// has already been given, or the rotations the intensities have actually confirmed.
const std::string preamble =
"The cell metric carries " + std::to_string(metric->n_operators) +
(sub_lattice
? " rotations where the intensities have confirmed " +
std::to_string(sg_search.point_group_order) +
", so the search was asked again on a SUB-lattice of the metric - the "
"largest one whose two-folds these intensities confirm one by one, carrying " +
std::to_string(cand.n_operators) + " of those rotations"
: " rotations where the " + std::string(gemmi::crystal_system_str(asked_system)) +
" lattice in hand has " + std::to_string(asked_rotations) +
", so the search was asked again on the metric's own cell");
logger.Info("{} (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}, centring {}, worst obliquity {:.3f} deg): {} (order "
"{} against {}){}",
preamble,
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, cand.centering,
cand.max_obliquity_deg,
s2.best_space_group.has_value() ? s2.best_space_group->xhm() : "nothing",
s2.point_group_order, sg_search.point_group_order,
higher ? " - adopted"
: !zone_refusal.empty() ? " - keeping the group already found; " + zone_refusal
: (point_group_higher
? " - keeping the group already found; the point group IS "
"higher, but no candidate of it carries the centring the "
"metric cell names, which this setting cannot test"
: (s2.refused_reason.empty()
? " - the intensities do not support it, keeping the "
"group already found"
: " - keeping the group already found; " + s2.refused_reason)));
if (higher) {
// The group of the lattice in hand, carried into the new setting: the subgroup of
// the twin-law merge, should this promotion stand on twin-like data or over a
// refusal by a twin gate (see twin_choice).
if (sg_search.best_space_group) {
gemmi::Mat33 q = reindex;
if (commit_reindex)
q = reindex.multiply(commit_reindex->inverse());
twin_metric_subgroup = GroupInReindexedSetting(*sg_search.best_space_group, q,
cand.centering);
if (sg_search.refused_by_twin_gate) {
twin_overruled_subgroup = twin_metric_subgroup;
twin_overruled_reason = sg_search.refused_reason;
}
}
sg_search = s2;
// Leave the candidate it displaced among the alternatives, so the reported
// "or ..." list stays the set of groups the data cannot separate.
if (chosen->number != s2.best_space_group->number) {
sg_search.alternatives.push_back(*s2.best_space_group);
std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) {
return a.number == chosen->number;
});
}
sg_search.best_space_group = chosen;
commit_reindex = reindex;
commit_cell = cand.conventional.GetUnitCell();
commit_lattice = end_msg.rotation_lattice->Multiply(reindex);
commit_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class,
.crystal_system = cand.system };
}
};
if (metric) {
// The metric's own class, where no search has been given it yet. Where one has -
// the centred-lattice test above reindexes into a metric candidate's cell and runs
// the full search there - asking it again would only re-run the same question.
//
// DIFFERENT, not merely greater. asked_rotations is what the class in hand CLAIMS,
// and a class can claim more rotations than the lattice carries: the character
// table names a class from a Niggli-reduced cell on relative tolerances, so a cell
// whose gamma is a degree and a half off 120 is named hexagonal while Le Page, which
// measures each two-fold's obliquity, finds four rotations and a C-centred
// orthorhombic cell. Asked only on ">" the comparison was then 4 against 12 and the
// ask stayed silent - so the crystal was searched under a holohedry its own lattice
// does not have, every operator of it was refused, and it fell to a subgroup, with
// the one mechanism built to rescue that disarmed by the over-call it exists to
// correct. Where the metric's group is not the group the class in hand names, the
// metric's own cell has not been asked about, whichever of the two is larger.
// Asking cannot demote: ask_metric_lattice adopts only on a STRICTLY higher point
// group, so an ask against a lower metric class costs one search and nothing else.
if (static_cast<size_t>(metric->n_operators) != asked_rotations)
ask_metric_lattice(*metric, false);
// That question is all-or-nothing, and it is asked at the TOP of the metric
// hierarchy: where the truth is an INTERMEDIATE class the answer is correctly "no",
// and the class in between is never asked about, because it is not expressible in
// the metric cell's basis. Measured on a pseudo-hexagonal monoclinic indexing whose
// metric group is hexagonal: three of its twelve operators close into 222 and
// correlate at 0.60 against 0.02 for the rest, and the conventional cell of that
// orthorhombic sub-lattice is the C-centred one the crystal is actually described on.
//
// So ask once more, on the sub-lattice the intensities themselves pick out: score
// every two-fold the metric offers against the P1 merge and keep only those that
// correlate, then let Le Page close what survives into a lattice. Nothing here
// decides a space group - it decides which lattice gets ASKED about, and the ask is
// the same call with the same gates as above. The number to beat here is what the
// intensities have CONFIRMED (the adopted point group's order), not what some
// search was allowed to enumerate: an enumeration that reached the metric's class
// and refused it says nothing about the classes below it.
if (static_cast<size_t>(metric->n_operators) > sg_search.point_group_order) {
// The two-folds are offered in the basis of the lattice handed to LePageLattice,
// which is the PRIMITIVE one; sm.merged is indexed on the lattice the run
// adopted. With a_prim[i] = sum_j P[i][j] a_idx[j] (CrystalLattice::Multiply
// combines basis rows) Miller indices map h_prim = P h_idx, so an operator's
// Miller-index matrix carries back as M_idx = P^-1 M_prim P. P is the identity
// for a primitive adopted lattice.
gemmi::Mat33 to_primitive;
if (end_msg.rotation_lattice_type->centering != 'P')
to_primitive = gemmi::rot_as_mat33(
gemmi::centred_to_primitive(end_msg.rotation_lattice_type->centering)).transpose();
const gemmi::Mat33 from_primitive = to_primitive.inverse();
// An operator counts as real on the same bound Stage A calls one present at -
// SearchSpaceGroupOptions::min_operator_cc, applied by OperatorCorrelation over
// the same reflection population - AND on the R contrast every promotion's added
// operators are held to (SearchSpaceGroupOptions::min_operator_r_contrast,
// against the same two ends the search above measured), so nothing new is
// calibrated here. The correlation alone cannot tell a pseudo-symmetric
// structure's near-operators from real ones: measured on a pseudo-cubic
// tetragonal crystal, two of the false cubic two-folds correlate at 0.31-0.33 and
// pass it, LePage closes them with the genuine ones into the whole cubic metric,
// and the tetragonal class between the two is never asked about. Their R reads
// 0.39 against 0.013 for the genuine operators and 0.49 for unrelated
// reflections - a contrast of 0.2 against the genuine operators' 1.0.
const double r_random = sg_search.random_pairing_r;
const double r_best = sg_search.global_best_operator_r;
// Read only where the search itself reads it: a best operator close to the
// random end leaves no scale to measure on (max_reference_over_random).
const bool contrast_known = std::isfinite(r_random) && std::isfinite(r_best)
&& r_best <= sg_opts.max_reference_over_random * r_random;
const OperatorScorer operator_correlation(sm.merged, sg_opts);
const auto keep_operator = [&](const gemmi::Mat33 &m_primitive) {
const gemmi::Mat33 m_indexed =
from_primitive.multiply(m_primitive).multiply(to_primitive);
const auto score = operator_correlation(m_indexed);
if (!score.has_value() || !score->present)
return false;
return !contrast_known
|| (r_random - score->r_stat) / (r_random - r_best) >= sg_opts.min_operator_r_contrast;
};
const auto sub = LePageLattice(primitive, LATTICE_MAX_OBLIQUITY_DEG, keep_operator);
if (sub && static_cast<size_t>(sub->n_operators) > sg_search.point_group_order
&& sub->n_operators < metric->n_operators)
ask_metric_lattice(*sub, true);
}
}
}
// A DOUBLED DESCRIPTION of a primitive lattice. gemmi states every setting as a change of
// basis from its reference one, and where that change has determinant < 1 the setting names
// a cell of 1/det times the reference volume. In the monoclinic family exactly two settings
// have it - C 1 1 2 and C 1 1 2_1 - and the reason is physical: with unique axis c the
// C-centring vector (1/2,1/2,0) has no component along c, lies wholly in the oblique a-b
// plane and reduces away, which is why the genuine c-unique centred settings are A/B/I 1 1 2.
// (The centred settings of P1, and the C/F settings of the tetragonal groups, are the same
// thing.) Such a group is never a distinct Bravais centring; it is a primitive lattice
// written on twice the cell.
//
// It wins because the candidate table is scored ON the doubled cell, where half the
// reflection positions are not reciprocal-lattice points at all and are absent BY
// CONSTRUCTION, so the centred candidate collects a centring evidence that is tautological.
// Refusing to enumerate those settings was measured and is worse, not better: the
// volume-preserving alternative in the same table keeps the by-construction absences as real
// observations and merges no better than P1. What the centred candidate gets RIGHT is
// discarding them.
//
// So take the win as a SIGNAL rather than as the answer. Halve the cell, put the lattice back
// in the primitive setting of the group's own crystal system, carry the reflections across -
// the by-construction absences map to non-integral indices and are dropped, which is exactly
// what the centred candidate was doing, so nothing that was merging is lost - and re-run the
// search there. The enumeration is untouched, so no candidate that JUDGES another moves and
// no chi2 reference changes; the doubled setting still wins its own table, and only the cell
// its answer is expressed on is corrected.
if (!commit_reindex && sg_search.best_space_group.has_value()
&& sg_search.best_space_group->number > 1
&& end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
const gemmi::SpaceGroup won = *sg_search.best_space_group;
constexpr double den3 = static_cast<double>(gemmi::Op::DEN) * gemmi::Op::DEN * gemmi::Op::DEN;
const double det = won.basisop().det_rot() / den3;
const int multiple = static_cast<int>(std::lround(1.0 / det)); // 2 for C 1 1 2_1, 4 for F 1
if (det < 0.99 && multiple >= 2 && multiple <= 4) {
// Reduce, then classify. The character table is reached through a Niggli reduction
// that takes whatever it is handed to be primitive, so the centred conventional cell
// has to be brought to its primitive one FIRST - asked on the doubled cell the same
// call answers with the same volume, merely permuted. Ask for the group's own system
// in 'P': the metric of a reducible lattice supports the doubled centred description
// too (that is how it got here), and an unrestricted search would hand it straight
// back.
const auto cand = LatticeSearchForClass(
end_msg.rotation_lattice->ToPrimitive(won.centring_type()),
won.crystal_system(), 'P');
// Change of basis from the doubled conventional setting straight to the reduced one,
// P[i][j] = conv_real[i] . doubled_reciprocal[j], as everywhere else here - except
// that its entries are multiples of 1/multiple rather than integers, which is the
// whole point of it.
gemmi::Mat33 reduce;
double reduce_res = 1.0;
if (cand) {
const Coord cv[3] = {cand->conventional.Vec0(), cand->conventional.Vec1(),
cand->conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
reduce_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reduce.a[i][j] = std::round(v * multiple) / multiple;
reduce_res = std::max(reduce_res, std::fabs(v - reduce.a[i][j]));
}
}
// hkl across the same change of basis. A reflection whose image is not integral is
// not a reciprocal-lattice point of the reduced cell - it is one of the positions the
// doubled description invented - and it is dropped rather than reindexed.
const auto reduce_hkl = [&reduce](auto &r) {
const double h = r.h, k = r.k, l = r.l;
const double n[3] = {reduce.a[0][0]*h + reduce.a[0][1]*k + reduce.a[0][2]*l,
reduce.a[1][0]*h + reduce.a[1][1]*k + reduce.a[1][2]*l,
reduce.a[2][0]*h + reduce.a[2][1]*k + reduce.a[2][2]*l};
for (const double x : n)
if (std::fabs(x - std::round(x)) > 1e-3)
return false;
r.h = static_cast<int32_t>(std::lround(n[0]));
r.k = static_cast<int32_t>(std::lround(n[1]));
r.l = static_cast<int32_t>(std::lround(n[2]));
return true;
};
if (!cand || reduce_res >= 0.05) {
logger.Warning("{} describes a cell of {}x the volume its reference setting does - "
"the lattice is a doubled description of a primitive one - but it "
"could not be reduced ({}). Keeping the doubled cell.",
won.short_name(), multiple,
cand ? "the change of basis is not exact" : "no primitive character fits");
} else {
std::vector<MergedReflection> merged_p;
merged_p.reserve(sm.merged.size());
for (auto m : sm.merged)
if (reduce_hkl(m))
merged_p.push_back(m);
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand->system;
o2.lattice_centring = cand->centering;
o2.cell = gemmi::UnitCell(cand->conventional.GetUnitCell());
const auto s2 = SearchSpaceGroup(merged_p, o2);
// Only ever a re-expression: the reduced cell has to support the same symmetry
// the doubled one found, in the primitive setting the reduction produced.
// Anything less and the doubled answer stands - it is the one that merges.
const bool holds = s2.best_space_group.has_value()
&& s2.point_group_order >= sg_search.point_group_order
&& s2.best_space_group->centring_type() == cand->centering;
if (!holds) {
logger.Warning("{} describes a cell of {}x the volume its reference setting "
"does, but the reduced primitive cell does not carry the same "
"symmetry ({}, order {} against {}). Keeping the doubled cell.",
won.short_name(), multiple,
s2.best_space_group ? s2.best_space_group->short_name() : "nothing",
s2.point_group_order, sg_search.point_group_order);
} else {
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(reduce);
ReflectionVector kept;
kept.reserve(io.reflections.size());
for (auto r : io.reflections)
if (reduce_hkl(r))
kept.push_back(r);
io.reflections = std::move(kept);
}
const size_t dropped = sm.merged.size() - merged_p.size();
result.consensus_cell = cand->conventional.GetUnitCell();
result.consensus_centering = cand->centering;
end_msg.unit_cell = result.consensus_cell;
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reduce);
end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand->centering,
.niggli_class = cand->niggli_class, .crystal_system = cand->system };
sg_search = s2;
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
const auto &uc = *result.consensus_cell;
logger.Info("{} names a cell of {}x the volume of its reference setting: the "
"lattice is a doubled description of a primitive one, and {} of "
"{} merged reflections were positions the doubled cell invented. "
"Reduced to a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f} and searched again there: {}",
won.short_name(), multiple, dropped, sm.merged.size(),
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
s2.best_space_group->short_name());
// The reduction halves the cell, so its change of basis is not integral and
// end_msg.reindex_matrix (int32) cannot carry it. The per-image reflections
// already written to the _process.h5 therefore stay in the doubled setting.
logger.Warning("The per-image reflections in the process file are in the "
"doubled setting the run integrated in; the cell, the merged "
"reflections and the space group are in the reduced one. The "
"change of basis between them is not integral, so it is not "
"recorded in the file.");
}
}
}
}
// The centre of symmetry of a glide group. Where the absences settle it - no group of the
// other kind predicts them (P2_1/c, Pbca; I-42d) - it is settled. Where a group with the same
// absences and the other answer exists (C2/c and Cc), the centrosymmetric one is written by
// default - a missed centre is the common error in small-molecule space-group assignment, not
// a spurious one - and gives way after the merge below only when the intensity distribution
// reads acentric on all three statistics AND the lattice rules twinning out, because a twinned
// centrosymmetric crystal reads exactly like an untwinned acentric one.
std::optional<gemmi::SpaceGroup> centre_partner;
if (sg_search.best_space_group.has_value() && !sg_search.best_space_group->is_sohncke()) {
const gemmi::SpaceGroup written = *sg_search.best_space_group;
for (const auto &c : sg_search.candidates)
if (c.selected && c.space_group.xhm() == written.xhm())
for (const auto &t : c.same_absences)
if (!centre_partner && t.is_centrosymmetric() != written.is_centrosymmetric())
centre_partner = t;
if (census_future.valid()) {
TimingMark census_mark(logger, "leftover-lattice census wait (centre partner)");
result.leftover_lattices = census_future.get();
}
if (result.consensus_cell.has_value() && result.leftover_lattices.has_value()) {
const auto &uc = *result.consensus_cell;
const gemmi::UnitCell gc(uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma);
size_t laue_rotations = 0;
for (const auto &op : written.operations().sym_ops)
if (op.det_rot() > 0)
++laue_rotations;
if (!written.is_centrosymmetric())
laue_rotations *= 2; // a group without a centre holds half its Laue class's rotations
result.centre_twinning_excluded =
gemmi::find_lattice_symmetry(gc, written.centring_type(), LATTICE_MAX_OBLIQUITY_DEG)
.sym_ops.size() == laue_rotations
&& std::none_of(result.leftover_lattices->lattices.begin(),
result.leftover_lattices->lattices.end(), [](const LeftoverLattice &l) {
return l.kind == LeftoverLatticeKind::TwinDomain;
});
}
if (!centre_partner)
result.space_group_centre = written.is_centrosymmetric() ? "IMPLIED_BY_ABSENCES"
: "ABSENT_BY_ABSENCES";
else
result.space_group_centre = "NOT_DETERMINED";
}
// Adopt the determined space group and re-scale/merge in it, so scaling uses symmetry
// equivalents and the statistics come out in the right symmetry. P1 stands when nothing
// is determined - and it has to stand as a MERGE of its own, not as whatever `sm` happens
// to hold: everything merged up to here was a search merge, which drops the ice rings and
// (on the Lorentz-filtered arm) most of the observations with them, is never resolution-cut
// and fits no correction surfaces, because its intensities exist only to be searched.
//
// The P1 fallback is a belt, not a brace. SearchSpaceGroup cannot hand back an unset group
// for a merge that has any reflections in it: finding no symmetry at all leaves it on the
// identity point group, whose representative IS P1, and it returns that; finding a point
// group but no screw or centering leaves the primitive symmorphic candidate, which has no
// predicted absences to be inconsistent with and is therefore always eligible; and an empty
// merge never reaches the search, because both merge engines throw on the resolution range
// before returning one. So the fallback buys nothing today - what it buys is that the
// promise above is kept HERE, rather than resting on eligibility gates in another file that
// a later change could tighten without anyone noticing what else was leaning on them.
{
gemmi::SpaceGroup sg = sg_search.best_space_group.value_or(gemmi::get_spacegroup_p1());
// xhm(), not short_name(): the search can now return a non-reference setting, and
// short_name() cannot tell P 1 2 1 from P 1 1 2.
logger.Info("Adopting space group {} (number {})", sg.xhm(), sg.number);
Note(fmt::format("space group {} (No. {})", sg.xhm(), sg.number));
// A GLIDE plane was confirmed, so the adopted group is not a Sohncke one. Say both:
// a crystal built from L-amino acids is chiral and cannot have a glide plane, so this
// is a statement that the sample is not a protein - and a reader who knows it IS one
// has to be able to see the Sohncke group the same run would otherwise have reported,
// without processing the images again. Both, always, is what the two fields are for.
if (sg_search.glide_space_group.has_value()) {
std::string zones;
for (const auto &z : sg_search.glide_zones)
zones += fmt::format("{}{} ({} absences at {:.2g} of the rest of their plane)",
zones.empty() ? "" : ", ", z.label, z.n_absent, z.mean_u);
const std::string msg = fmt::format(
"Glide plane(s) confirmed from zonal absences - {}. The group is {}, which is "
"NOT a Sohncke group: a crystal of chiral molecules (any protein) cannot have "
"a glide plane, so this says the sample is not one. The best Sohncke group for "
"the same data is {} - re-run with -S {} to process in it.",
zones, sg.xhm(),
sg_search.sohncke_space_group.has_value()
? sg_search.sohncke_space_group->xhm() : std::string("none"),
sg_search.sohncke_space_group.has_value()
? sg_search.sohncke_space_group->xhm() : std::string("P 1"));
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
}
// An axis whose screw these data never tested. The group just adopted is then the
// member of a set that claims no screw, picked by convention because a reflection
// file must name one group - and a line saying only "Adopting space group P 6" reads
// as a determination. Say which axis is open and why, here, where the adoption is.
if (!sg_search.undetermined_screws.empty()) {
std::string axes, why;
for (const auto &u : sg_search.undetermined_screws) {
axes += (axes.empty() ? "" : ", ") + std::string(1, u.axis);
why += fmt::format(
"{}the {} row {}", why.empty() ? "" : "; ", u.row_label,
u.n_observed == 0
? std::string("holds no reflections in this merge at all - it was "
"never recorded")
: u.n_control == 0
? fmt::format("holds {} reflection(s) but no control class to "
"judge them against", u.n_observed)
: fmt::format("does not separate the candidates ({} absent, {} "
"control)", u.n_absent, u.n_control));
}
std::string alts;
for (const auto &alt : sg_search.alternatives)
alts += " or " + alt.xhm();
const std::string msg = fmt::format(
"The screw along {} is UNDETERMINED: {}. These data say {}{} and nothing "
"here chooses between them; {} is written because a reflection file must "
"carry one group. To settle it, record the missing row.",
axes, why, sg.xhm(), alts, sg.xhm());
logger.Warning("{}", msg);
stats_text << " " << msg << "\n\n";
}
// The group has to be able to DESCRIBE the cell it is adopted on, and where the
// promotion was made HERE it cannot. A two-fold the intensities confirm is one the spot
// positions never offered - the Bravais class is decided on the unrefined indexing
// candidate, and the free refinement can walk into a class afterwards - so no fit under
// this group's constraint has ever been run, and what goes to the report, the master
// file and the MTZ is the free fit's metric re-expressed in the group's setting: a
// C 1 2 1 whose alpha is 88.5, which no monoclinic refinement could have produced. Ask
// the spots what cell the constraint supports; where they refuse it, report the metric
// the group fixes rather than one it cannot describe. A lattice already refined under
// this class leaves only float noise behind and is left alone, which is what the bound
// tests - so a de-novo run whose indexing found the class behaves exactly as it did.
{
std::optional<CrystalLattice> &adopted =
commit_reindex ? commit_lattice : end_msg.rotation_lattice;
// The band between the two bounds is where the cell is merely UNCONSTRAINED. Above
// the upper one it is the wrong cell for this group - the axes permuted, the fold
// running along an axis the group does not put it on - and nothing here can repair
// that: a constrained refit has no cell to find and the metric projection would
// manufacture one, turning a visible mismatch into a plausible-looking lie. That
// case is refused where the group comes in (MAX_METRIC_VIOLATION), and is left
// exactly as it is here.
const double violation = adopted.has_value()
? MetricViolation(adopted->GetUnitCell(), sg) : 0.0;
if (adopted.has_value() && violation > MAX_CONSTRAINED_METRIC_NOISE
&& violation <= MAX_METRIC_VIOLATION) {
const UnitCell before = adopted->GetUnitCell();
const auto refit = indexer->RefineRotationLatticeConstrained(*adopted,
sg.crystal_system());
// A refit is usable only when the constraint it was written in is the one this
// group needs: XtalResidual parameterises monoclinic with the unique axis along
// b, and the search can adopt a group that puts it elsewhere (C 1 1 2), whose
// metric such a fit does not satisfy. Ask the group, rather than enumerate the
// settings - a fit made in the right constraint lands at float noise.
const bool fits_group = refit.has_value()
&& MetricViolation(refit->lattice.GetUnitCell(), sg)
<= MAX_CONSTRAINED_METRIC_NOISE;
// ...and the spots have to keep it. Half is the bar the indexer's own
// pseudo-symmetry guard uses for this very comparison: a constrained cell that
// indexes half of what the free one did is not describing these images, and the
// metric and the intensities are then saying different things - which is a
// geometry error to go and find, not a cell to report.
const bool spots_keep_it = refit.has_value()
&& refit->indexed_fraction > 0.5f * refit->indexed_fraction_before;
if (fits_group && spots_keep_it) {
*adopted = refit->lattice;
const UnitCell uc = adopted->GetUnitCell();
logger.Info("{} was adopted on a cell no fit under it had produced (the "
"promotion was made after integration, on the freely refined "
"metric). Re-refined under the constraint: a={:.3f} b={:.3f} "
"c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, from a={:.3f} "
"b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f} - it "
"indexes {:.1f} % of the accumulated spots against {:.1f} % for "
"the free cell.",
sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
before.a, before.b, before.c, before.alpha, before.beta,
before.gamma, 100.0 * refit->indexed_fraction,
100.0 * refit->indexed_fraction_before);
} else {
*adopted = SymmetrizeMetric(*adopted, sg);
const UnitCell uc = adopted->GetUnitCell();
const std::string msg = fmt::format(
"The cell {} was adopted on is one that group cannot describe - "
"a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, "
"refined FREE, because the symmetry was confirmed from the intensities "
"after integration and no constrained fit had produced this lattice. {} "
"The reported cell is the nearest metric the group fixes (alpha={:.2f} "
"beta={:.2f} gamma={:.2f}); the reflections are unchanged. A deviation "
"this size is not refinement noise - the geometry the images were "
"integrated at is the place to look.",
sg.xhm(), before.a, before.b, before.c, before.alpha, before.beta,
before.gamma,
refit.has_value()
? (fits_group
? fmt::format("Re-refining under the constraint indexes {:.1f} % "
"of the accumulated spots against {:.1f} % free, so "
"it was refused.",
100.0 * refit->indexed_fraction,
100.0 * refit->indexed_fraction_before)
: "The constrained refinement is not written in this group's "
"setting, so it could not be asked.")
: "There was no rotation spot cloud to re-refine against.",
uc.alpha, uc.beta, uc.gamma);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
}
// Carry the corrected lattice into the cell that is reported and merged on.
// Where a reindex is pending the block below does that and re-ingests; where
// there is none, this is the only place it happens.
if (commit_reindex) {
commit_cell = adopted->GetUnitCell();
} else {
result.consensus_cell = adopted->GetUnitCell();
end_msg.unit_cell = result.consensus_cell;
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(),
result.consensus_cell, static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
}
}
}
// A reindex is committed only when the intensity re-test confirmed a higher symmetry in a
// centred setting: bring the integrated reflections + cell into that setting and re-ingest
// the rotation merge, so the final scaling/merging folds the equivalents correctly.
if (commit_reindex) {
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(*commit_reindex);
for (auto &r : io.reflections)
reindex_hkl(r, *commit_reindex);
}
result.consensus_cell = commit_cell;
result.consensus_centering = commit_lattice_type->centering;
end_msg.unit_cell = commit_cell;
// Keep the lattice metadata (hence the master-file UB matrix and indexed-lattice
// vectors) in the same conventional setting as the cell, reflections and space group.
end_msg.rotation_lattice = commit_lattice;
end_msg.rotation_lattice_type = commit_lattice_type;
record_reindex(*commit_reindex);
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
}
experiment_.SetSpaceGroup(sg);
end_msg.space_group_name = sg.xhm();
end_msg.space_group_number = static_cast<uint64_t>(sg.number);
result.space_group = sg;
decisions_mark.Stop();
phase("Re-scaling in space group " + sg.short_name());
auto adopted = scale_and_merge(sg.short_name(), false);
join_pre_promotion();
sm = std::move(adopted);
// The intensity statistics for the centre, on this merge - the full one, resolution-cut and
// corrected, in the adopted group. Read on every run and reported, Sohncke groups included,
// so a chiral small molecule's reader sees them too; they decide only between a glide
// group and its partner with the same absences (see above).
result.centre_statistics = AnalyzeCentreOfSymmetry(sm.merged, sg);
if (centre_partner && sg.is_centrosymmetric() && result.centre_statistics->reads_acentric
&& result.centre_twinning_excluded) {
logger.Info("Centre of symmetry: {} and {} predict the same absences; the intensity "
"distribution reads acentric (<|L|> f={:.2f}, <|E^2-1|> f={:.2f}, N(0.1) "
"f={:.2f}) and the lattice rules twinning out, so {} is written",
sg.xhm(), centre_partner->xhm(), result.centre_statistics->mean_abs_l.f,
result.centre_statistics->mean_abs_e2_minus_1.f,
result.centre_statistics->n_z_01.f, centre_partner->xhm());
result.space_group_centre = "ABSENT_BY_STATISTICS";
std::replace_if(sg_search.alternatives.begin(), sg_search.alternatives.end(),
[&](const gemmi::SpaceGroup &a) { return a.number == centre_partner->number; },
sg);
sg = *centre_partner;
sg_search.best_space_group = sg;
sg_search.glide_space_group = sg;
experiment_.SetSpaceGroup(sg);
end_msg.space_group_name = sg.xhm();
end_msg.space_group_number = static_cast<uint64_t>(sg.number);
result.space_group = sg;
sm = scale_and_merge(sg.short_name(), false);
}
// A promotion the operator correlations CONFIRMED and Stage A then refused, put to
// the merge. Every gate that can refuse one is a ratio against another operator or
// against the noise floor, and both references are properties of how the crystal was
// mounted rather than of its symmetry: a 2-fold within a degree of the spindle
// records its mates on the same detector pixel half a turn later, carries no
// geometry-dependent systematic at all, and by being that clean makes every other
// operator look bad; and the best-agreeing operator anywhere can be one of the
// operators under test, which collapses the ratio on worse data and raises it on
// better. So ask the data the question those ratios stand in for - merge under the
// refused group and see whether the reflections it folds together agree.
//
// R_meas is multiplicity-corrected, so a merge that averaged non-equivalent
// reflections has to inflate it, and the error model is refitted per merge, so ISa
// says whether the extra multiplicity was bought with systematic disagreement. Over
// six refused promotions the two signs separate completely: the genuine ones leave
// both flat or better, the false ones inflate R_meas by 21-99% and drop ISa. CC1/2
// and <I/sigma> do NOT separate them - both RISE on a false promotion too, which
// gains multiplicity either way - so the rule is read on R_meas and ISa and nothing
// else.
//
// Both arms are merged over the SAME resolution range, the one the adopted group's
// own merge chose: left to pick their own cutoffs the two are not the same
// measurement and the separation muddies to a couple of percent with the sign lost.
// Rotation only - it is the rotation merge that can be re-cut between merges.
if (rsm && sg_search.refused_point_group_representative.has_value()) {
// Which group the comparison arm merges in. The point group is the refused one -
// passed as its representative, because gemmi calls both P321 and P312 "32" - and
// the absence classes are the adopted group's, because Stage B is asked for the
// group on the adopted merge, where those classes are already empty. That is what
// makes the two arms differ by the added ROTATIONS alone.
SearchSpaceGroupOptions hi_opts = sg_opts;
hi_opts.fixed_point_group = sg_search.refused_point_group_representative;
hi_opts.merge_isa = result.error_model_isa;
if (result.consensus_cell.has_value())
hi_opts.cell = gemmi::UnitCell(*result.consensus_cell);
if (end_msg.rotation_lattice_type.has_value())
hi_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
auto hi_search = SearchSpaceGroup(sm.merged, hi_opts);
// ...and its CENTRING is taken rather than read. The adopted group has already
// removed its centring-absent class from this merge, so that class holds no data,
// every candidate of the point group is equally consistent with it, and the
// tie-break takes the lowest-numbered - which is the primitive one. Merging in
// that would hand the higher arm the centring-absent reflections as if they were
// data and lose the comparison on noise. Same move the metric-lattice ask makes.
//
// Its SCREWS are taken the same way, for the same reason: the adopted group's
// screw-absent axial reflections are not in this merge either, so on those rows
// the higher search has nothing to judge, its candidates tie, and the lowest-
// numbered - the one claiming no screw - is named. Measured on a P4_2 2_1 2
// crystal adopted as P2_1 2_1 2_1 and promoted by the twin-immune zone: all three
// axial rows read "no control class" and P 4 2 2 was written. So among the
// candidates the higher search cannot separate, the ones that drop an axial
// absence the adopted group claims are dropped; the axial reflections the adopted
// group left present are in this merge and stay the higher search's to decide.
const gemmi::GroupOps lo_ops = sg.operations();
const auto keeps_screws = [&](const gemmi::SpaceGroup &hi) {
const gemmi::GroupOps hi_ops = hi.operations();
for (int axis = 0; axis < 3; ++axis)
for (int n = 1; n <= 6; ++n) {
gemmi::Op::Miller hkl{{0, 0, 0}};
hkl[axis] = n;
if (lo_ops.is_systematically_absent(hkl) && !hi_ops.is_systematically_absent(hkl))
return false;
}
return true;
};
if (hi_search.best_space_group.has_value()) {
std::vector<gemmi::SpaceGroup> tied{*hi_search.best_space_group};
tied.insert(tied.end(), hi_search.alternatives.begin(), hi_search.alternatives.end());
const auto kept = std::find_if(tied.begin(), tied.end(), [&](const gemmi::SpaceGroup &c) {
return c.centring_type() == sg.centring_type() && keeps_screws(c);
});
if (kept != tied.end()) {
hi_search.best_space_group = *kept;
if (kept->is_sohncke())
hi_search.sohncke_space_group = *kept;
hi_search.alternatives.clear();
for (const auto &c : tied)
if (c.number != kept->number
&& (c.centring_type() != sg.centring_type() || keeps_screws(c)))
hi_search.alternatives.push_back(c);
for (auto &c : hi_search.candidates)
if (c.selected && c.space_group.centring_type() == sg.centring_type())
c.selected = keeps_screws(c.space_group);
// A row the adopted group decided is not open, whatever this merge holds of it.
std::erase_if(hi_search.undetermined_screws, [&](const auto &u) {
return lo_ops.is_systematically_absent({{u.row[0], u.row[1], u.row[2]}});
});
}
}
std::optional<gemmi::SpaceGroup> hi_sg;
if (hi_search.best_space_group.has_value()
&& hi_search.best_space_group->centring_type() == sg.centring_type())
hi_sg = hi_search.best_space_group;
else
for (const auto &alt : hi_search.alternatives)
if (alt.centring_type() == sg.centring_type()) { hi_sg = alt; break; }
// A refusal is arbitrated by the twin-immune zone of the added operators
// (TwinZoneVerdict) where it is decisive, and only otherwise by the merge
// comparison below: that comparison reads R_meas and ISa of two merges whose
// per-frame scales have settled, and the lower group, with the same scales fitted
// against fewer equivalents, then agrees with itself the better of the two on a
// genuine promotion as much as on a twin (measured: a genuine 2 -> 222 step reading
// R_meas 0.094 -> 0.103, ISa 12.3 -> 10.9). The zone does not depend on the scales.
// Read on the adopted group's own merge: the zone reflections are centric in the
// refused group and acentric in the adopted one, so they are distinct reflections
// of this merge, at full multiplicity and production scale - where the P1 search
// merge of a weak crystal has no shell with signal enough to read a zone on.
// The pseudo-translation is carried into the basis this merge is indexed on. It was
// measured on the search merge, before the change of basis committed above re-indexed
// the reflections, and the zone spends it assigning each reflection a phase class by
// cos(2 pi h.u) - which is a phase class only if h and u share a basis. Left in the
// old one it names classes at random and normalises the zone by noise, silently: the
// numbers stay the size they were. The reindex is (h,k,l)_new = M (h,k,l)_old, so h.u
// is preserved by u_new = (M^T)^-1 u_old - the ordinary transformation of a
// real-space fractional coordinate. Where nothing was committed this is the identity.
TranslationalNCSResult zone_tncs = pre_tncs;
if (commit_reindex && zone_tncs.detected) {
const gemmi::Vec3 u = commit_reindex->transpose().inverse().multiply(
gemmi::Vec3(pre_tncs.vector_frac[0], pre_tncs.vector_frac[1],
pre_tncs.vector_frac[2]));
zone_tncs.vector_frac = {u.x - std::floor(u.x), u.y - std::floor(u.y),
u.z - std::floor(u.z)};
}
const int zone = hi_sg.has_value()
? TwinZoneVerdict(sm.merged, gemmi::UnitCell(*result.consensus_cell), hi_search,
sg_search, result, TranslationalNCSVectorForLTest(zone_tncs)) : 0;
if (zone < 0) {
const std::string msg = fmt::format(
"Point group {} was refused ({}), and the twin-immune zone of its added "
"operators reads acentric: they relate unequal intensities on the very "
"reflections a twin law cannot touch, so the refusal stands and the "
"operator is recorded as a twin law. Keeping {}.",
sg_search.refused_point_group_hm, sg_search.refused_reason, sg.xhm());
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
sg_search.refused_reason += ", and its twin-immune zone reads acentric";
hi_sg.reset();
} else if (zone > 0) {
const std::string msg = fmt::format(
"Point group {} was confirmed by the operator correlations and then "
"refused ({}), but the twin-immune zone of its added operators reads "
"centric - a twin law cannot produce that on the reflections it leaves "
"untouched - so {} is adopted.",
sg_search.refused_point_group_hm, sg_search.refused_reason, hi_sg->xhm());
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
if (sg_search.refused_by_twin_gate) {
twin_overruled_subgroup = sg;
twin_overruled_reason = sg_search.refused_reason;
}
sg_search = hi_search;
if (hi_search.best_space_group->number != hi_sg->number) {
sg_search.alternatives.push_back(*hi_search.best_space_group);
std::erase_if(sg_search.alternatives,
[&](const gemmi::SpaceGroup &a) { return a.number == hi_sg->number; });
}
sg_search.best_space_group = *hi_sg;
result.space_group = *hi_sg;
end_msg.space_group_name = hi_sg->xhm();
end_msg.space_group_number = static_cast<uint64_t>(hi_sg->number);
promoted_point_group = true;
experiment_.SetSpaceGroup(*hi_sg);
phase("Re-scaling in space group " + hi_sg->short_name());
sm = scale_and_merge(hi_sg->short_name(), false);
hi_sg.reset();
}
if (hi_sg.has_value()) {
// scale_and_merge reports the error model of whatever it merged, and the two
// comparison merges are not the run's answer, so it goes back if they lose.
const double em_isa = result.error_model_isa;
const double em_isa_asymptotic = result.error_model_isa_asymptotic;
const bool em_isa_resolved = result.error_model_isa_resolved;
const double em_a = result.error_model_a;
const double em_b = result.error_model_b;
const auto res_fit = result.resolution_fit_A;
const auto manual_d_min =
experiment_.GetScalingSettings().GetHighResolutionLimit_A();
phase("Merging under the refused point group "
+ sg_search.refused_point_group_hm);
rsm->SetDMinLimit(sm.statistics.overall.d_min);
const auto low = scale_and_merge(sg.short_name() + ", pinned", false);
const double low_isa = result.error_model_isa;
experiment_.SetSpaceGroup(*hi_sg);
const auto high = scale_and_merge(hi_sg->short_name() + ", pinned", false);
const double high_isa = result.error_model_isa;
rsm->SetDMinLimit(manual_d_min);
const double low_r = low.statistics.overall.r_meas;
const double high_r = high.statistics.overall.r_meas;
const auto mult = [](const MergeStatisticsShell &o) {
return o.unique_reflections > 0
? static_cast<double>(o.total_observations) / o.unique_reflections
: 0.0;
};
const std::string measured = fmt::format(
"R_meas {:.4f} -> {:.4f}, ISa {:.2f} -> {:.2f}, multiplicity {:.2f} "
"-> {:.2f}, both merged to {:.2f} A",
low_r, high_r, low_isa, high_isa, mult(low.statistics.overall),
mult(high.statistics.overall), sm.statistics.overall.d_min);
// Neither statistic may get worse. A NaN on either side answers "worse",
// which is the recoverable direction: the run keeps the group it already has.
if (high_r <= low_r && high_isa >= low_isa) {
const std::string msg = fmt::format(
"Point group {} was confirmed by the operator correlations and "
"then refused ({}), so both groups were merged and compared: {}. "
"Merging under it does not degrade the data, which folding "
"non-equivalent reflections together cannot manage, so {} is "
"adopted.",
sg_search.refused_point_group_hm, sg_search.refused_reason,
measured, hi_sg->xhm());
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
if (sg_search.refused_by_twin_gate) {
twin_overruled_subgroup = sg;
twin_overruled_reason = sg_search.refused_reason;
}
sg_search = hi_search;
// Leave the candidate it displaced among the alternatives, so the
// reported "or ..." list stays the set of groups the data cannot
// separate, each named once - as the metric-lattice ask does.
if (hi_search.best_space_group->number != hi_sg->number) {
sg_search.alternatives.push_back(*hi_search.best_space_group);
std::erase_if(sg_search.alternatives,
[&](const gemmi::SpaceGroup &a) {
return a.number == hi_sg->number;
});
}
sg_search.best_space_group = *hi_sg;
result.space_group = *hi_sg;
end_msg.space_group_name = hi_sg->xhm();
end_msg.space_group_number = static_cast<uint64_t>(hi_sg->number);
promoted_point_group = true;
phase("Re-scaling in space group " + hi_sg->short_name());
sm = scale_and_merge(hi_sg->short_name(), false);
} else {
logger.Info("Point group {} was refused ({}), and merging under it "
"agrees: {}. Keeping {}.", sg_search.refused_point_group_hm,
sg_search.refused_reason, measured, sg.xhm());
sg_search.refused_reason +=
", and merging under it makes the data worse (" + measured + ")";
experiment_.SetSpaceGroup(sg);
result.error_model_isa = em_isa;
result.error_model_isa_asymptotic = em_isa_asymptotic;
result.error_model_isa_resolved = em_isa_resolved;
result.error_model_a = em_a;
result.error_model_b = em_b;
result.resolution_fit_A = res_fit;
}
}
}
// After the arbitration, so a promoted run is not told that the metric hosts more
// symmetry than a group it no longer has.
WarnIfMetricHostsMoreSymmetry(result.consensus_cell,
result.space_group.value_or(gemmi::get_spacegroup_p1()));
}
result.space_group_search = sg_search;
// Was a twin law possibly absorbed? Where a twin gate refused the adopted group and was
// overruled, or where a group above P1 was adopted on data that read twin-like before the
// search (the pre-promotion L-test), the merge in the subgroup is written beside the answer
// too, with the law - so a refinement program can take the twin hypothesis without a re-run.
// The answer itself is not touched by any of this.
const bool pre_twin_like = result.pre_promotion_twinning && result.pre_promotion_twinning->twinning_suspected;
// A predicted model can test the twin hypothesis itself (--mr's twin comparison), so with one the
// merge in the indexed lattice's group is made wherever a metric re-ask promoted from it.
const bool model_tests_twin = !config_.mr_model_path.empty() && twin_metric_subgroup.has_value();
if (!geometry_prepass && result.space_group && sg_search.point_group_order > 1
&& (twin_overruled_subgroup || pre_twin_like || model_tests_twin)) {
twin_choice = ChooseTwinSubgroup(*result.space_group, sg_search,
twin_overruled_subgroup ? twin_overruled_subgroup
: twin_metric_subgroup);
twin_choice_group_number = result.space_group->number;
twin_for_mr_only = !twin_overruled_subgroup && !pre_twin_like;
twin_choice_reason = twin_overruled_subgroup
? fmt::format("a twin gate refused it ({}) and was overruled", twin_overruled_reason)
: pre_twin_like
? fmt::format("the merge read twin-like before the space-group search ({})",
TwinningVerdictLine(*result.pre_promotion_twinning))
: std::string("it was promoted from the indexed lattice's group, which the predicted model "
"can test");
if (twin_choice)
logger.Info("Twin hypothesis: {} was adopted, but {} - the merge in {} is written too, "
"with twin law {}", result.space_group->xhm(), twin_choice_reason,
twin_choice->subgroup.xhm(), twin_choice->law.as_hkl().triplet());
}
} else {
// A space group was fixed by the user; surface it so the viewer/CLI can still show it.
result.space_group = experiment_.GetGemmiSpaceGroup();
// ...and record it, exactly as the search arm above does with the group it determined, or
// the _process.h5 comes out with a cell but no group and re-merging it defaults to P1.
if (const auto &sg = experiment_.GetGemmiSpaceGroup(); sg.has_value()) {
end_msg.space_group_name = sg->xhm();
end_msg.space_group_number = static_cast<uint64_t>(sg->number);
}
}
search_mark.Stop();
TimingMark adoption_mark(logger, "adoption (setting, reference, guard)");
// The files are written on other axes than the ones the space group was determined on in two
// cases: to match something the user gave (a reference MTZ, a model, -C, -S), and to write the
// standard setting. Both are relabellings made after every decision. The group's label goes
// with the axes: the one experiment_ holds and the one the _process.h5 records (end_msg),
// which differ where a model's enantiomorph was taken as a label only.
const auto relabel_group = [&](const gemmi::Op &cob, const gemmi::SpaceGroup &moved) {
const char old_centring = experiment_.GetSpaceGroupOrP1().centring_type();
experiment_.SetSpaceGroup(moved);
result.space_group = moved;
if (end_msg.space_group_name.has_value())
if (const gemmi::SpaceGroup *recorded = gemmi::find_spacegroup_by_name(*end_msg.space_group_name))
if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(*recorded, cob)) {
end_msg.space_group_name = m->xhm();
end_msg.space_group_number = static_cast<uint64_t>(m->number);
}
if (moved.centring_type() != old_centring) {
result.consensus_centering = moved.centring_type();
if (end_msg.rotation_lattice_type.has_value())
end_msg.rotation_lattice_type->centering = moved.centring_type();
result.rotation_lattice_type = end_msg.rotation_lattice_type;
}
// What the report names beside the group, on the same axes.
if (result.space_group_search.has_value()) {
auto &sgs = *result.space_group_search;
if (sgs.sohncke_space_group.has_value())
if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(*sgs.sohncke_space_group, cob))
sgs.sohncke_space_group = *m;
for (auto &alt : sgs.alternatives)
if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(alt, cob))
alt = *m;
for (auto &u : sgs.undetermined_screws) {
const auto row = HklOperator(cob).apply_to_hkl({{u.row[0], u.row[1], u.row[2]}});
const int n_axes = (row[0] != 0) + (row[1] != 0) + (row[2] != 0);
if (n_axes != 1)
continue;
const int axis = row[0] != 0 ? 0 : row[1] != 0 ? 1 : 2;
u.row = {std::abs(row[0]), std::abs(row[1]), std::abs(row[2])};
u.axis = "abc"[axis];
u.row_label = std::array<const char *, 3>{"h00", "0k0", "00l"}[axis];
}
}
};
// The indices the finalist ledger's P1 merge below (made from the scaler's own copy of the
// observations) still has to be put through to reach the written frame, and the ones the
// report states: every relabelling since the group was determined.
gemmi::Op merge_to_written = gemmi::Op::identity();
gemmi::Op determined_to_written = gemmi::Op::identity();
// `relabelled`: the merged reflections are already in the new setting, free set included - made by the
// validation started on the forecast from the same reflections the same way - so only the rest moves.
const auto relabel_output = [&](const gemmi::Op &cob, const char *source,
std::vector<MergedReflection> *relabelled = nullptr) {
result.setting_source = source;
if (cob == gemmi::Op::identity())
return;
const gemmi::SpaceGroup *moved = SpaceGroupInBasis(experiment_.GetSpaceGroupOrP1(), cob);
if (moved == nullptr) {
logger.Warning("No setting of {} is reached by the change of basis {}; the files keep "
"the axes the group was determined on",
experiment_.GetSpaceGroupOrP1().xhm(), IndexTriplet(HklOperator(cob)));
return;
}
const gemmi::Op hkl = HklOperator(cob);
if (relabelled == nullptr)
sm.merged = ReindexMergedIntoAsu(sm.merged, hkl, *moved,
experiment_.GetScalingSettings().GetMergeFriedel());
else
sm.merged = std::move(*relabelled);
reseat_integrated(cob);
relabel_group(cob, *moved);
// The free set is a function of the written index (RfreeFlags.h), so that every dataset of
// a crystal form written on the same axes carries the same one: drawn again on these axes.
if (relabelled == nullptr)
AssignRfreeFlags(sm.merged, *moved, experiment_.GetScalingSettings().GetRfreeFraction(), 500,
result.consensus_cell, config_.nthreads);
merge_to_written = merge_to_written.combine(hkl);
determined_to_written = determined_to_written.combine(hkl);
result.setting_operator = IndexTriplet(determined_to_written);
const auto &uc = *result.consensus_cell;
logger.Info("Written in {} (setting chosen by {}): reindexed by {} to a={:.3f} b={:.3f} c={:.3f} "
"alpha={:.2f} beta={:.2f} gamma={:.2f}", moved->xhm(), source,
IndexTriplet(hkl), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma);
};
// A reference MTZ (rotation): put the merge into the reference's frame before anything is
// matched against it - its intensities and its free-R flags are keyed by its own indices. The
// candidates are every description of the data's lattice on the reference's axes, and each of
// the data's alternative indexings (twin laws) on top; the one whose intensities correlate
// best with the reference is taken, the reflections are re-seated and re-merged in it, and the
// group becomes the reference's own, in its own setting. Stills resolve the twin-law ambiguity
// per image with ReindexAmbiguityResolver instead.
if (rsm && !config_.reference_data.empty() && result.consensus_cell
&& experiment_.GetSpaceGroupNumber().has_value()) {
const gemmi::SpaceGroup data_sg = experiment_.GetSpaceGroupOrP1();
const gemmi::SpaceGroup ref_sg = config_.reference_space_group.value_or(data_sg);
const auto choice = ChooseReferenceSetting(sm.merged, *result.consensus_cell, data_sg,
config_.reference_data, config_.reference_cell, ref_sg);
const gemmi::Op hkl = HklOperator(choice.cob);
result.reference_operator = IndexTriplet(hkl);
result.setting_source = "REFERENCE";
logger.Info("Reference: {} candidate frame(s) scored; CC with the reference {:.3f} as indexed, "
"{:.3f} under {}", choice.n_candidates, choice.identity_cc, choice.cc,
result.reference_operator);
// The reference's group where the run merged in the same group type, in the reference's
// setting; otherwise (-S overrode it) the run's own group on the reference's axes.
const gemmi::SpaceGroup *moved = data_sg.number == ref_sg.number ? &ref_sg
: SpaceGroupInBasis(data_sg, choice.cob);
if (moved != nullptr && !(choice.cob == gemmi::Op::identity() && moved->xhm() == data_sg.xhm())) {
if (!(choice.cob == gemmi::Op::identity()))
reseat_integrated(choice.cob);
relabel_group(choice.cob, *moved);
// The engine is about to be remade in the reference's frame, and the twin-law merge is
// made from it: its subgroup moves with the data.
if (twin_choice && !MoveTwinSubgroupChoice(*twin_choice, choice.cob))
twin_choice.reset();
determined_to_written = determined_to_written.combine(hkl);
result.setting_operator = IndexTriplet(determined_to_written);
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
phase("Re-merging in the reference's frame");
sm = scale_and_merge(moved->short_name(), false);
const auto &uc = *result.consensus_cell;
logger.Info("Reference: re-merged in its frame, {}: a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} "
"beta={:.2f} gamma={:.2f}", moved->xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta,
uc.gamma);
}
} else if (!geometry_prepass && config_.reference_data.empty() && result.consensus_cell
&& experiment_.GetGemmiSpaceGroup().has_value()) {
// No reference: the standard setting, or the one the user asked for with -S / -C. A model
// that fits can still move it below.
const gemmi::SpaceGroup sg = *experiment_.GetGemmiSpaceGroup();
const gemmi::SpaceGroup *target =
user_fixed_sg_ && user_fixed_sg_->number == sg.number ? &*user_fixed_sg_ : nullptr;
const std::optional<UnitCell> &target_cell = user_cell_;
relabel_output(ChooseOutputSetting(*result.consensus_cell, sg, target, target_cell),
target_cell ? "CELL"
: target != nullptr && !target->is_reference_setting() ? "SPACE_GROUP" : "STANDARD");
}
// A pass RunAllPasses is about to throw away and run again. Its report is never printed and
// its files are overwritten seconds later, so everything below that only fills those in would
// be spent on an answer nobody reads. Every trigger of a re-run is settled by here - the
// integrator's background-starvation count (measured at integration), the fixed-group lattice
// conflict (at indexing) and the search merge the quality guard reads (at the first merge) - so
// a pass can tell that it is being superseded, which RunAllPasses cannot until it returns.
// Only the refined passes ask: quality_guard_pass1_ holds the header-geometry pass they are
// judged against and is set only while they run. The supercell guard is deliberately not asked
// here - it compares this pass's cell against pass 1's rather than reading this pass's own
// numbers - so a pass it re-runs still writes a report first.
//
// Nothing below this point is read by a later pass: the analyses are report-only (see each),
// the files are rewritten by the pass that supersedes this one, and the one place that touches
// experiment_ - the model frame - is undone by the SetSpaceGroup at the top of RunPipeline.
//
// So is a canonical pass the geometry walk is going to replace, where that answer is in by now
// (GeometryWalkTakesRound). A refined pass run beside the rotation-scale walk is superseded too
// where the walk overturns it (see RunAllPasses); it waits here for the walk's answer.
superseded =
(!cancelled_ && quality_guard_pass1_ && quality_guard_pass1_->has_merge_statistics
&& ((bragg_adaptive_ && bkg_starved_fraction_
&& *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION)
|| !RefinedPassIsWorse(*quality_guard_pass1_, result).empty()))
|| (lattice_arm_check_ && lattice_arm_check_->lost) // only with a _process.h5 open, see above
|| GeometryWalkTakesRound(result, p.postrefine_ahead)
|| (rotation_scale_check_ && rotation_scale_check_->Overturns());
if (superseded) {
logger.Info("This pass is going to be re-run, so it makes no report and writes no files.");
Note("this pass is to be re-run (see the log); it writes nothing");
} else if (!geometry_prepass) {
Note(fmt::format("resolution {:.2f} A{}", sm.statistics.overall.d_min,
result.resolution_fit_A
? fmt::format(" (CC1/2 fall-off at {:.2f} A)", *result.resolution_fit_A)
: std::string()));
Step("Analyses");
}
adoption_mark.Stop();
TimingMark analyses_mark(logger, "analyses");
// --finalist-ledger (see where it is read, below) is folded from a merge of its own: the same
// observations in P1. It needs every node of the lattice integrated, which a fixed centred group
// does not do.
const bool p1_integration_complete =
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
const bool finalist_ledger = config_.finalist_ledger && rsm && result.consensus_cell && write_files
&& config_.write_merged && !geometry_prepass && !superseded
&& p1_integration_complete && is_rotation;
// The ledger's P1 merge is made from here on, beside the analyses. Nothing here reads what it
// writes or writes what it reads: the engine merges its own copy of the observations, in a group
// of its own rather than the experiment's, and writes no per-frame scale back onto the outcomes;
// and nothing up to take_ledger_p1 calls the engine, changes the outcomes it was ingested from,
// or changes the experiment. Its log lines are held and printed where it is taken.
// The twin-law merge (see twin_choice), from the same engine as the answer's, to the resolution the
// answer was cut at. Like the ledger's below, it merges the engine's own copy of the observations in
// a group of its own and writes no per-frame scale back, so nothing the answer is made of sees it.
std::optional<RotationScaleMerge::Result> twin_merge;
if (twin_choice && rsm && result.consensus_cell && write_files && config_.write_merged
&& !geometry_prepass && !superseded) {
TimingMark mark(logger, "twin-law merge");
phase("Merging in " + twin_choice->subgroup.short_name() + " (twin hypothesis)");
rsm->SetWriteBackPerFrameScale(false);
rsm->SetExportScaledFulls(false);
rsm->SetFrenchWilson(false);
const auto manual_d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A();
rsm->SetDMinLimit(sm.statistics.overall.d_min);
twin_merge = rsm->Run(/*for_search=*/false, /*full_stats=*/true,
/*measure_cc_before_corrections=*/false, &twin_choice->subgroup);
rsm->SetDMinLimit(manual_d_min);
rsm->SetWriteBackPerFrameScale(true);
}
std::optional<RotationScaleMerge::Result> ledger_p1;
Logger ledger_log = Logger::Buffered();
std::future<RotationScaleMerge::Result> ledger_ahead;
if (finalist_ledger) {
rsm->SetWriteBackPerFrameScale(false);
rsm->SetExportScaledFulls(false);
rsm->SetFrenchWilson(false);
rsm->SetLogger(ledger_log);
ledger_ahead = std::async(std::launch::async, [&rsm] {
return rsm->Run(/*for_search=*/false, /*full_stats=*/true,
/*measure_cc_before_corrections=*/false, &gemmi::get_spacegroup_p1());
});
}
auto take_ledger_p1 = [&] {
if (!ledger_ahead.valid()) return;
TimingMark wait_mark(logger, "finalist ledger merge wait");
ledger_p1 = ledger_ahead.get();
wait_mark.Stop();
rsm->SetLogger(logger);
rsm->SetWriteBackPerFrameScale(true);
ledger_log.ReplayInto(logger);
};
const auto &twin_sg_opt = experiment_.GetGemmiSpaceGroup();
const gemmi::SpaceGroup *twin_sg = twin_sg_opt ? &*twin_sg_opt : nullptr;
// Diffraction anisotropy (see where it is reported, below), made beside the analyses that come
// before it: it reads the merge, the integrated observations and the per-frame scales the merge
// wrote back, none of which they change, and most of it is gathering the observations.
std::future<decltype(sm.statistics.anisotropy)> anisotropy;
if (!geometry_prepass && !superseded && result.consensus_cell) {
AnisotropyRunInfo aniso_run;
if (sm.statistics.sweep_quality.measured && sm.statistics.sweep_quality.sweep_deg > 0.0f)
aniso_run.observed_rotation_deg = sm.statistics.sweep_quality.sweep_deg;
aniso_run.dose_term_in_scale_model = experiment_.GetScalingSettings().GetCorrectionSurfaces();
aniso_run.radiation_damage_relative_b = sm.statistics.radiation_damage_delta_b;
const float wedge_deg = experiment_.GetGoniometer() ? experiment_.GetGoniometer()->GetWedge_deg() : 0.0f;
anisotropy = std::async(std::launch::async,
[&, aniso_run, wedge_deg, cell = *result.consensus_cell,
rotation = experiment_.IsRotationIndexing()] {
return AnalyzeAnisotropy(sm.merged,
ScaledObservations(indexer->GetIntegrationOutcome(), rotation, twin_sg,
wedge_deg, 0.5, config_.nthreads),
cell, twin_sg, aniso_run);
});
}
// Not on the geometry pre-pass, nor on a superseded one: the analysis goes into that pass's
// statistics text and its written reflections, and neither survives the run. The promotion flag
// below is a different thing - it is what the SEARCH did, the second pass reads it, and it is
// set either way.
if (!geometry_prepass && !superseded) {
// Translational pseudo-symmetry first: its verdict is an input to the twinning analysis,
// which must not read its own L-test when a non-half-integer pseudo-translation has biased
// it. Reporting only - no reflection and no merge depends on the outcome.
if (result.consensus_cell)
result.tncs = AnalyzeTranslationalNCS(sm.merged, gemmi::UnitCell(*result.consensus_cell),
twin_sg, 30, 0, config_.nthreads);
if (result.tncs.detected)
logger.Warning("Translational pseudo-symmetry: Patterson off-origin peak {:.1f}% of the "
"origin at ({:.3f}, {:.3f}, {:.3f}), {:.1f} A - declare it to the "
"molecular-replacement program",
result.tncs.peak_percent, result.tncs.vector_frac[0],
result.tncs.vector_frac[1], result.tncs.vector_frac[2],
result.tncs.vector_length_A);
// Without the reflections the lattice centring extinguishes, as the pre-promotion test
// above is measured: the merged ASU of a centred group still holds keys in the absent
// class (the merge writes what it was given), they are noise rather than intensities,
// and an L-test partner step that lands on one pairs a real reflection with an absent.
// This is the number the user reads, so it is the one that must be treated.
const char twin_centring = twin_sg ? twin_sg->centring_type()
: end_msg.rotation_lattice_type.has_value()
? end_msg.rotation_lattice_type->centering : 'P';
const std::optional<gemmi::UnitCell> twin_cell =
result.consensus_cell ? std::optional<gemmi::UnitCell>(*result.consensus_cell) : std::nullopt;
result.twinning = AnalyzeTwinning(sm.merged, twin_sg, 20,
TranslationalNCSVectorForLTest(result.tncs), twin_centring,
twin_cell ? &*twin_cell : nullptr);
// Set on the struct the text below is rendered from. AnalyzeTwinning returns a fresh result
// with this flag clear, so setting it only afterwards left the statistics text asserting
// "No twinning: the Laue class is holohedral" as a fact while the report, rendering the same
// struct once the flag was set, said the opposite - two twin verdicts from one run, with the
// flat one landing on exactly the case the flag exists for. (Assigning it before the call
// would not do: the call replaces the whole struct.)
result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group;
// A refusal that stood on the twin-immune zone names the law: the first operator the
// refused group adds, at the fraction its own disagreement implies (a lower bound).
if (twin_sg && result.space_group_search
&& result.space_group_search->refused_point_group_representative
&& result.twin_zone_refusal_stood) {
const auto &sgs = *result.space_group_search;
const gemmi::GroupOps hi = sgs.refused_point_group_representative->operations();
const gemmi::GroupOps lo = twin_sg->operations();
for (const auto &op : hi.derive_symmorphic().sym_ops) {
if (op.det_rot() <= 0) continue;
const bool in_lower = std::any_of(lo.sym_ops.begin(), lo.sym_ops.end(),
[&](const gemmi::Op &o) { return o.rot == op.rot; });
if (in_lower) continue;
result.twinning.twin_law_operator = gemmi::Op{op.rot, {0, 0, 0}, 'x'}.as_hkl().triplet();
break;
}
// The fraction: the refused operators' own H where the H gate measured it, else the
// L-test of the merge before the search (a twin fraction that does not depend on the
// promotion), else the perfect twin every operator statistic reads as symmetry.
result.twinning.twin_law_fraction =
std::isfinite(sgs.refused_twin_fraction) ? sgs.refused_twin_fraction
: result.pre_promotion_twinning
&& result.pre_promotion_twinning->twin_fraction_source != TwinFractionSource::None
? result.pre_promotion_twinning->estimated_twin_fraction : 0.5;
}
if (result.tncs.measurable)
stats_text << TranslationalNCSToText(result.tncs) << "\n";
stats_text << TwinningAnalysisToText(result.twinning) << "\n";
}
// Mark the conclusion as non-authoritative when the Laue class was reached by a promotion the
// search itself made, so the text cannot claim "no twin law exists" on its own say-so.
result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group;
// How the crystal sat on the spindle. Reported always on rotation data (it is a property of
// the mounting, which the user can change). The angle to the nearest symmetry axis is kept
// as a descriptive key, but the WARNING is decided by the exact orbit computation: the old
// any-proper-axis-within-15-deg rule was wrong in both directions - an aligned in-plane
// 2-fold of a dihedral group is repaired by the principal axis, a cubic group is never
// severe in any orientation, and a lone diad PERPENDICULAR to the spindle is severe while
// no axis is anywhere near it.
if (experiment_.IsRotationIndexing() && twin_sg && end_msg.rotation_lattice.has_value()) {
if (const auto gonio = experiment_.GetGoniometer()) {
const auto closest = ClosestSymmetryAxisToSpindle(*twin_sg, *end_msg.rotation_lattice,
gonio->GetAxis());
if (closest.has_value()) {
const auto [angle, order] = *closest;
result.spindle_symmetry_axis_deg = angle;
result.spindle_symmetry_axis_order = order;
stats_text << "Closest symmetry axis to the spindle: " << order << "-fold at "
<< std::fixed << std::setprecision(1) << angle << " deg\n";
}
// The measured resolution of THIS run sets the cone: past merging the question is
// what these data lost, not what the detector could have reached.
double d_min = 0;
for (const auto &r : sm.merged)
if (std::isfinite(r.d) && r.d > 0 && (d_min == 0 || r.d < d_min))
d_min = r.d;
if (const auto loss = SpindleUnrepairedFraction(
*twin_sg, *end_msg.rotation_lattice, gonio->GetAxis(),
experiment_.GetWavelength_A(), d_min)) {
result.spindle_lost_unique_fraction = loss->lost_unique_fraction;
end_msg.spindle_lost_unique_fraction =
static_cast<float>(loss->lost_unique_fraction);
stats_text << "Unique reflections the mounting made unmeasurable: "
<< std::fixed << std::setprecision(2)
<< 100.0 * loss->lost_unique_fraction << "% (of the "
<< std::setprecision(2) << 100.0 * (1.0 - std::cos(loss->theta_max_deg * PI / 180.0))
<< "% a sweep's blind cone holds at " << std::setprecision(2)
<< d_min << " A)\n\n";
// Warn when the point group recovers less than half the cone - the same half
// that fixes the per-image trigger threshold. The loss is a coherent cap about
// the spindle direction, which costs a map more than the same percentage lost
// at random.
if (loss->cone_fraction >= 0.5) {
const std::string msg = fmt::format(
"The mounting makes {:.1f}% of unique reflections unmeasurable: the "
"measured point group cannot map that part of the sweep's blind cone "
"onto measured territory, however long the sweep runs. The loss is a "
"coherent cap about the spindle direction rather than a scatter, so "
"overall completeness may still look reasonable while the gap sits at "
"the spindle direction. A second sweep about a different axis, or "
"re-mounting, recovers it.",
100.0 * loss->lost_unique_fraction);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
result.warnings.push_back({PathologyCode::SPINDLE_CAP, msg});
}
} else {
stats_text << "\n";
}
}
}
// Indexing-ambiguity (alternative-indexing) advisory. When the lattice metric symmetry exceeds
// the Laue symmetry the crystal can be validly indexed in several hands related by twin-law
// operators. For an obvious merohedral case (P3/P4/P6...) users expect this; but a PSEUDO-merohedral
// metric (e.g. a C2 crystal whose beta makes it pseudo-F-orthorhombic) is easy to miss, so surface it. The
// reindex operator is applied directly to (h,k,l), so its triplet reads as an h,k,l transform.
if (result.consensus_cell && twin_sg != nullptr) {
const auto twin_ops = ReindexAmbiguityOperators(*result.consensus_cell, *twin_sg, 2.0);
if (!twin_ops.empty()) {
// The merge is folded under the adopted group, and a twin law sends a reflection
// outside that group's asymmetric unit - so the pairs are formed on the merge
// expanded to P1 under the group's own operations (each reflection once per
// operation; the H statistic is a median and the correlation a ratio, so the
// repetition changes neither).
std::vector<MergedReflection> expanded;
const auto expanded_merge = [&]() -> const std::vector<MergedReflection> & {
if (expanded.empty()) {
const gemmi::GroupOps ops = twin_sg->operations();
expanded.reserve(sm.merged.size() * ops.sym_ops.size());
for (const auto &r : sm.merged)
for (const auto &o : ops.sym_ops) {
const auto hkl = o.apply_to_hkl({{r.h, r.k, r.l}});
MergedReflection e = r;
e.h = hkl[0]; e.k = hkl[1]; e.l = hkl[2];
expanded.push_back(e);
}
}
return expanded;
};
std::optional<OperatorScorer> law_correlation;
std::string laws;
for (const auto &op : twin_ops) {
std::string t = op.triplet();
std::replace(t.begin(), t.end(), 'x', 'h');
std::replace(t.begin(), t.end(), 'y', 'k');
std::replace(t.begin(), t.end(), 'z', 'l');
laws += (laws.empty() ? "" : " ; ") + t;
// What each law reads on the merge: the disagreement H of the reflections it
// relates (Yeates (1997) Methods Enzymol. 276, 344-358: a twin of fraction a reads
// (1-2a)/2, so a >= 0.5 - H), their correlation and their intensity-weighted R.
// Report-only, so a user sees which law a twin refinement would need.
gemmi::Mat33 m;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
m[j][i] = static_cast<double>(op.rot[i][j]) / gemmi::Op::DEN;
SearchSpaceGroupOptions law_opts;
law_opts.nthreads = static_cast<size_t>(std::max(1, config_.nthreads));
if (!law_correlation)
law_correlation.emplace(expanded_merge(), law_opts);
if (const auto s = (*law_correlation)(m); s.has_value())
result.twin_law_evidence.push_back(fmt::format(
"{}: {} pairs, H {:.3f} (twin fraction {:.2f} or more), R {:.3f}, CC {:.3f}",
t, s->n_pairs, s->h_stat, std::max(0.0, 0.5 - s->h_stat), s->r_stat, s->cc));
else
result.twin_law_evidence.push_back(fmt::format("{}: too few pairs to read", t));
}
for (const auto &e : result.twin_law_evidence)
logger.Info("Twin law {}", e);
// A model can resolve it too, but whether it does is only known once the model has
// been fitted (validation step below), so the warning stands here exactly as in a run
// without a model, and is withdrawn there if the model's indexing probe decided it.
// On serial stills with -C and -S the model is already in `ambiguity_reference`.
if (!ambiguity_reference && experiment_.IsRotationIndexing()) {
// One sweep has one orientation matrix, so every frame is in the same hand and the
// merge mixes nothing. The hand is arbitrary, which matters only against other
// data: the report states it, and it is not a condition of this data set.
logger.Info("Indexing ambiguity: this cell / space group admits alternative indexing "
"(reindex operator(s): {}); one sweep is indexed in one hand, arbitrary "
"against other data sets", laws);
result.indexing_ambiguity_operators = laws;
} else if (!ambiguity_reference) {
const std::string msg = fmt::format(
"Indexing ambiguity: this cell / space group admits alternative indexing "
"(reindex operator(s): {}). Serial-stills crystals are indexed in one hand at random, "
"and rugnux can only break this against an external reference. WITHOUT one the merge "
"mixes the hands and CC1/2 is degraded; a reference MTZ (-z) or a model (--model, "
"which needs -C and -S here) resolves it.", laws);
logger.Warning("{}", msg);
ambiguity_warning_text = " !! " + msg + "\n\n";
stats_text << ambiguity_warning_text;
result.warnings.push_back({PathologyCode::INDEXING_AMBIGUITY, msg});
} else {
logger.Info("Indexing ambiguity present (reindex operator(s): {}); resolved "
"against the supplied reference.", laws);
}
}
}
// Dataset-wide Wilson B-factor estimate (like XDS's WILSON LINE B). Diagnostic only - it is not
// fed back into scaling; it just lands in the printed statistics, the mmCIF, and the log, none
// of which the geometry pre-pass or a superseded pass produces.
if (!geometry_prepass && !superseded) {
const GlobalWilsonB wilson = CalcGlobalWilsonB(sm.merged);
sm.statistics.wilson_b = wilson.b;
sm.statistics.wilson_b_correlation = wilson.correlation;
sm.statistics.wilson_b_d_low_A = wilson.d_low_A;
sm.statistics.wilson_b_d_high_A = wilson.d_high_A;
if (std::isfinite(wilson.b) && wilson.b > 0.0)
logger.Info("Wilson B-factor estimate: {:.2f} A^2 (correlation {:.3f}, {} shells, fitted over "
"{:.2f}-{:.2f} A)", wilson.b, wilson.correlation, wilson.n_shells, wilson.d_low_A,
wilson.d_high_A);
// Diffraction anisotropy. It describes the data and corrects no intensity; its tensor is
// used only as the Wilson prior of the amplitudes, below. The tensor and its resolution signature come from
// the merged intensities; the error bar the verdict is gated on has to come from the
// unmerged observations, because a merge has exact Laue symmetry by construction and the
// tensor directions the symmetry forbids - the only place a dataset measures its own
// systematic error - are identically zero in it.
if (anisotropy.valid()) {
sm.statistics.anisotropy = anisotropy.get();
stats_text << AnisotropyToText(sm.statistics.anisotropy) << "\n";
// The amplitudes are the one place the tensor is used: French-Wilson with the Wilson
// prior of each reflection taken along its own direction. The intensities and
// everything measured on them above are left as they are.
FrenchWilsonOptions fw_opts;
fw_opts.num_threads = config_.nthreads;
fw_opts.anisotropy_hkl = AnisotropyTensorHKL(sm.statistics.anisotropy,
gemmi::UnitCell(*result.consensus_cell));
ApplyFrenchWilson(sm.merged, experiment_.GetSpaceGroupOrP1(), fw_opts);
} else if (rsm) {
// No tensor: the isotropic prior. The rotation merge leaves its amplitudes to here.
FrenchWilsonOptions fw_opts;
fw_opts.num_threads = config_.nthreads;
ApplyFrenchWilson(sm.merged, experiment_.GetSpaceGroupOrP1(), fw_opts);
}
}
stats_text << sm.statistics;
result.merge_statistics_text = stats_text.str();
result.has_merge_statistics = true;
result.merge_statistics = sm.statistics;
result.reference_merge_statistics = sm.reference_statistics;
result.reference_isa = sm.reference_isa;
// Per-image form of the sweep-quality ranges, for the _process.h5: one code per image, 0 where
// the image is in no flagged range, plus the vocabulary the codes index. Only filled when the
// diagnostic ran, so absent datasets mean "not looked for" rather than "all clean". The
// disposition rides alongside as a second per-image array: what the reason SAW and what became
// of the frame are different statements, and a consumer needs both.
if (sm.statistics.sweep_quality.measured) {
const auto &sq = sm.statistics.sweep_quality;
end_msg.sweep_quality.assign(end_msg.max_image_number, 0);
for (const auto &r : sq.ranges)
for (int64_t i = std::max<int64_t>(0, r.first_image);
i <= r.last_image && i < static_cast<int64_t>(end_msg.sweep_quality.size()); ++i)
end_msg.sweep_quality[i] = static_cast<uint8_t>(r.reason) + 1;
for (int r = 0; r <= static_cast<int>(SweepQualityReason::InconsistentWithMerge); ++r)
end_msg.sweep_quality_reasons.emplace_back(
SweepQualityReasonCode(static_cast<SweepQualityReason>(r)));
end_msg.frame_disposition.assign(
sq.frame_disposition.begin(),
sq.frame_disposition.begin()
+ std::min<int64_t>(end_msg.max_image_number, sq.frame_disposition.size()));
for (int d = 0; d <= static_cast<int>(FrameDisposition::Rejected); ++d)
end_msg.frame_disposition_codes.emplace_back(
FrameDispositionCode(static_cast<FrameDisposition>(d)));
}
{
// Stride rather than take the head: the merged list is ordered by hkl, so the first N
// reflections are one corner of reciprocal space and would not show the intensity range.
constexpr size_t MAX_POINTS = 4000;
const size_t n = sm.merged.size();
const size_t step = std::max<size_t>(1, n / MAX_POINTS);
result.merged_i_sigma.reserve(std::min(n, MAX_POINTS + 1));
for (size_t i = 0; i < n; i += step)
if (std::isfinite(sm.merged[i].I) && std::isfinite(sm.merged[i].sigma)
&& sm.merged[i].sigma > 0.0f)
result.merged_i_sigma.emplace_back(sm.merged[i].I, sm.merged[i].sigma);
}
result.has_reference = !config_.reference_data.empty();
// Inherit the campaign's shared R-free test set from the reference MTZ (overriding the
// per-hkl hash that the merge assigned), so every dataset flags the same free reflections - but
// only where the merge, now in the reference's frame, is recognisably the same crystal form on
// the same axes. A reference that is not would hand each reflection the flag of an unrelated
// one, and a count of matched indices cannot see that: a permuted orthorhombic cell matches
// nearly every index, each to the wrong reflection.
if (!config_.reference_data.empty() && !sm.merged.empty()) {
const auto match = MatchReference(sm.merged, config_.reference_data, experiment_.GetSpaceGroupOrP1());
result.reference_cc = match.cc;
result.reference_matched_fraction = match.matched_fraction;
const bool matches = match.cc >= MIN_REFERENCE_CC
&& match.matched_fraction >= MIN_REFERENCE_MATCHED_FRACTION;
if (config_.reference_has_free_flags && matches) {
const size_t matched = ApplyReferenceFreeFlags(sm.merged, experiment_.GetSpaceGroupOrP1(),
config_.reference_data);
result.reference_free_flags_inherited = true;
logger.Info("R-free flags: inherited the reference test set ({} of {} merged reflections "
"matched; CC with the reference {:.3f})", matched, sm.merged.size(), match.cc);
} else if (!matches) {
const std::string msg = fmt::format(
"The reference MTZ does not match these data: CC {:.3f} (at least {:.2f} expected) "
"over {:.0f}% of the merged reflections in its resolution range (at least {:.0f}%). "
"It is not the same crystal form on the same axes, so {}.",
match.cc, MIN_REFERENCE_CC, 100.0 * match.matched_fraction,
100.0 * MIN_REFERENCE_MATCHED_FRACTION,
config_.reference_has_free_flags
? "its free-R flags were NOT inherited - the merge carries its own test set"
: "its scaling and CCref describe a different crystal");
logger.Warning("{}", msg);
result.warnings.push_back({PathologyCode::REFERENCE_MISMATCH, msg});
}
}
// Radiation-damage report (rotation): the per-image scale CC-to-merge and mosaicity across the
// sweep (the per-image scaling the rotation merge already fits, binned by frame = dose). A
// per-image CC that falls, and/or a mosaicity that rises, with frame number is the classic
// radiation-damage signature - a data-quality-vs-dose read complementary to the fitted decay
// correction. The full per-image table is written to <prefix>_plot.txt for detail. Both this
// and the sweep-quality report below are text, so a superseded pass makes neither.
if (experiment_.IsRotationIndexing() && !superseded) {
const auto &outs = indexer->GetIntegrationOutcome();
const int nf = static_cast<int>(outs.size());
constexpr int nb = 10;
std::array<double, nb> cc_sum{}, mos_sum{};
std::array<int, nb> cc_n{}, mos_n{};
for (int f = 0; f < nf; ++f) {
const int b = nf > 1 ? std::min(nb - 1, f * nb / nf) : 0;
if (outs[f].image_scale_cc && std::isfinite(*outs[f].image_scale_cc)) {
cc_sum[b] += *outs[f].image_scale_cc; cc_n[b]++;
}
if (outs[f].mosaicity_deg && std::isfinite(*outs[f].mosaicity_deg)) {
mos_sum[b] += *outs[f].mosaicity_deg; mos_n[b]++;
}
}
int tot = 0;
for (int n : cc_n) tot += n;
if (tot >= 20) {
std::ostringstream os;
os << fmt::format("Radiation-damage report (per-image scale over {} frames):\n", nf);
os << " dose <CC to merge> <mosaicity deg>\n";
for (int b = 0; b < nb; ++b) {
if (cc_n[b] == 0 && mos_n[b] == 0) continue;
os << fmt::format(" {:3d}-{:3d}% {:>7} {:>7}\n", b * 10, (b + 1) * 10,
cc_n[b] ? fmt::format("{:.3f}", cc_sum[b] / cc_n[b]) : std::string("-"),
mos_n[b] ? fmt::format("{:.3f}", mos_sum[b] / mos_n[b]) : std::string("-"));
}
if (cc_n[0] > 0 && cc_n[nb - 1] > 0) {
const double cc0 = cc_sum[0] / cc_n[0], cc9 = cc_sum[nb - 1] / cc_n[nb - 1];
os << fmt::format(" => per-image CC to merge {:.3f} (first 10%) -> {:.3f} (last 10%){}",
cc0, cc9,
cc9 < cc0 - 0.05 ? " (falling: radiation damage / crystal decay)" : "");
}
// Relative B-factor across the sweep (measured before any decay correction, against the
// low-dose start): the resolution-dependent complement to the CC/mosaicity read above.
// Radiation damage fades the high-resolution intensity, so only a POSITIVE change is dose.
// A curve no straight line describes gets no first->last number at all, and a batch whose
// data could not measure one prints "-" rather than a value.
if (!sm.statistics.radiation_damage_b_batch.empty()) {
if (std::isfinite(sm.statistics.radiation_damage_delta_b))
os << fmt::format("\n => relative B-factor change over run = {:+.2f} A^2 (first->last){}",
sm.statistics.radiation_damage_delta_b,
sm.statistics.radiation_damage_delta_b > 5.0
? " (significant dose-dependent scaling: radiation damage)" : "");
else
os << "\n => relative B-factor: the per-batch curve below is not a trend, so no "
"first->last number describes it - dose does not come back, so whatever moved "
"here was not dose; see the sweep-quality report";
os << fmt::format("\n per-batch relative-B (A^2, {:.0f} deg/batch):",
sm.statistics.radiation_damage_batch_deg);
for (float bb : sm.statistics.radiation_damage_b_batch)
os << (std::isfinite(bb) ? fmt::format(" {:.1f}", bb) : std::string(" -"));
}
logger.Info("{}", os.str());
result.radiation_damage_text = os.str();
}
// Sweep-quality report: the stretches of the sweep over which the crystal delivered much
// less than the rest of the run, what each one looks like, what became of it and what
// keeping it costs the merged intensities. Frame numbers are processed-image ordinals,
// inclusive, as in <prefix>_plot.txt.
const auto &sq = sm.statistics.sweep_quality;
if (sq.measured) {
const int n_sweep = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected;
auto pct = [&](int n) { return n_sweep > 0 ? 100.0 * n / n_sweep : 0.0; };
std::ostringstream os;
os << fmt::format("Sweep quality over {:.0f} deg (per-image scale with the incident flux, "
"which varied {:.2f}x, already divided out):\n",
sq.sweep_deg, sq.flux_peak_to_trough);
os << fmt::format(" {} frames: {} merged ({:.1f}%), {} downgraded ({:.1f}%), "
"{} rejected ({:.1f}%, {:.1f} deg)\n",
n_sweep, sq.frames_merged, pct(sq.frames_merged),
sq.frames_downgraded, pct(sq.frames_downgraded),
sq.frames_rejected, pct(sq.frames_rejected), sq.rejected_deg);
if (sq.ranges.empty()) {
os << " no stretch of the sweep is materially worse than the run";
} else {
os << " frames rotation diagnosis severity scale CC indexed disposition dCC1/2\n";
for (const auto &r : sq.ranges)
os << fmt::format(" {:<17s} {:6.1f} deg {:<16s} {:5.2f} {:5.2f} {:5.2f} {:4.0f}% {:<11s} {}\n",
fmt::format("{}-{}", r.first_image, r.last_image), r.rotation_deg,
SweepQualityReasonText(r.reason), r.severity, r.mean_relative_scale,
r.mean_relative_cc, 100.0 * r.indexed_fraction,
FrameDispositionCode(r.disposition),
std::isfinite(r.delta_cc_half)
? fmt::format("{:+.4f} +/- {:.4f}", r.delta_cc_half,
r.delta_cc_half_se)
: std::string("-"));
os << " => severity is the fraction of the run's typical diffracting power missing "
"over the range; dCC1/2 is what keeping it costs the merged intensities";
if (sq.modulation_peak_to_trough >= 1.05f)
os << fmt::format("\n => once-per-revolution modulation of the per-image scale: "
"{:.1f}x peak to trough", sq.modulation_peak_to_trough);
}
if (!sq.delta_cc_half_batch.empty()) {
os << fmt::format("\n => delta-CC1/2 per {:.0f} deg:", sq.delta_cc_half_batch_deg);
for (float d : sq.delta_cc_half_batch)
os << (std::isfinite(d) ? fmt::format(" {:+.3f}", d) : std::string(" -"));
}
logger.Info("{}", os.str());
}
}
analyses_mark.Stop();
// Model validation runs BEFORE the reflection files are written, because it is what settles the
// frame they are written in: the enantiomorph, which merged intensities cannot choose, and -
// with no reference MTZ - the alternative indexing. Both are relabelings of the same
// measurements, so the statistics computed above are unaffected; what changes is that the file,
// the R-factors and the maps then all describe one indexing instead of two.
std::future<void> placed_model_file;
// The twin-law merge made ready to be written: on the axes the answer is written on (every
// relabelling since it was merged), with the answer's own free set - read off the answer's merge
// by the adopted group's asymmetric unit, which both twin mates of a pair fall in - then in the
// subgroup's standard setting, with its French-Wilson amplitudes and the law's fraction. Made once
// the model, which can relabel the answer, is done with it; and before the model is asked about
// the hypothesis itself (below).
struct TwinFile {
std::vector<MergedReflection> merged;
UnitCell cell;
gemmi::SpaceGroup space_group;
gemmi::Op law;
TwinLawFraction fraction;
};
std::optional<TwinFile> twin_file;
const auto prepare_twin_file = [&] {
if (twin_file || !twin_merge || !twin_choice)
return;
const bool friedel = experiment_.GetScalingSettings().GetMergeFriedel();
const gemmi::SpaceGroup adopted = experiment_.GetSpaceGroupOrP1();
TwinSubgroupChoice c = *twin_choice;
std::vector<MergedReflection> merged = twin_merge->merged;
if (!(merge_to_written == gemmi::Op::identity())) {
if (!MoveTwinSubgroupChoice(c, merge_to_written.inverse())) {
logger.Warning("Twin hypothesis: {} has no setting on the axes the answer is written on; "
"its merge is not written", twin_choice->subgroup.xhm());
return;
}
merged = ReindexMergedIntoAsu(merged, merge_to_written, c.subgroup, friedel);
}
// A subgroup of the group written - which the model can relabel to its enantiomorph, a change
// of label only, so the subgroup's label follows it.
const auto rotations_in = [](const gemmi::SpaceGroup &sub, const gemmi::SpaceGroup &group) {
const gemmi::GroupOps g = group.operations();
for (const auto &op : sub.operations().sym_ops)
if (std::none_of(g.sym_ops.begin(), g.sym_ops.end(),
[&](const gemmi::Op &o) { return o.rot == op.rot; }))
return false;
return true;
};
if (adopted.number != twin_choice_group_number && c.subgroup.is_enantiomorphic()) {
gemmi::GroupOps e = c.subgroup.operations();
e.change_basis_forward(c.subgroup.change_of_hand_op());
if (const gemmi::SpaceGroup *other = gemmi::find_spacegroup_by_ops(e))
c.subgroup = *other;
}
if (!rotations_in(c.subgroup, adopted)) {
logger.Warning("Twin hypothesis: {} is not a subgroup of {} as written; its merge is not written",
c.subgroup.xhm(), adopted.xhm());
return;
}
AssignRfreeFlags(merged, c.subgroup, experiment_.GetScalingSettings().GetRfreeFraction(), 500,
result.consensus_cell, config_.nthreads);
ApplyReferenceFreeFlags(merged, adopted, sm.merged);
const TwinLawFraction fraction = EstimateTwinLawFraction(merged, c.subgroup, c.law, friedel);
UnitCell cell = *result.consensus_cell;
// The subgroup's reference setting first: a subgroup of a centred group can be a primitive
// group written on a multiple of its cell - C 1 1 2_1 is P 1 2_1 1 on twice the volume - which
// no unimodular relabelling reaches. The reflections the centring extinguished have no index on
// the primitive cell and are dropped; the rest move exactly.
if (!(c.subgroup.basisop() == gemmi::Op::identity())) {
const gemmi::Op to_reference = c.subgroup.basisop().inverse();
const gemmi::Op m = HklOperator(to_reference);
const gemmi::SpaceGroup before = c.subgroup;
if (MoveTwinSubgroupChoice(c, to_reference)) {
std::erase_if(merged, [&](const MergedReflection &r) {
const auto h = m.apply_to_hkl_without_division({{r.h, r.k, r.l}});
return h[0] % gemmi::Op::DEN != 0 || h[1] % gemmi::Op::DEN != 0 || h[2] % gemmi::Op::DEN != 0;
});
merged = ReindexMergedIntoAsu(merged, m, c.subgroup, friedel);
cell = CellInBasis(cell, to_reference);
} else {
c.subgroup = before;
}
}
const gemmi::Op standard = ChooseOutputSetting(cell, c.subgroup);
if (!(standard == gemmi::Op::identity())) {
const gemmi::SpaceGroup before = c.subgroup;
if (MoveTwinSubgroupChoice(c, standard)) {
merged = ReindexMergedIntoAsu(merged, HklOperator(standard), c.subgroup, friedel);
cell = CellInBasis(cell, standard);
} else {
c.subgroup = before;
}
}
FrenchWilsonOptions fw_opts;
fw_opts.num_threads = config_.nthreads;
ApplyFrenchWilson(merged, c.subgroup, fw_opts);
twin_file = TwinFile{std::move(merged), cell, c.subgroup, c.law, fraction};
auto &t = result.twin_subgroup_merge.emplace();
t.reason = twin_choice_reason;
t.adopted_space_group = adopted.xhm();
t.space_group = c.subgroup.xhm();
t.file_prefix = config_.output_prefix + "_twin_" + c.subgroup.short_name();
t.twin_law = c.law.as_hkl().triplet();
t.twin_fraction = fraction.fraction;
t.twin_fraction_pairs = fraction.pairs;
t.other_subgroups = c.others;
logger.Info("Twin hypothesis: {} in {}, twin law {} at a fraction of at least {:.2f} ({} pairs) -> {}.mtz/.cif",
adopted.xhm(), c.subgroup.xhm(), t.twin_law, fraction.fraction, fraction.pairs,
t.file_prefix);
};
if (result.consensus_cell && write_files && !config_.model_path.empty() && !geometry_prepass
&& !superseded) {
phase("Validating against model");
Step("Model validation");
TimingMark mark(logger, "model validation");
const auto data_sg = experiment_.GetGemmiSpaceGroup();
// With a reference MTZ the merohedral indexing was already resolved against it (rotation
// merge / stills scaling), so trust that; only probe indexing by R-free when model-only.
// The merge's own shell bounds, so CC(model, data) is reported on the shells CC1/2 was.
std::vector<float> report_shell_d_min;
for (const auto &sh : sm.statistics.shells)
report_shell_d_min.push_back(sh.d_min);
// The validation in the model's setting further down depends on this one only through the
// setting and the indexing it settles, and both are forecast well before this one has finished
// (ModelFrameForecast). So it is started on that forecast, beside this one - on the next card
// where there is one - and kept only where the decision is the one forecast; its log is held,
// and its files wait, until then. On any other decision it is dropped, unwritten. The relabelling
// it runs on is the one AdoptModelFrame and relabel_output below make, made here on a copy.
// A model asserting the other enantiomorph is not forecast: whether the label is taken is
// only known at the end, from the anomalous map.
struct Ahead {
gemmi::Op change_of_basis_op, indexing_op;
// The reflections it validates against, relabelled as the files would be: where it is kept,
// they are the written ones, and the relabelling below is not made a second time.
std::vector<MergedReflection> merged;
Logger log = Logger::Buffered();
std::future<ModelValidationResult> run;
std::promise<bool> write; // destroyed before `run`, so a run left waiting on it is released
} ahead;
ModelValidationSchedule first_schedule;
first_schedule.on_forecast = [&](const ModelFrameForecast &f) {
if (ahead.run.valid() || !config_.reference_data.empty() || !data_sg.has_value()
|| !f.may_run_beside || f.enantiomorph_candidate
|| f.change_of_basis_op == gemmi::Op::identity())
return;
ahead.change_of_basis_op = f.change_of_basis_op;
ahead.indexing_op = f.indexing_op;
ahead.run = std::async(std::launch::async,
[&, merged = sm.merged, sg = experiment_.GetSpaceGroupOrP1(), cell = *result.consensus_cell,
friedel = experiment_.GetScalingSettings().GetMergeFriedel(),
rfree_fraction = experiment_.GetScalingSettings().GetRfreeFraction(),
wavelength = experiment_.GetWavelength_A(),
gate = ahead.write.get_future().share()]() mutable {
if (const int32_t cards = get_gpu_count(); cards > 0)
pin_gpu(1 % cards);
if (!(ahead.indexing_op == gemmi::Op::identity())) {
merged = ReindexMergedIntoAsu(merged, ahead.indexing_op, sg, friedel);
cell = CellInBasis(cell, HklOperator(ahead.indexing_op));
}
gemmi::Op to_model = ahead.change_of_basis_op;
to_model.tran = {0, 0, 0};
const gemmi::Op cob = to_model.inverse();
const gemmi::SpaceGroup *in = &sg;
if (const gemmi::SpaceGroup *moved = SpaceGroupInBasis(sg, cob)) {
merged = ReindexMergedIntoAsu(merged, HklOperator(cob), *moved, friedel);
cell = CellInBasis(cell, cob);
AssignRfreeFlags(merged, *moved, rfree_fraction, 500, cell, config_.nthreads);
in = moved;
}
ModelValidationSchedule schedule;
schedule.write_gate = gate;
ahead.merged = std::move(merged);
return ValidateAgainstModel(ahead.merged, cell, config_.model_path, config_.output_prefix, ahead.log,
in, /*probe_indexing_ambiguity=*/false,
static_cast<size_t>(config_.nthreads), wavelength,
report_shell_d_min, schedule);
});
};
// Whether the validation started on the forecast is the one kept: the decision is the one forecast.
// Asked by the first validation before it makes its maps, which are then left to that one, and
// again below with the verdict in hand - the same answer, as nothing it reads changes between.
const auto forecast_kept = [&](const ModelValidationResult &v) {
return ahead.run.valid() && config_.reference_data.empty() && v.model_fits
&& !(v.change_of_basis_op == gemmi::Op::identity()) && !v.adopted_model_enantiomorph
&& v.change_of_basis_op == ahead.change_of_basis_op && v.indexing_op == ahead.indexing_op;
};
first_schedule.maps_superseded = forecast_kept;
// The ledger's P1 merge (started above) is taken before anything below acts on what the
// validation decides: relabelling the outcomes and the group is what it must not see happen.
// Every such relabelling reaches the P1 merge afterwards, through merge_to_written.
// The merge is done with its engine - nothing below reads it but the finalist ledger's P1 merge,
// which runs on it beside the validation - so the device memory it holds goes back before the
// validation decides, from the card's total, whether its GPU path fits.
if (!ledger_ahead.valid())
rsm.reset();
ModelValidationResult validation =
ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
config_.output_prefix, logger, data_sg ? &*data_sg : nullptr,
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
static_cast<size_t>(config_.nthreads), experiment_.GetWavelength_A(),
report_shell_d_min, first_schedule);
take_ledger_p1();
// A model that was asked for and could not be used has to say so where anyone will see
// it. Without this the run ends successfully with no R-free, no maps and nothing in the
// report - indistinguishable from a run that was never given --model at all.
result.model_validation = validation;
if (!validation.failure_reason.empty())
result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED,
"Model validation did not run: " + validation.failure_reason});
// The ambiguity warning above was issued before the model was read; take it back only
// where the model actually settled the indexing.
if (validation.indexing_decided)
result.indexing_ambiguity_operators.clear();
if (validation.indexing_decided && !ambiguity_warning_text.empty()) {
std::erase_if(result.warnings, [](const ReportWarning &w) {
return w.code == PathologyCode::INDEXING_AMBIGUITY;
});
const auto pos = result.merge_statistics_text.find(ambiguity_warning_text);
if (pos != std::string::npos)
result.merge_statistics_text.erase(pos, ambiguity_warning_text.size());
logger.Info("Indexing ambiguity resolved against the model; warning withdrawn");
}
bool indexing_moved_cell = false;
if (data_sg.has_value()) {
const gemmi::SpaceGroup *adopted =
AdoptModelFrame(validation, sm.merged, *data_sg,
experiment_.GetScalingSettings().GetMergeFriedel(), logger);
if (adopted->number != data_sg->number) {
// The written reflections and the report that describes them, but deliberately not
// end_msg: the _process.h5 already holds the per-image reflections as they were
// integrated, and a file whose recorded group did not match its own reflections
// would mis-merge on a later --mode scale. An enantiomorphic pair merges identically
// anyway - same Laue class, same absences - so nothing is lost by leaving it.
experiment_.SetSpaceGroup(*adopted);
result.space_group = *adopted;
}
// The unmerged export below comes from the integrated observations, not from the merge,
// so it follows the same relabeling - otherwise the two files would describe different
// indexings of one dataset. The enantiomorph is a label, so only the indexing moves it.
// The cell moves with the indices: an alternative indexing that is only a near-symmetry of
// the lattice (b within 2% of c, beta within a degree of 90) permutes the axes or flips an
// angle, and the written cell has to be the one the written indices are on.
const gemmi::Op &op = validation.indexing_op;
if (!(op == gemmi::Op::identity())) {
const UnitCell before = *result.consensus_cell;
reseat_integrated(HklOperator(op));
indexing_moved_cell = !(*result.consensus_cell == before);
merge_to_written = merge_to_written.combine(op);
determined_to_written = determined_to_written.combine(op);
result.setting_operator = IndexTriplet(determined_to_written);
}
}
// A model that fits and is written on other axes than the data - the depositor's setting,
// an I-centred cell, an unreduced triclinic one - takes the files into its own setting,
// so the reflections and the deposited coordinates can be used together as they are. The
// validation is then made again on the relabelled data, so the maps and the placed model
// come out on the same axes as the reflections; what it had decided is kept, since the
// second one, facing a model already in the data's setting, has nothing left to decide.
// The same where the model's indexing moved the cell (a near-symmetry, above): the first
// validation placed the model in the cell as it was before.
const bool remake = config_.reference_data.empty() && validation.model_fits
&& (!(validation.change_of_basis_op == gemmi::Op::identity()) || indexing_moved_cell);
const bool forecast_held = forecast_kept(validation);
if (ahead.run.valid() && !forecast_held) {
ahead.write.set_value(false);
ahead.run.wait();
}
if (remake) {
gemmi::Op to_model = validation.change_of_basis_op;
to_model.tran = {0, 0, 0};
ModelValidationResult remade;
if (forecast_held) {
ahead.write.set_value(true);
remade = ahead.run.get();
relabel_output(to_model.inverse(), "MODEL", &ahead.merged);
ahead.log.ReplayInto(logger);
} else {
if (!(to_model == gemmi::Op::identity()))
relabel_output(to_model.inverse(), "MODEL");
const auto sg_now = experiment_.GetGemmiSpaceGroup();
remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
config_.output_prefix, logger,
sg_now ? &*sg_now : nullptr,
/*probe_indexing_ambiguity=*/false,
static_cast<size_t>(config_.nthreads),
experiment_.GetWavelength_A(), report_shell_d_min);
}
if (remade.ok) {
KeepModelVerdict(remade, validation);
validation = std::move(remade);
result.model_validation = validation;
} else if (forecast_held) {
logger.Warning("Model validation: the validation in the model's setting failed ({}), and the "
"first left its maps to it - no maps were written", remade.failure_reason);
}
}
// The coordinates that go with the maps just written, in the frame the reflection files
// beside them carry. Written whenever the validation ran, not only where the rigid-body
// step moved the model: it is also re-fractionalized into the data cell and may be
// relabelled to the model's enantiomorph, so "did not move" is not "identical to the input
// file". A rejected model is scored, placed and mapped like any other, and that is the case
// where someone most wants to see the model in the density, so it gets its file too.
// Written beside the reflection files below: nothing after this reads it.
// The model asked about the twin hypothesis too, where one was written: scored against the
// subgroup's merge, plainly and as twinned by its law (the twin target, at a scanned fraction),
// against the answer's merge as above. The maps and the placed model come from whichever merge
// the model fits better on R-free; the answer's files are the answer's either way - the model
// chooses the maps, never the merge (see docs/RUGNUX_REPORT.md).
prepare_twin_file();
bool maps_from_twin = false;
if (twin_file && validation.ok) {
const double answer_r_free = validation.r_free;
const auto twin_r_free = [](const ModelValidationResult &v) {
return v.twin_target ? std::min(v.r_free, v.r_free_twin) : v.r_free;
};
ModelValidationSchedule twin_schedule;
twin_schedule.maps_superseded = [&](const ModelValidationResult &v) {
return !(twin_r_free(v) < answer_r_free);
};
phase("Validating against model (twin hypothesis)");
auto twin_validation = ValidateAgainstModel(
twin_file->merged, twin_file->cell, config_.model_path, config_.output_prefix, logger,
&twin_file->space_group, /*probe_indexing_ambiguity=*/config_.reference_data.empty(),
static_cast<size_t>(config_.nthreads), experiment_.GetWavelength_A(), {}, twin_schedule,
&twin_file->law);
maps_from_twin = twin_validation.ok && twin_r_free(twin_validation) < answer_r_free;
// The subgroup file follows its own validation's indexing, as the answer follows its own:
// the file, the R-factors and the maps then describe one indexing.
if (twin_validation.ok) {
const gemmi::Op op = twin_validation.indexing_op;
twin_file->space_group = *AdoptModelFrame(twin_validation, twin_file->merged,
twin_file->space_group,
experiment_.GetScalingSettings().GetMergeFriedel(),
logger);
if (!(op == gemmi::Op::identity())) {
twin_file->law = op.inverse().combine(twin_file->law).combine(op);
twin_file->cell = CellInBasis(twin_file->cell, HklOperator(op));
}
result.twin_subgroup_merge->twin_law = twin_file->law.as_hkl().triplet();
result.twin_subgroup_merge->space_group = twin_file->space_group.xhm();
}
logger.Info("Model validation, twin hypothesis: R-free {:.4f} against the answer's merge in {}, "
"{:.4f} against the merge in {}{} - the maps and the placed model come from {}",
answer_r_free, experiment_.GetSpaceGroupOrP1().xhm(), twin_validation.r_free,
twin_file->space_group.xhm(),
twin_validation.twin_target
? fmt::format(" and {:.4f} as twinned at a fraction of {:.2f}",
twin_validation.r_free_twin, twin_validation.twin_fraction)
: std::string(),
maps_from_twin ? "the merge in " + twin_file->space_group.xhm() : "the answer's merge");
if (maps_from_twin && twin_validation.placed_model)
placed_model_file = std::async(std::launch::async,
[placed = twin_validation.placed_model, cell = twin_file->cell,
sg = twin_file->space_group, prefix = config_.output_prefix, &logger] {
WritePlacedModel(*placed, cell, sg, prefix, logger);
});
result.twin_subgroup_merge->maps_from_this_merge = maps_from_twin;
result.twin_subgroup_merge->model_validation = std::move(twin_validation);
}
if (validation.placed_model && !maps_from_twin)
placed_model_file = std::async(std::launch::async,
[placed = validation.placed_model, cell = *result.consensus_cell,
sg = experiment_.GetSpaceGroupOrP1(), prefix = config_.output_prefix, &logger] {
WritePlacedModel(*placed, cell, sg, prefix, logger);
});
}
prepare_twin_file();
// --mr (experimental): the model placed by molecular replacement, as many copies as the cell holds, in the
// groups the answer's merge cannot tell apart and in the twin hypothesis' subgroup. It takes the
// enantiomorph label into the answer, as --model does, and nothing else: a different group it
// chooses goes to files of its own beside the answer's, <prefix>_mr, from the merge it was chosen on.
struct ModelPriorFile {
std::vector<MergedReflection> merged;
UnitCell cell;
gemmi::SpaceGroup space_group;
std::optional<gemmi::Op> law;
TwinLawFraction fraction;
std::shared_ptr<gemmi::Structure> placed;
};
std::optional<ModelPriorFile> model_prior_file;
if (result.consensus_cell && write_files && !config_.mr_model_path.empty() && !geometry_prepass
&& !superseded) {
phase("Placing the predicted model");
Step("Molecular replacement");
TimingMark mark(logger, "model prior");
std::vector<MRMerge> merges{{"MERGE", &sm.merged, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}};
if (twin_file)
merges.push_back({"TWIN_SUBGROUP_MERGE", &twin_file->merged, twin_file->cell, twin_file->space_group,
twin_file->law});
MolecularReplacementResult mr = RunMolecularReplacement(config_.mr_model_path, merges, logger,
static_cast<size_t>(config_.nthreads));
if (!mr.ok)
result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED,
"The --mr model could not be searched for: " + mr.failure_reason});
if (mr.chosen >= 0) {
const MRCandidate &c = mr.candidates[mr.chosen];
if (mr.decision == "ENANTIOMORPH")
CheckEnantiomorphHand(mr, sm.merged, experiment_.GetSpaceGroupOrP1(), logger,
static_cast<size_t>(config_.nthreads));
if (mr.decision == "ENANTIOMORPH") {
// A change of label only (see ValidateAgainstModel): the two groups index identically.
experiment_.SetSpaceGroup(c.space_group);
result.space_group = c.space_group;
if (twin_file && twin_file->space_group.is_enantiomorphic()) {
gemmi::GroupOps e = twin_file->space_group.operations();
e.change_basis_forward(twin_file->space_group.change_of_hand_op());
if (const gemmi::SpaceGroup *other = gemmi::find_spacegroup_by_ops(e))
twin_file->space_group = *other;
}
logger.Info("Molecular replacement: the answer is relabelled {}, the enantiomorph the placed model chose "
"(fit {:+.1f} sd over its null, anomalous map at its atoms not inverted, {:.3f} s)",
c.space_group.xhm(), mr.hand_check.fit_sigma, mr.hand_check.seconds);
} else if (mr.decision == "SPACE_GROUP") {
ModelPriorFile f{sm.merged, *result.consensus_cell, c.space_group, std::nullopt, {}, c.placed};
const gemmi::GroupOps gops = c.space_group.operations();
std::erase_if(f.merged, [&](const MergedReflection &r) {
return gops.is_systematically_absent({{r.h, r.k, r.l}});
});
model_prior_file = std::move(f);
} else if (mr.decision == "TWIN_SUBGROUP" && twin_file) {
model_prior_file = ModelPriorFile{twin_file->merged, twin_file->cell, c.space_group, twin_file->law,
twin_file->fraction, c.placed};
}
}
if (model_prior_file)
result.model_prior_file_prefix = config_.output_prefix + "_mr";
if (twin_for_mr_only && mr.decision != "TWIN_SUBGROUP") {
twin_file.reset();
result.twin_subgroup_merge.reset();
}
result.model_prior = std::move(mr);
}
// The unmerged MTZ (written with the other output files, see there) is built from here on,
// beside the reflection files: the indexing and the group it is written in are final once the
// model has been read, and nothing after this changes the integrated observations it reads. It
// gets its own copy of the experiment, which it points into for its space group.
if (full && write_files && !geometry_prepass && !superseded && result.consensus_cell
&& config_.export_unmerged) {
unmerged_experiment = experiment_;
unmerged_mtz = std::async(std::launch::async,
[&outcomes = indexer->GetIntegrationOutcome(),
cell = *result.consensus_cell, &x = *unmerged_experiment,
&provenance = config_.provenance, nthreads = config_.nthreads] {
return UnmergedMtz(outcomes, cell, x, true, provenance, nthreads);
});
}
// Not on the geometry pre-pass, and not on a pass that is going to be re-run. Pass 1 exists to
// choose the space group and post-refine the geometry; the pass after it remakes these files
// seconds later at the refined geometry, and that is the answer anyone reads. Writing them
// twice costs 0.6 s of a 15 s run - a fifth of the merged mmCIF on a large crystal - for a file
// that is superseded before the run ends. The quality guard is unaffected: it reads the search
// merge, which is made well above this.
if (result.consensus_cell && write_files && config_.write_merged && !geometry_prepass
&& !superseded) {
phase("Writing reflections");
Step("Writing files");
TimingMark writing_mark(logger, "writing reflections");
const ErrorModelReport em_report{
result.error_model_isa > 0 && result.error_model_isa_resolved
? fmt::format("{:.2f}", result.error_model_isa) : "?",
result.error_model_isa_asymptotic > 0 && result.error_model_isa_resolved
? fmt::format("{:.2f}", result.error_model_isa_asymptotic) : std::string(),
result.error_model_a > 0 ? fmt::format("{:.3f}", result.error_model_a) : std::string(),
result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string()};
// The fulls carry the index they were measured at in the scaler's frame; every relabelling
// the merged reflections went through since (merge_to_written) takes them to the written one.
if (!(merge_to_written == gemmi::Op::identity()))
for (auto &f : sm.scaled_fulls) {
const gemmi::Op::Miller h = merge_to_written.apply_to_hkl({{f.h, f.k, f.l}});
f.h = h[0]; f.k = h[1]; f.l = h[2];
}
WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics,
em_report, result.twinning, config_.provenance, config_.output_prefix,
static_cast<size_t>(std::max(1, config_.nthreads)), &sm.scaled_fulls);
if (keep_merged_reflections_)
result.merged_reflections = sm.merged;
// The twin hypothesis beside it (see prepare_twin_file): the reflection files only, with the
// law and its fraction in the mmCIF's _pdbx_reflns_twin.
if (twin_file) {
DiffractionExperiment twin_x = experiment_;
twin_x.SetSpaceGroup(twin_file->space_group);
TwinningAnalysisResult law;
law.twin_law_operator = twin_file->law.as_hkl().triplet();
law.twin_law_fraction = std::isfinite(twin_file->fraction.fraction) ? twin_file->fraction.fraction : 0.5;
const ErrorModelReport twin_em{
twin_merge->isa > 0 && twin_merge->isa_resolved ? fmt::format("{:.2f}", twin_merge->isa) : "?",
std::string(), twin_merge->error_model_a > 0 ? fmt::format("{:.3f}", twin_merge->error_model_a)
: std::string(),
twin_merge->error_model_b > 0 ? fmt::format("{:.4e}", twin_merge->error_model_b)
: std::string()};
const std::string prefix = result.twin_subgroup_merge->file_prefix;
WriteMtzReflections(twin_file->merged, twin_file->cell, twin_x, config_.provenance, prefix + ".mtz");
WriteMmcifReflections(twin_file->merged, twin_file->cell, twin_x, twin_merge->statistics, twin_em,
law, config_.provenance, prefix + ".cif", static_cast<size_t>(std::max(1, config_.nthreads)));
}
// The predicted model's choice (see model_prior_file): its reflections, from the merge it was
// chosen on and labelled with the group it chose, and the model as it placed it.
if (model_prior_file) {
const ModelPriorFile &f = *model_prior_file;
DiffractionExperiment mr_x = experiment_;
mr_x.SetSpaceGroup(f.space_group);
TwinningAnalysisResult law;
const MergeStatistics *statistics = &sm.statistics;
ErrorModelReport mr_em = em_report;
if (f.law) {
law.twin_law_operator = f.law->as_hkl().triplet();
law.twin_law_fraction = std::isfinite(f.fraction.fraction) ? f.fraction.fraction : 0.5;
statistics = &twin_merge->statistics;
mr_em = ErrorModelReport{
twin_merge->isa > 0 && twin_merge->isa_resolved ? fmt::format("{:.2f}", twin_merge->isa) : "?",
std::string(), twin_merge->error_model_a > 0 ? fmt::format("{:.3f}", twin_merge->error_model_a)
: std::string(),
twin_merge->error_model_b > 0 ? fmt::format("{:.4e}", twin_merge->error_model_b)
: std::string()};
}
const std::string prefix = result.model_prior_file_prefix;
WriteMtzReflections(f.merged, f.cell, mr_x, config_.provenance, prefix + ".mtz");
WriteMmcifReflections(f.merged, f.cell, mr_x, *statistics, mr_em, law, config_.provenance, prefix + ".cif",
static_cast<size_t>(std::max(1, config_.nthreads)));
if (f.placed)
WritePlacedModel(*f.placed, f.cell, f.space_group, prefix, logger);
}
writing_mark.Stop();
// FINALIST LEDGER (--finalist-ledger), report-only and deliberately behind a flag: these
// numbers are an instrument, and reading them as a decision before they are calibrated is
// the one way this can do harm.
//
// It is folded from the P1 merge started above, because that is the substrate the whole
// space-group audit says is missing and nothing else in the run has: the same observations
// at full resolution, production-scaled with the correction surfaces fitted, and folded
// under NO symmetry - so every hypothesis can be folded from it on equal terms. It also
// carries half-set intensities, which is what lets the random-noise R floor be formed - the
// only reference available to the first step out of P1, and the reason this cannot be done
// from the stored MTZ afterwards. Say why there is none: a user who gave the flag and got
// no table would otherwise have to read this function to find out whether the instrument
// declined, or the crystal simply had nothing to report.
if (config_.finalist_ledger && !finalist_ledger)
logger.Warning("No finalist ledger: it is folded from a P1 merge, and this run makes none ({}).",
!is_rotation ? "stills - the ledger is rotation-only for now"
: "a fixed centred space group's absences were never predicted, so a P1 "
"merge would be missing whole centring classes");
if (finalist_ledger) {
TimingMark mark(logger, "finalist ledger");
take_ledger_p1();
// The scaler still holds the observations in the indexing they were merged in, so every
// relabelling since (the written setting, the model's indexing) is applied to this merge
// too, so it is read on the axes of the merged output and of the cell it is read with.
std::vector<MergedReflection> p1 = std::move(ledger_p1->merged);
if (!(merge_to_written == gemmi::Op::identity()))
p1 = ReindexMergedIntoAsu(p1, merge_to_written, gemmi::get_spacegroup_p1(),
experiment_.GetScalingSettings().GetMergeFriedel());
// Twin-immune zone evidence for each 2-fold the adopted group holds, read on this merge
// because it is folded under no symmetry, at full resolution and production-scaled.
// Report-only: nothing reads it back.
const auto determined = experiment_.GetGemmiSpaceGroup();
if (determined.has_value() && determined->number > 1) {
// The pseudo-translation the zone is normalised by is measured on THIS merge, not
// taken from `result.tncs`. That one is fitted on the adopted group's merge, and
// where the pseudo-translation is not compatible with the adopted symmetry that
// merge has already averaged across its own phase classes, so the vector fitted
// there is poorly determined - measured on one crystal, the correlation between E^2
// and cos(2 pi h.u) reads 0.04 on the group merge against 0.25 here, with the
// optimum elsewhere entirely, and a vector that names the wrong classes normalises
// the zone by noise. It is also the basis this merge is indexed on: a change of
// basis committed by the search leaves a vector measured before it pointing
// somewhere else.
const auto p1_tncs = AnalyzeTranslationalNCS(
p1, gemmi::UnitCell(*result.consensus_cell), nullptr, 30, 0, config_.nthreads);
result.twin_immune_zones = AnalyzeTwinImmuneZones(
p1, gemmi::UnitCell(*result.consensus_cell), *determined,
TranslationalNCSVectorForLTest(p1_tncs), config_.nthreads,
result.space_group_search ? result.space_group_search->twin_fraction_outside : 0.0);
if (!result.twin_immune_zones->zones.empty())
logger.Info("{}", TwinImmuneZonesToText(*result.twin_immune_zones));
}
SearchSpaceGroupOptions lopt;
lopt.nthreads = static_cast<size_t>(std::max(1, config_.nthreads));
lopt.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
lopt.cell = gemmi::UnitCell(*result.consensus_cell);
lopt.enumerate_all_settings = true;
lopt.enumerate_all_rotation_sets = true;
lopt.merge_isa = ledger_p1->isa; // the error model this merge was scaled with
const auto led = SearchSpaceGroup(p1, lopt);
logger.Info("Finalist ledger on the full-resolution P1 merge ({} reflections):\n{}",
p1.size(), FinalistLedgerToText(led));
}
}
if (placed_model_file.valid())
placed_model_file.get();
return false;
}
Reference in New Issue View Git Blame Copy Permalink