// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" #include "../image_analysis/bragg_integration/BraggStencil.h" // BRAGG_FOOTPRINT_NSIGMA #include "../image_analysis/hot_pixels/HotPixels.h" #include "../image_analysis/SensorAbsorption.h" #include "DiagnosticOutput.h" #include "RugnuxFirstPass.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../reader/JFJochHDF5Reader.h" #include "../common/JFJochMath.h" #include "../common/ParallelFor.h" #include "../common/Logger.h" #include "../common/AzimuthalIntegrationMapping.h" #include "../common/AzimuthalIntegrationProfile.h" #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" #include "../image_analysis/indexing/AnalyzeIndexing.h" #include "../image_analysis/indexing/HarmonicContamination.h" #include "../image_analysis/geom_refinement/BeamCenterFFT.h" #include "../image_analysis/geom_refinement/BeamCenterFromBackground.h" #include "../image_analysis/geom_refinement/GeometryRefiner.h" #include "../image_analysis/indexing/IndexerThreadPool.h" #include "../image_analysis/spot_finding/ImageSpotFinderCPU.h" #include "../image_analysis/spot_finding/SpotUtils.h" #include "gemmi/twin.hpp" #include "../image_analysis/azint/AzIntEngineCPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h" #ifdef JFJOCH_USE_CUDA #include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h" #include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h" #endif #include "../image_analysis/scale_merge/Merge.h" #include "../image_analysis/scale_merge/RfreeFlags.h" #include "../image_analysis/scale_merge/RotationScaleMerge.h" #include "../image_analysis/scale_merge/ResolutionCutoff.h" #include "../image_analysis/scale_merge/ReindexAmbiguity.h" #include "../image_analysis/scale_merge/ScalingResult.h" #include "../image_analysis/scale_merge/SearchSpaceGroup.h" #include "../image_analysis/geom_refinement/PostRefine.h" #include "../image_analysis/lattice_search/LatticeSearch.h" #include "../image_analysis/lattice_search/LePageLattice.h" #include "../image_analysis/scale_merge/AnisotropyAnalysis.h" #include "../image_analysis/scale_merge/FrenchWilson.h" #include "../image_analysis/scale_merge/TwinningAnalysis.h" #include "../image_analysis/scale_merge/TranslationalNCS.h" #include "../image_analysis/scale_merge/HKLKey.h" #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" #include "../image_analysis/scale_merge/SublatticeRelabel.h" #include "../image_analysis/WriteReflections.h" #include "../image_analysis/bragg_integration/CalcISigma.h" #include "../common/Definitions.h" #include "../common/CorrelationCoefficient.h" #include #include #include "RugnuxPipeline.h" #include "../image_analysis/scale_merge/CellPopulation.h" using namespace rugnux_internal; namespace { // How precisely the measured beam centre has to be placed before the post-refinement takes it as a // second place the centre is believed to be (the bound on its beam move). The ceiling is the larger // of this and what the geometry asks of the centre (BeamCenterNeed_pxl). constexpr float MAX_BEAM_CENTER_SIGMA_PXL = 1.0f; // The twin-immune zone of the operators `higher` adds over `lower`, read on a P1 merge, as a // verdict: +1 the added operators are real (the zone is centric with margin), -1 they are a twin // law or a pseudo-symmetry (acentric with margin), 0 undecided - fewer than two orders apart // (the immunity argument holds for one coset), no zone reflections, or within the margin. // // The margin is the screw-axis convention (SearchSpaceGroupOptions::min_screw_absence_evidence): // the same log-likelihood-ratio units, and the same bound of 20 nats. Read over three batteries // the zones of genuine promotions sit at +60 nats and above and the zones behind the known // twins at -290 nats and below, with genuine trigonal crystals at -46 and -206 nats whose // promotion no gate refused - twinned by a law other than the 2-folds, which reaches their zone. // Read at the twin fraction those other laws show (twin_fraction_outside) the -206 one reads // +209, and the search itself now asks the zone of a promotion on a merge that reads twinned // (SearchSpaceGroup's twin-immune zone test); here it is read where a gate refused. // // Read on the CALIBRATED evidence: the zone's less what the acentric control shows the // normalisation to add per reflection (TwinImmuneZoneResult::control_excess_per_reflection). The // raw evidence rescued two refused 622 promotions - a 6/m crystal and a 312 one, H 4.7x and 5.2x // the parent's, R_meas 0.09 -> 0.34 under the higher group - whose zones read centric only // because everything did: a 67 A^2 anisotropy and a pseudo-translation had the control at // +0.10 nats per reflection, the zone at the same, and +124 nats over 947 reflections was the // normalisation's. Calibrated they read -115 and -86; the genuine promotions measured keep +140 // and above. int TwinZoneVerdict(const std::vector &p1_merged, const gemmi::UnitCell &cell, const SearchSpaceGroupResult &higher, const SearchSpaceGroupResult &lower, ProcessResult &result, const std::array *tncs_vector = nullptr) { constexpr double DECISIVE_NATS = 20.0; if (!higher.point_group_representative || !lower.point_group_representative) return 0; if (higher.point_group_order != 2 * lower.point_group_order) return 0; const auto z = AnalyzeTwinImmuneZone(p1_merged, cell, *higher.point_group_representative, *lower.point_group_representative, tncs_vector, 1, higher.twin_fraction_outside); Logger logger("Rugnux"); if (z.zones.empty() || z.zones[0].n == 0) { logger.Info("Twin-immune zone {} over {}: no zone reflections in the shells with signal " "({} merged reflections read) - undecided", higher.point_group_hm, lower.point_group_hm, p1_merged.size()); return 0; } const auto &zone = z.zones[0]; // Where the control reads more compressed than any twin can make it (zones_ambiguous), something // else - the overlapping spots of a long axis - compresses the zone as well, and a genuine zone // reads acentric: no verdict either way, as SearchSpaceGroup's own zone gate reads it. A 622 // crystal with a very long axis, 95% of its spots overlapping and its control at 0.513, read its // genuine 2-folds at -319 nats and had the refusal of 622 stand on it. const int verdict = z.zones_ambiguous ? 0 : zone.calibrated_evidence_nats <= -DECISIVE_NATS ? -1 : zone.calibrated_evidence_nats >= DECISIVE_NATS ? +1 : 0; const std::string text = fmt::format( "{} over {}: operators {}; n {} <|E^2-1|> {:.3f} +- {:.3f} (control {:.3f}, {:+.3f} nats per " "reflection above acentric), centric read twinned at {:.2f}, evidence {:+.1f} nats, calibrated " "{:+.1f} ({:.2f}-{:.2f} A{}) -> {}", higher.point_group_hm, lower.point_group_hm, zone.operators, zone.n, zone.mean_abs_e2_minus_1, zone.standard_error, z.control.mean_abs_e2_minus_1, z.control_excess_per_reflection, z.twin_fraction, zone.evidence_nats, zone.calibrated_evidence_nats, z.d_max_A, z.d_min_A, z.tncs_normalised ? ", normalised per pseudo-translation class" : "", z.zones_ambiguous ? "undecided - the control is compressed beyond a perfect twin, so genuine symmetry would read as the zone does" : verdict < 0 ? "acentric, the added operators are a twin law or a pseudo-symmetry" : verdict > 0 ? "centric, the added operators are real" : "undecided"); // Recorded only where the zone DECIDED something. An undecided read decides nothing - the // rule stays as it was - and recording it had the report announce "a twin gate refused this // promotion, and the twin-immune zone decided it" over a line that says the opposite. if (verdict == 0) { logger.Info("Twin-immune zone {}", text); return 0; } result.twin_zone_decision = text; // The verdict as a VALUE, for the report to act on. Every caller acts on a decisive read - // the refusal stands or it is overturned - so recording it here records the decision, where // matching on the text records only the last comparison the run happened to make. result.twin_zone_refusal_stood = verdict < 0; return verdict; } // Angle between the crystal's symmetry axes and the spindle. A rotation sweep never records the // reflections whose reciprocal vector lies close to the spindle - the blind cusp - and normally // symmetry fills it in from an equivalent elsewhere in reciprocal space. It cannot when a symmetry // axis IS the spindle: the cusp then maps onto itself, every reflection in it is equivalent only to // other reflections in it, and the hole stays empty however long the sweep runs. Returns the // smallest angle in degrees between any proper rotation axis of the space group and the goniometer // axis, with the order of the axis that achieves it. std::optional> ClosestSymmetryAxisToSpindle(const gemmi::SpaceGroup &sg, const CrystalLattice &lattice, const Coord &spindle) { const double spindle_len = spindle.Length(); if (spindle_len < 1e-9) return std::nullopt; std::optional> best; for (const auto &op : sg.operations().derive_symmorphic().sym_ops) { if (op.rot == gemmi::Op::identity().rot) continue; // Order of the rotation, then project onto its invariant direction by summing its powers: // (1/n) sum_k W^k annihilates everything except the axis. const gemmi::Op w{op.rot, {0, 0, 0}, op.notation}; int order = 0; gemmi::Op cur = gemmi::Op::identity(); std::array, 3> sum{}; for (int k = 0; k < 6; ++k) { for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) sum[i][j] += static_cast(cur.rot[i][j]) / gemmi::Op::DEN; cur = cur.combine(w).wrap(); ++order; if (cur.rot == gemmi::Op::identity().rot) break; } if (order < 2) continue; // Any non-degenerate column of the projector is the axis in fractional direct coordinates. for (int c = 0; c < 3; ++c) { const Coord axis = lattice.Vec0() * static_cast(sum[0][c]) + lattice.Vec1() * static_cast(sum[1][c]) + lattice.Vec2() * static_cast(sum[2][c]); const double len = axis.Length(); if (len < 1e-6 * lattice.Vec0().Length()) continue; const double cosang = std::abs((axis * spindle) / (len * spindle_len)); const double ang = std::acos(std::min(1.0, cosang)) * 180.0 / PI; if (!best.has_value() || ang < best->first) best = std::make_pair(ang, order); break; } } return best; } std::vector SortedRotations(const gemmi::GroupOps &g) { std::vector r; for (const auto &op : g.sym_ops) r.push_back(op.rot); std::sort(r.begin(), r.end()); return r; } // The rotations of a model's group on the data's axes: one set for each change of basis that // carries the model's cell onto the data's (CellMappingOperators). Empty where the two cells // describe different lattices. std::vector> ModelRotationsOnDataAxes(const gemmi::SpaceGroup &model_group, const gemmi::UnitCell &model_cell, const gemmi::UnitCell &data_cell) { std::vector> out; for (const gemmi::Op &op : CellMappingOperators(model_cell, data_cell)) { gemmi::GroupOps g = model_group.operations(); g.change_basis_forward(op); const auto r = SortedRotations(g); if (std::find(out.begin(), out.end(), r) == out.end()) out.push_back(r); } return out; } // The subgroup a twin-law merge is made in (_twin_), and the law it carries, where the // adopted group may have absorbed a twin law. Candidates are the index-2 subgroups of `group` // (IndexTwoTwinSubgroups); where a decision of the search named one - the group a refused promotion // was adopted over, or the indexed lattice's group a metric re-ask promoted from - only that one, // and otherwise those the search confirmed as point-group finalists (the ledger's eligible rows). // Given the rotations of a model's group (`model`, ModelRotationsOnDataAxes), only a subgroup with // exactly those rotations instead: the merge the model's own symmetry asks for. // Among them, the one whose law agrees worst with the intensities (lowest Stage A CC): the // two-fold most likely to be the twin law. The rest are named for the report. struct TwinSubgroupChoice { gemmi::SpaceGroup subgroup; gemmi::Op law; std::vector others; }; std::optional ChooseTwinSubgroup(const gemmi::SpaceGroup &group, const SearchSpaceGroupResult &search, const std::optional &named, const std::vector> &model = {}) { const auto rotations = [](const gemmi::SpaceGroup &g) { return SortedRotations(g.operations()); }; const auto law_cc = [&](const gemmi::Op &law) { const std::string t = law.as_hkl().triplet('h'); for (const auto &s : search.operator_scores) if (s.op_triplet_hkl == t) return s.cc; return std::numeric_limits::infinity(); }; std::optional best; double best_cc = 0.0; std::vector names; for (const auto &t : IndexTwoTwinSubgroups(group)) { if (t.subgroup.operations().sym_ops.size() < 2) continue; // P1: a twin of it is not what a promotion absorbs const auto rots = rotations(t.subgroup); const bool qualifies = !model.empty() ? std::find(model.begin(), model.end(), rots) != model.end() : named ? rotations(*named) == rots : std::any_of(search.point_group_ledger.begin(), search.point_group_ledger.end(), [&](const PointGroupLedgerEntry &e) { return e.eligible && e.representative && rotations(*e.representative) == rots; }); if (!qualifies) continue; names.push_back(t.subgroup.xhm()); const gemmi::Op *law = &t.laws.front(); for (const auto &l : t.laws) if (law_cc(l) < law_cc(*law)) law = &l; if (!best || law_cc(*law) < best_cc) { best = TwinSubgroupChoice{t.subgroup, *law, {}}; best_cc = law_cc(*law); } } if (best) for (const auto &n : names) if (n != best->subgroup.xhm()) best->others.push_back(n); return best; } // A twin-law choice on other axes: the group by SpaceGroupInBasis, the law conjugated by the // Miller-index operator the change of basis moves the reflections by. False where no setting in // GEMMI's table names the moved group. bool MoveTwinSubgroupChoice(TwinSubgroupChoice &c, const gemmi::Op &cob) { const gemmi::SpaceGroup *moved = SpaceGroupInBasis(c.subgroup, cob); if (moved == nullptr) return false; const gemmi::Op m = HklOperator(cob); c.subgroup = *moved; c.law = m.inverse().combine(c.law).combine(m); // h' = m h: back by m, the law, forward by m return true; } // `group` carried into the setting a Miller-index reindex q (h' = q h) takes the merge to, with the // centring that setting describes - the move the metric re-ask makes with the lattice. Empty where // no setting in GEMMI's table names the result. std::optional GroupInReindexedSetting(const gemmi::SpaceGroup &group, const gemmi::Mat33 &q, char centring) { try { gemmi::Op hkl = gemmi::Op::identity(); for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) hkl.rot[i][j] = static_cast(std::lround(q[j][i])) * gemmi::Op::DEN; // apply_to_hkl uses rot^T gemmi::Op cob = hkl.inverse(); for (int i = 0; i < 12; ++i) for (int j = 0; j < 12; ++j) for (int k = 0; k < 12; ++k) { cob.tran = {i * gemmi::Op::DEN / 12, j * gemmi::Op::DEN / 12, k * gemmi::Op::DEN / 12}; gemmi::GroupOps gops = group.operations(); gops.change_basis_forward(cob); gops.cen_ops = gemmi::centring_vectors(centring); if (const gemmi::SpaceGroup *found = gemmi::find_spacegroup_by_ops(gops)) return *found; } } catch (const std::exception &) { } return std::nullopt; } } // The molecular replacement of a live run started on the geometry pre-pass's merge (Rugnux.h): the merge it // searches - the pre-pass's P1 merge averaged into a group of its own search, on the axes the answer's files // would be written on - and the search, its log held until the answer's merge reads it. struct Rugnux::EarlyMolecularReplacement { std::vector merged; UnitCell cell; gemmi::SpaceGroup space_group; Logger log = Logger::Buffered(); std::future run; }; // A P1 merge averaged into the asymmetric unit of `sg`, each reflection weighted by 1/sigma^2; the // reflections `sg` makes absent left out. static std::vector AverageIntoGroup(const std::vector &p1, const gemmi::SpaceGroup &sg, const UnitCell &cell) { const HKLKeyGenerator key_gen(/*merge_friedel=*/true, sg); std::map, std::pair> sums; // sum of w I, sum of w for (const MergedReflection &r : p1) { if (!std::isfinite(r.I) || !(r.sigma > 0.0f) || key_gen.IsSystematicallyAbsent(r.h, r.k, r.l)) continue; const HKLKey key = key_gen(r.h, r.k, r.l); const double w = 1.0 / (static_cast(r.sigma) * r.sigma); auto &[wi, ws] = sums[{key.h, key.k, key.l}]; wi += w * r.I; ws += w; } const gemmi::UnitCell uc(cell); std::vector out; for (const auto &[hkl, sum] : sums) { MergedReflection m; m.h = hkl[0]; m.k = hkl[1]; m.l = hkl[2]; m.I = static_cast(sum.first / sum.second); m.sigma = static_cast(1.0 / std::sqrt(sum.second)); m.d = static_cast(uc.calculate_d({{m.h, m.k, m.l}})); out.push_back(m); } return out; } // Proper rotations of a crystal system's holohedry - what a lattice of that class carries, and the // bound SearchSpaceGroup caps its point-group candidates at. static size_t HolohedryRotations(gemmi::CrystalSystem system) { switch (system) { case gemmi::CrystalSystem::Triclinic: return 1; case gemmi::CrystalSystem::Monoclinic: return 2; case gemmi::CrystalSystem::Orthorhombic: return 4; case gemmi::CrystalSystem::Trigonal: return 6; case gemmi::CrystalSystem::Tetragonal: return 8; case gemmi::CrystalSystem::Hexagonal: return 12; case gemmi::CrystalSystem::Cubic: return 24; } return 1; } // The lattice METRIC can carry more rotational symmetry than the group the intensities supported. // That is not by itself a mistake - a pseudo-symmetric metric is ordinary, and only the intensities // can say whether the extra rotations are real - but it is where the corpus's symmetry under-calls // live, so the run says it out loud rather than leaving the user to notice. It decides nothing: no // threshold, no promotion, no demotion. // // The metric symmetry is Le Page's, and its obliquity is the continuous measure of how nearly a // two-fold holds; 1 degree is the middle of the band over which the answer is stable. // Following Le Page (1982) J. Appl. Cryst. 15, 255-259 static void WarnIfMetricHostsMoreSymmetry(const std::optional &cell, const gemmi::SpaceGroup &sg) { if (!cell) return; Logger logger("Rugnux"); const gemmi::UnitCell gc(cell->a, cell->b, cell->c, cell->alpha, cell->beta, cell->gamma); const size_t metric_rotations = gemmi::find_lattice_symmetry(gc, sg.centring_type(), LATTICE_MAX_OBLIQUITY_DEG).sym_ops.size(); const size_t group_rotations = sg.operations().sym_ops.size(); if (metric_rotations <= group_rotations) return; // The proper rotations of each holohedry, which is what find_lattice_symmetry returns. const char *metric_system = "a higher"; switch (metric_rotations) { case 2: metric_system = "monoclinic"; break; case 4: metric_system = "orthorhombic"; break; case 6: metric_system = "trigonal"; break; case 8: metric_system = "tetragonal"; break; case 12: metric_system = "hexagonal"; break; case 24: metric_system = "cubic"; break; default: break; } logger.Warning("The cell metric is {} - it admits {} rotations where {} has {}. Whether the extra " "ones were tested against the intensities depends on the lattice class the search " "was given: where that class is lower than the metric, they were never enumerated, " "so their absence here is not evidence against them. This is a note, not a " "correction. Where the higher symmetry is real, merging in it would raise " "multiplicity and completeness.", metric_system, metric_rotations, sg.xhm(), group_rotations); } bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { auto &logger = p.logger; auto &result = p.result; auto &observer = p.observer; auto &geometry_prepass = p.geometry_prepass; auto &full = p.full; auto &images_to_process = p.images_to_process; auto &prepass_images = p.prepass_images; auto &write_output = p.write_output; auto &plots = p.plots; auto *const writer = p.sink; // a _process.h5 open: that file is written whole by every pass that opens it auto &indexer = p.indexer; auto &ambiguity_reference = p.ambiguity_reference; auto &superseded = p.superseded; auto &census_future = p.census_future; auto &start_time = p.start_time; auto &end_msg = p.end_msg; auto &unmerged_mtz = p.unmerged_mtz; // Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress // bar reflects what is happening instead of freezing on one label. auto phase = [&](const std::string &p) { if (observer) observer->OnPhase(p); }; phase("Scaling and merging"); Step(geometry_prepass ? "Scaling and post-refinement" : "Scaling and merging"); // Ice-ring handling (--detect-ice-rings): flag reflections sitting on a hexagonal-ice powder // ring. Their integrated intensity is contaminated by the strong, variable ice background, so // they drag the per-image scale fit. Flag them here so scaling (the per-image G fit and the // fulls refit) skips them, while the combine, the merge and the statistics keep them - dropping // them outright guts low/mid-resolution completeness on crystals that merge them fine. // The eleven hexagonal-ice bands cover 16-26% of the unique reflections at typical resolutions // whether or not the crystal has any ice, so handling ice unconditionally taxes clean data for // nothing. Gate the flagging on the run's own measured ice strength (1 = no ice; the strongest // ring over the smooth radial background, measured on the peak-excluded profile so Bragg peaks // cannot pose as ice). The score is logged either way, so a run that gates off still reports why. // Ice is looked for on TWO channels, because it arrives in two forms. Fine-grained ice is a // smooth powder ring and shows up in the radial profile (ice_ring_score). Ice in large // crystallites arrives as discrete reflections instead, leaves the profile flat, and is only // visible as a pile-up of found SPOTS on the ring positions - measured against the same q width // of ice-free flanks beside them, so the fall-off of spot density with resolution cancels. On // the rotation battery the two channels barely overlap: the crystals with smooth ice read // 2.1-2.4 / ~1.0, the ones whose spots are mostly ice read ~1.1 / 3.8-14.6, and a clean crystal // reads 1.04 on both. Either channel alone is blind to half the cases. const auto ice_score = plots.GetIceRingScore(); const auto ice_spot_ratio = plots.GetIceRingSpotRatio(); const float ice_min_score = experiment_.GetScalingSettings().GetIceMinScore(); const float ice_min_spot_ratio = experiment_.GetScalingSettings().GetIceMinSpotRatio(); const bool ice_present = (!ice_score.has_value() || *ice_score >= ice_min_score) || (ice_min_spot_ratio > 0.0f && ice_spot_ratio.has_value() && *ice_spot_ratio >= ice_min_spot_ratio); result.ice_ring_score = ice_score; result.ice_spot_ratio = ice_spot_ratio; const bool flag_ice = experiment_.IsDetectIceRings() && ice_present; const float ice_width = config_.spot_finding.ice_ring_width_Q_recipA; // Overloads, for the report: the integrated reflections a saturated signal pixel made unmeasurable. // Counted before anything below drops or flags one. The merge leaves them out (a rotation merge // drops the whole rocking event) - they are the strongest reflections, mostly at low resolution, // and what a model's low-resolution terms rest on. The ice flags are set in the same sweep. Integer // counts, one image per task: the totals do not depend on the split. if (experiment_.IsRotationIndexing()) { const auto [at_start, at_end] = TrimBlankSweepEnds(indexer->GetIntegrationOutcome(), config_.nthreads); result.blank_start_frames = at_start; result.blank_end_frames = at_end; if (at_start + at_end > 0) logger.Info("Sweep ends: left out {} frame(s) at the start and {} at the end that recorded no " "diffraction - a shutter or beam that was not on them", at_start, at_end); } // Stills given a cell: only the frames whose refined cell belongs to its population are merged // (see GivenCellPopulation) - a second crystal form, fitted to the same cell, is left out. if (!experiment_.IsRotationIndexing() && stills_user_cell_) { auto &outcomes = indexer->GetIntegrationOutcome(); const double given = CrystalLattice(*stills_user_cell_).CalcVolume(); std::vector rel; for (const auto &o : outcomes) if (!o.reflections.empty()) rel.push_back(o.latt.CalcVolume() / given - 1.0); const auto [low, high] = GivenCellPopulation(rel); std::string all; for (double r : rel) all += fmt::format(" {:.4f}", r); logger.Debug("Given cell: refined volumes relative to it:{}", all); size_t below = 0, above = 0; for (auto &o : outcomes) { if (o.reflections.empty()) continue; const double r = o.latt.CalcVolume() / given - 1.0; if (r < low || r > high) { o.reflections.clear(); (r < low ? below : above)++; } } if (below + above > 0) { logger.Info("Given cell: {} of {} indexed frames refined to a cell outside its population " "({:+.1f}% to {:+.1f}% in volume) and are left out of the merge - {} smaller, {} larger", below + above, rel.size(), 100 * low, 100 * high, below, above); // The cell written is the mean over the frames merged, not over the other form's as well. result.consensus_cell = indexer->GetConsensusUnitCell(); end_msg.unit_cell = result.consensus_cell; } } size_t ice_flagged = 0, ice_total = 0; { auto &outcomes = indexer->GetIntegrationOutcome(); std::atomic n_all{0}, n_low{0}, n_over{0}, n_over_low{0}, n_ice{0}; ParallelFor(static_cast(outcomes.size()), config_.nthreads, [&](int o) { int64_t all = 0, low = 0, over = 0, over_low = 0, on_ice = 0; for (auto &r : outcomes[o].reflections) { const bool low_res = r.d >= OVERLOAD_LOW_RES_A; ++all; low += low_res; if (r.overloaded) { ++over; over_low += low_res; } if (flag_ice) { r.on_ice_ring = IsOnIceRing(r.d, ice_width); on_ice += r.on_ice_ring; } } n_all += all; n_low += low; n_over += over; n_over_low += over_low; n_ice += on_ice; }); result.integrated_reflections += n_all; result.integrated_reflections_low_res += n_low; result.overloaded_reflections += n_over; result.overloaded_reflections_low_res += n_over_low; ice_flagged = n_ice; ice_total = n_all; if (result.integrated_reflections > 0) logger.Info("Overloads: {} of {} integrated reflections ({:.3f}%) held a saturated pixel, {:.3f}% of " "the {} coarser than {:.0f} A", result.overloaded_reflections, result.integrated_reflections, 100.0 * result.overloaded_reflections / result.integrated_reflections, result.integrated_reflections_low_res > 0 ? 100.0 * result.overloaded_reflections_low_res / result.integrated_reflections_low_res : 0.0, result.integrated_reflections_low_res, OVERLOAD_LOW_RES_A); } if (experiment_.IsDetectIceRings() && !ice_present) { logger.Info("Ice-ring handling: measured ice score {:.2f} < {:.2f} and spot ratio {:.2f} < " "{:.2f}, no ice detected - ice-ring handling skipped entirely", *ice_score, ice_min_score, ice_spot_ratio.value_or(NAN), ice_min_spot_ratio); } else if (flag_ice) { result.ice_rings_detected = true; result.ice_flagged_fraction = ice_total > 0 ? static_cast(ice_flagged) / ice_total : 0.0; logger.Info("Ice-ring handling: ice score {:.2f} (gate {:.2f}), spot ratio {:.2f} (gate " "{:.2f}); flagged {} of {} reflections on ice rings (half-width {:.3f} A^-1); " "excluded from scaling, kept for merging", ice_score.value_or(NAN), ice_min_score, ice_spot_ratio.value_or(NAN), ice_min_spot_ratio, ice_flagged, ice_total, ice_width); } // Scale the images and merge. Factored so it can run twice: first in P1 to give the // space-group search a merged dataset, then again in the determined space group so the // scaling sees symmetry equivalents and the final statistics are in the right symmetry. // (With a user-fixed space group it simply runs once, already in that symmetry.) struct ScaleMergeResult { std::vector merged; MergeStatistics statistics; // CC1/2 as measured before the correction surfaces (rotation) - the quantity the two-pass // quality guard in Run() compares the passes on. Equal to the reported CC1/2 wherever no // surface was fitted, which is every merge on the stills path. double cc_half_before_corrections = NAN; // The reference-range table and its ISa (--report-resolution); see ProcessResult. std::optional reference_statistics; double reference_isa = 0.0; // The merge's fulls unmerged, for the .hkl (rotation, written merges only - see below). std::vector scaled_fulls; }; // The reference path computes each image's G once (per-image scaling against the // reference); the scaling loop below is skipped, so G is stable across the two passes. // Smoothing it more than once would compound the correction, so do it only on the first // pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly. // Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single // allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame // partials, scales the fulls and merges; the whole hot path runs on the GPU when one is present). // Ingested once here (after ice flagging) and reused across both space-group passes; a fixed // (forced) mosaicity is handled by the recompute. The decay + absorption correction surfaces run as // post-scale-fulls stages (on by default, see ScalingSettings::CorrectionSurfaces); external- // reference scaling, the stills -B (per-image B-factor) and wedge refinement are unsupported here. // Re-seat the integrated reflections into a different setting of the SAME lattice, for the two // arms below. The change of basis is read straight off the two lattices: P[i][j] = // conv_real[i] . indexed_reciprocal[j] is the coefficient of the indexed cell's j-th axis in the // new cell's i-th axis, and indices transform with the axes, so it is also the matrix that takes // hkl across. Rounding it to integers - and declining when the residual is large - keeps it exact // even though the refined cell is not exactly LatticeSearch's Niggli cell. // // The per-image reflections and lattices are already in the _process.h5 by now - they go to file // as each image is processed, long before the group is known - so re-seating them here leaves the // stored ones in the OLD setting beside a cell in the new one. Accumulate the change of basis // (composed, in case more than one fires) and hand it to the writer, so the file says which setting // its per-image data are in and a reader can bring them across. gemmi::Mat33 stored_to_final; // identity until something below re-seats const auto record_reindex = [&](const gemmi::Mat33 &m) { stored_to_final = m.multiply(stored_to_final); std::array v{}; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) v[3 * i + j] = static_cast(std::lround(stored_to_final.a[i][j])); end_msg.reindex_matrix = v; }; const auto reindex_into = [&](const LatticeSearchResult &cand) { const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()}; const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(), end_msg.rotation_lattice->Cstar()}; gemmi::Mat33 reindex; double reindex_res = 0.0; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) { const double v = cv[i] * rs[j]; reindex.a[i][j] = std::round(v); reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j])); } if (reindex_res >= 0.1) return false; for (auto &io : indexer->GetIntegrationOutcome()) { io.latt = io.latt.Multiply(reindex); for (auto &r : io.reflections) { const double h = r.h, k = r.k, l = r.l; r.h = static_cast(std::lround(reindex.a[0][0]*h + reindex.a[0][1]*k + reindex.a[0][2]*l)); r.k = static_cast(std::lround(reindex.a[1][0]*h + reindex.a[1][1]*k + reindex.a[1][2]*l)); r.l = static_cast(std::lround(reindex.a[2][0]*h + reindex.a[2][1]*k + reindex.a[2][2]*l)); } } result.consensus_cell = cand.conventional.GetUnitCell(); result.consensus_centering = cand.centering; end_msg.unit_cell = result.consensus_cell; end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reindex); end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand.centering, .niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system }; result.rotation_lattice_type = end_msg.rotation_lattice_type; record_reindex(reindex); return true; }; // The same for a change of basis of the SAME cell - a relabelling of its axes (CrystalSetting.h): // the integrated reflections, their lattices and the cell go across together. const auto reseat_integrated = [&](const gemmi::Op &cob) { const gemmi::Mat33 basis = BasisMatrix(cob); const gemmi::Op hkl = HklOperator(cob); // One image per task: each touches only its own outcome. auto &outcomes = indexer->GetIntegrationOutcome(); ParallelFor(static_cast(outcomes.size()), config_.nthreads, [&](int o) { auto &io = outcomes[o]; io.latt = io.latt.Multiply(basis); for (auto &r : io.reflections) { const gemmi::Op::Miller h = hkl.apply_to_hkl({{r.h, r.k, r.l}}); r.h = h[0]; r.k = h[1]; r.l = h[2]; } }); result.consensus_cell = CellInBasis(*result.consensus_cell, cob); end_msg.unit_cell = result.consensus_cell; if (end_msg.rotation_lattice.has_value()) end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(basis); record_reindex(basis); }; // A space group the USER fixed carries its own Bravais lattice, and that is the setting its // reflections have to be indexed in - but the indexer answers to the metric, not to the group, and // LatticeSearch hands back the MOST symmetric setting the metric supports. A tetragonal-P lattice // therefore never comes back C-centred orthorhombic even though the same lattice has that setting, // and -S C222 on one merges with the C absence rule applied in the wrong frame: measured, half the // observations thrown away (2.14 M -> 1.08 M) and ISa 28.3 -> 24.6. Ask the character table for the // group's own class instead and re-seat the reflections into it. // // Two ways an indexed lattice can fail to carry the fixed group's Bravais lattice, and one // re-seating for both. The CENTRING can differ, as above. Or the centring can agree while the // METRIC does not - the axes permuted, a=37.909 b=78.031 c=77.594 declared P 41 21 2, which // needs a=b with the 4-fold along c. That one is invisible to a centring test and merges // anyway, in a frame the reflections are not in: measured, --mode scale -S P41212 on such a // run reports COMPLETENESS= 195.3. // A fixed group of lower symmetry than the lattice but with the same centring (-S P21 on an // orthorhombic-P lattice, say) satisfies both tests where it stands, merges correctly, and is // left alone. So is a group the RUN determined - it was determined FROM this lattice, and the // arm above and the check below are its handling; a de-novo run must behave exactly as it did. if (user_fixed_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) { const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_; const char indexed_centering = end_msg.rotation_lattice_type->centering; const double violation = end_msg.unit_cell ? MetricViolation(*end_msg.unit_cell, fixed_sg) : 0.0; if (indexed_centering != fixed_sg.centring_type() || violation > MAX_METRIC_VIOLATION) { // A trigonal-P group sits on a hexagonal-P lattice; every other system names its own. const auto want_system = fixed_sg.crystal_system() == gemmi::CrystalSystem::Trigonal && fixed_sg.centring_type() == 'P' ? gemmi::CrystalSystem::Hexagonal : fixed_sg.crystal_system(); // Search the PRIMITIVE cell. rotation_lattice is the conventional one and its centring // is carried beside it, but the character table is reached through a Niggli reduction // that takes whatever it is handed to be primitive - so a centred conventional cell // reduces as a lattice with the wrong point set. This arm is entered exactly when the // centrings differ, which includes every centred indexed lattice, so the distinction is // not academic: a C-centred orthorhombic conventional cell read as primitive is itself a // perfectly good oP metric, and -S P212121 on one would match, reindex by the identity // and merge in a group whose absence rule deletes half the reflections that exist. // The other lattice arithmetic in this file goes through ToPrimitive for the same reason. const auto cand = LatticeSearchForClass( end_msg.rotation_lattice->ToPrimitive(indexed_centering), want_system, fixed_sg.centring_type()); if (cand && reindex_into(*cand)) { const auto &uc = *result.consensus_cell; logger.Info("Reindexed the indexed {}-centred lattice (metric violation {:.3f} against " "the group's own operators) into the {}-centred setting the fixed space " "group {} needs: a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} " "gamma={:.2f}", indexed_centering, violation, cand->centering, fixed_sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); } } } // ...and which of its settings. The centring and the metric are now the group's, but where the // group's axes are not all alike - P 21 21 2 has its pure two-fold on c - the metric cannot // say which axis of the cell is which: an orthorhombic cell ordered by length puts the pure // axis wherever its length puts it, and merging there applies the screw absences to the wrong // rows and writes a wrong label. The absences say which, so they are asked, on every axial row // and zone of the integrated reflections (partials summed). A group all of whose settings on // this cell predict the same absences is left exactly where it is. if (user_fixed_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value() && result.consensus_cell.has_value() && end_msg.rotation_lattice_type->centering == user_fixed_sg_->centring_type() && MetricViolation(*result.consensus_cell, *user_fixed_sg_) <= MAX_METRIC_VIOLATION) { std::map, std::pair> summed; // I and sigma^2 per index for (const auto &io : indexer->GetIntegrationOutcome()) for (const auto &r : io.reflections) if ((r.h == 0 || r.k == 0 || r.l == 0) && std::isfinite(r.I) && std::isfinite(r.sigma) && r.sigma > 0.0f && !r.overloaded) { auto &sum = summed[{r.h, r.k, r.l}]; sum.first += r.I; sum.second += static_cast(r.sigma) * r.sigma; } std::vector axial; axial.reserve(summed.size()); for (const auto &[hkl, sum] : summed) axial.push_back({hkl, sum.first / std::sqrt(sum.second)}); const gemmi::Op cob = SeatGroupByAbsences(*result.consensus_cell, *user_fixed_sg_, axial); if (!(cob == gemmi::Op::identity())) { reseat_integrated(cob); const auto &uc = *result.consensus_cell; logger.Info("The fixed space group {} names its symmetry elements by axis, and the absences " "on the {} axial and zonal reflections put them on other axes of this cell than " "the indexing did: reindexed by {} to a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} " "beta={:.2f} gamma={:.2f}", user_fixed_sg_->xhm(), axial.size(), IndexTriplet(HklOperator(cob)), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); } } // The lattice a pass indexed has to carry the Bravais lattice of the group it merges in. Only a // group the USER fixed can fail that - one the pass determined for itself was determined FROM // this lattice - and the re-seating above has already had its chance to find a setting that // does. What comes out otherwise is not a merely suboptimal answer: the absence rule is applied // across a frame the reflections are not in, and the statistics stop being arithmetic (measured: // 173.5% complete on a triclinic-P lattice merged in C2, and an undefined R_meas on an F-centred // cubic one merged in a trigonal-P group). // // The two-pass second pass is flagged and not adopted rather than refused: its lattice comes from // a de-novo re-index at the post-refined geometry, and the first pass - whose lattice did carry // the group - is still there to go back to. A pass with nothing to fall back on refuses instead; // the group is the user's assertion, and quietly determining a different one would answer a // question that was not asked. Name the cell that WAS indexed so the user can act on it. if (user_fixed_sg_ && end_msg.rotation_lattice_type.has_value() && end_msg.unit_cell.has_value()) { const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_; const bool centring_wrong = end_msg.rotation_lattice_type->centering != fixed_sg.centring_type(); // The same two tests as the re-seating above, asked again now that it has had its chance. const double violation = MetricViolation(*end_msg.unit_cell, fixed_sg); if (centring_wrong || violation > MAX_METRIC_VIOLATION) { const auto &uc = *end_msg.unit_cell; const std::string why = centring_wrong ? fmt::format("its lattice is {}-centred and this crystal indexes as {}-centred", fixed_sg.centring_type(), end_msg.rotation_lattice_type->centering) : fmt::format("this crystal's cell is not one the group's own operators leave " "invariant (metric violation {:.3f}, tolerance {:.3f}) - the symmetry " "axes do not run along the axes the group puts them on", violation, MAX_METRIC_VIOLATION); if (prepass_result_) { logger.Warning("Two-pass: the second pass indexed a lattice the fixed space group {} " "cannot describe - {} - so this pass will not be adopted", fixed_sg.xhm(), why); result.lattice_conflicts_with_fixed_sg = true; } else { throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, fmt::format( "The space group {} was fixed for this run, but {}: the crystal indexes as a " "{}-centred {} lattice (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} " "gamma={:.2f}), and no setting of that lattice carries the group's. Merging in {} " "would apply its symmetry to reflections that are not in its frame, so nothing it " "reported would describe this crystal - a completeness above 100% is the usual " "symptom. Re-run without a fixed space group to have it determined from the data, " "or fix one whose lattice this crystal has.", fixed_sg.xhm(), why, end_msg.rotation_lattice_type->centering, gemmi::crystal_system_str(end_msg.rotation_lattice_type->crystal_system), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, fixed_sg.xhm())); } } } const auto &rot_ss = experiment_.GetScalingSettings(); const bool is_rotation = experiment_.IsRotationIndexing(); // rotation indexing -> rotation scaling/merge std::optional rsm; std::optional guard_merge; // what the quality guard reads, where not the search merge std::optional subset_merge; // the pre-pass subset's, for a beam-centre arm (below) std::optional prepass_postrefine_obs; // The canonical pass's post-refinement gather, run beside the chain (see there): set once it has // read the reflections, which nothing may change before. std::shared_ptr> postrefine_gathered; std::shared_future postrefine_gathered_future; // ... and it starts once the scaling engine has its copy of them: both read every reflection, and the // engine's ingest is what the chain waits for. Released after the ingest, or as this pass ends. std::promise engine_ingested; const std::shared_future engine_ingested_future = engine_ingested.get_future().share(); struct ReleaseGather { std::promise &p; bool done = false; void operator()() { if (!done) { done = true; p.set_value(); } } ~ReleaseGather() { (*this)(); } } release_gather{engine_ingested}; // The rotation geometry post-refinement (see its call sites below for what it is for), in two // parts. prepare_post_refine reads the pass - the outcomes, the indexer, the scaling engine - and // returns the fit with everything it needs; fit_post_refine reads only that, so the canonical pass // can run it beside its merge chain (below). struct PostRefineJob { RotationIndexerResult rot; PostRefineSettings prs; PostRefineObservations obs; }; // gather = false leaves the job's observations for the caller to gather. const auto prepare_post_refine = [&](size_t max_images = SIZE_MAX, bool gather = true) -> std::optional { const auto &outcomes = indexer->GetIntegrationOutcome(); if (geometry_prepass) { prepass_mosaicity_.assign(outcomes.size(), NAN); std::vector fitted; for (size_t o = 0; o < outcomes.size(); ++o) if (outcomes[o].mosaicity_deg) { prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg; fitted.push_back(*outcomes[o].mosaicity_deg); } // A pre-pass of part of the sweep has fitted nothing for the frames after it, and they are // predicted at the median of what it did fit, as the merge fills its own frames without a // fit. Left at the frame's own estimate alone, the second half of a small-molecule sweep // predicted a fifth fewer reflections and merged at R_meas 7.2 % against 5.2 %. if (!fitted.empty() && outcomes.size() < static_cast(images_to_process)) { std::nth_element(fitted.begin(), fitted.begin() + fitted.size() / 2, fitted.end()); prepass_mosaicity_.resize(images_to_process, fitted[fitted.size() / 2]); } } if (result.consensus_cell.has_value()) { const auto rot = indexer->FinalizeRotationIndexing(); indexer->RethrowIndexerError(); if (rot && rot->axis) { if (geometry_prepass) prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback PostRefineSettings prs; prs.crystal_system = rot->search_result.system; prs.num_threads = static_cast(config_.nthreads); prs.refine_geometry = true; // Only a pass still at the file's distance can hold it as a hypothesis; a pass // that has walked off it (the geometry walk in RunAllPasses) is past that question. prs.distance_at_header = !std::isfinite(header_distance_mm_) || std::fabs(experiment_.GetDetectorDistance_mm() - header_distance_mm_) < 1e-3 * header_distance_mm_; // The centre the pre-scan measured, where the fit was precise enough to be used as // a centre at all (MAX_BEAM_CENTER_SIGMA_PXL). It does // not move the geometry - it only tells the bound on step B's beam move where else // the centre is independently believed to be. JoinBeamCenterCapture(); if (measured_beam_center_ && measured_beam_center_->sigma_pxl <= std::max(MAX_BEAM_CENTER_SIGMA_PXL, BeamCenterNeed_pxl(experiment_))) prs.measured_beam_px = std::array{ static_cast(measured_beam_center_->beam_x_pxl), static_cast(measured_beam_center_->beam_y_pxl)}; // The first pass's own geometry, not the header's. The outcomes being post-refined // were integrated at it, and step B holds the tilt fixed while it fits the beam and // the distance - so handing it the header tilt fits a beam that has to absorb a tilt // error the images do not have, and the answer describes neither geometry. // // On the pre-pass the observations were gathered right after Ingest (see there) and // the scaling engine is finished - nothing after this point reads it on that pass // (the search and the reference re-merge run only on the canonical one) - so it is // freed BEFORE the fit: the engine holds about twice the partial payload again, // and with it gone the fit no longer sits on the pass's memory peak. An engine that // read the reflections in its merges left the gather to here, and from here nothing // on the pass reads them either: each image's is handed back as it is gathered. const bool release = geometry_prepass && rsm && config_.reference_data.empty(); if (prepass_postrefine_obs || release) rsm.reset(); auto pr_obs = prepass_postrefine_obs ? std::move(*prepass_postrefine_obs) : gather ? GatherPostRefineObservations(indexer->GetIntegrationOutcome(), config_.nthreads, /*release_reflections=*/release, logger, max_images) : PostRefineObservations{}; prepass_postrefine_obs.reset(); return PostRefineJob{*rot, prs, std::move(pr_obs)}; } } return std::nullopt; }; const auto fit_post_refine = [this, &result = p.result, geometry_prepass = p.geometry_prepass] (PostRefineJob job, Logger &logger, std::vector ¬es) { const auto &rot = job.rot; const auto pr = PostRefineRotationGeometry(std::move(job.obs), *rot.axis, rot.geom, rot.lattice, job.prs, logger); result.post_refine = pr; if (pr.ok && pr.detector_refined) { // Store, don't apply: this pass continues to write its own (header-geometry) output; // Run() applies the refined geometry before the second pass. prepass_detector_geometry_ = std::array{ static_cast(pr.beam_x_after_px), static_cast(pr.beam_y_after_px), static_cast(pr.distance_after_mm), rot.geom.GetPoniRot1_rad(), rot.geom.GetPoniRot2_rad()}; logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, " "beam {:.2f},{:.2f} px, tilt {:.4f},{:.4f} deg) - {}", pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px, rot.geom.GetPoniRot1_rad() * 180.0 / PI, rot.geom.GetPoniRot2_rad() * 180.0 / PI, geometry_prepass ? "it will drive the second integration pass" : "what the fit still asks for at the geometry " "this pass ran at"); notes.push_back(fmt::format("{}: {:.2f} mm, beam {:.1f}, {:.1f}", geometry_prepass ? "geometry refined" : "post-refinement asks", pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px)); } else { logger.Info("Two-pass: geometry post-refine committed no detector change{}", geometry_prepass ? " - the second pass reproduces the first" : ""); notes.push_back("geometry post-refinement: no detector change committed"); } // The goniometer rotation scale in pr is only a fit: RunAllPasses walks it to its // fixed point and decides it on the validation frames of the whole sweep // (WalkRotationScale). It does not ride on pr.ok - the crystals that have a stage // fault are exactly the ones whose cell and detector steps do NOT pass, because the // angle error is what their residual is made of. }; const auto post_refine_geometry = [&] { TimingMark mark(logger, "post-refinement"); if (auto job = prepare_post_refine()) { std::vector notes; fit_post_refine(std::move(*job), logger, notes); for (const auto ¬e : notes) Note(note); } }; // On a CANONICAL pass the post-refinement only measures (see below), and it reads nothing the // scaling engine produces: the reflections' hkl, I, sigma and positions are final once the // images are integrated, and the engine writes back per-frame fields only. So it is gathered // here, beside the engine's ingest, and fitted beside the merges that follow. // The quality guard judges a refined pass against the pre-pass on the search merge, and a // pre-pass that integrated only the first part of the sweep merged only that part. Against // it a merge of the whole sweep holds more reflections, more signal and more of the axial // rows for no reason but the frames, and the guard could never find it worse. So the guard // reads this pass's own integration of the same frames, merged the way the pre-pass merged // them. It only reads the outcomes (the per-frame scale is not written back), and so does the // post-refinement, so it runs beside it; its log lines are held and replayed where it is used. const bool merge_first_part = is_rotation && quality_guard_pass1_ && prepass_images < images_to_process && !experiment_.GetRefineRotationWedgeInScaling() && !rot_ss.GetRotationWedgeForScaling().has_value(); // With two GPUs or more the guard's merge has an engine of its own on the second card, and the // main engine does not wait for it: its ingest and first merge run beside the guard's merge. With // one, the guard's merge runs first and the main engine waits for the card (see below). The // guard reads the outcomes only while it ingests - the first merge writes per-frame fields back // into them (mosaicity among them, which the ingest reads) - so that merge holds its write-back // until the guard has ingested, and nothing else waits. Neither merge reads anything the other // writes, and on cards of one model a merge gives the same bits on either, so this changes only // when the guard runs. const bool guard_on_second_gpu = get_gpu_count() >= 2; std::promise guard_ingested; Logger guard_log = Logger::Buffered(); const auto merge_guard = [&] { TimingMark mark(guard_log, "merge quality guard (pre-pass frames)"); RotationScaleMerge first_part_rsm(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, guard_log); first_part_rsm.SetFrameLimit(prepass_images); first_part_rsm.SetWriteBackPerFrameScale(false); if (guard_on_second_gpu) first_part_rsm.SetGpuDevice(1); { // Set however the ingest ends: a guard that failed rethrows from its future, later. struct Ingested { std::promise &p; ~Ingested() { p.set_value(); } } ingested{guard_ingested}; first_part_rsm.Ingest(); } first_part_rsm.SetExportScaledFulls(false); first_part_rsm.SetFrenchWilson(false); auto r = first_part_rsm.Run(!experiment_.GetGemmiSpaceGroup().has_value(), /*full_stats=*/false, /*measure_cc_before_corrections=*/true); return ScaleMergeResult{std::move(r.merged), std::move(r.statistics), r.cc_half_before_corrections}; }; // What the host holds from here on (HostMemory.h): the reflections, and beside them the engines below. // The counts are known now, before anything is ingested, so each concurrency below is decided on them: // where two pieces of work would not fit the host budget together, one waits for the other. None of // them reads anything another writes while it runs (see each), so only the order changes. size_t n_reflections = 0, n_reflections_guard = 0; { const auto &outcomes = indexer->GetIntegrationOutcome(); for (size_t o = 0; o < outcomes.size(); ++o) { n_reflections += outcomes[o].reflections.size(); if (static_cast(o) < prepass_images) n_reflections_guard += outcomes[o].reflections.size(); } } const size_t reflection_bytes = n_reflections * sizeof(Reflection); const auto host_gb = [](size_t bytes) { return static_cast(bytes) / 1e9; }; std::future guard_ahead; if (merge_first_part && !(postrefine_probe_ && !geometry_prepass && postrefine_probe_only_)) { // The guard's merge runs beside the main engine's ingest. Where an engine of the guard's frames and // one of all the frames would not fit the budget together - counted at the most an engine of that // many reflections can hold - the guard merges first, on its own, and the ingest starts after it. const bool on_gpu = get_gpu_count() > 0; const size_t beside = reflection_bytes + RotationScaleMerge::HostBytesBound(n_reflections_guard, on_gpu) + RotationScaleMerge::HostBytesBound(n_reflections, on_gpu); if (beside <= host_budget_ || guard_on_second_gpu) guard_ahead = std::async(std::launch::async, [&] { if (guard_on_second_gpu) pin_gpu(1); // the card and its NUMA node, as an image-loop worker has them return merge_guard(); }); else { logger.Info("Host memory: the quality guard's merge and this pass's ingest need up to {:.1f} GB side by " "side, over the {:.1f} GB budget - the guard merges first", host_gb(beside), host_gb(host_budget_)); guard_merge = merge_guard(); } } // The scaling engine's ingest reads the outcomes and nothing else either, so its host half runs // beside the post-refinement too; with one GPU the device half waits for the guard to hand the // card back. Logger engine_log = Logger::Buffered(); std::future engine_ahead; if (is_rotation && postrefine_probe_ && !geometry_prepass && !postrefine_probe_only_ && !experiment_.GetRefineRotationWedgeInScaling() && !rot_ss.GetRotationWedgeForScaling().has_value()) engine_ahead = std::async(std::launch::async, [&] { rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, engine_log, config_.observation_dump); rsm->IngestHost(); }); // The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the // comparison reads the held-out residual of a post-refinement on the frames the demoted arm // integrated, so that fit runs here, before the merge, and the pass stops if the demoted arm // realised the lower residual. The fit only measures and leaves nothing on the run; otherwise the // pass goes on exactly as the canonical pass it is, measuring its own post-refinement over all // the frames where it measures one anyway (postrefine_probe_). const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass && is_rotation && !postrefine_probe_only_; if (lattice_arm_asked) { LatticeArmCheck &arm = *lattice_arm_check_; arm.asked = true; const auto arm_fit = [&](size_t max_images) -> std::optional { TimingMark mark(logger, "post-refinement (metric-symmetry arm)"); auto job = prepare_post_refine(max_images); if (!job) return std::nullopt; return PostRefineRotationGeometry(std::move(job->obs), *job->rot.axis, job->rot.geom, job->rot.lattice, job->prs, logger); }; arm.promoted = arm_fit(arm.prepass_frames ? static_cast(prepass_images) : SIZE_MAX); if (!cancelled_ && arm.promoted && arm.demoted) { arm.lost = HeldOutResidualFell(*arm.promoted, *arm.demoted); // Decided on the start of the sweep, a near tie is re-checked on all of it once the run is // over (RefinedPasses), against this pass's own fit over every frame, taken while its // reflections are still here. The margin is the residual's distance from the line // HeldOutResidualFell draws, against the noise that draws it. const double noise = std::hypot(arm.promoted->held_out_before_se, arm.demoted->held_out_before_se); const double margin = arm.promoted->held_out_before - arm.demoted->held_out_before - noise; arm.near_tie = arm.prepass_frames && std::isfinite(noise) && std::abs(margin) <= FirstPassRun::NEAR_TIE_SIGMA * noise; logger.Info("Metric-symmetry check: on the first {} images the held-out residual is {:.3e} +- {:.1e} " "as the promoted class against {:.3e} +- {:.1e} as the demoted one{}", arm.prepass_frames ? prepass_images : images_to_process, arm.promoted->held_out_before, arm.promoted->held_out_before_se, arm.demoted->held_out_before, arm.demoted->held_out_before_se, arm.near_tie ? " - a near tie, re-checked on all the frames after the run" : ""); if (arm.near_tie) arm.promoted_all = arm_fit(SIZE_MAX); } if (arm.lost) { if (census_future.valid()) census_future.wait(); result.processing_time_s = std::chrono::duration( std::chrono::steady_clock::now() - start_time).count(); logger.Info("Metric-symmetry check: {} images integrated and post-refined in {:.2f} s - the " "demoted class realises the lower held-out residual, so this pass does not merge", result.images_processed, result.processing_time_s); if (!writer) return true; } } if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ && !(lattice_arm_asked && lattice_arm_check_->lost)) { // Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined // it), so it runs beside the chain, and so does the gather before it: the gather reads only // the reflections, which nothing changes until the chain re-indexes them in place once the // space group is settled - and the chain waits for the gather (postrefine_gathered) before // its search. The speculative probe the fit may start runs beside the chain too; the probe's // copy of the run is taken now, as the chain goes on to change the run's members. TimingMark mark(logger, "post-refinement observations"); if (auto job = prepare_post_refine(SIZE_MAX, /*gather=*/false)) { std::shared_ptr probe_run; if (speculate_geometry_probe_) probe_run = std::make_shared(*this); postrefine_gathered = std::make_shared>(); postrefine_gathered_future = postrefine_gathered->get_future().share(); p.postrefine_ahead = std::async(std::launch::async, [this, &p, fit_post_refine, job = std::move(*job), probe_run, gathered = postrefine_gathered, engine_ingested_future]() mutable { { // Set however the gather ends: a gather that failed rethrows from the future, later. struct Gathered { std::promise &p; ~Gathered() { p.set_value(); } } done{*gathered}; engine_ingested_future.wait(); TimingMark gather_mark(p.postrefine_log, "post-refinement gather"); job.obs = GatherPostRefineObservations(p.indexer->GetIntegrationOutcome(), config_.nthreads, /*release_reflections=*/false, p.postrefine_log); } TimingMark fit_mark(p.postrefine_log, "post-refinement"); fit_post_refine(std::move(job), p.postrefine_log, p.postrefine_notes); fit_mark.Stop(); if (probe_run && !cancelled_ && prepass_detector_geometry_ && p.result.post_refine && p.result.post_refine->detector_refined && !p.result.post_refine->large_move) StartSpeculativeGeometryProbe(probe_run, *prepass_detector_geometry_); }); } } if (postrefine_probe_ && !geometry_prepass && is_rotation && postrefine_probe_only_) { post_refine_geometry(); if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine && result.post_refine->detector_refined && !result.post_refine->large_move) StartSpeculativeGeometryProbe(std::make_shared(*this), *prepass_detector_geometry_); // A pass the lattice arms run to compare two Bravais classes has now produced the only // thing the comparison reads - the held-out residual of the fit just measured. Stop here // rather than scale, search, fit surfaces and merge for a result nothing looks at. result.processing_time_s = std::chrono::duration( std::chrono::steady_clock::now() - start_time).count(); logger.Info("Probe pass: {} images integrated and post-refined in {:.2f} s " "(the comparison reads the post-refinement only, so this pass does not merge)", result.images_processed, result.processing_time_s); return true; } // A pass RunAllPasses re-runs at the fixed radius for a starved background ring: its merge, // search and files are all replaced by the re-run's, and what anything reads from it - the // post-refinement and validation evidence above, and bkg_starved_fraction_ - is in hand. The // pass after it starts from the group the run was asked for (top of RunPipeline), so the // space group this pass would have determined is not carried either. Not with a _process.h5 // writer open: that file is written whole by every pass that opens it. if (rerun_if_starved_ && !writer && !cancelled_ && bragg_adaptive_ && bkg_starved_fraction_ && *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION) { if (census_future.valid()) census_future.wait(); result.processing_time_s = std::chrono::duration( std::chrono::steady_clock::now() - start_time).count(); logger.Info("Starved pass: {} images integrated in {:.2f} s - {:.2f}% of the reflections " "lost their background ring, so this pass is re-run at the fixed radius and " "does not merge", result.images_processed, result.processing_time_s, 100.0 * *bkg_starved_fraction_); return true; } // The all-observation arm of the space-group search (below) is the same ingest merged with the // Lorentz filter off. No merge depends on the merges made before it (RotationScaleMerge::RunArm), // so it is merged on a copy of the engine, started here and read where the search needs it: beside // the P1 merge, the re-scaled search merge and the first arm. Started whenever the search will run; // a search that turns out not to need it (no point group from the first arm) does not read it. // Declared before the future, which waits for the merge when it goes, so the engine outlives it. std::unique_ptr rsm_all; Logger all_arm_log = Logger::Buffered(); std::future all_arm_ahead; const auto start_all_arm = [&rsm_all, &all_arm_log] { return std::async(std::launch::async, [&rsm_all, &all_arm_log] { pin_gpu(rsm_all->GetGpuDevice()); // the card and its NUMA node, as the guard's merge has them TimingMark mark(all_arm_log, "merge P1, all observations"); auto r = rsm_all->Run(/*for_search=*/true, /*full_stats=*/true, /*measure_cc_before_corrections=*/false); // This engine merges once: what is left for it to do is the write-back of its // per-frame scales, so its observations and its merge's working arrays go now, // rather than through the search the others are about to run. rsm_all->ReleaseAllButPerFrameScale(); return r; }); }; if (is_rotation) { if (experiment_.GetRefineRotationWedgeInScaling() || rot_ss.GetRotationWedgeForScaling().has_value()) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Rotation scaling/merging (RotationScaleMerge) does not support " "wedge refinement"); if (merge_first_part) logger.Info("Two-pass: the quality guard reads this pass's merge of the first {} images, " "the ones the pre-pass integrated", prepass_images); // A geometry pre-pass that a beam-centre arm follows merges the pre-pass subset as well - the // frames that arm integrates first - for the two centres to be compared on the same frames // (RunAllPasses). The other frames are taken out of the outcomes while its engine ingests, and // it is merged before this pass's own engine is on the card. The run made again compares // the two on all the frames. if (geometry_prepass && beam_center_alternative_ && !measured_centre_arm_ && !decide_all_late_) { TimingMark mark(logger, "search merge (pre-pass subset)"); auto &outcomes = indexer->GetIntegrationOutcome(); const std::vector subset = PrepassSubset(prepass_images); std::vector held(outcomes.size()); const auto swap_others = [&] { for (size_t o = 0; o < outcomes.size(); ++o) if (o >= subset.size() || !subset[o]) std::swap(held[o], outcomes[o]); }; RotationScaleMerge subset_rsm(experiment_, outcomes, result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, logger); subset_rsm.SetWriteBackPerFrameScale(false); swap_others(); try { subset_rsm.Ingest(); } catch (...) { swap_others(); throw; } swap_others(); subset_rsm.SetExportScaledFulls(false); subset_rsm.SetFrenchWilson(false); auto r = subset_rsm.Run(!experiment_.GetGemmiSpaceGroup().has_value(), /*full_stats=*/false, /*measure_cc_before_corrections=*/true); subset_merge = ScaleMergeResult{std::move(r.merged), std::move(r.statistics), r.cc_half_before_corrections}; } // A reference MTZ is allowed for rotation: it fixes the space group / cell (on the CLI) and // resolves the indexing ambiguity (below), but is NOT used to scale - the rotation merge stays // self-consistent. if (engine_ahead.valid()) engine_ahead.get(); else { rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, engine_log, config_.observation_dump); rsm->IngestHost(); } // With one card the guard's merge has the card to itself first. if (merge_first_part && !guard_on_second_gpu) { if (!guard_merge) guard_merge = guard_ahead.valid() ? guard_ahead.get() : merge_guard(); guard_log.ReplayInto(logger); } rsm->IngestDevice(); const int32_t cards = get_gpu_count(); // On a card of its own from three cards up (card 0 has this engine, card 1 the quality guard's // and the speculative passes). Otherwise on the last card, beside the engine already there - // this one, or the guard's, which merges at most as many observations - and only where two // engines of this size fit the card's memory, judged on its total, not on what happens to be // free: the answer must not depend on the moment. What else stays on the card meanwhile - the // context, the run's indexers - was measured at up to 2.3 GB; a speculative pass running beside // holds more, but a merge that does not fit beside it waits for it to end. On a 16 GB card that // is the data sets up to about 90 M observations at high multiplicity; the large ones merge // their arms one after the other. const int32_t all_arm_device = std::min(cards - 1, 2); constexpr size_t CARD_BYTES_RESERVED = size_t(3) << 30; const bool all_arm = !geometry_prepass && !experiment_.GetGemmiSpaceGroup().has_value() && rsm->GetSearchMinZeta() > 0.0 && cards > 0 && rsm->OnGPU() && (cards >= 3 || 2 * rsm->DeviceBytes() + CARD_BYTES_RESERVED <= get_gpu_total_memory(all_arm_device)); // What the merge holds of the card from here on - this engine and the arm's - said before the // post-refinement is let go: a speculative probe it starts runs beside the merge only where both // fit the card (GPUMerge). Counted as if on one card. if (p.gpu_merge) { const size_t merge_bytes = rsm->OnGPU() ? (all_arm ? 2 : 1) * rsm->DeviceBytes() : 0; logger.Debug("Merge on the GPU: {:.2f} GB", merge_bytes / 1e9); p.gpu_merge->Holds(merge_bytes); } release_gather(); engine_log.ReplayInto(logger); rsm->SetLogger(logger); // The canonical pass's post-refinement gathers (from the moment the engine has ingested) and fits // beside the merges that follow. Where its gather and the selection it ranks would not fit the host // budget beside the reflections, this engine and its first merge, it is done before that merge. const size_t host_engine = reflection_bytes + rsm->HostIngestBytes(); const size_t host_merge = rsm->HostMergeBytes(); constexpr size_t POSTREFINE_RANKED_BYTES = 16; // PostRefineRotationGeometry's ranking, per partial const size_t host_postrefine = n_reflections * (sizeof(PostRefinePartial) + POSTREFINE_RANKED_BYTES); bool postrefine_beside = p.postrefine_ahead.valid(); if (postrefine_beside && host_engine + host_merge + host_postrefine > host_budget_) { logger.Info("Host memory: the post-refinement ({:.1f} GB) does not fit beside the merge ({:.1f} GB, " "{:.1f} GB held) within the {:.1f} GB budget - it is done before the merge", host_gb(host_postrefine), host_gb(host_merge), host_gb(host_engine), host_gb(host_budget_)); p.postrefine_ahead.wait(); postrefine_beside = false; } bool all_arm_beside = false; if (all_arm) { rsm_all = rsm->CloneIngest(all_arm_device, all_arm_log); rsm_all->SetSearchMinZeta(0.0); rsm_all->SetExportScaledFulls(false); // as scale_and_merge sets them for a search merge rsm_all->SetFrenchWilson(false); // ... and beside this engine's merges only where two merges fit the host budget as well, // with the post-refinement where it is still running; otherwise it merges where the search // first reads it, after this engine's merges, on its own. const size_t host_beside = host_engine + 2 * host_merge + (postrefine_beside ? host_postrefine : 0); all_arm_beside = host_beside <= host_budget_; if (!all_arm_beside) logger.Info("Host memory: two P1 merges side by side need about {:.1f} GB, over the {:.1f} GB " "budget - the all-observation merge is made after this engine's", host_gb(host_beside), host_gb(host_budget_)); } if (rsm_all) { // Its observations are copied from this engine's card arrays, not staged from the // reflections a second time. rsm_all->IngestDevice(&*rsm); if (all_arm_beside) all_arm_ahead = start_all_arm(); } } // The geometry pre-pass reads the per-image reflections exactly twice more: they were just // ingested above, and the post-refinement at the end of the pass reads a narrow // (hkl, frame, I, sigma, observed position) view of them. Nothing else in the pre-pass // touches them - its reflections are never written, and the scaling engine reads back only // per-frame fields - so take the post-refine's view NOW and hand each image's vector back as // it is read: on a fine-sliced long axis the vectors are gigabytes, held to the end of the // pass for those 32 bytes a partial. A run with a reference MTZ keeps them: resolving the // indexing ambiguity re-reads them (and re-ingests). So does an engine that scales on the host, // which reads them in its merges (RotationScaleMerge::ReadsReflections): there the view is taken // once the engine is done with them (prepare_post_refine). if (geometry_prepass && rsm && config_.reference_data.empty() && !rsm->ReadsReflections()) prepass_postrefine_obs = GatherPostRefineObservations( indexer->GetIntegrationOutcome(), config_.nthreads, /*release_reflections=*/true, logger); // What a rotation merge leaves in the result, taken in the order the merges are made - which, for // the all-observation arm merged beside the others, is where the search reads it. const auto take_merge = [&](const std::string &label, bool for_search, RotationScaleMerge::Result r) -> ScaleMergeResult { result.error_model_isa = r.isa; result.error_model_isa_asymptotic = r.isa_asymptotic; result.error_model_isa_resolved = r.isa_resolved; if (r.resolution_fit_A) result.resolution_fit_A = r.resolution_fit_A; result.error_model_a = r.error_model_a; result.error_model_b = r.error_model_b; result.scaling_iterations_partials = r.scaling_iterations_partials; result.scaling_iterations_fulls = r.scaling_iterations_fulls; result.scaling_converged = r.scaling_converged; if (for_search) result.search_scaling_converged = r.scaling_converged; Note(fmt::format("merge in {}: {} unique", label, r.merged.size())); return ScaleMergeResult{std::move(r.merged), std::move(r.statistics), r.cc_half_before_corrections, std::move(r.reference_statistics), r.reference_isa, std::move(r.scaled_fulls)}; }; // measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a // merge of its own - only the first merge, which the two-pass quality guard reads. auto scale_and_merge = [&](const std::string &label, bool for_search, bool measure_cc = false) -> ScaleMergeResult { TimingMark mark(logger, "merge " + label); if (rsm) { phase("Scale/combine/merge (" + label + ")"); // The geometry pre-pass merges only to choose the space group and to give the quality // guard something to judge the second pass against; its reflections are never written, // so the parts of the merge that only fill in an output file are skipped there. // Only a merge that may be written hands its fulls back: they are a copy as long as the // fulls, and the search merges and the pre-pass never write a .hkl. rsm->SetExportScaledFulls(!for_search && !geometry_prepass && write_output && config_.write_merged); // Amplitudes are made once, on the merge that is written, where the anisotropy tensor // their prior uses is known (see below) - not on every merge the tail makes on its way. rsm->SetFrenchWilson(false); return take_merge(label, for_search, rsm->Run(for_search, /*full_stats=*/!geometry_prepass, /*measure_cc_before_corrections=*/measure_cc)); } // Stills (rotation goes through RotationScaleMerge above): self-scale each image against the // running merge with ScaleOnTheFly (fixed partiality), then merge directly. This runs even // with an external reference: the reference is used only to break the per-image indexing // ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor (scaling each // image against a foreign dataset injects cross-dataset systematics and is a worse reference // than the data's own merge). One pass: the per-image scale G is the exact one-pass solution, // so iterating would only rebuild the reference from the freshly-scaled (noisy) data and // degrade weak stills (measured CC1/2 collapse at the default 3 iters). { phase("Scaling images (" + label + ")"); TimingMark scale_mark(logger, "stills scaling images " + label); auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads, result.consensus_cell); indexer->ScaleAllImages(merge_result); } // Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal // orientation tilt against the running merge and recompute each reflection's partiality + scale // correction (no re-integration). Never for the P1 search pass. if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) { phase("Partiality post-refine (" + label + ")"); TimingMark refine_mark(logger, "stills partiality post-refine " + label); StillsPartialityRefine refiner(experiment_); const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads, result.consensus_cell); logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt); // The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally // when the image has no prior CC - which is exactly the state the guard above leaves. So // the guard was protecting nothing: measured on a jet dataset, of 367 images it left // unscaled only 8 were still unscaled in the output, and 53 reached the merge at or below // a fiftieth of the run median, the worst at a 4525th. Whatever produced the final scale, // it is the final scale that has to be credible. ScaleOnTheFly::RejectCollapsedScales(indexer->GetIntegrationOutcome()); } const std::vector &merge_input = indexer->GetIntegrationOutcome(); phase("Merging"); TimingMark merge_mark(logger, "stills error model and merge " + label); MergeOnTheFly merge_engine(experiment_); merge_engine.Threads(config_.nthreads); // For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged // intensities and the error model, so the space-group search sees clean data; the final // in-symmetry merge keeps them so completeness is not lost. merge_engine.ExcludeIceRings(for_search); if (result.consensus_cell.has_value()) merge_engine.ReferenceCell(*result.consensus_cell); // Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells) // most indexed crystals do not correlate with the true structure; --min-image-cc drops the // crystals whose per-image CC to the reference is below the limit, so the merge keeps only // the coherent (real) ones. Set before the error model so the model, the merge and the // reported statistics are all fitted over the same set of images. The P1 search pass keeps // every image: its job is to find the symmetry, not to produce final intensities. merge_engine.FilterByImageCC(!for_search && experiment_.GetScalingSettings().GetMinCCForImage() > 0.0); merge_engine.RefineErrorModel(merge_input); if (merge_engine.ErrorModelActive()) { // XDS convention, so the pair reads directly against a CORRECT.LP; the stills ISa is // already the whole-range 1/b, there being no asymptote tier on this path. const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB()); logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}", em.a, em.b, em.isa, merge_engine.ErrorModelChi2()); } merge_engine.AddImages(merge_input); ScaleMergeResult out; out.merged = merge_engine.ExportReflections(); merge_mark.Stop(); // Automatic high-resolution cutoff (post-merge): trim the written reflections + reported // shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full // range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A // manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the // space-group search still sees the full range. (Rotation is cut inside RotationScaleMerge.) const auto &cut_ss = experiment_.GetScalingSettings(); const std::optional effective_d_min = ApplyResolutionCutoff( out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(), cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A); phase("Computing statistics"); TimingMark stats_mark(logger, "stills statistics " + label); out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min); // The same kept reflections binned over the reference range (--report-resolution); the // stills error model is fitted over the whole range already, so no ISa of its own. if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search) out.reference_statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, std::nullopt, range); { // stills: no asymptote tier, and its ISa already IS the whole-range 1/b const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB()); result.error_model_isa = em.isa; result.error_model_a = em.a; result.error_model_b = em.b; } out.cc_half_before_corrections = out.statistics.overall.cc_half; // no surfaces on this path logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label); Note(fmt::format("merge in {}: {} unique", label, out.merged.size())); return out; }; // First pass: P1 when searching, or directly the user-fixed space group. A guard merging beside it // (above) may still be reading the outcomes, so this merge holds its write-back until it has. const bool guard_beside = merge_first_part && guard_ahead.valid(); if (guard_beside) rsm->SetWriteBackPerFrameScale(false); // Stills: a crystal indexed in a lattice that contains the run's - an axis multiple, or the run's in // another setting - is put on the run's cell before anything is scaled, rather than left out. if (!rsm && result.consensus_cell) { const auto relabel = RelabelOnSublattice(indexer->GetIntegrationOutcome(), *result.consensus_cell, experiment_.GetIndexingSettings()); logger.Info("Put {} crystal(s) indexed in a lattice containing the run's on the run's cell: {} " "reflections at its nodes kept, {} between them dropped", relabel.frames, relabel.reflections_kept, relabel.reflections_dropped); } const auto initial_sg = experiment_.GetGemmiSpaceGroup(); auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value(), /*measure_cc=*/true); if (guard_beside) { guard_ingested.get_future().wait(); rsm->WriteBackPerFrameScale(); rsm->SetWriteBackPerFrameScale(true); } // The two-pass quality guard judges the second pass against the first on THIS merge, which both // passes make in the same terms - P1 (or the group the user fixed), full range, no correction // surfaces - where their final merges need not even be in the same group. See RunAllPasses. Where // the pre-pass integrated only part of the sweep, it is the same merge of those frames (above); // a guard merging beside the chain is read where the pass first needs it (below). const auto record_search_merge = [&](const ScaleMergeResult &counted, ProcessResult &into) { const auto &o = counted.statistics.overall; into.search_merge_completeness_measured = o.possible_unique_reflections > 0; into.search_merge_completeness = into.search_merge_completeness_measured ? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0; into.search_merge_cc_half = counted.cc_half_before_corrections; if (!counted.statistics.shells.empty()) { into.search_merge_inner_cc_half = counted.statistics.shells.front().cc_half; into.search_merge_inner_reflections = counted.statistics.shells.front().unique_reflections; } // ... and the reflections it holds on the principal axial rows, which the guard needs // separately because CC1/2 cannot express them (see RunAllPasses). Only the low-order // part is counted: a screw is read off the strong start of its row - the evidence gate // wants four absences in one zone, and on a 70 A axis that is already at 7 A - while the // high-order axial reflections are as numerous as they are uninformative and would dilute // exactly the difference this number exists to show. // // Both counts are taken over the reflections this pass's own lattice centring allows, and // that is what makes them the same measurement on the two sides. Each pass merges on its // OWN conventional cell, so a pass that finds a centring the other did not carries the // centring-extinguished half (two thirds, R; three quarters, F) of that cell's reflections // as measured noise: measured, two crystals compare merges holding 2.0x the reflections of // the other, with different axial rows, for no reason but the setting. What both settings // describe is the primitive lattice, and these are its reflections. const auto centring_allows = [c = result.consensus_centering.value_or('P')] (const MergedReflection &r) { switch (c) { // rhombohedral is described obverse everywhere in this code case 'A': return (r.k + r.l) % 2 == 0; case 'B': return (r.h + r.l) % 2 == 0; case 'C': return (r.h + r.k) % 2 == 0; case 'I': return (r.h + r.k + r.l) % 2 == 0; case 'F': return (r.h + r.k) % 2 == 0 && (r.h + r.l) % 2 == 0; case 'R': return (-r.h + r.k + r.l) % 3 == 0; default: return true; } }; constexpr float AXIAL_ROW_LOW_ORDER_D_A = 4.0f; into.search_merge_axial_reflections = std::count_if( counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) { return ((r.h == 0) + (r.k == 0) + (r.l == 0)) == 2 && r.d > AXIAL_ROW_LOW_ORDER_D_A && centring_allows(r); }); // The signal the pass measured, and the size of the merge it measured it in, which is // what the guard decides on. constexpr float STRONG_I_OVER_SIGMA = 2.0f; into.search_merge_strong_reflections = std::count_if( counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) { return r.sigma > 0 && r.I / r.sigma >= STRONG_I_OVER_SIGMA && centring_allows(r); }); into.search_merge_negative_reflections = std::count_if( counted.merged.begin(), counted.merged.end(), [&](const MergedReflection &r) { return r.sigma > 0 && r.I / r.sigma <= -STRONG_I_OVER_SIGMA && centring_allows(r); }); into.search_merge_reflections = std::count_if(counted.merged.begin(), counted.merged.end(), centring_allows); }; if (!guard_beside) record_search_merge(guard_merge ? *guard_merge : sm, result); if (subset_merge) { ProcessResult subset_view; record_search_merge(*subset_merge, subset_view); result.prepass_subset_merge = SearchSignal(subset_view); } // Rotation two-pass geometry pre-pass: this first scale/merge has now fitted a frame-order-smoothed // mosaicity and written it back onto the per-frame outcomes. Capture it for the second pass's Bragg // prediction (so prediction uses the smoothed value, not a fresh per-image estimate) and post-refine + // apply the detector geometry from this integration. That is all the pre-pass is for: everything // the second pass consumes from it is fitted here, and the space group is left to the second pass // (see the search gate below). // postrefine_probe_ brings the post-refinement into a CANONICAL pass as well, where the // pre-pass moved the geometry: the fit it committed was made on reflections integrated at the // geometry BEFORE the move, and whether it has finished walking can only be asked of // reflections integrated at the geometry after it. The probe measures - it applies nothing and // leaves the smoothed mosaicity and the pre-pass lattice alone, which stay pass 1's for every // pass that consumes them; RunAllPasses decides what to do with the answer. if (geometry_prepass && rsm && experiment_.IsRotationIndexing()) post_refine_geometry(); // Live, the molecular replacement of a prediction (or of --mr) starts here, on this pass's merge, and runs // beside the second pass on a card of its own where there is one, with a quarter of the threads. It is a // hypothesis: the answer's merge takes its placement over only where it holds there (CarryMolecularReplacement) // and searches itself otherwise. Its group is a quick one of its own, from this merge's search to the // resolution the placements are scored at - the pre-pass decides no group for the run - so the merge searched is // this one averaged into that group, on the axes the answer's files would be written on. if (geometry_prepass && config_.fast_live && config_.model && result.consensus_cell && config_.reference_data.empty() && (config_.force_mr || PrepareSearchModel(*config_.model).predicted)) { early_mr_ = std::make_shared(); early_mr_->run = std::async(std::launch::async, [early = early_mr_.get(), p1 = sm.merged, cell = *result.consensus_cell, fixed_sg = initial_sg, lattice = end_msg.rotation_lattice_type, isa = result.error_model_isa, model = config_.model, rfree_fraction = experiment_.GetScalingSettings().GetRfreeFraction(), nthreads = static_cast(std::max(1, config_.nthreads / 4))]() mutable { GPUWorkBeside beside; if (const int32_t cards = get_gpu_count(); cards > 0) pin_gpu(1 % cards); gemmi::SpaceGroup sg; if (fixed_sg) { sg = *fixed_sg; } else { SearchSpaceGroupOptions o; o.nthreads = nthreads; o.merge_friedel = true; o.d_min_limit_A = 3.0; if (lattice) { o.lattice_system = lattice->crystal_system; o.lattice_centring = lattice->centering; } o.cell = gemmi::UnitCell(cell); o.enumerate_all_settings = true; o.enumerate_all_rotation_sets = true; o.merge_isa = isa; const SearchSpaceGroupResult found = SearchSpaceGroup(p1, o); if (found.best_space_group) sg = *found.best_space_group; else if (found.point_group_representative) sg = *found.point_group_representative; else { MolecularReplacementResult none; none.failure_reason = "no space group from the pre-pass merge"; return none; } } const gemmi::Op cob = ChooseOutputSetting(cell, sg); if (const gemmi::SpaceGroup *moved = SpaceGroupInBasis(sg, cob); !(cob == gemmi::Op::identity()) && moved != nullptr) { p1 = ReindexReflections(p1, HklOperator(cob)); cell = CellInBasis(cell, cob); sg = *moved; } early->merged = AverageIntoGroup(p1, sg, cell); early->cell = cell; early->space_group = sg; AssignRfreeFlags(early->merged, sg, rfree_fraction, 500, cell, nthreads); FrenchWilsonOptions fw_opts; fw_opts.num_threads = nthreads; fw_opts.anisotropy_hkl = AnisotropyPriorHKL(early->merged, gemmi::UnitCell(cell), &sg); ApplyFrenchWilson(early->merged, sg, fw_opts); early->log.Info("Molecular replacement on the geometry pre-pass's merge, in {} from its own search, beside " "the second pass", sg.xhm()); return RunMolecularReplacement(*model, {{"geometry pre-pass's merge", &early->merged, cell, sg}}, early->log, nthreads); }); } // For a de-novo search, gate the P1 scaling+merge (the pass that feeds the space-group search) at // >= 1 over thin resolution shells: noise-dominated high-res shells otherwise mislead // the determination, where a single artifact (a zinger the box sum integrates whole, or a // weak-reflection profile-fit runaway) has an astronomical resolution-normalised E and wrecks the // intensity-correlation / second-moment statistics. The final in-symmetry merge keeps the full // range (only this P1 search pass is cut); a manual --scaling-high-resolution, if coarser, wins. // The geometry pre-pass does not determine the space group. Its integration is at the header // geometry, which is the thing the second pass exists to improve on, and the symmetry evidence // moves with the geometry: measured on a rotation crystal, 0.33 mm of distance and 0.65 px of // beam centre took the screw-axis evidence from -19 to +311 nats, where merging the same // integration in symmetry instead of P1 moved it 2.5. Deciding on the worse integration also // fails in one direction only - every statistic the search weighs is a ratio of the candidate // merge's scatter to the subgroup's, so poorly-integrated data inflate them and REFUSE real // symmetry (corpus-wide the misses run ten under-calls to one over-call). So the search runs on // the canonical pass, at the refined geometry, and the pre-pass skips it along with the // in-symmetry re-merge that only exists to serve it. const bool search_space_group = !geometry_prepass && !initial_sg.has_value(); double d_min_search = 0.0; if (search_space_group) { std::vector> rs; // (d, I/sigma) over the full P1 merge rs.reserve(sm.merged.size()); for (const auto &m : sm.merged) if (std::isfinite(m.I) && std::isfinite(m.sigma) && m.sigma > 0.0f && std::isfinite(m.d) && m.d > 0.0f) rs.emplace_back(m.d, m.I / m.sigma); if (rs.size() >= 400) { std::sort(rs.begin(), rs.end(), [](const auto &a, const auto &b) { return a.first > b.first; }); // low -> high res const int bins = std::clamp(static_cast(rs.size() / 100), 1, 40); const size_t per = (rs.size() + bins - 1) / static_cast(bins); // The shell means, then their best NON-INCREASING fit (pool adjacent violators, weighted // by shell size). The cut is where the FIT drops under 1, not where the first shell // does: on a merge whose is flat near 1 - its ISa has collapsed - the first // single shell to dip under 1 is a coin toss that moved the cut from 0.85 to 1.19 A // between two compiler flags on one crystal, and the coarser cut left its glide zones // too few absences to be judged. The fit moves by as much as its input does. std::vector mean, weight; for (size_t lo = 0; lo < rs.size(); lo += per) { const size_t hi = std::min(rs.size(), lo + per); double sum = 0.0; for (size_t j = lo; j < hi; ++j) sum += rs[j].second; mean.push_back(sum / static_cast(hi - lo)); weight.push_back(static_cast(hi - lo)); } std::vector block_mean, block_weight; std::vector block_size; for (size_t b = 0; b < mean.size(); ++b) { block_mean.push_back(mean[b]); block_weight.push_back(weight[b]); block_size.push_back(1); while (block_mean.size() > 1 && block_mean[block_mean.size() - 2] < block_mean.back()) { const size_t k = block_mean.size() - 1; const double w = block_weight[k - 1] + block_weight[k]; block_mean[k - 1] = (block_mean[k - 1] * block_weight[k - 1] + block_mean[k] * block_weight[k]) / w; block_weight[k - 1] = w; block_size[k - 1] += block_size[k]; block_mean.pop_back(); block_weight.pop_back(); block_size.pop_back(); } } std::vector fit; for (size_t k = 0; k < block_mean.size(); ++k) fit.insert(fit.end(), block_size[k], block_mean[k]); for (size_t b = 0; b < fit.size(); ++b) { const size_t lo = b * per; if (fit[b] < 1.0) { // Cut the noise-dominated high-res shells. When even the lowest-res shell fails, // keep that shell alone rather than abandoning the cut. The bound is absolute // while the merged I/sigma it tests saturates at the merge's own ISa (times the // square root of the multiplicity), so a merge whose ISa has collapsed towards 1 // can fail it in every shell - and that is exactly the merge the search must not // be handed whole, since the noise which drove ISa down is what the cut exists to // remove. Abandoning it feeds back: more noise into the search, a weaker operator // statistic, and a lost symmetry. if (b > 0) d_min_search = rs[lo].first; else if (per >= 400) d_min_search = rs[per - 1].first; break; } } } const double d_manual = experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0); if (rsm && d_min_search > d_manual) { logger.Info("Space-group search: re-scaling the P1 pass at {:.2f} A ( >= 1)", d_min_search); rsm->SetDMinLimit(d_min_search); sm = scale_and_merge("P1, space-group search", true); rsm->SetDMinLimit(d_manual > 0.0 ? std::optional(d_manual) : std::nullopt); } } std::ostringstream stats_text; // The indexing-ambiguity paragraph as it went into stats_text, so that a model which then // settles the ambiguity can withdraw it; empty where there was none. std::string ambiguity_warning_text; // True when the point group below was chosen BY THE SEARCH (rather than given by the user) and is // above triclinic - i.e. a promotion was made, which is what makes a later "the Laue class is // holohedral so there is no twin law" conclusion circular. bool promoted_point_group = false; // The twin-law merge (_twin_): where the adopted group may have absorbed a twin law, // the subgroup a second merge is made in and the law it carries, in the scaler's frame, and why. // The decisions below that name a subgroup leave it here: the group a refused promotion was // adopted over (a twin gate overruled), and the indexed lattice's group a metric re-ask promoted // from. Report and an extra file only - nothing the run writes as its answer reads any of it. std::optional twin_overruled_subgroup, twin_metric_subgroup; std::string twin_overruled_reason; std::optional twin_choice; std::string twin_choice_reason; // The twin hypothesis' merge made only for the model to test: written, and reported, only where it chooses it. bool twin_for_mr_only = false; // The model's own subgroup, where it is not the one above (which the data chose): a merge of its own, // made only for the model to test. std::optional model_twin_choice; // The --model's kind, from the file alone: a prediction has no placement of its own and goes straight to // molecular replacement; an experimental structure's group, where it has fewer rotations than the merge's, // asks for the twin hypothesis' merge. bool model_predicted = false; size_t model_rotations = 0; std::optional model_group; gemmi::UnitCell model_cell; if (config_.model && write_output && !geometry_prepass) { const SearchModel m = PrepareSearchModel(*config_.model); model_predicted = m.failure_reason.empty() && m.predicted; if (const gemmi::SpaceGroup *sg = m.st.find_spacegroup(); sg != nullptr && m.st.cell.is_crystal()) { model_rotations = sg->operations().sym_ops.size(); model_group = *sg; model_cell = m.st.cell; } } int twin_choice_group_number = 0; // the adopted group's, when the choice was made // A canonical pass the geometry walk is going to replace stops here, where the answer is // already in (GeometryWalkTakesRound); it is asked again below, before the analyses. Not with a // _process.h5 open: that file is written whole by every pass that opens it. if (guard_beside) { guard_merge = guard_ahead.get(); guard_log.ReplayInto(logger); record_search_merge(*guard_merge, result); } if (!writer && GeometryWalkTakesRound(result, p.postrefine_ahead)) { logger.Info("The geometry walk will re-integrate at the geometry the post-refinement asks for, " "so this pass stops before its space-group search"); return true; } if (postrefine_gathered_future.valid()) postrefine_gathered_future.wait(); // the search re-indexes the reflections the gather reads TimingMark search_mark(logger, "space-group search"); if (search_space_group) { // The engine's next merge is in the group the search picks; until then its last merge's // working arrays only sit beside the searches' own (gigabytes on a large cell in P1). if (rsm) rsm->ReleaseMergeScratch(); SearchSpaceGroupOptions sg_opts; sg_opts.nthreads = static_cast(std::max(1, config_.nthreads)); sg_opts.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel(); sg_opts.d_min_limit_A = std::max( d_min_search, experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0)); // Constrain the search to subgroups of the lattice (metric) symmetry found by rotation // indexing. Centering is not constrained here - it is determined from the absences. if (end_msg.rotation_lattice_type.has_value()) { sg_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system; sg_opts.lattice_centring = end_msg.rotation_lattice_type->centering; } // The cell the reflections are indexed on, which is what lets the search offer a candidate // in a setting other than gemmi's reference one: a setting names the symmetry directions by // axis, so a candidate the cell cannot host is refused before it is scored. if (result.consensus_cell.has_value()) sg_opts.cell = gemmi::UnitCell(*result.consensus_cell); sg_opts.enumerate_all_settings = true; sg_opts.enumerate_all_rotation_sets = true; // The error model this merge was scaled with, so the search's "genuinely present" cut can be // read on the counting sigma instead of on the merged one (SearchSpaceGroupOptions::merge_isa). // Set per merge, not once: the two arms below are scaled separately and their ISa differ by up // to a factor of six. 0 (b not measurable) leaves the cut as the caller set it. sg_opts.merge_isa = result.error_model_isa; // The all-observation merge made beside everything since the ingest (see rsm_all), and its // search started beside this one's: a search reads its merge and its options and nothing // else. Whether the all-observation arm is wanted at all is known only from the first arm // (below), so nothing of it reaches the run before then. A merge that failed beside the // others is made again where it is wanted, in series, as it would have been without it. std::optional all_merge; std::future all_search_ahead; if (rsm_all && !all_arm_ahead.valid()) all_arm_ahead = start_all_arm(); // not beside the merges above (the host budget): now if (all_arm_ahead.valid()) { TimingMark wait_mark(logger, "wait for the all-observation merge"); try { all_merge = all_arm_ahead.get(); } catch (const std::exception &e) { logger.Warning("Space-group search: the all-observation merge beside the others failed ({}); " "it is made again in series", e.what()); } wait_mark.Stop(); } if (all_merge) { SearchSpaceGroupOptions all_opts = sg_opts; all_opts.merge_isa = all_merge->isa; // as take_merge leaves it below all_search_ahead = std::async(std::launch::async, [&merged = all_merge->merged, all_opts] { return SearchSpaceGroup(merged, all_opts); }); } TimingMark first_arm_mark(logger, "SearchSpaceGroup (first arm)"); auto sg_search = SearchSpaceGroup(sm.merged, sg_opts); first_arm_mark.Stop(); // The Lorentz-filtered arm's merge, kept past the point where the all-observation one // replaces it. The metric re-ask below is a search like any other and answers to the same // two-arm rule: a promotion is refused by the most damning statistic it can be shown, and // only one of the two merges has to be starved for that to happen. std::vector merged_filtered; // ...and that arm's OWN ISa. sg_opts.merge_isa is re-set below for the all-observation // merge, and the two differ by up to a factor of six, so the re-ask has to keep this one // or it judges the filtered merge's "genuinely present" cut on the other arm's error model. double merged_filtered_isa = 0.0; // Second opinion from a merge that keeps only well-measured observations (see // RotationScaleMerge::search_min_zeta). Where the two disagree the all-observation merge // decides. The filter earns its place because a reflection that crosses the Ewald sphere // near-tangentially is measured worst, and a handful of bad measurements can make a real // operator look like a twin law; but it discards 40-80% of the observations, which starves // the operator correlations. Both failure directions are real - the filtered arm has been // seen to lose an operator the full merge confirms, and, at geometries away from the ones // this comparison was first checked on, to confirm one the full merge refuses. So the two // are split rather than ranked: whichever arm finds the LARGER point group decides which // rotations are real, because a promotion is refused by the most damning statistic it can // be shown and only one arm has to be starved for that to happen, while the ABSENCES are // always judged on the all-observation merge, which is the only one holding the weak // reflections a systematic absence is made of. The vetoes are NOT kept per arm: a promotion // one arm refused on its twin gates (H ratio, added-operator R) is adopted when the other arm // confirms it, because the higher order wins. That held on one real twin at three frame // ranges, where both arms refused, but an H3 partial twin has been seen refused by the // all-observation arm on both twin gates and promoted to R32 by the filtered one. So the // losing arm's refusal is logged and carried to the report below; the rule is unchanged. if (rsm && rsm->GetSearchMinZeta() > 0.0 && !sg_search.point_group_hm.empty()) { ScaleMergeResult sm_all; SearchSpaceGroupResult alt; merged_filtered_isa = sg_opts.merge_isa; // the filtered arm's, before it is replaced if (all_merge) { TimingMark all_arm_mark(logger, "SearchSpaceGroup (all-observation arm, wait)"); alt = all_search_ahead.get(); all_arm_mark.Stop(); phase("Scale/combine/merge (P1, all observations)"); all_arm_log.ReplayInto(logger); rsm_all->WriteBackPerFrameScale(); sm_all = take_merge("P1, all observations", true, std::move(*all_merge)); sg_opts.merge_isa = result.error_model_isa; // this arm's own error model } else { const double zeta = rsm->GetSearchMinZeta(); rsm->SetSearchMinZeta(0.0); sm_all = scale_and_merge("P1, all observations", true); rsm->SetSearchMinZeta(zeta); rsm->ReleaseMergeScratch(); // as before the first arm's search sg_opts.merge_isa = result.error_model_isa; // this arm's own error model TimingMark all_arm_mark(logger, "SearchSpaceGroup (all-observation arm)"); alt = SearchSpaceGroup(sm_all.merged, sg_opts); all_arm_mark.Stop(); } // The order of the point group each arm confirmed. Read from the search, not from the // space group it went on to pick: Stage B leaves best_space_group unset when no // candidate has enough absences to be eligible, which is common on the filtered arm // precisely because the filter removes the weak reflections the absences live in. An // arm that confirmed 422 would then have scored 0 and lost to a P2 arm - a demotion, // reported as "taking the higher symmetry". const auto order_of = [](const SearchSpaceGroupResult &r) { return r.point_group_order; }; logger.Info("Space-group search: all-observation merge -> {} (order {}), " "Lorentz-filtered -> {} (order {})", alt.point_group_hm.empty() ? "?" : alt.point_group_hm, order_of(alt), sg_search.point_group_hm.empty() ? "?" : sg_search.point_group_hm, order_of(sg_search)); // The screw/centering determination in particular comes from the merge with all the // observations, because systematic absences are decided by the WEAK reflections and the // filter throws most of them away. Preferring the filtered arm on a tie cost four // crystals their screw axes (P2(1) read as P2) for exactly that reason, with the point // group and every intensity statistic identical. // Checked while both arms are still intact - below, the all-observation arm is adopted // by value and the two become the same result. // Which point group, by its symmorphic representative rather than by name - gemmi calls // both P321 and P312 "32", so two arms that disagree about WHICH 2-folds are real read // as agreeing. By the representative's full symbol rather than its NUMBER, for the same // reason one level down: P 1 2 1 and P 1 1 2 are both number 3 and put their 2-fold on // different axes. Empty when no point group was chosen. const auto pg_of = [](const SearchSpaceGroupResult &r) { return r.point_group_representative.has_value() ? r.point_group_representative->xhm() : std::string(); }; if (order_of(alt) > 0 && order_of(alt) == order_of(sg_search) && pg_of(alt) != pg_of(sg_search)) { // Same order, different symmetry: the two merges disagree about WHICH operators are // real, and neither is higher, so there is nothing to prefer. Say so instead of // picking silently - the two imply different molecular-replacement searches, and // trying both is cheap next to processing the data again. const std::string a = alt.best_space_group.has_value() ? alt.best_space_group->short_name() : alt.point_group_hm; const std::string b = sg_search.best_space_group.has_value() ? sg_search.best_space_group->short_name() : sg_search.point_group_hm; const std::string msg = fmt::format( "Space group is AMBIGUOUS between {} and {} (point groups {} and {}, the same " "order): the merge of all observations and the merge of only the well-measured " "ones each support one of them, and neither is the higher symmetry, so the data " "do not decide. Processing continues in {} (the all-observation choice); " "molecular replacement in both is what settles it, and a re-run with -S {} " "forces the other one.", a, b, alt.point_group_hm, sg_search.point_group_hm, a, b); logger.Warning("{}", msg); stats_text << " !! " << msg << "\n\n"; result.warnings.push_back({PathologyCode::SYMMETRY_AMBIGUITY, msg}); } // The arm whose lower answer is about to be overridden, and why it stopped there. A // refusal by a consistency gate is the case that matters: the higher group is adopted // anyway, and without this nothing in the output says a gate ever fired. const auto overridden = [&](const SearchSpaceGroupResult &lower, const char *lower_name, const char *higher_name) { std::string msg; if (!lower.refused_point_group_hm.empty()) msg = fmt::format("the {} merge REFUSED {} ({}); overridden, because the {} merge " "confirmed the higher point group", lower_name, lower.refused_point_group_hm, lower.refused_reason, higher_name); else msg = fmt::format("the {} merge did not confirm the operators of the higher point " "group (no gate refused it); the {} merge did", lower_name, higher_name); logger.Warning("Space-group search: {}", msg); result.space_group_arm_override = msg; }; // A refusal is not simply outvoted by the other arm's confirmation: the gates read // the operators' own disagreement, which a partial twin at a fraction the other arm's // coverage happens to hide passes there (an H3 partial twin promoted to R32 by the // filtered arm; a P31 one to P3121 by the all-observation arm). What decides is the // twin-immune zone of the added operators, read on the all-observation P1 merge // (TwinZoneVerdict): acentric with margin and the refusal stands, on all observations; // centric with margin and the higher group is taken as before; undecided and the rule // stays as it was. int zone_verdict = 0; if (order_of(alt) > order_of(sg_search)) { logger.Info("Space-group search: all-observation merge supports {} where the " "Lorentz-filtered one supports {} - taking the higher symmetry", alt.point_group_hm, sg_search.point_group_hm); overridden(sg_search, "Lorentz-filtered", "all-observation"); zone_verdict = !sg_search.refused_point_group_hm.empty() ? TwinZoneVerdict(sm_all.merged, gemmi::UnitCell(*result.consensus_cell), alt, sg_search, result) : 0; if (zone_verdict < 0) { logger.Warning("Space-group search: the Lorentz-filtered merge's refusal of {} " "STANDS - the twin-immune zone of its added operators reads " "acentric; keeping {}, judged on all observations", sg_search.refused_point_group_hm, sg_search.point_group_hm); sg_opts.fixed_point_group = sg_search.point_group_representative; auto lower = SearchSpaceGroup(sm_all.merged, sg_opts); sg_opts.fixed_point_group.reset(); lower.refused_point_group_hm = sg_search.refused_point_group_hm; lower.refused_reason = sg_search.refused_reason + ", and its twin-immune zone reads acentric"; lower.refused_point_group_representative = sg_search.refused_point_group_representative; lower.refused_by_twin_gate = true; lower.refused_twin_fraction = sg_search.refused_twin_fraction; alt = std::move(lower); } } if (order_of(alt) < order_of(sg_search) && !alt.refused_point_group_hm.empty() && TwinZoneVerdict(sm_all.merged, gemmi::UnitCell(*result.consensus_cell), sg_search, alt, result) < 0) { logger.Warning("Space-group search: the all-observation merge's refusal of {} STANDS " "- the twin-immune zone of its added operators reads acentric; keeping {}", alt.refused_point_group_hm, alt.point_group_hm); overridden(alt, "all-observation", "Lorentz-filtered"); result.space_group_arm_override += " - but the refusal stands (see TWIN_ZONE_DECISION)"; alt.refused_reason += ", and its twin-immune zone reads acentric"; sg_search = alt; } else if (order_of(alt) < order_of(sg_search)) { // The filtered arm found symmetry the all-observation arm did not. Discarding it // was costing real point groups: over the corpus the two arms disagree on eight // crystals and the higher one is right on all eight, because the filter removes // the near-tangential observations that make a genuine operator look like a twin, // and a promotion is refused by the most damning statistic it can be shown. // // But the two halves of the answer do not come from the same place. The filtered // merge is deliberately starved, and ABSENCES live in the weak reflections it // throws away - adopting its result whole recovers the point group and loses the // screw (a P6_5 crystal comes back P6). So the point group is taken from the arm // that found it and the absences are re-judged here, on every observation. That // is what fixed_point_group is for: Stage A still runs and still reports what it // would have refused, only the choice is overridden. overridden(alt, "all-observation", "Lorentz-filtered"); sg_opts.fixed_point_group = sg_search.point_group_representative; auto split = SearchSpaceGroup(sm_all.merged, sg_opts); sg_opts.fixed_point_group.reset(); logger.Info("Space-group search: Lorentz-filtered merge supports {} where the " "all-observation one supports {} - taking the higher symmetry, and " "judging its absences on all observations -> {}", sg_search.point_group_hm, alt.point_group_hm, split.best_space_group.has_value() ? split.best_space_group->xhm() : split.point_group_hm); sg_search = std::move(split); } else { sg_search = alt; } // The all-observation merge is the one carried forward: it is what the // pre-promotion twinning numbers below are measured on, and the filtered merge is the // second opinion reported above. merged_filtered = std::move(sm.merged); sm = std::move(sm_all); } // The arm merged beside the search is done with - or was not needed - before anything below // changes the experiment it reads, and its card memory goes back before the merges below. if (all_search_ahead.valid()) all_search_ahead.wait(); all_merge.reset(); rsm_all.reset(); // The walk's indexing probes, which the fit starts as it ends, run beside the search; where // their answer came in during its arms, a pass the walk replaces stops here, before the // decisions and the merge in the adopted group. Nothing waits for it. if (!writer && GeometryWalkTakesRound(result, p.postrefine_ahead)) { logger.Info("The geometry walk will re-integrate at the geometry the post-refinement asks for, " "so this pass stops after its space-group search arms"); return true; } // Everything from here to the adopted group's merge: the lattice re-asks and the choices // between the candidates the arms left. TimingMark decisions_mark(logger, "space-group decisions"); // Twinning evidence measured BEFORE any promotion, on the P1/subgroup merge the search was // given. Reported alongside the post-adoption analysis: once a higher Laue class has been // adopted, its own twinning test can only say "no twin law exists within this class", which // is circular - the promotion is what a twin would have caused. These numbers are not. // // Measured as the post-adoption test is: with the pseudo-translation of this same merge // declared, so the L-test can be repaired for it, and - the merge being in P1 - without the // reflections the lattice centring extinguishes, which an R lattice in its hexagonal cell // still carries (two thirds of them, and every axis step lands on one). // // Nothing reads these before the adopted group's merge below, and nothing writes this merge // until that merge replaces it, so they are measured beside it rather than ahead of it. The // cell is taken now: a change of basis committed below replaces it. const char lattice_centring = end_msg.rotation_lattice_type.has_value() ? end_msg.rotation_lattice_type->centering : 'P'; auto pre_promotion = std::async(std::launch::async, [&merged = sm.merged, cell = result.consensus_cell, lattice_centring, nthreads = config_.nthreads] { TranslationalNCSResult tncs; if (cell) tncs = AnalyzeTranslationalNCS(merged, gemmi::UnitCell(*cell), nullptr, 30, 0, nthreads); auto twinning = AnalyzeTwinning(merged, nullptr, 20, TranslationalNCSVectorForLTest(tncs), lattice_centring); return std::make_pair(tncs, twinning); }); TranslationalNCSResult pre_tncs; const auto join_pre_promotion = [&] { auto [tncs, twinning] = pre_promotion.get(); pre_tncs = tncs; result.pre_promotion_tncs_measured = tncs.measurable; result.pre_promotion_twinning = twinning; if (twinning.twinning_suspected) logger.Warning("Twinning indicators BEFORE the space-group decision (subgroup merge): " "<|L|> = {:.3f}, second moment = {:.3f} (untwinned 0.500 / 2.00) - if the " "search promoted the point group, treat the promotion with suspicion", twinning.mean_abs_l, twinning.second_moment); }; promoted_point_group = !sg_search.point_group_hm.empty() && sg_search.point_group_hm != "1"; if (!sg_search.refused_point_group_hm.empty()) logger.Warning("Higher symmetry {} was confirmed by the operator correlations but refused: " "{}. Processing in the lower symmetry (the recoverable direction).", sg_search.refused_point_group_hm, sg_search.refused_reason); // Miller-index reindex under a change of basis: (h,k,l)_conv = reindex * (h,k,l)_prim. const auto reindex_hkl = [](auto &r, const gemmi::Mat33 &m) { const double h = r.h, k = r.k, l = r.l; r.h = static_cast(std::lround(m.a[0][0] * h + m.a[0][1] * k + m.a[0][2] * l)); r.k = static_cast(std::lround(m.a[1][0] * h + m.a[1][1] * k + m.a[1][2] * l)); r.l = static_cast(std::lround(m.a[2][0] * h + m.a[2][1] * k + m.a[2][2] * l)); }; // Intensity-based centred-lattice test (XDS CORRECT / POINTLESS style). When the indexer // demoted to a primitive/triclinic cell but the metric is pseudo-symmetric, the point-group // search above tested no operators (its candidate set is bounded by the indexer's lattice). // A genuinely centred lattice and a merely pseudo-symmetric triclinic one are indistinguishable // from spot POSITIONS - the 2-fold is an INTENSITY property - so decide it here, after P1 // integration: reindex the P1-merged data to the metric candidate's conventional setting and // re-run the search. The I(h) vs I(Rh) correlation confirms a real centred lattice (high CC -> // commit the centring) and rejects a pseudo-symmetric triclinic (low CC -> stays P1, no reindex). std::optional commit_reindex; std::optional commit_cell; std::optional commit_lattice; std::optional commit_lattice_type; // Only a demoted (triclinic) indexing result reaches here with a primitive cell - that is both // the case the point-group search could not test (a triclinic holohedry offers no operators) // and the case where LatticeSearch's Niggli reduction is valid (it assumes a primitive input). const bool searched_trivial = !sg_search.best_space_group || sg_search.best_space_group->number == 1; const bool demoted_triclinic = end_msg.rotation_lattice_type.has_value() && end_msg.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Triclinic; if (searched_trivial && demoted_triclinic && end_msg.rotation_lattice.has_value()) { // The metric only generates the candidate; the intensity CC gate below makes the decision // (a false pseudo-symmetry is rejected by the correlation). const auto cand = LatticeSearch(*end_msg.rotation_lattice); // Exact integer reindex from the indexed setting straight to the candidate conventional // setting: P[i][j] = conv_real[i] . rot_reciprocal[j]. Derived from the two lattices, so it // holds even if the refined indexed cell is not exactly LatticeSearch's Niggli cell (whose // primitive_reduced->conventional reindex would otherwise be applied in the wrong setting). const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()}; const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(), end_msg.rotation_lattice->Cstar()}; gemmi::Mat33 reindex; double reindex_res = 0.0; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) { const double v = cv[i] * rs[j]; reindex.a[i][j] = std::round(v); reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j])); } if (cand.system != gemmi::CrystalSystem::Triclinic && reindex_res < 0.1) { std::vector merged_c = sm.merged; for (auto &m : merged_c) reindex_hkl(m, reindex); SearchSpaceGroupOptions o2 = sg_opts; o2.lattice_system = cand.system; o2.lattice_centring = cand.centering; o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell()); const auto s2 = SearchSpaceGroup(merged_c, o2); if (s2.best_space_group.has_value() && s2.best_space_group->number > 1) { // The centring cannot be confirmed from absences here (integrated in the primitive // cell, so the centring-absent reflections do not exist) - it is metric-determined. // Among the point-group-equivalent candidates the intensities cannot separate // (e.g. P2 / P21 / C2), pick the one matching the metric candidate's centring. gemmi::SpaceGroup chosen = *s2.best_space_group; if (chosen.centring_type() != cand.centering) for (const auto &alt : s2.alternatives) if (alt.centring_type() == cand.centering) { chosen = alt; break; } // Commit only if the adopted centring matches the metric candidate whose conventional // cell we reindex into - otherwise the space group and the written cell disagree. if (chosen.centring_type() == cand.centering) { sg_search = s2; // Swap the metric-matching candidate into the answer and leave the one it // displaced among the alternatives, so the reported "or ..." list stays the // set of groups the data cannot separate, each named once. if (chosen.number != s2.best_space_group->number) { sg_search.alternatives.push_back(*s2.best_space_group); std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) { return a.number == chosen.number; }); } sg_search.best_space_group = chosen; commit_reindex = reindex; commit_cell = cand.conventional.GetUnitCell(); commit_lattice = end_msg.rotation_lattice->Multiply(reindex); commit_lattice_type = LatticeMessage{ .centering = cand.centering, .niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system }; // Say so at the point of decision. The point group here is confirmed // from the intensities, but the centering is not and cannot be - the // reflections it extinguishes were never measured. (The report warns // about it too, from the candidate's centering_untested flag, which // survives into the pass that is written.) // Only for a CENTRED candidate: a primitive one extinguishes nothing, so // there is no untested absence class to warn about. Before the enumeration // was widened this branch could only ever adopt a centred group, so the // condition was implicit. if (cand.centering != 'P') logger.Warning("The {} centering of {} comes from the lattice METRIC, " "not from the intensities: the crystal is indexed on " "the primitive sub-cell, so the reflections a " "{}-centred lattice extinguishes were never predicted " "or integrated. The point group ({}) IS confirmed, by " "the operator correlations.", cand.centering, chosen.short_name(), cand.centering, s2.point_group_hm); } } } } // A CLASS DECIDED BY ROUNDING. LatticeSearch names the Bravais class by Niggli-reducing the // indexed cell and looking the reduced cell up in the 44 lattice characters. For a lattice // whose Buerger cells straddle the Niggli type-I/type-II boundary that lookup is a coin // flip - an F-centred cubic lattice has reduced forms on both sides, the two sides carry // different characters, and which side the reduction lands on is decided by the last digits // of whatever refinement produced the cell rather than by the data. Measured on such a // lattice: 43% cubic F, 36% tetragonal I, 21% orthorhombic I, and the split is flat over a // factor of ten in the noise. The class then caps the point-group search // (SearchSpaceGroupOptions::lattice_system), so from the I-centred sub-cell the cubic // three-fold is never enumerated and the run reports that nothing was refused - which is // accurate, because nothing was asked. // // Le Page's two-fold search has no such key: it measures each rotation's obliquity on the // lattice itself, in a primitive basis, and the conventional cell is then built from the // axes of the rotations it found. Where it carries more rotations than the adopted class's // holohedry, offer its conventional cell as a SECOND lattice candidate - reindex the merge // into it and run the same search there, with every gate live. This only ever ASKS. The // reindex is committed if and only if the search in the new setting confirms a strictly // higher point group, and the centring the new cell describes, on these intensities; a // pseudo-symmetric metric fails that and the answer already in hand stands. // // Asked whether or not the centred-lattice test above already committed. What that test // adopts is a lattice class of its own, and a class it settled from a triclinic indexing is // exactly the kind that can still be short of the metric's - so the question here is not // answered by its having run. The two cannot disagree: the ask below only ever replaces the // answer in hand with a STRICTLY higher point group, and it writes all four commit_ values // together, so the group, the cell and the lattice type stay one answer. if (end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) { TimingMark metric_mark(logger, "metric lattice re-ask"); // The merge is indexed on the lattice the indexer named, whatever has been committed // since, so the metric is measured on the primitive form of THAT one. const CrystalLattice primitive = end_msg.rotation_lattice->ToPrimitive(end_msg.rotation_lattice_type->centering); // The lattice class a search has already been given - the indexer's, or the one the // centred-lattice test above committed. It bounds what any search so far could // ENUMERATE, which is what says whether the metric's own class is a new question. const gemmi::CrystalSystem asked_system = commit_lattice_type ? commit_lattice_type->crystal_system : end_msg.rotation_lattice_type->crystal_system; const size_t asked_rotations = HolohedryRotations(asked_system); const auto metric = LePageLattice(primitive); // Ask the intensities about ONE lattice the metric offers: reindex the merge into its // conventional setting and run the same search there, with every gate of that search // live. It commits if and only if the search confirms a strictly higher point group and // the centring the new cell describes. Called at most twice - the metric's own cell, // then the sub-lattice below - and only the ask that adopts sets commit_reindex, so the // second never runs against an answer the first already took. const auto ask_metric_lattice = [&](const LePageResult &cand, bool sub_lattice) { // Exact integer reindex from the indexed setting to the candidate's conventional // one, P[i][j] = conv_real[i] . rot_reciprocal[j], as in the centred-lattice test // above. const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()}; const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(), end_msg.rotation_lattice->Cstar()}; gemmi::Mat33 reindex; double reindex_res = 0.0; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) { const double v = cv[i] * rs[j]; reindex.a[i][j] = std::round(v); reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j])); } if (reindex_res >= 0.1) return; SearchSpaceGroupOptions o2 = sg_opts; o2.lattice_system = cand.system; o2.lattice_centring = cand.centering; o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell()); const auto reindexed = [&](const std::vector &in) { std::vector out = in; for (auto &m : out) reindex_hkl(m, reindex); return out; }; const auto merged_all = reindexed(sm.merged); auto s2 = SearchSpaceGroup(merged_all, o2); // The same two-arm rule the search above follows: the Lorentz-filtered merge can // confirm an operator the all-observation one refuses, because the filter removes // the near-tangential measurements that make a genuine operator look like a twin. // Where it finds the larger point group, the absences are still judged on all // observations, with the point group pinned. // And the same twin-immune zone verdict on a refusal the other arm would outvote: // a P31 partial twin whose 32 the rule above had refused on the zone was promoted // by this ask's filtered arm - the same override, one copy later. if (!merged_filtered.empty()) { SearchSpaceGroupOptions of = o2; of.merge_isa = merged_filtered_isa; // this arm's own error model auto filtered = SearchSpaceGroup(reindexed(merged_filtered), of); const auto refusal_stands = [&](const SearchSpaceGroupResult &higher, SearchSpaceGroupResult &lower) { if (higher.point_group_order <= lower.point_group_order || lower.refused_point_group_hm.empty() || TwinZoneVerdict(merged_all, *o2.cell, higher, lower, result) >= 0) return false; logger.Warning("Space-group search (metric lattice): the refusal of {} STANDS - the " "twin-immune zone of its added operators reads acentric; keeping {}", lower.refused_point_group_hm, lower.point_group_hm); lower.refused_reason += ", and its twin-immune zone reads acentric"; return true; }; if (filtered.point_group_order > s2.point_group_order && !refusal_stands(filtered, s2)) { o2.fixed_point_group = filtered.point_group_representative; s2 = SearchSpaceGroup(merged_all, o2); o2.fixed_point_group.reset(); } else if (refusal_stands(s2, filtered)) { o2.fixed_point_group = filtered.point_group_representative; auto lower = SearchSpaceGroup(merged_all, o2); o2.fixed_point_group.reset(); lower.refused_point_group_hm = filtered.refused_point_group_hm; lower.refused_reason = filtered.refused_reason; lower.refused_point_group_representative = filtered.refused_point_group_representative; lower.refused_by_twin_gate = true; lower.refused_twin_fraction = filtered.refused_twin_fraction; s2 = std::move(lower); } } // The centring cannot be read from the absences in this setting: reindexing // into the metric candidate's conventional cell leaves its centring-absent class // holding no data at all, so among the point-group-equivalent candidates the // intensities cannot separate, the tie-break takes the lowest number, and that is // the primitive one. Pick the candidate matching the metric's own centring out of // the alternatives first, exactly as the centred-lattice test above does - // otherwise a promotion the intensities DID confirm is refused for naming the // same lattice in its equivalent primitive setting. std::optional chosen = s2.best_space_group; if (chosen.has_value() && chosen->centring_type() != cand.centering) for (const auto &alt : s2.alternatives) if (alt.centring_type() == cand.centering) { chosen = alt; break; } const bool point_group_higher = s2.best_space_group.has_value() && s2.point_group_order > sg_search.point_group_order; bool higher = point_group_higher && chosen->centring_type() == cand.centering; // A promotion over a refusal is arbitrated by the twin-immune zone of the added // operators (TwinZoneVerdict), here as where the two arms disagree: where a twin // gate of the search in hand refused a higher group, this ask reopens the question // that gate answered, for whichever of the refused group's operators the new // lattice holds, and its own search cannot answer it - the gates there are relative // too, and a twin law or a pseudo-symmetry the refused group's other operators // diluted passes them alone. Measured on a monoclinic crystal on a hexagonal metric, // a pseudo-merohedral twin: 622 refused on the added-operator R, the C-centred // orthorhombic sub-lattice then asked, its two in-plane two-folds at R 0.061 against // 0.045 for the genuine screw, and C 2 2 21 adopted with the zone of those two-folds // reading acentric. The group in hand is carried into the candidate's setting for // it: h -> Q h with Q the reindex from the basis it was found in (read where GEMMI's // table names the result, as every setting the search enumerates is). std::string zone_refusal; if (higher && sg_search.refused_by_twin_gate && sg_search.point_group_representative) { gemmi::Mat33 q = reindex; if (commit_reindex) q = reindex.multiply(commit_reindex->inverse()); const gemmi::Mat33 q_inv = q.inverse(); gemmi::GroupOps lower_ops; lower_ops.cen_ops = gemmi::centring_vectors(cand.centering); for (const auto &op : sg_search.point_group_representative->operations().sym_ops) { // apply_to_hkl uses rot^T, so the Miller-index matrix of op is rot^T / DEN. gemmi::Mat33 m; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) m.a[i][j] = static_cast(op.rot[j][i]) / gemmi::Op::DEN; const gemmi::Mat33 m_new = q.multiply(m).multiply(q_inv); gemmi::Op op_new = gemmi::Op::identity(); for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) op_new.rot[i][j] = static_cast(std::lround(m_new[j][i])) * gemmi::Op::DEN; lower_ops.sym_ops.push_back(op_new); } if (const gemmi::SpaceGroup *lower_sg = gemmi::find_spacegroup_by_ops(lower_ops)) { SearchSpaceGroupResult lower = sg_search; lower.point_group_representative = *lower_sg; if (TwinZoneVerdict(merged_all, *o2.cell, s2, lower, result) < 0) { zone_refusal = sg_search.refused_point_group_hm + " was refused (" + sg_search.refused_reason + "), and the twin-immune zone of the " "operators this group adds reads acentric"; higher = false; } } } const auto &uc = cand.conventional.GetUnitCell(); // The two asks are told apart by what they were asked against: the class a search // has already been given, or the rotations the intensities have actually confirmed. const std::string preamble = "The cell metric carries " + std::to_string(metric->n_operators) + (sub_lattice ? " rotations where the intensities have confirmed " + std::to_string(sg_search.point_group_order) + ", so the search was asked again on a SUB-lattice of the metric - the " "largest one whose two-folds these intensities confirm one by one, carrying " + std::to_string(cand.n_operators) + " of those rotations" : " rotations where the " + std::string(gemmi::crystal_system_str(asked_system)) + " lattice in hand has " + std::to_string(asked_rotations) + ", so the search was asked again on the metric's own cell"); logger.Info("{} (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} " "gamma={:.2f}, centring {}, worst obliquity {:.3f} deg): {} (order " "{} against {}){}", preamble, uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, cand.centering, cand.max_obliquity_deg, s2.best_space_group.has_value() ? s2.best_space_group->xhm() : "nothing", s2.point_group_order, sg_search.point_group_order, higher ? " - adopted" : !zone_refusal.empty() ? " - keeping the group already found; " + zone_refusal : (point_group_higher ? " - keeping the group already found; the point group IS " "higher, but no candidate of it carries the centring the " "metric cell names, which this setting cannot test" : (s2.refused_reason.empty() ? " - the intensities do not support it, keeping the " "group already found" : " - keeping the group already found; " + s2.refused_reason))); if (higher) { // The group of the lattice in hand, carried into the new setting: the subgroup of // the twin-law merge, should this promotion stand on twin-like data or over a // refusal by a twin gate (see twin_choice). if (sg_search.best_space_group) { gemmi::Mat33 q = reindex; if (commit_reindex) q = reindex.multiply(commit_reindex->inverse()); twin_metric_subgroup = GroupInReindexedSetting(*sg_search.best_space_group, q, cand.centering); if (sg_search.refused_by_twin_gate) { twin_overruled_subgroup = twin_metric_subgroup; twin_overruled_reason = sg_search.refused_reason; } } sg_search = s2; // Leave the candidate it displaced among the alternatives, so the reported // "or ..." list stays the set of groups the data cannot separate. if (chosen->number != s2.best_space_group->number) { sg_search.alternatives.push_back(*s2.best_space_group); std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) { return a.number == chosen->number; }); } sg_search.best_space_group = chosen; commit_reindex = reindex; commit_cell = cand.conventional.GetUnitCell(); commit_lattice = end_msg.rotation_lattice->Multiply(reindex); commit_lattice_type = LatticeMessage{ .centering = cand.centering, .niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system }; } }; if (metric) { // The metric's own class, where no search has been given it yet. Where one has - // the centred-lattice test above reindexes into a metric candidate's cell and runs // the full search there - asking it again would only re-run the same question. // // DIFFERENT, not merely greater. asked_rotations is what the class in hand CLAIMS, // and a class can claim more rotations than the lattice carries: the character // table names a class from a Niggli-reduced cell on relative tolerances, so a cell // whose gamma is a degree and a half off 120 is named hexagonal while Le Page, which // measures each two-fold's obliquity, finds four rotations and a C-centred // orthorhombic cell. Asked only on ">" the comparison was then 4 against 12 and the // ask stayed silent - so the crystal was searched under a holohedry its own lattice // does not have, every operator of it was refused, and it fell to a subgroup, with // the one mechanism built to rescue that disarmed by the over-call it exists to // correct. Where the metric's group is not the group the class in hand names, the // metric's own cell has not been asked about, whichever of the two is larger. // Asking cannot demote: ask_metric_lattice adopts only on a STRICTLY higher point // group, so an ask against a lower metric class costs one search and nothing else. if (static_cast(metric->n_operators) != asked_rotations) ask_metric_lattice(*metric, false); // That question is all-or-nothing, and it is asked at the TOP of the metric // hierarchy: where the truth is an INTERMEDIATE class the answer is correctly "no", // and the class in between is never asked about, because it is not expressible in // the metric cell's basis. Measured on a pseudo-hexagonal monoclinic indexing whose // metric group is hexagonal: three of its twelve operators close into 222 and // correlate at 0.60 against 0.02 for the rest, and the conventional cell of that // orthorhombic sub-lattice is the C-centred one the crystal is actually described on. // // So ask once more, on the sub-lattice the intensities themselves pick out: score // every two-fold the metric offers against the P1 merge and keep only those that // correlate, then let Le Page close what survives into a lattice. Nothing here // decides a space group - it decides which lattice gets ASKED about, and the ask is // the same call with the same gates as above. The number to beat here is what the // intensities have CONFIRMED (the adopted point group's order), not what some // search was allowed to enumerate: an enumeration that reached the metric's class // and refused it says nothing about the classes below it. if (static_cast(metric->n_operators) > sg_search.point_group_order) { // The two-folds are offered in the basis of the lattice handed to LePageLattice, // which is the PRIMITIVE one; sm.merged is indexed on the lattice the run // adopted. With a_prim[i] = sum_j P[i][j] a_idx[j] (CrystalLattice::Multiply // combines basis rows) Miller indices map h_prim = P h_idx, so an operator's // Miller-index matrix carries back as M_idx = P^-1 M_prim P. P is the identity // for a primitive adopted lattice. gemmi::Mat33 to_primitive; if (end_msg.rotation_lattice_type->centering != 'P') to_primitive = gemmi::rot_as_mat33( gemmi::centred_to_primitive(end_msg.rotation_lattice_type->centering)).transpose(); const gemmi::Mat33 from_primitive = to_primitive.inverse(); // An operator counts as real on the same bound Stage A calls one present at - // SearchSpaceGroupOptions::min_operator_cc, applied by OperatorCorrelation over // the same reflection population - AND on the R contrast every promotion's added // operators are held to (SearchSpaceGroupOptions::min_operator_r_contrast, // against the same two ends the search above measured), so nothing new is // calibrated here. The correlation alone cannot tell a pseudo-symmetric // structure's near-operators from real ones: measured on a pseudo-cubic // tetragonal crystal, two of the false cubic two-folds correlate at 0.31-0.33 and // pass it, LePage closes them with the genuine ones into the whole cubic metric, // and the tetragonal class between the two is never asked about. Their R reads // 0.39 against 0.013 for the genuine operators and 0.49 for unrelated // reflections - a contrast of 0.2 against the genuine operators' 1.0. const double r_random = sg_search.random_pairing_r; const double r_best = sg_search.global_best_operator_r; // Read only where the search itself reads it: a best operator close to the // random end leaves no scale to measure on (max_reference_over_random). const bool contrast_known = std::isfinite(r_random) && std::isfinite(r_best) && r_best <= sg_opts.max_reference_over_random * r_random; const OperatorScorer operator_correlation(sm.merged, sg_opts); const auto keep_operator = [&](const gemmi::Mat33 &m_primitive) { const gemmi::Mat33 m_indexed = from_primitive.multiply(m_primitive).multiply(to_primitive); const auto score = operator_correlation(m_indexed); if (!score.has_value() || !score->present) return false; return !contrast_known || (r_random - score->r_stat) / (r_random - r_best) >= sg_opts.min_operator_r_contrast; }; const auto sub = LePageLattice(primitive, LATTICE_MAX_OBLIQUITY_DEG, keep_operator); if (sub && static_cast(sub->n_operators) > sg_search.point_group_order && sub->n_operators < metric->n_operators) ask_metric_lattice(*sub, true); } } } // A DOUBLED DESCRIPTION of a primitive lattice. gemmi states every setting as a change of // basis from its reference one, and where that change has determinant < 1 the setting names // a cell of 1/det times the reference volume. In the monoclinic family exactly two settings // have it - C 1 1 2 and C 1 1 2_1 - and the reason is physical: with unique axis c the // C-centring vector (1/2,1/2,0) has no component along c, lies wholly in the oblique a-b // plane and reduces away, which is why the genuine c-unique centred settings are A/B/I 1 1 2. // (The centred settings of P1, and the C/F settings of the tetragonal groups, are the same // thing.) Such a group is never a distinct Bravais centring; it is a primitive lattice // written on twice the cell. // // It wins because the candidate table is scored ON the doubled cell, where half the // reflection positions are not reciprocal-lattice points at all and are absent BY // CONSTRUCTION, so the centred candidate collects a centring evidence that is tautological. // Refusing to enumerate those settings was measured and is worse, not better: the // volume-preserving alternative in the same table keeps the by-construction absences as real // observations and merges no better than P1. What the centred candidate gets RIGHT is // discarding them. // // So take the win as a SIGNAL rather than as the answer. Halve the cell, put the lattice back // in the primitive setting of the group's own crystal system, carry the reflections across - // the by-construction absences map to non-integral indices and are dropped, which is exactly // what the centred candidate was doing, so nothing that was merging is lost - and re-run the // search there. The enumeration is untouched, so no candidate that JUDGES another moves and // no chi2 reference changes; the doubled setting still wins its own table, and only the cell // its answer is expressed on is corrected. if (!commit_reindex && sg_search.best_space_group.has_value() && sg_search.best_space_group->number > 1 && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) { const gemmi::SpaceGroup won = *sg_search.best_space_group; constexpr double den3 = static_cast(gemmi::Op::DEN) * gemmi::Op::DEN * gemmi::Op::DEN; const double det = won.basisop().det_rot() / den3; const int multiple = static_cast(std::lround(1.0 / det)); // 2 for C 1 1 2_1, 4 for F 1 if (det < 0.99 && multiple >= 2 && multiple <= 4) { // Reduce, then classify. The character table is reached through a Niggli reduction // that takes whatever it is handed to be primitive, so the centred conventional cell // has to be brought to its primitive one FIRST - asked on the doubled cell the same // call answers with the same volume, merely permuted. Ask for the group's own system // in 'P': the metric of a reducible lattice supports the doubled centred description // too (that is how it got here), and an unrestricted search would hand it straight // back. const auto cand = LatticeSearchForClass( end_msg.rotation_lattice->ToPrimitive(won.centring_type()), won.crystal_system(), 'P'); // Change of basis from the doubled conventional setting straight to the reduced one, // P[i][j] = conv_real[i] . doubled_reciprocal[j], as everywhere else here - except // that its entries are multiples of 1/multiple rather than integers, which is the // whole point of it. gemmi::Mat33 reduce; double reduce_res = 1.0; if (cand) { const Coord cv[3] = {cand->conventional.Vec0(), cand->conventional.Vec1(), cand->conventional.Vec2()}; const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(), end_msg.rotation_lattice->Cstar()}; reduce_res = 0.0; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) { const double v = cv[i] * rs[j]; reduce.a[i][j] = std::round(v * multiple) / multiple; reduce_res = std::max(reduce_res, std::fabs(v - reduce.a[i][j])); } } // hkl across the same change of basis. A reflection whose image is not integral is // not a reciprocal-lattice point of the reduced cell - it is one of the positions the // doubled description invented - and it is dropped rather than reindexed. const auto reduce_hkl = [&reduce](auto &r) { const double h = r.h, k = r.k, l = r.l; const double n[3] = {reduce.a[0][0]*h + reduce.a[0][1]*k + reduce.a[0][2]*l, reduce.a[1][0]*h + reduce.a[1][1]*k + reduce.a[1][2]*l, reduce.a[2][0]*h + reduce.a[2][1]*k + reduce.a[2][2]*l}; for (const double x : n) if (std::fabs(x - std::round(x)) > 1e-3) return false; r.h = static_cast(std::lround(n[0])); r.k = static_cast(std::lround(n[1])); r.l = static_cast(std::lround(n[2])); return true; }; if (!cand || reduce_res >= 0.05) { logger.Warning("{} describes a cell of {}x the volume its reference setting does - " "the lattice is a doubled description of a primitive one - but it " "could not be reduced ({}). Keeping the doubled cell.", won.short_name(), multiple, cand ? "the change of basis is not exact" : "no primitive character fits"); } else { std::vector merged_p; merged_p.reserve(sm.merged.size()); for (auto m : sm.merged) if (reduce_hkl(m)) merged_p.push_back(m); SearchSpaceGroupOptions o2 = sg_opts; o2.lattice_system = cand->system; o2.lattice_centring = cand->centering; o2.cell = gemmi::UnitCell(cand->conventional.GetUnitCell()); const auto s2 = SearchSpaceGroup(merged_p, o2); // Only ever a re-expression: the reduced cell has to support the same symmetry // the doubled one found, in the primitive setting the reduction produced. // Anything less and the doubled answer stands - it is the one that merges. const bool holds = s2.best_space_group.has_value() && s2.point_group_order >= sg_search.point_group_order && s2.best_space_group->centring_type() == cand->centering; if (!holds) { logger.Warning("{} describes a cell of {}x the volume its reference setting " "does, but the reduced primitive cell does not carry the same " "symmetry ({}, order {} against {}). Keeping the doubled cell.", won.short_name(), multiple, s2.best_space_group ? s2.best_space_group->short_name() : "nothing", s2.point_group_order, sg_search.point_group_order); } else { for (auto &io : indexer->GetIntegrationOutcome()) { io.latt = io.latt.Multiply(reduce); ReflectionVector kept; kept.reserve(io.reflections.size()); for (auto r : io.reflections) if (reduce_hkl(r)) kept.push_back(r); io.reflections = std::move(kept); } const size_t dropped = sm.merged.size() - merged_p.size(); result.consensus_cell = cand->conventional.GetUnitCell(); result.consensus_centering = cand->centering; end_msg.unit_cell = result.consensus_cell; end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reduce); end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand->centering, .niggli_class = cand->niggli_class, .crystal_system = cand->system }; sg_search = s2; if (rsm) { rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, logger, config_.observation_dump); rsm->Ingest(); } const auto &uc = *result.consensus_cell; logger.Info("{} names a cell of {}x the volume of its reference setting: the " "lattice is a doubled description of a primitive one, and {} of " "{} merged reflections were positions the doubled cell invented. " "Reduced to a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} " "gamma={:.2f} and searched again there: {}", won.short_name(), multiple, dropped, sm.merged.size(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, s2.best_space_group->short_name()); // The reduction halves the cell, so its change of basis is not integral and // end_msg.reindex_matrix (int32) cannot carry it. The per-image reflections // already written to the _process.h5 therefore stay in the doubled setting. logger.Warning("The per-image reflections in the process file are in the " "doubled setting the run integrated in; the cell, the merged " "reflections and the space group are in the reduced one. The " "change of basis between them is not integral, so it is not " "recorded in the file."); } } } } // The centre of symmetry of a glide group. Where the absences settle it - no group of the // other kind predicts them (P2_1/c, Pbca; I-42d) - it is settled. Where a group with the same // absences and the other answer exists (C2/c and Cc), the centrosymmetric one is written by // default - a missed centre is the common error in small-molecule space-group assignment, not // a spurious one - and gives way after the merge below only when the intensity distribution // reads acentric on all three statistics AND the lattice rules twinning out, because a twinned // centrosymmetric crystal reads exactly like an untwinned acentric one. std::optional centre_partner; if (sg_search.best_space_group.has_value() && !sg_search.best_space_group->is_sohncke()) { const gemmi::SpaceGroup written = *sg_search.best_space_group; for (const auto &c : sg_search.candidates) if (c.selected && c.space_group.xhm() == written.xhm()) for (const auto &t : c.same_absences) if (!centre_partner && t.is_centrosymmetric() != written.is_centrosymmetric()) centre_partner = t; if (census_future.valid()) { TimingMark census_mark(logger, "leftover-lattice census wait (centre partner)"); result.leftover_lattices = census_future.get(); } if (result.consensus_cell.has_value() && result.leftover_lattices.has_value()) { const auto &uc = *result.consensus_cell; const gemmi::UnitCell gc(uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); size_t laue_rotations = 0; for (const auto &op : written.operations().sym_ops) if (op.det_rot() > 0) ++laue_rotations; if (!written.is_centrosymmetric()) laue_rotations *= 2; // a group without a centre holds half its Laue class's rotations result.centre_twinning_excluded = gemmi::find_lattice_symmetry(gc, written.centring_type(), LATTICE_MAX_OBLIQUITY_DEG) .sym_ops.size() == laue_rotations && std::none_of(result.leftover_lattices->lattices.begin(), result.leftover_lattices->lattices.end(), [](const LeftoverLattice &l) { return l.kind == LeftoverLatticeKind::TwinDomain; }); } if (!centre_partner) result.space_group_centre = written.is_centrosymmetric() ? "IMPLIED_BY_ABSENCES" : "ABSENT_BY_ABSENCES"; else result.space_group_centre = "NOT_DETERMINED"; } // Adopt the determined space group and re-scale/merge in it, so scaling uses symmetry // equivalents and the statistics come out in the right symmetry. P1 stands when nothing // is determined - and it has to stand as a MERGE of its own, not as whatever `sm` happens // to hold: everything merged up to here was a search merge, which drops the ice rings and // (on the Lorentz-filtered arm) most of the observations with them, is never resolution-cut // and fits no correction surfaces, because its intensities exist only to be searched. // // The P1 fallback is a belt, not a brace. SearchSpaceGroup cannot hand back an unset group // for a merge that has any reflections in it: finding no symmetry at all leaves it on the // identity point group, whose representative IS P1, and it returns that; finding a point // group but no screw or centering leaves the primitive symmorphic candidate, which has no // predicted absences to be inconsistent with and is therefore always eligible; and an empty // merge never reaches the search, because both merge engines throw on the resolution range // before returning one. So the fallback buys nothing today - what it buys is that the // promise above is kept HERE, rather than resting on eligibility gates in another file that // a later change could tighten without anyone noticing what else was leaning on them. { gemmi::SpaceGroup sg = sg_search.best_space_group.value_or(gemmi::get_spacegroup_p1()); // xhm(), not short_name(): the search can now return a non-reference setting, and // short_name() cannot tell P 1 2 1 from P 1 1 2. logger.Info("Adopting space group {} (number {})", sg.xhm(), sg.number); Note(fmt::format("space group {} (No. {})", sg.xhm(), sg.number)); // A GLIDE plane was confirmed, so the adopted group is not a Sohncke one. Say both: // a crystal built from L-amino acids is chiral and cannot have a glide plane, so this // is a statement that the sample is not a protein - and a reader who knows it IS one // has to be able to see the Sohncke group the same run would otherwise have reported, // without processing the images again. Both, always, is what the two fields are for. if (sg_search.glide_space_group.has_value()) { std::string zones; for (const auto &z : sg_search.glide_zones) zones += fmt::format("{}{} ({} absences at {:.2g} of the rest of their plane)", zones.empty() ? "" : ", ", z.label, z.n_absent, z.mean_u); const std::string msg = fmt::format( "Glide plane(s) confirmed from zonal absences - {}. The group is {}, which is " "NOT a Sohncke group: a crystal of chiral molecules (any protein) cannot have " "a glide plane, so this says the sample is not one. The best Sohncke group for " "the same data is {} - re-run with -S {} to process in it.", zones, sg.xhm(), sg_search.sohncke_space_group.has_value() ? sg_search.sohncke_space_group->xhm() : std::string("none"), sg_search.sohncke_space_group.has_value() ? sg_search.sohncke_space_group->xhm() : std::string("P 1")); logger.Info("{}", msg); stats_text << " " << msg << "\n\n"; } // An axis whose screw these data never tested. The group just adopted is then the // member of a set that claims no screw, picked by convention because a reflection // file must name one group - and a line saying only "Adopting space group P 6" reads // as a determination. Say which axis is open and why, here, where the adoption is. if (!sg_search.undetermined_screws.empty()) { std::string axes, why; for (const auto &u : sg_search.undetermined_screws) { axes += (axes.empty() ? "" : ", ") + std::string(1, u.axis); why += fmt::format( "{}the {} row {}", why.empty() ? "" : "; ", u.row_label, u.n_observed == 0 ? std::string("holds no reflections in this merge at all - it was " "never recorded") : u.n_control == 0 ? fmt::format("holds {} reflection(s) but no control class to " "judge them against", u.n_observed) : fmt::format("does not separate the candidates ({} absent, {} " "control)", u.n_absent, u.n_control)); } std::string alts; for (const auto &alt : sg_search.alternatives) alts += " or " + alt.xhm(); const std::string msg = fmt::format( "The screw along {} is UNDETERMINED: {}. These data say {}{} and nothing " "here chooses between them; {} is written because a reflection file must " "carry one group. To settle it, record the missing row.", axes, why, sg.xhm(), alts, sg.xhm()); logger.Warning("{}", msg); stats_text << " " << msg << "\n\n"; } // The group has to be able to DESCRIBE the cell it is adopted on, and where the // promotion was made HERE it cannot. A two-fold the intensities confirm is one the spot // positions never offered - the Bravais class is decided on the unrefined indexing // candidate, and the free refinement can walk into a class afterwards - so no fit under // this group's constraint has ever been run, and what goes to the report, the master // file and the MTZ is the free fit's metric re-expressed in the group's setting: a // C 1 2 1 whose alpha is 88.5, which no monoclinic refinement could have produced. Ask // the spots what cell the constraint supports; where they refuse it, report the metric // the group fixes rather than one it cannot describe. A lattice already refined under // this class leaves only float noise behind and is left alone, which is what the bound // tests - so a de-novo run whose indexing found the class behaves exactly as it did. { std::optional &adopted = commit_reindex ? commit_lattice : end_msg.rotation_lattice; // The band between the two bounds is where the cell is merely UNCONSTRAINED. Above // the upper one it is the wrong cell for this group - the axes permuted, the fold // running along an axis the group does not put it on - and nothing here can repair // that: a constrained refit has no cell to find and the metric projection would // manufacture one, turning a visible mismatch into a plausible-looking lie. That // case is refused where the group comes in (MAX_METRIC_VIOLATION), and is left // exactly as it is here. const double violation = adopted.has_value() ? MetricViolation(adopted->GetUnitCell(), sg) : 0.0; if (adopted.has_value() && violation > MAX_CONSTRAINED_METRIC_NOISE && violation <= MAX_METRIC_VIOLATION) { const UnitCell before = adopted->GetUnitCell(); const auto refit = indexer->RefineRotationLatticeConstrained(*adopted, sg.crystal_system()); // A refit is usable only when the constraint it was written in is the one this // group needs: XtalResidual parameterises monoclinic with the unique axis along // b, and the search can adopt a group that puts it elsewhere (C 1 1 2), whose // metric such a fit does not satisfy. Ask the group, rather than enumerate the // settings - a fit made in the right constraint lands at float noise. const bool fits_group = refit.has_value() && MetricViolation(refit->lattice.GetUnitCell(), sg) <= MAX_CONSTRAINED_METRIC_NOISE; // ...and the spots have to keep it. Half is the bar the indexer's own // pseudo-symmetry guard uses for this very comparison: a constrained cell that // indexes half of what the free one did is not describing these images, and the // metric and the intensities are then saying different things - which is a // geometry error to go and find, not a cell to report. const bool spots_keep_it = refit.has_value() && refit->indexed_fraction > 0.5f * refit->indexed_fraction_before; if (fits_group && spots_keep_it) { *adopted = refit->lattice; const UnitCell uc = adopted->GetUnitCell(); logger.Info("{} was adopted on a cell no fit under it had produced (the " "promotion was made after integration, on the freely refined " "metric). Re-refined under the constraint: a={:.3f} b={:.3f} " "c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, from a={:.3f} " "b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f} - it " "indexes {:.1f} % of the accumulated spots against {:.1f} % for " "the free cell.", sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, before.a, before.b, before.c, before.alpha, before.beta, before.gamma, 100.0 * refit->indexed_fraction, 100.0 * refit->indexed_fraction_before); } else { *adopted = SymmetrizeMetric(*adopted, sg); const UnitCell uc = adopted->GetUnitCell(); const std::string msg = fmt::format( "The cell {} was adopted on is one that group cannot describe - " "a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, " "refined FREE, because the symmetry was confirmed from the intensities " "after integration and no constrained fit had produced this lattice. {} " "The reported cell is the nearest metric the group fixes (alpha={:.2f} " "beta={:.2f} gamma={:.2f}); the reflections are unchanged. A deviation " "this size is not refinement noise - the geometry the images were " "integrated at is the place to look.", sg.xhm(), before.a, before.b, before.c, before.alpha, before.beta, before.gamma, refit.has_value() ? (fits_group ? fmt::format("Re-refining under the constraint indexes {:.1f} % " "of the accumulated spots against {:.1f} % free, so " "it was refused.", 100.0 * refit->indexed_fraction, 100.0 * refit->indexed_fraction_before) : "The constrained refinement is not written in this group's " "setting, so it could not be asked.") : "There was no rotation spot cloud to re-refine against.", uc.alpha, uc.beta, uc.gamma); logger.Warning("{}", msg); stats_text << " !! " << msg << "\n\n"; } // Carry the corrected lattice into the cell that is reported and merged on. // Where a reindex is pending the block below does that and re-ingests; where // there is none, this is the only place it happens. if (commit_reindex) { commit_cell = adopted->GetUnitCell(); } else { result.consensus_cell = adopted->GetUnitCell(); end_msg.unit_cell = result.consensus_cell; if (rsm) { rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, logger, config_.observation_dump); rsm->Ingest(); } } } } // A reindex is committed only when the intensity re-test confirmed a higher symmetry in a // centred setting: bring the integrated reflections + cell into that setting and re-ingest // the rotation merge, so the final scaling/merging folds the equivalents correctly. if (commit_reindex) { for (auto &io : indexer->GetIntegrationOutcome()) { io.latt = io.latt.Multiply(*commit_reindex); for (auto &r : io.reflections) reindex_hkl(r, *commit_reindex); } result.consensus_cell = commit_cell; result.consensus_centering = commit_lattice_type->centering; end_msg.unit_cell = commit_cell; // Keep the lattice metadata (hence the master-file UB matrix and indexed-lattice // vectors) in the same conventional setting as the cell, reflections and space group. end_msg.rotation_lattice = commit_lattice; end_msg.rotation_lattice_type = commit_lattice_type; record_reindex(*commit_reindex); if (rsm) { rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, logger, config_.observation_dump); rsm->Ingest(); } } experiment_.SetSpaceGroup(sg); end_msg.space_group_name = sg.xhm(); end_msg.space_group_number = static_cast(sg.number); result.space_group = sg; decisions_mark.Stop(); phase("Re-scaling in space group " + sg.short_name()); auto adopted = scale_and_merge(sg.short_name(), false); join_pre_promotion(); sm = std::move(adopted); // The intensity statistics for the centre, on this merge - the full one, resolution-cut and // corrected, in the adopted group. Read on every run and reported, Sohncke groups included, // so a chiral small molecule's reader sees them too; they decide only between a glide // group and its partner with the same absences (see above). result.centre_statistics = AnalyzeCentreOfSymmetry(sm.merged, sg); if (centre_partner && sg.is_centrosymmetric() && result.centre_statistics->reads_acentric && result.centre_twinning_excluded) { logger.Info("Centre of symmetry: {} and {} predict the same absences; the intensity " "distribution reads acentric (<|L|> f={:.2f}, <|E^2-1|> f={:.2f}, N(0.1) " "f={:.2f}) and the lattice rules twinning out, so {} is written", sg.xhm(), centre_partner->xhm(), result.centre_statistics->mean_abs_l.f, result.centre_statistics->mean_abs_e2_minus_1.f, result.centre_statistics->n_z_01.f, centre_partner->xhm()); result.space_group_centre = "ABSENT_BY_STATISTICS"; std::replace_if(sg_search.alternatives.begin(), sg_search.alternatives.end(), [&](const gemmi::SpaceGroup &a) { return a.number == centre_partner->number; }, sg); sg = *centre_partner; sg_search.best_space_group = sg; sg_search.glide_space_group = sg; experiment_.SetSpaceGroup(sg); end_msg.space_group_name = sg.xhm(); end_msg.space_group_number = static_cast(sg.number); result.space_group = sg; sm = scale_and_merge(sg.short_name(), false); } // A promotion the operator correlations CONFIRMED and Stage A then refused, put to // the merge. Every gate that can refuse one is a ratio against another operator or // against the noise floor, and both references are properties of how the crystal was // mounted rather than of its symmetry: a 2-fold within a degree of the spindle // records its mates on the same detector pixel half a turn later, carries no // geometry-dependent systematic at all, and by being that clean makes every other // operator look bad; and the best-agreeing operator anywhere can be one of the // operators under test, which collapses the ratio on worse data and raises it on // better. So ask the data the question those ratios stand in for - merge under the // refused group and see whether the reflections it folds together agree. // // R_meas is multiplicity-corrected, so a merge that averaged non-equivalent // reflections has to inflate it, and the error model is refitted per merge, so ISa // says whether the extra multiplicity was bought with systematic disagreement. Over // six refused promotions the two signs separate completely: the genuine ones leave // both flat or better, the false ones inflate R_meas by 21-99% and drop ISa. CC1/2 // and do NOT separate them - both RISE on a false promotion too, which // gains multiplicity either way - so the rule is read on R_meas and ISa and nothing // else. // // Both arms are merged over the SAME resolution range, the one the adopted group's // own merge chose: left to pick their own cutoffs the two are not the same // measurement and the separation muddies to a couple of percent with the sign lost. // Rotation only - it is the rotation merge that can be re-cut between merges. if (rsm && sg_search.refused_point_group_representative.has_value()) { // Which group the comparison arm merges in. The point group is the refused one - // passed as its representative, because gemmi calls both P321 and P312 "32" - and // the absence classes are the adopted group's, because Stage B is asked for the // group on the adopted merge, where those classes are already empty. That is what // makes the two arms differ by the added ROTATIONS alone. SearchSpaceGroupOptions hi_opts = sg_opts; hi_opts.fixed_point_group = sg_search.refused_point_group_representative; hi_opts.merge_isa = result.error_model_isa; if (result.consensus_cell.has_value()) hi_opts.cell = gemmi::UnitCell(*result.consensus_cell); if (end_msg.rotation_lattice_type.has_value()) hi_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system; auto hi_search = SearchSpaceGroup(sm.merged, hi_opts); // ...and its CENTRING is taken rather than read. The adopted group has already // removed its centring-absent class from this merge, so that class holds no data, // every candidate of the point group is equally consistent with it, and the // tie-break takes the lowest-numbered - which is the primitive one. Merging in // that would hand the higher arm the centring-absent reflections as if they were // data and lose the comparison on noise. Same move the metric-lattice ask makes. // // Its SCREWS are taken the same way, for the same reason: the adopted group's // screw-absent axial reflections are not in this merge either, so on those rows // the higher search has nothing to judge, its candidates tie, and the lowest- // numbered - the one claiming no screw - is named. Measured on a P4_2 2_1 2 // crystal adopted as P2_1 2_1 2_1 and promoted by the twin-immune zone: all three // axial rows read "no control class" and P 4 2 2 was written. So among the // candidates the higher search cannot separate, the ones that drop an axial // absence the adopted group claims are dropped; the axial reflections the adopted // group left present are in this merge and stay the higher search's to decide. const gemmi::GroupOps lo_ops = sg.operations(); const auto keeps_screws = [&](const gemmi::SpaceGroup &hi) { const gemmi::GroupOps hi_ops = hi.operations(); for (int axis = 0; axis < 3; ++axis) for (int n = 1; n <= 6; ++n) { gemmi::Op::Miller hkl{{0, 0, 0}}; hkl[axis] = n; if (lo_ops.is_systematically_absent(hkl) && !hi_ops.is_systematically_absent(hkl)) return false; } return true; }; if (hi_search.best_space_group.has_value()) { std::vector tied{*hi_search.best_space_group}; tied.insert(tied.end(), hi_search.alternatives.begin(), hi_search.alternatives.end()); const auto kept = std::find_if(tied.begin(), tied.end(), [&](const gemmi::SpaceGroup &c) { return c.centring_type() == sg.centring_type() && keeps_screws(c); }); if (kept != tied.end()) { hi_search.best_space_group = *kept; if (kept->is_sohncke()) hi_search.sohncke_space_group = *kept; hi_search.alternatives.clear(); for (const auto &c : tied) if (c.number != kept->number && (c.centring_type() != sg.centring_type() || keeps_screws(c))) hi_search.alternatives.push_back(c); for (auto &c : hi_search.candidates) if (c.selected && c.space_group.centring_type() == sg.centring_type()) c.selected = keeps_screws(c.space_group); // A row the adopted group decided is not open, whatever this merge holds of it. std::erase_if(hi_search.undetermined_screws, [&](const auto &u) { return lo_ops.is_systematically_absent({{u.row[0], u.row[1], u.row[2]}}); }); } } std::optional hi_sg; if (hi_search.best_space_group.has_value() && hi_search.best_space_group->centring_type() == sg.centring_type()) hi_sg = hi_search.best_space_group; else for (const auto &alt : hi_search.alternatives) if (alt.centring_type() == sg.centring_type()) { hi_sg = alt; break; } // A refusal is arbitrated by the twin-immune zone of the added operators // (TwinZoneVerdict) where it is decisive, and only otherwise by the merge // comparison below: that comparison reads R_meas and ISa of two merges whose // per-frame scales have settled, and the lower group, with the same scales fitted // against fewer equivalents, then agrees with itself the better of the two on a // genuine promotion as much as on a twin (measured: a genuine 2 -> 222 step reading // R_meas 0.094 -> 0.103, ISa 12.3 -> 10.9). The zone does not depend on the scales. // Read on the adopted group's own merge: the zone reflections are centric in the // refused group and acentric in the adopted one, so they are distinct reflections // of this merge, at full multiplicity and production scale - where the P1 search // merge of a weak crystal has no shell with signal enough to read a zone on. // The pseudo-translation is carried into the basis this merge is indexed on. It was // measured on the search merge, before the change of basis committed above re-indexed // the reflections, and the zone spends it assigning each reflection a phase class by // cos(2 pi h.u) - which is a phase class only if h and u share a basis. Left in the // old one it names classes at random and normalises the zone by noise, silently: the // numbers stay the size they were. The reindex is (h,k,l)_new = M (h,k,l)_old, so h.u // is preserved by u_new = (M^T)^-1 u_old - the ordinary transformation of a // real-space fractional coordinate. Where nothing was committed this is the identity. TranslationalNCSResult zone_tncs = pre_tncs; if (commit_reindex && zone_tncs.detected) { const gemmi::Vec3 u = commit_reindex->transpose().inverse().multiply( gemmi::Vec3(pre_tncs.vector_frac[0], pre_tncs.vector_frac[1], pre_tncs.vector_frac[2])); zone_tncs.vector_frac = {u.x - std::floor(u.x), u.y - std::floor(u.y), u.z - std::floor(u.z)}; } const int zone = hi_sg.has_value() ? TwinZoneVerdict(sm.merged, gemmi::UnitCell(*result.consensus_cell), hi_search, sg_search, result, TranslationalNCSVectorForLTest(zone_tncs)) : 0; if (zone < 0) { const std::string msg = fmt::format( "Point group {} was refused ({}), and the twin-immune zone of its added " "operators reads acentric: they relate unequal intensities on the very " "reflections a twin law cannot touch, so the refusal stands and the " "operator is recorded as a twin law. Keeping {}.", sg_search.refused_point_group_hm, sg_search.refused_reason, sg.xhm()); logger.Info("{}", msg); stats_text << " " << msg << "\n\n"; sg_search.refused_reason += ", and its twin-immune zone reads acentric"; hi_sg.reset(); } else if (zone > 0) { const std::string msg = fmt::format( "Point group {} was confirmed by the operator correlations and then " "refused ({}), but the twin-immune zone of its added operators reads " "centric - a twin law cannot produce that on the reflections it leaves " "untouched - so {} is adopted.", sg_search.refused_point_group_hm, sg_search.refused_reason, hi_sg->xhm()); logger.Info("{}", msg); stats_text << " " << msg << "\n\n"; if (sg_search.refused_by_twin_gate) { twin_overruled_subgroup = sg; twin_overruled_reason = sg_search.refused_reason; } sg_search = hi_search; if (hi_search.best_space_group->number != hi_sg->number) { sg_search.alternatives.push_back(*hi_search.best_space_group); std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) { return a.number == hi_sg->number; }); } sg_search.best_space_group = *hi_sg; result.space_group = *hi_sg; end_msg.space_group_name = hi_sg->xhm(); end_msg.space_group_number = static_cast(hi_sg->number); promoted_point_group = true; experiment_.SetSpaceGroup(*hi_sg); phase("Re-scaling in space group " + hi_sg->short_name()); sm = scale_and_merge(hi_sg->short_name(), false); hi_sg.reset(); } if (hi_sg.has_value()) { // scale_and_merge reports the error model of whatever it merged, and the two // comparison merges are not the run's answer, so it goes back if they lose. const double em_isa = result.error_model_isa; const double em_isa_asymptotic = result.error_model_isa_asymptotic; const bool em_isa_resolved = result.error_model_isa_resolved; const double em_a = result.error_model_a; const double em_b = result.error_model_b; const auto res_fit = result.resolution_fit_A; const auto manual_d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A(); phase("Merging under the refused point group " + sg_search.refused_point_group_hm); rsm->SetDMinLimit(sm.statistics.overall.d_min); const auto low = scale_and_merge(sg.short_name() + ", pinned", false); const double low_isa = result.error_model_isa; experiment_.SetSpaceGroup(*hi_sg); const auto high = scale_and_merge(hi_sg->short_name() + ", pinned", false); const double high_isa = result.error_model_isa; rsm->SetDMinLimit(manual_d_min); const double low_r = low.statistics.overall.r_meas; const double high_r = high.statistics.overall.r_meas; const auto mult = [](const MergeStatisticsShell &o) { return o.unique_reflections > 0 ? static_cast(o.total_observations) / o.unique_reflections : 0.0; }; const std::string measured = fmt::format( "R_meas {:.4f} -> {:.4f}, ISa {:.2f} -> {:.2f}, multiplicity {:.2f} " "-> {:.2f}, both merged to {:.2f} A", low_r, high_r, low_isa, high_isa, mult(low.statistics.overall), mult(high.statistics.overall), sm.statistics.overall.d_min); // Neither statistic may get worse. A NaN on either side answers "worse", // which is the recoverable direction: the run keeps the group it already has. if (high_r <= low_r && high_isa >= low_isa) { const std::string msg = fmt::format( "Point group {} was confirmed by the operator correlations and " "then refused ({}), so both groups were merged and compared: {}. " "Merging under it does not degrade the data, which folding " "non-equivalent reflections together cannot manage, so {} is " "adopted.", sg_search.refused_point_group_hm, sg_search.refused_reason, measured, hi_sg->xhm()); logger.Info("{}", msg); stats_text << " " << msg << "\n\n"; if (sg_search.refused_by_twin_gate) { twin_overruled_subgroup = sg; twin_overruled_reason = sg_search.refused_reason; } sg_search = hi_search; // Leave the candidate it displaced among the alternatives, so the // reported "or ..." list stays the set of groups the data cannot // separate, each named once - as the metric-lattice ask does. if (hi_search.best_space_group->number != hi_sg->number) { sg_search.alternatives.push_back(*hi_search.best_space_group); std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) { return a.number == hi_sg->number; }); } sg_search.best_space_group = *hi_sg; result.space_group = *hi_sg; end_msg.space_group_name = hi_sg->xhm(); end_msg.space_group_number = static_cast(hi_sg->number); promoted_point_group = true; phase("Re-scaling in space group " + hi_sg->short_name()); sm = scale_and_merge(hi_sg->short_name(), false); } else { logger.Info("Point group {} was refused ({}), and merging under it " "agrees: {}. Keeping {}.", sg_search.refused_point_group_hm, sg_search.refused_reason, measured, sg.xhm()); sg_search.refused_reason += ", and merging under it makes the data worse (" + measured + ")"; experiment_.SetSpaceGroup(sg); result.error_model_isa = em_isa; result.error_model_isa_asymptotic = em_isa_asymptotic; result.error_model_isa_resolved = em_isa_resolved; result.error_model_a = em_a; result.error_model_b = em_b; result.resolution_fit_A = res_fit; } } } // After the arbitration, so a promoted run is not told that the metric hosts more // symmetry than a group it no longer has. WarnIfMetricHostsMoreSymmetry(result.consensus_cell, result.space_group.value_or(gemmi::get_spacegroup_p1())); } // A screw no axial reflection here could have shown (SearchSpaceGroupResult::unreached_screws), where the // group that search chose is the one adopted: the screw is open, as on a row never recorded. Report only - // nothing above read it. if (!sg_search.unreached_screws.empty() && result.space_group && !sg_search.candidates.empty() && sg_search.candidates.front().space_group.xhm() == result.space_group->xhm()) { std::string axes, alts; for (const auto &u : sg_search.unreached_screws) { axes += (axes.empty() ? "" : ", ") + std::string(1, u.axis); sg_search.undetermined_screws.push_back(u); } for (const auto &g : sg_search.unreached_screw_groups) { alts += " or " + g.xhm(); sg_search.alternatives.push_back(g); } logger.Warning("The screw along {} is UNDETERMINED: its axial row holds too few reflections a screw would " "extinguish to show one, however weak they are, and none that contradicts it. These data " "say {}{}; {} is written because a reflection file must carry one group.", axes, result.space_group->xhm(), alts, result.space_group->xhm()); } result.space_group_search = sg_search; // Was a twin law possibly absorbed? Where a twin gate refused the adopted group and was // overruled, or where a group above P1 was adopted on data that read twin-like before the // search (the pre-promotion L-test), the merge in the subgroup is written beside the answer // too, with the law - so a refinement program can take the twin hypothesis without a re-run. // The answer itself is not touched by any of this. const bool pre_twin_like = result.pre_promotion_twinning && result.pre_promotion_twinning->twinning_suspected; // A model can test the twin hypothesis itself (the twin validation, molecular replacement's twin // comparison), so with one the merge in the subgroup is made wherever a metric re-ask promoted from the // indexed lattice's group, or the model is in a group with fewer rotations than the adopted one. const bool model_tests_twin = config_.model && (twin_metric_subgroup.has_value() || (!model_predicted && model_rotations > 0 && model_rotations < result.space_group->operations().sym_ops.size())); if (!geometry_prepass && result.space_group && sg_search.point_group_order > 1 && (twin_overruled_subgroup || pre_twin_like || model_tests_twin)) { twin_choice = ChooseTwinSubgroup(*result.space_group, sg_search, twin_overruled_subgroup ? twin_overruled_subgroup : twin_metric_subgroup); twin_choice_group_number = result.space_group->number; twin_for_mr_only = !twin_overruled_subgroup && !pre_twin_like; // The model's group, where it is an index-2 subgroup of the adopted one on the data's axes, is the // twin hypothesis it asks to be tested in: the merge in another subgroup averages reflections the // model's symmetry does not relate. Made in place of the subgroup above where that one was only for // the model; beside it where the data asked for theirs, which the model does not change. if (model_tests_twin && model_group) { model_twin_choice = ChooseTwinSubgroup( *result.space_group, sg_search, std::nullopt, ModelRotationsOnDataAxes(*model_group, model_cell, gemmi::UnitCell(*result.consensus_cell))); if (model_twin_choice && twin_choice && SortedRotations(model_twin_choice->subgroup.operations()) == SortedRotations(twin_choice->subgroup.operations())) model_twin_choice.reset(); if (model_twin_choice && twin_for_mr_only) { twin_choice = std::move(model_twin_choice); model_twin_choice.reset(); } if (model_twin_choice) logger.Info("Twin hypothesis: the model's group {} asks for the merge in {}, with twin law {}, " "made for the model to test", model_group->xhm(), model_twin_choice->subgroup.xhm(), model_twin_choice->law.as_hkl().triplet()); } twin_choice_reason = twin_overruled_subgroup ? fmt::format("a twin gate refused it ({}) and was overruled", twin_overruled_reason) : pre_twin_like ? fmt::format("the merge read twin-like before the space-group search ({})", TwinningVerdictLine(*result.pre_promotion_twinning)) : std::string("the model can test it"); if (twin_choice) logger.Info("Twin hypothesis: {} was adopted, but {} - the merge in {} is written too, " "with twin law {}", result.space_group->xhm(), twin_choice_reason, twin_choice->subgroup.xhm(), twin_choice->law.as_hkl().triplet()); } } else { // A space group was fixed by the user; surface it so the viewer/CLI can still show it. result.space_group = experiment_.GetGemmiSpaceGroup(); // ...and record it, exactly as the search arm above does with the group it determined, or // the _process.h5 comes out with a cell but no group and re-merging it defaults to P1. if (const auto &sg = experiment_.GetGemmiSpaceGroup(); sg.has_value()) { end_msg.space_group_name = sg->xhm(); end_msg.space_group_number = static_cast(sg->number); } } search_mark.Stop(); TimingMark adoption_mark(logger, "adoption (setting, reference, guard)"); // The files are written on other axes than the ones the space group was determined on in two // cases: to match something the user gave (a reference MTZ, a model, -C, -S), and to write the // standard setting. Both are relabellings made after every decision. The group's label goes // with the axes: the one experiment_ holds and the one the _process.h5 records (end_msg), // which differ where a model's enantiomorph was taken as a label only. const auto relabel_group = [&](const gemmi::Op &cob, const gemmi::SpaceGroup &moved) { const char old_centring = experiment_.GetSpaceGroupOrP1().centring_type(); experiment_.SetSpaceGroup(moved); result.space_group = moved; if (end_msg.space_group_name.has_value()) if (const gemmi::SpaceGroup *recorded = gemmi::find_spacegroup_by_name(*end_msg.space_group_name)) if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(*recorded, cob)) { end_msg.space_group_name = m->xhm(); end_msg.space_group_number = static_cast(m->number); } if (moved.centring_type() != old_centring) { result.consensus_centering = moved.centring_type(); if (end_msg.rotation_lattice_type.has_value()) end_msg.rotation_lattice_type->centering = moved.centring_type(); result.rotation_lattice_type = end_msg.rotation_lattice_type; } // What the report names beside the group, on the same axes. if (result.space_group_search.has_value()) { auto &sgs = *result.space_group_search; if (sgs.sohncke_space_group.has_value()) if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(*sgs.sohncke_space_group, cob)) sgs.sohncke_space_group = *m; for (auto &alt : sgs.alternatives) if (const gemmi::SpaceGroup *m = SpaceGroupInBasis(alt, cob)) alt = *m; for (auto &u : sgs.undetermined_screws) { const auto row = HklOperator(cob).apply_to_hkl({{u.row[0], u.row[1], u.row[2]}}); const int n_axes = (row[0] != 0) + (row[1] != 0) + (row[2] != 0); if (n_axes != 1) continue; const int axis = row[0] != 0 ? 0 : row[1] != 0 ? 1 : 2; u.row = {std::abs(row[0]), std::abs(row[1]), std::abs(row[2])}; u.axis = "abc"[axis]; u.row_label = std::array{"h00", "0k0", "00l"}[axis]; } } }; // The indices the finalist ledger's P1 merge below (made from the scaler's own copy of the // observations) still has to be put through to reach the written frame, and the ones the // report states: every relabelling since the group was determined. gemmi::Op merge_to_written = gemmi::Op::identity(); gemmi::Op determined_to_written = gemmi::Op::identity(); // `relabelled`: the merged reflections are already in the new setting, free set included - made by the // validation started on the forecast from the same reflections the same way - so only the rest moves. const auto relabel_output = [&](const gemmi::Op &cob, const char *source, std::vector *relabelled = nullptr) { result.setting_source = source; if (cob == gemmi::Op::identity()) return; const gemmi::SpaceGroup *moved = SpaceGroupInBasis(experiment_.GetSpaceGroupOrP1(), cob); if (moved == nullptr) { logger.Warning("No setting of {} is reached by the change of basis {}; the files keep " "the axes the group was determined on", experiment_.GetSpaceGroupOrP1().xhm(), IndexTriplet(HklOperator(cob))); return; } const gemmi::Op hkl = HklOperator(cob); if (relabelled == nullptr) sm.merged = ReindexMergedIntoAsu(sm.merged, hkl, *moved, experiment_.GetScalingSettings().GetMergeFriedel()); else sm.merged = std::move(*relabelled); reseat_integrated(cob); relabel_group(cob, *moved); // The free set is a function of the written index (RfreeFlags.h), so that every dataset of // a crystal form written on the same axes carries the same one: drawn again on these axes. if (relabelled == nullptr) AssignRfreeFlags(sm.merged, *moved, experiment_.GetScalingSettings().GetRfreeFraction(), 500, result.consensus_cell, config_.nthreads); merge_to_written = merge_to_written.combine(hkl); determined_to_written = determined_to_written.combine(hkl); result.setting_operator = IndexTriplet(determined_to_written); const auto &uc = *result.consensus_cell; logger.Info("Written in {} (setting chosen by {}): reindexed by {} to a={:.3f} b={:.3f} c={:.3f} " "alpha={:.2f} beta={:.2f} gamma={:.2f}", moved->xhm(), source, IndexTriplet(hkl), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); }; // A reference MTZ (rotation): put the merge into the reference's frame before anything is // matched against it - its intensities and its free-R flags are keyed by its own indices. The // candidates are every description of the data's lattice on the reference's axes, and each of // the data's alternative indexings (twin laws) on top; the one whose intensities correlate // best with the reference is taken, the reflections are re-seated and re-merged in it, and the // group becomes the reference's own, in its own setting. Stills resolve the twin-law ambiguity // per image with ReindexAmbiguityResolver instead. if (rsm && !config_.reference_data.empty() && result.consensus_cell && experiment_.GetSpaceGroupNumber().has_value()) { const gemmi::SpaceGroup data_sg = experiment_.GetSpaceGroupOrP1(); const gemmi::SpaceGroup ref_sg = config_.reference_space_group.value_or(data_sg); const auto choice = ChooseReferenceSetting(sm.merged, *result.consensus_cell, data_sg, config_.reference_data, config_.reference_cell, ref_sg); const gemmi::Op hkl = HklOperator(choice.cob); result.reference_operator = IndexTriplet(hkl); result.setting_source = "REFERENCE"; logger.Info("Reference: {} candidate frame(s) scored; CC with the reference {:.3f} as indexed, " "{:.3f} under {}", choice.n_candidates, choice.identity_cc, choice.cc, result.reference_operator); // The reference's group where the run merged in the same group type, in the reference's // setting; otherwise (-S overrode it) the run's own group on the reference's axes. const gemmi::SpaceGroup *moved = data_sg.number == ref_sg.number ? &ref_sg : SpaceGroupInBasis(data_sg, choice.cob); if (moved != nullptr && !(choice.cob == gemmi::Op::identity() && moved->xhm() == data_sg.xhm())) { if (!(choice.cob == gemmi::Op::identity())) reseat_integrated(choice.cob); relabel_group(choice.cob, *moved); // The engine is about to be remade in the reference's frame, and the twin-law merge is // made from it: its subgroup moves with the data. if (twin_choice && !MoveTwinSubgroupChoice(*twin_choice, choice.cob)) twin_choice.reset(); if (model_twin_choice && !MoveTwinSubgroupChoice(*model_twin_choice, choice.cob)) model_twin_choice.reset(); determined_to_written = determined_to_written.combine(hkl); result.setting_operator = IndexTriplet(determined_to_written); rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell, static_cast(config_.scaling_iter), config_.nthreads, logger, config_.observation_dump); rsm->Ingest(); phase("Re-merging in the reference's frame"); sm = scale_and_merge(moved->short_name(), false); const auto &uc = *result.consensus_cell; logger.Info("Reference: re-merged in its frame, {}: a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} " "beta={:.2f} gamma={:.2f}", moved->xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); } } else if (!geometry_prepass && config_.reference_data.empty() && result.consensus_cell && experiment_.GetGemmiSpaceGroup().has_value()) { // No reference: the standard setting, or the one the user asked for with -S / -C. A model // that fits can still move it below. const gemmi::SpaceGroup sg = *experiment_.GetGemmiSpaceGroup(); const gemmi::SpaceGroup *target = user_fixed_sg_ && user_fixed_sg_->number == sg.number ? &*user_fixed_sg_ : nullptr; const std::optional &target_cell = user_cell_; relabel_output(ChooseOutputSetting(*result.consensus_cell, sg, target, target_cell), target_cell ? "CELL" : target != nullptr && !target->is_reference_setting() ? "SPACE_GROUP" : "STANDARD"); } // A pass RunAllPasses is about to throw away and run again. Its report is never printed and // its files are overwritten seconds later, so everything below that only fills those in would // be spent on an answer nobody reads. Every trigger of a re-run is settled by here - the // integrator's background-starvation count (measured at integration), the fixed-group lattice // conflict (at indexing) and the search merge the quality guard reads (at the first merge) - so // a pass can tell that it is being superseded, which RunAllPasses cannot until it returns. // Only the refined passes ask: quality_guard_pass1_ holds the header-geometry pass they are // judged against and is set only while they run. The supercell guard is deliberately not asked // here - it compares this pass's cell against pass 1's rather than reading this pass's own // numbers - so a pass it re-runs still writes a report first. // // Nothing below this point is read by a later pass: the analyses are report-only (see each), // the files are rewritten by the pass that supersedes this one, and the one place that touches // experiment_ - the model frame - is undone by the SetSpaceGroup at the top of RunPipeline. // // So is a canonical pass the geometry walk is going to replace, where that answer is in by now // (GeometryWalkTakesRound). A refined pass run beside the rotation-scale walk is superseded too // where the walk overturns it (see RunAllPasses); it waits here for the walk's answer. superseded = (!cancelled_ && quality_guard_pass1_ && quality_guard_pass1_->has_merge_statistics && ((bragg_adaptive_ && bkg_starved_fraction_ && *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION) || !RefinedPassIsWorse(*quality_guard_pass1_, result).empty())) || (lattice_arm_check_ && lattice_arm_check_->lost) // only with a _process.h5 open, see above || GeometryWalkTakesRound(result, p.postrefine_ahead) || (rotation_scale_check_ && rotation_scale_check_->Overturns()); if (superseded) { logger.Info("This pass is going to be re-run, so it makes no report and writes no files."); Note("this pass is to be re-run (see the log); it writes nothing"); } else if (!geometry_prepass) { Note(fmt::format("resolution {:.2f} A{}", sm.statistics.overall.d_min, result.resolution_fit_A ? fmt::format(" (CC1/2 fall-off at {:.2f} A)", *result.resolution_fit_A) : std::string())); Step("Analyses"); } adoption_mark.Stop(); TimingMark analyses_mark(logger, "analyses"); // The answer's merge is made and in `sm`; a merge below (twin law, ledger) refills the arrays. if (rsm) rsm->ReleaseMergeScratch(); // --finalist-ledger (see where it is read, below) is folded from a merge of its own: the same // observations in P1. It needs every node of the lattice integrated, which a fixed centred group // does not do. const bool p1_integration_complete = search_space_group || (indexer && indexer->GetPredictionCentring() == 'P'); const bool finalist_ledger = config_.finalist_ledger && rsm && result.consensus_cell && write_output && config_.write_merged && !geometry_prepass && !superseded && p1_integration_complete && is_rotation; // The ledger's P1 merge is made from here on, beside the analyses. Nothing here reads what it // writes or writes what it reads: the engine merges its own copy of the observations, in a group // of its own rather than the experiment's, and writes no per-frame scale back onto the outcomes; // and nothing up to take_ledger_p1 calls the engine, changes the outcomes it was ingested from, // or changes the experiment. Its log lines are held and printed where it is taken. // The twin-law merge (see twin_choice), from the same engine as the answer's, to the resolution the // answer was cut at. Like the ledger's below, it merges the engine's own copy of the observations in // a group of its own and writes no per-frame scale back, so nothing the answer is made of sees it. // The model's own subgroup (model_twin_choice) the same way. const auto twin_law_merge = [&](const TwinSubgroupChoice &c) -> std::optional { if (!rsm || !result.consensus_cell || !write_output || !config_.write_merged || geometry_prepass || superseded) return std::nullopt; TimingMark mark(logger, "twin-law merge"); phase("Merging in " + c.subgroup.short_name() + " (twin hypothesis)"); rsm->SetWriteBackPerFrameScale(false); rsm->SetExportScaledFulls(false); rsm->SetFrenchWilson(false); const auto manual_d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A(); rsm->SetDMinLimit(sm.statistics.overall.d_min); auto merge = rsm->Run(/*for_search=*/false, /*full_stats=*/true, /*measure_cc_before_corrections=*/false, &c.subgroup); rsm->SetDMinLimit(manual_d_min); rsm->SetWriteBackPerFrameScale(true); return merge; }; std::optional twin_merge, model_twin_merge; if (twin_choice) twin_merge = twin_law_merge(*twin_choice); if (model_twin_choice) model_twin_merge = twin_law_merge(*model_twin_choice); std::optional ledger_p1; Logger ledger_log = Logger::Buffered(); std::future ledger_ahead; if (finalist_ledger) { rsm->SetWriteBackPerFrameScale(false); rsm->SetExportScaledFulls(false); rsm->SetFrenchWilson(false); rsm->SetLogger(ledger_log); ledger_ahead = std::async(std::launch::async, [&rsm] { return rsm->Run(/*for_search=*/false, /*full_stats=*/true, /*measure_cc_before_corrections=*/false, &gemmi::get_spacegroup_p1()); }); } auto take_ledger_p1 = [&] { if (!ledger_ahead.valid()) return; TimingMark wait_mark(logger, "finalist ledger merge wait"); ledger_p1 = ledger_ahead.get(); wait_mark.Stop(); rsm->SetLogger(logger); rsm->SetWriteBackPerFrameScale(true); ledger_log.ReplayInto(logger); }; const auto &twin_sg_opt = experiment_.GetGemmiSpaceGroup(); const gemmi::SpaceGroup *twin_sg = twin_sg_opt ? &*twin_sg_opt : nullptr; // Diffraction anisotropy (see where it is reported, below), made beside the analyses that come // before it: it reads the merge, the integrated observations and the per-frame scales the merge // wrote back, none of which they change, and most of it is gathering the observations. std::future anisotropy; if (!geometry_prepass && !superseded && result.consensus_cell) { AnisotropyRunInfo aniso_run; if (sm.statistics.sweep_quality.measured && sm.statistics.sweep_quality.sweep_deg > 0.0f) aniso_run.observed_rotation_deg = sm.statistics.sweep_quality.sweep_deg; aniso_run.dose_term_in_scale_model = experiment_.GetScalingSettings().GetCorrectionSurfaces(); aniso_run.radiation_damage_relative_b = sm.statistics.radiation_damage_delta_b; const float wedge_deg = experiment_.GetGoniometer() ? experiment_.GetGoniometer()->GetWedge_deg() : 0.0f; anisotropy = std::async(std::launch::async, [&, aniso_run, wedge_deg, cell = *result.consensus_cell, rotation = experiment_.IsRotationIndexing()] { return AnalyzeAnisotropy(sm.merged, ScaledObservations(indexer->GetIntegrationOutcome(), rotation, twin_sg, wedge_deg, 0.5, config_.nthreads), cell, twin_sg, aniso_run); }); } // fast_live, cut (the broker's deadline or a cancel): the model is left out, and the files are written // as a run without one writes them. Asked again before the search below. const auto model_cut = [&] { if (!config_.fast_live || !cancelled_) return false; Note("cut: the model is left out"); return true; }; const bool model_run = result.consensus_cell && write_output && config_.model && !geometry_prepass && !superseded && !model_cut(); // The merge's own shell bounds, so CC(model, data) is reported on the shells CC1/2 was. std::vector report_shell_d_min; for (const auto &sh : sm.statistics.shells) report_shell_d_min.push_back(sh.d_min); // Nothing indexes after the merge, so the image pass's indexers - an FFT indexer keeps its transform // buffers and histograms on the card, 0.8 GB for the two of a rotation run - go before the model is // validated or searched for, once the leftover-lattice census, the last to use them, has ended. if (model_run) { if (census_future.valid()) { TimingMark census_mark(logger, "leftover-lattice census wait (model)"); census_future.wait(); } p.indexer_pool.reset(); // The model's GPU budget is 80% of the card less what the merge still holds, so from here the merge // holds the card: a speculative probe not started yet waits for it to end (GPUMerge), and one let // beside it is waited for. That is a probe pass of seconds: this long means it is stuck, and the // model goes on as it would have without it. if (p.gpu_merge) p.gpu_merge->Holds(get_gpu_total_memory(0)); TimingMark beside_mark(logger, "GPU work beside wait (model)"); wait_for_gpu_work_beside(std::chrono::minutes(10)); } // The model's first validation reads of the merge only its amplitudes, which the analyses below make // last, from the anisotropy tensor - a fit to the merge alone (AnisotropyPriorHKL). So it starts here, // beside the analyses, on a copy of the merge given the same amplitudes: the same French-Wilson on the // same tensor. Its log is held, and printed where it is taken; nothing it decides is acted on before. std::vector model_merged; const auto data_sg = experiment_.GetGemmiSpaceGroup(); // With a reference MTZ the merohedral indexing was already resolved against it (rotation // merge / stills scaling), so trust that; only probe indexing by R-free when model-only. // The validation in the model's setting further down depends on this one only through the // setting and the indexing it settles, and both are forecast well before this one has finished // (ModelFrameForecast). So it is started on that forecast, beside this one - on the next card // where there is one - and kept only where the decision is the one forecast; its log is held, // and its files wait, until then. On any other decision it is dropped, unwritten. The relabelling // it runs on is the one AdoptModelFrame and relabel_output below make, made here on a copy. // A model asserting the other enantiomorph is not forecast: whether the label is taken is // only known at the end, from the anomalous map. struct Ahead { gemmi::Op change_of_basis_op, indexing_op; // The reflections it validates against, relabelled as the files would be: where it is kept, // they are the written ones, and the relabelling below is not made a second time. std::vector merged; Logger log = Logger::Buffered(); std::future run; std::promise write; // destroyed before `run`, so a run left waiting on it is released } ahead; // The merge is done with its engine - nothing below reads it but the finalist ledger's P1 merge, // which runs on it beside the validation - so the device memory it holds goes back before the // validation decides, from the card's total, what goes on its GPU. Where the ledger's merge keeps // it, what it holds is counted against the card by every validation here, the same for each, so // that one started on the forecast decides as the one it stands in for would. if (model_run && !ledger_ahead.valid()) rsm.reset(); const size_t merge_on_card = rsm && rsm->OnGPU() ? rsm->DeviceBytes() : 0; ModelValidationSchedule first_schedule; first_schedule.gpu_resident_bytes = merge_on_card; first_schedule.on_forecast = [&](const ModelFrameForecast &f) { if (ahead.run.valid() || !config_.reference_data.empty() || !data_sg.has_value() || !f.may_run_beside || f.enantiomorph_candidate || f.change_of_basis_op == gemmi::Op::identity()) return; ahead.change_of_basis_op = f.change_of_basis_op; ahead.indexing_op = f.indexing_op; ahead.run = std::async(std::launch::async, [&, merged = model_merged, sg = experiment_.GetSpaceGroupOrP1(), cell = *result.consensus_cell, friedel = experiment_.GetScalingSettings().GetMergeFriedel(), rfree_fraction = experiment_.GetScalingSettings().GetRfreeFraction(), wavelength = experiment_.GetWavelength_A(), gate = ahead.write.get_future().share(), beside = f.gpu_bytes]() mutable { if (const int32_t cards = get_gpu_count(); cards > 0) pin_gpu(1 % cards); if (!(ahead.indexing_op == gemmi::Op::identity())) { merged = ReindexMergedIntoAsu(merged, ahead.indexing_op, sg, friedel); cell = CellInBasis(cell, HklOperator(ahead.indexing_op)); } gemmi::Op to_model = ahead.change_of_basis_op; to_model.tran = {0, 0, 0}; const gemmi::Op cob = to_model.inverse(); const gemmi::SpaceGroup *in = &sg; if (const gemmi::SpaceGroup *moved = SpaceGroupInBasis(sg, cob)) { merged = ReindexMergedIntoAsu(merged, HklOperator(cob), *moved, friedel); cell = CellInBasis(cell, cob); AssignRfreeFlags(merged, *moved, rfree_fraction, 500, cell, config_.nthreads); in = moved; } ModelValidationSchedule schedule; schedule.maps_gate = gate; schedule.fit_test = false; // the verdict is the first validation's (KeepModelVerdict) schedule.gpu_resident_bytes = merge_on_card; schedule.gpu_beside_bytes = beside; ahead.merged = std::move(merged); return ValidateAgainstModel(ahead.merged, cell, *config_.model, ahead.log, in, /*probe_indexing_ambiguity=*/false, static_cast(config_.nthreads), wavelength, report_shell_d_min, schedule); }); }; // Whether the validation started on the forecast is the one kept: the decision is the one forecast. // Asked by the first validation before it makes its maps, which are then left to that one, and // again below with the verdict in hand - the same answer, as nothing it reads changes between. const auto forecast_kept = [&](const ModelValidationResult &v) { return ahead.run.valid() && config_.reference_data.empty() && v.model_fits && !(v.change_of_basis_op == gemmi::Op::identity()) && !v.adopted_model_enantiomorph && v.change_of_basis_op == ahead.change_of_basis_op && v.indexing_op == ahead.indexing_op; }; first_schedule.maps_superseded = forecast_kept; Logger model_validation_log = Logger::Buffered(); std::future model_validation_run; // last: it is joined before what it reads goes if (model_run && !model_predicted && !config_.force_mr) { model_merged = sm.merged; model_validation_run = std::async(std::launch::async, [&, device = get_gpu()] { set_gpu(device); TimingMark mark(model_validation_log, "model validation (beside the analyses)"); FrenchWilsonOptions fw_opts; fw_opts.num_threads = config_.nthreads; const gemmi::UnitCell cell(*result.consensus_cell); fw_opts.anisotropy_hkl = AnisotropyPriorHKL(model_merged, cell, twin_sg); ApplyFrenchWilson(model_merged, experiment_.GetSpaceGroupOrP1(), fw_opts); return ValidateAgainstModel(model_merged, *result.consensus_cell, *config_.model, model_validation_log, data_sg ? &*data_sg : nullptr, /*probe_indexing_ambiguity=*/config_.reference_data.empty(), static_cast(config_.nthreads), experiment_.GetWavelength_A(), report_shell_d_min, first_schedule); }); } // Not on the geometry pre-pass, nor on a superseded one: the analysis goes into that pass's // statistics text and its written reflections, and neither survives the run. The promotion flag // below is a different thing - it is what the SEARCH did, the second pass reads it, and it is // set either way. if (!geometry_prepass && !superseded) { // Translational pseudo-symmetry first: its verdict is an input to the twinning analysis, // which must not read its own L-test when a non-half-integer pseudo-translation has biased // it. Reporting only - no reflection and no merge depends on the outcome. if (result.consensus_cell) result.tncs = AnalyzeTranslationalNCS(sm.merged, gemmi::UnitCell(*result.consensus_cell), twin_sg, 30, 0, config_.nthreads); if (result.tncs.detected) logger.Warning("Translational pseudo-symmetry: Patterson off-origin peak {:.1f}% of the " "origin at ({:.3f}, {:.3f}, {:.3f}), {:.1f} A - declare it to the " "molecular-replacement program", result.tncs.peak_percent, result.tncs.vector_frac[0], result.tncs.vector_frac[1], result.tncs.vector_frac[2], result.tncs.vector_length_A); // Without the reflections the lattice centring extinguishes, as the pre-promotion test // above is measured: the merged ASU of a centred group still holds keys in the absent // class (the merge writes what it was given), they are noise rather than intensities, // and an L-test partner step that lands on one pairs a real reflection with an absent. // This is the number the user reads, so it is the one that must be treated. const char twin_centring = twin_sg ? twin_sg->centring_type() : end_msg.rotation_lattice_type.has_value() ? end_msg.rotation_lattice_type->centering : 'P'; const std::optional twin_cell = result.consensus_cell ? std::optional(*result.consensus_cell) : std::nullopt; result.twinning = AnalyzeTwinning(sm.merged, twin_sg, 20, TranslationalNCSVectorForLTest(result.tncs), twin_centring, twin_cell ? &*twin_cell : nullptr); // Set on the struct the text below is rendered from. AnalyzeTwinning returns a fresh result // with this flag clear, so setting it only afterwards left the statistics text asserting // "No twinning: the Laue class is holohedral" as a fact while the report, rendering the same // struct once the flag was set, said the opposite - two twin verdicts from one run, with the // flat one landing on exactly the case the flag exists for. (Assigning it before the call // would not do: the call replaces the whole struct.) result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group; // A refusal that stood on the twin-immune zone names the law: the first operator the // refused group adds, at the fraction its own disagreement implies (a lower bound). if (twin_sg && result.space_group_search && result.space_group_search->refused_point_group_representative && result.twin_zone_refusal_stood) { const auto &sgs = *result.space_group_search; const gemmi::GroupOps hi = sgs.refused_point_group_representative->operations(); const gemmi::GroupOps lo = twin_sg->operations(); for (const auto &op : hi.derive_symmorphic().sym_ops) { if (op.det_rot() <= 0) continue; const bool in_lower = std::any_of(lo.sym_ops.begin(), lo.sym_ops.end(), [&](const gemmi::Op &o) { return o.rot == op.rot; }); if (in_lower) continue; result.twinning.twin_law_operator = gemmi::Op{op.rot, {0, 0, 0}, 'x'}.as_hkl().triplet(); break; } // The fraction: the refused operators' own H where the H gate measured it, else the // L-test of the merge before the search (a twin fraction that does not depend on the // promotion), else the perfect twin every operator statistic reads as symmetry. result.twinning.twin_law_fraction = std::isfinite(sgs.refused_twin_fraction) ? sgs.refused_twin_fraction : result.pre_promotion_twinning && result.pre_promotion_twinning->twin_fraction_source != TwinFractionSource::None ? result.pre_promotion_twinning->estimated_twin_fraction : 0.5; } if (result.tncs.measurable) stats_text << TranslationalNCSToText(result.tncs) << "\n"; stats_text << TwinningAnalysisToText(result.twinning) << "\n"; } // Mark the conclusion as non-authoritative when the Laue class was reached by a promotion the // search itself made, so the text cannot claim "no twin law exists" on its own say-so. result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group; // How the crystal sat on the spindle. Reported always on rotation data (it is a property of // the mounting, which the user can change). The angle to the nearest symmetry axis is kept // as a descriptive key, but the WARNING is decided by the exact orbit computation: the old // any-proper-axis-within-15-deg rule was wrong in both directions - an aligned in-plane // 2-fold of a dihedral group is repaired by the principal axis, a cubic group is never // severe in any orientation, and a lone diad PERPENDICULAR to the spindle is severe while // no axis is anywhere near it. if (experiment_.IsRotationIndexing() && twin_sg && end_msg.rotation_lattice.has_value()) { if (const auto gonio = experiment_.GetGoniometer()) { const auto closest = ClosestSymmetryAxisToSpindle(*twin_sg, *end_msg.rotation_lattice, gonio->GetAxis()); if (closest.has_value()) { const auto [angle, order] = *closest; result.spindle_symmetry_axis_deg = angle; result.spindle_symmetry_axis_order = order; stats_text << "Closest symmetry axis to the spindle: " << order << "-fold at " << std::fixed << std::setprecision(1) << angle << " deg\n"; } // The measured resolution of THIS run sets the cone: past merging the question is // what these data lost, not what the detector could have reached. double d_min = 0; for (const auto &r : sm.merged) if (std::isfinite(r.d) && r.d > 0 && (d_min == 0 || r.d < d_min)) d_min = r.d; if (const auto loss = SpindleUnrepairedFraction( *twin_sg, *end_msg.rotation_lattice, gonio->GetAxis(), experiment_.GetWavelength_A(), d_min)) { result.spindle_lost_unique_fraction = loss->lost_unique_fraction; end_msg.spindle_lost_unique_fraction = static_cast(loss->lost_unique_fraction); stats_text << "Unique reflections the mounting made unmeasurable: " << std::fixed << std::setprecision(2) << 100.0 * loss->lost_unique_fraction << "% (of the " << std::setprecision(2) << 100.0 * (1.0 - std::cos(loss->theta_max_deg * PI / 180.0)) << "% a sweep's blind cone holds at " << std::setprecision(2) << d_min << " A)\n\n"; // Warn when the point group recovers less than half the cone - the same half // that fixes the per-image trigger threshold. The loss is a coherent cap about // the spindle direction, which costs a map more than the same percentage lost // at random. if (loss->cone_fraction >= 0.5) { const std::string msg = fmt::format( "The mounting makes {:.1f}% of unique reflections unmeasurable: the " "measured point group cannot map that part of the sweep's blind cone " "onto measured territory, however long the sweep runs. The loss is a " "coherent cap about the spindle direction rather than a scatter, so " "overall completeness may still look reasonable while the gap sits at " "the spindle direction. A second sweep about a different axis, or " "re-mounting, recovers it.", 100.0 * loss->lost_unique_fraction); logger.Warning("{}", msg); stats_text << " !! " << msg << "\n\n"; result.warnings.push_back({PathologyCode::SPINDLE_CAP, msg}); } } else { stats_text << "\n"; } } } // Indexing-ambiguity (alternative-indexing) advisory. When the lattice metric symmetry exceeds // the Laue symmetry the crystal can be validly indexed in several hands related by twin-law // operators. For an obvious merohedral case (P3/P4/P6...) users expect this; but a PSEUDO-merohedral // metric (e.g. a C2 crystal whose beta makes it pseudo-F-orthorhombic) is easy to miss, so surface it. The // reindex operator is applied directly to (h,k,l), so its triplet reads as an h,k,l transform. if (result.consensus_cell && twin_sg != nullptr) { const auto twin_ops = ReindexAmbiguityOperators(*result.consensus_cell, *twin_sg, 2.0); if (!twin_ops.empty()) { // The merge is folded under the adopted group, and a twin law sends a reflection // outside that group's asymmetric unit - so the pairs are formed on the merge // expanded to P1 under the group's own operations (each reflection once per // operation; the H statistic is a median and the correlation a ratio, so the // repetition changes neither). std::vector expanded; const auto expanded_merge = [&]() -> const std::vector & { if (expanded.empty()) { const gemmi::GroupOps ops = twin_sg->operations(); expanded.reserve(sm.merged.size() * ops.sym_ops.size()); for (const auto &r : sm.merged) for (const auto &o : ops.sym_ops) { const auto hkl = o.apply_to_hkl({{r.h, r.k, r.l}}); MergedReflection e = r; e.h = hkl[0]; e.k = hkl[1]; e.l = hkl[2]; expanded.push_back(e); } } return expanded; }; std::optional law_correlation; std::string laws; for (const auto &op : twin_ops) { std::string t = op.triplet(); std::replace(t.begin(), t.end(), 'x', 'h'); std::replace(t.begin(), t.end(), 'y', 'k'); std::replace(t.begin(), t.end(), 'z', 'l'); laws += (laws.empty() ? "" : " ; ") + t; // What each law reads on the merge: the disagreement H of the reflections it // relates (Yeates (1997) Methods Enzymol. 276, 344-358: a twin of fraction a reads // (1-2a)/2, so a >= 0.5 - H), their correlation and their intensity-weighted R. // Report-only, so a user sees which law a twin refinement would need. gemmi::Mat33 m; for (int i = 0; i < 3; ++i) for (int j = 0; j < 3; ++j) m[j][i] = static_cast(op.rot[i][j]) / gemmi::Op::DEN; SearchSpaceGroupOptions law_opts; law_opts.nthreads = static_cast(std::max(1, config_.nthreads)); if (!law_correlation) law_correlation.emplace(expanded_merge(), law_opts); if (const auto s = (*law_correlation)(m); s.has_value()) result.twin_law_evidence.push_back(fmt::format( "{}: {} pairs, H {:.3f} (twin fraction {:.2f} or more), R {:.3f}, CC {:.3f}", t, s->n_pairs, s->h_stat, std::max(0.0, 0.5 - s->h_stat), s->r_stat, s->cc)); else result.twin_law_evidence.push_back(fmt::format("{}: too few pairs to read", t)); } for (const auto &e : result.twin_law_evidence) logger.Info("Twin law {}", e); // A model can resolve it too, but whether it does is only known once the model has // been fitted (validation step below), so the warning stands here exactly as in a run // without a model, and is withdrawn there if the model's indexing probe decided it. // On serial stills with -C and -S the model is already in `ambiguity_reference`. if (!ambiguity_reference && experiment_.IsRotationIndexing()) { // One sweep has one orientation matrix, so every frame is in the same hand and the // merge mixes nothing. The hand is arbitrary, which matters only against other // data: the report states it, and it is not a condition of this data set. logger.Info("Indexing ambiguity: this cell / space group admits alternative indexing " "(reindex operator(s): {}); one sweep is indexed in one hand, arbitrary " "against other data sets", laws); result.indexing_ambiguity_operators = laws; } else if (!ambiguity_reference) { const std::string msg = fmt::format( "Indexing ambiguity: this cell / space group admits alternative indexing " "(reindex operator(s): {}). Serial-stills crystals are indexed in one hand at random, " "and rugnux can only break this against an external reference. WITHOUT one the merge " "mixes the hands and CC1/2 is degraded; a reference MTZ (-z) or a model (--model, " "which needs -C and -S here) resolves it.", laws); logger.Warning("{}", msg); ambiguity_warning_text = " !! " + msg + "\n\n"; stats_text << ambiguity_warning_text; result.warnings.push_back({PathologyCode::INDEXING_AMBIGUITY, msg}); } else { logger.Info("Indexing ambiguity present (reindex operator(s): {}); resolved " "against the supplied reference.", laws); } } } // Dataset-wide Wilson B-factor estimate (like XDS's WILSON LINE B). Diagnostic only - it is not // fed back into scaling; it just lands in the printed statistics, the mmCIF, and the log, none // of which the geometry pre-pass or a superseded pass produces. if (!geometry_prepass && !superseded) { const GlobalWilsonB wilson = CalcGlobalWilsonB(sm.merged); sm.statistics.wilson_b = wilson.b; sm.statistics.wilson_b_correlation = wilson.correlation; sm.statistics.wilson_b_d_low_A = wilson.d_low_A; sm.statistics.wilson_b_d_high_A = wilson.d_high_A; if (std::isfinite(wilson.b) && wilson.b > 0.0) logger.Info("Wilson B-factor estimate: {:.2f} A^2 (correlation {:.3f}, {} shells, fitted over " "{:.2f}-{:.2f} A)", wilson.b, wilson.correlation, wilson.n_shells, wilson.d_low_A, wilson.d_high_A); // Diffraction anisotropy. It describes the data and corrects no intensity; its tensor is // used only as the Wilson prior of the amplitudes, below. The tensor and its resolution signature come from // the merged intensities; the error bar the verdict is gated on has to come from the // unmerged observations, because a merge has exact Laue symmetry by construction and the // tensor directions the symmetry forbids - the only place a dataset measures its own // systematic error - are identically zero in it. if (anisotropy.valid()) { sm.statistics.anisotropy = anisotropy.get(); stats_text << AnisotropyToText(sm.statistics.anisotropy) << "\n"; // The amplitudes are the one place the tensor is used: French-Wilson with the Wilson // prior of each reflection taken along its own direction. The intensities and // everything measured on them above are left as they are. FrenchWilsonOptions fw_opts; fw_opts.num_threads = config_.nthreads; fw_opts.anisotropy_hkl = AnisotropyTensorHKL(sm.statistics.anisotropy, gemmi::UnitCell(*result.consensus_cell)); ApplyFrenchWilson(sm.merged, experiment_.GetSpaceGroupOrP1(), fw_opts); } else if (rsm) { // No tensor: the isotropic prior. The rotation merge leaves its amplitudes to here. FrenchWilsonOptions fw_opts; fw_opts.num_threads = config_.nthreads; ApplyFrenchWilson(sm.merged, experiment_.GetSpaceGroupOrP1(), fw_opts); } } stats_text << sm.statistics; result.merge_statistics_text = stats_text.str(); result.has_merge_statistics = true; result.merge_statistics = sm.statistics; result.reference_merge_statistics = sm.reference_statistics; result.reference_isa = sm.reference_isa; // Per-image form of the sweep-quality ranges, for the _process.h5: one code per image, 0 where // the image is in no flagged range, plus the vocabulary the codes index. Only filled when the // diagnostic ran, so absent datasets mean "not looked for" rather than "all clean". The // disposition rides alongside as a second per-image array: what the reason SAW and what became // of the frame are different statements, and a consumer needs both. if (sm.statistics.sweep_quality.measured) { const auto &sq = sm.statistics.sweep_quality; end_msg.sweep_quality.assign(end_msg.max_image_number, 0); for (const auto &r : sq.ranges) for (int64_t i = std::max(0, r.first_image); i <= r.last_image && i < static_cast(end_msg.sweep_quality.size()); ++i) end_msg.sweep_quality[i] = static_cast(r.reason) + 1; for (int r = 0; r <= static_cast(SweepQualityReason::InconsistentWithMerge); ++r) end_msg.sweep_quality_reasons.emplace_back( SweepQualityReasonCode(static_cast(r))); end_msg.frame_disposition.assign( sq.frame_disposition.begin(), sq.frame_disposition.begin() + std::min(end_msg.max_image_number, sq.frame_disposition.size())); for (int d = 0; d <= static_cast(FrameDisposition::Rejected); ++d) end_msg.frame_disposition_codes.emplace_back( FrameDispositionCode(static_cast(d))); } { // Stride rather than take the head: the merged list is ordered by hkl, so the first N // reflections are one corner of reciprocal space and would not show the intensity range. constexpr size_t MAX_POINTS = 4000; const size_t n = sm.merged.size(); const size_t step = std::max(1, n / MAX_POINTS); result.merged_i_sigma.reserve(std::min(n, MAX_POINTS + 1)); for (size_t i = 0; i < n; i += step) if (std::isfinite(sm.merged[i].I) && std::isfinite(sm.merged[i].sigma) && sm.merged[i].sigma > 0.0f) result.merged_i_sigma.emplace_back(sm.merged[i].I, sm.merged[i].sigma); } result.has_reference = !config_.reference_data.empty(); // Inherit the campaign's shared R-free test set from the reference MTZ (overriding the // per-hkl hash that the merge assigned), so every dataset flags the same free reflections - but // only where the merge, now in the reference's frame, is recognisably the same crystal form on // the same axes. A reference that is not would hand each reflection the flag of an unrelated // one, and a count of matched indices cannot see that: a permuted orthorhombic cell matches // nearly every index, each to the wrong reflection. if (!config_.reference_data.empty() && !sm.merged.empty()) { const auto match = MatchReference(sm.merged, config_.reference_data, experiment_.GetSpaceGroupOrP1()); result.reference_cc = match.cc; result.reference_matched_fraction = match.matched_fraction; const bool matches = match.cc >= MIN_REFERENCE_CC && match.matched_fraction >= MIN_REFERENCE_MATCHED_FRACTION; if (config_.reference_has_free_flags && matches) { const size_t matched = ApplyReferenceFreeFlags(sm.merged, experiment_.GetSpaceGroupOrP1(), config_.reference_data); result.reference_free_flags_inherited = true; logger.Info("R-free flags: inherited the reference test set ({} of {} merged reflections " "matched; CC with the reference {:.3f})", matched, sm.merged.size(), match.cc); } else if (!matches) { const std::string msg = fmt::format( "The reference MTZ does not match these data: CC {:.3f} (at least {:.2f} expected) " "over {:.0f}% of the merged reflections in its resolution range (at least {:.0f}%). " "It is not the same crystal form on the same axes, so {}.", match.cc, MIN_REFERENCE_CC, 100.0 * match.matched_fraction, 100.0 * MIN_REFERENCE_MATCHED_FRACTION, config_.reference_has_free_flags ? "its free-R flags were NOT inherited - the merge carries its own test set" : "its scaling and CCref describe a different crystal"); logger.Warning("{}", msg); result.warnings.push_back({PathologyCode::REFERENCE_MISMATCH, msg}); } } // Radiation-damage report (rotation): the per-image scale CC-to-merge and mosaicity across the // sweep (the per-image scaling the rotation merge already fits, binned by frame = dose). A // per-image CC that falls, and/or a mosaicity that rises, with frame number is the classic // radiation-damage signature - a data-quality-vs-dose read complementary to the fitted decay // correction. The full per-image table is written to _plot.txt for detail. Both this // and the sweep-quality report below are text, so a superseded pass makes neither. if (experiment_.IsRotationIndexing() && !superseded) { const auto &outs = indexer->GetIntegrationOutcome(); const int nf = static_cast(outs.size()); constexpr int nb = 10; std::array cc_sum{}, mos_sum{}; std::array cc_n{}, mos_n{}; for (int f = 0; f < nf; ++f) { const int b = nf > 1 ? std::min(nb - 1, f * nb / nf) : 0; if (outs[f].image_scale_cc && std::isfinite(*outs[f].image_scale_cc)) { cc_sum[b] += *outs[f].image_scale_cc; cc_n[b]++; } if (outs[f].mosaicity_deg && std::isfinite(*outs[f].mosaicity_deg)) { mos_sum[b] += *outs[f].mosaicity_deg; mos_n[b]++; } } int tot = 0; for (int n : cc_n) tot += n; if (tot >= 20) { std::ostringstream os; os << fmt::format("Radiation-damage report (per-image scale over {} frames):\n", nf); os << " dose \n"; for (int b = 0; b < nb; ++b) { if (cc_n[b] == 0 && mos_n[b] == 0) continue; os << fmt::format(" {:3d}-{:3d}% {:>7} {:>7}\n", b * 10, (b + 1) * 10, cc_n[b] ? fmt::format("{:.3f}", cc_sum[b] / cc_n[b]) : std::string("-"), mos_n[b] ? fmt::format("{:.3f}", mos_sum[b] / mos_n[b]) : std::string("-")); } if (cc_n[0] > 0 && cc_n[nb - 1] > 0) { const double cc0 = cc_sum[0] / cc_n[0], cc9 = cc_sum[nb - 1] / cc_n[nb - 1]; os << fmt::format(" => per-image CC to merge {:.3f} (first 10%) -> {:.3f} (last 10%){}", cc0, cc9, cc9 < cc0 - 0.05 ? " (falling: radiation damage / crystal decay)" : ""); } // Relative B-factor across the sweep (measured before any decay correction, against the // low-dose start): the resolution-dependent complement to the CC/mosaicity read above. // Radiation damage fades the high-resolution intensity, so only a POSITIVE change is dose. // A curve no straight line describes gets no first->last number at all, and a batch whose // data could not measure one prints "-" rather than a value. if (!sm.statistics.radiation_damage_b_batch.empty()) { if (std::isfinite(sm.statistics.radiation_damage_delta_b)) os << fmt::format("\n => relative B-factor change over run = {:+.2f} A^2 (first->last){}", sm.statistics.radiation_damage_delta_b, sm.statistics.radiation_damage_delta_b > 5.0 ? " (significant dose-dependent scaling: radiation damage)" : ""); else os << "\n => relative B-factor: the per-batch curve below is not a trend, so no " "first->last number describes it - dose does not come back, so whatever moved " "here was not dose; see the sweep-quality report"; os << fmt::format("\n per-batch relative-B (A^2, {:.0f} deg/batch):", sm.statistics.radiation_damage_batch_deg); for (float bb : sm.statistics.radiation_damage_b_batch) os << (std::isfinite(bb) ? fmt::format(" {:.1f}", bb) : std::string(" -")); } logger.Info("{}", os.str()); result.radiation_damage_text = os.str(); } // Sweep-quality report: the stretches of the sweep over which the crystal delivered much // less than the rest of the run, what each one looks like, what became of it and what // keeping it costs the merged intensities. Frame numbers are processed-image ordinals, // inclusive, as in _plot.txt. const auto &sq = sm.statistics.sweep_quality; if (sq.measured) { const int n_sweep = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected; auto pct = [&](int n) { return n_sweep > 0 ? 100.0 * n / n_sweep : 0.0; }; std::ostringstream os; os << fmt::format("Sweep quality over {:.0f} deg (per-image scale with the incident flux, " "which varied {:.2f}x, already divided out):\n", sq.sweep_deg, sq.flux_peak_to_trough); os << fmt::format(" {} frames: {} merged ({:.1f}%), {} downgraded ({:.1f}%), " "{} rejected ({:.1f}%, {:.1f} deg)\n", n_sweep, sq.frames_merged, pct(sq.frames_merged), sq.frames_downgraded, pct(sq.frames_downgraded), sq.frames_rejected, pct(sq.frames_rejected), sq.rejected_deg); if (sq.ranges.empty()) { os << " no stretch of the sweep is materially worse than the run"; } else { os << " frames rotation diagnosis severity scale CC indexed disposition dCC1/2\n"; for (const auto &r : sq.ranges) os << fmt::format(" {:<17s} {:6.1f} deg {:<16s} {:5.2f} {:5.2f} {:5.2f} {:4.0f}% {:<11s} {}\n", fmt::format("{}-{}", r.first_image, r.last_image), r.rotation_deg, SweepQualityReasonText(r.reason), r.severity, r.mean_relative_scale, r.mean_relative_cc, 100.0 * r.indexed_fraction, FrameDispositionCode(r.disposition), std::isfinite(r.delta_cc_half) ? fmt::format("{:+.4f} +/- {:.4f}", r.delta_cc_half, r.delta_cc_half_se) : std::string("-")); os << " => severity is the fraction of the run's typical diffracting power missing " "over the range; dCC1/2 is what keeping it costs the merged intensities"; if (sq.modulation_peak_to_trough >= 1.05f) os << fmt::format("\n => once-per-revolution modulation of the per-image scale: " "{:.1f}x peak to trough", sq.modulation_peak_to_trough); } if (!sq.delta_cc_half_batch.empty()) { os << fmt::format("\n => delta-CC1/2 per {:.0f} deg:", sq.delta_cc_half_batch_deg); for (float d : sq.delta_cc_half_batch) os << (std::isfinite(d) ? fmt::format(" {:+.3f}", d) : std::string(" -")); } logger.Info("{}", os.str()); } } analyses_mark.Stop(); // Model validation runs BEFORE the reflection files are made, because it is what settles the // frame they are written in: the enantiomorph, which merged intensities cannot choose, and - // with no reference MTZ - the alternative indexing. Both are relabelings of the same // measurements, so the statistics computed above are unaffected; what changes is that the file, // the R-factors and the maps then all describe one indexing instead of two. // // A validation's maps go to the outputs as it returns: at or _mr, the ones the last // validation there made. const auto take_maps = [](std::shared_ptr &at, ModelValidationResult &v) { if (v.maps) at = std::move(v.maps); }; // The twin-law merge made ready to be written: on the axes the answer is written on (every // relabelling since it was merged), with the answer's own free set - read off the answer's merge // by the adopted group's asymmetric unit, which both twin mates of a pair fall in - then in the // subgroup's standard setting, with its French-Wilson amplitudes and the law's fraction. Made once // the model, which can relabel the answer, is done with it; and before the model is asked about // the hypothesis itself (below). struct TwinFile { std::vector merged; UnitCell cell; gemmi::SpaceGroup space_group; gemmi::Op law; TwinLawFraction fraction; const RotationScaleMerge::Result *merge = nullptr; // the merge it was made from, for its statistics }; // The model's own subgroup's merge (model_twin_choice), made ready the same way, and reported in the // model's section only: written, as _mr, only where the model chooses it. std::optional twin_file, model_twin_file; std::optional model_twin_report; const auto make_twin_file = [&](const TwinSubgroupChoice &choice, const RotationScaleMerge::Result &twin_merge, const std::string &file_suffix, ProcessResult::TwinSubgroupMerge &t) -> std::optional { const bool friedel = experiment_.GetScalingSettings().GetMergeFriedel(); const gemmi::SpaceGroup adopted = experiment_.GetSpaceGroupOrP1(); TwinSubgroupChoice c = choice; std::vector merged = twin_merge.merged; if (!(merge_to_written == gemmi::Op::identity())) { if (!MoveTwinSubgroupChoice(c, merge_to_written.inverse())) { logger.Warning("Twin hypothesis: {} has no setting on the axes the answer is written on; " "its merge is not written", choice.subgroup.xhm()); return std::nullopt; } merged = ReindexMergedIntoAsu(merged, merge_to_written, c.subgroup, friedel); } // A subgroup of the group written - which the model can relabel to its enantiomorph, a change // of label only, so the subgroup's label follows it. const auto rotations_in = [](const gemmi::SpaceGroup &sub, const gemmi::SpaceGroup &group) { const gemmi::GroupOps g = group.operations(); for (const auto &op : sub.operations().sym_ops) if (std::none_of(g.sym_ops.begin(), g.sym_ops.end(), [&](const gemmi::Op &o) { return o.rot == op.rot; })) return false; return true; }; if (adopted.number != twin_choice_group_number && c.subgroup.is_enantiomorphic()) { gemmi::GroupOps e = c.subgroup.operations(); e.change_basis_forward(c.subgroup.change_of_hand_op()); if (const gemmi::SpaceGroup *other = gemmi::find_spacegroup_by_ops(e)) c.subgroup = *other; } if (!rotations_in(c.subgroup, adopted)) { logger.Warning("Twin hypothesis: {} is not a subgroup of {} as written; its merge is not written", c.subgroup.xhm(), adopted.xhm()); return std::nullopt; } AssignRfreeFlags(merged, c.subgroup, experiment_.GetScalingSettings().GetRfreeFraction(), 500, result.consensus_cell, config_.nthreads); ApplyReferenceFreeFlags(merged, adopted, sm.merged); const TwinLawFraction fraction = EstimateTwinLawFraction(merged, c.subgroup, c.law, friedel); UnitCell cell = *result.consensus_cell; // The subgroup's reference setting first: a subgroup of a centred group can be a primitive // group written on a multiple of its cell - C 1 1 2_1 is P 1 2_1 1 on twice the volume - which // no unimodular relabelling reaches. The reflections the centring extinguished have no index on // the primitive cell and are dropped; the rest move exactly. if (!(c.subgroup.basisop() == gemmi::Op::identity())) { const gemmi::Op to_reference = c.subgroup.basisop().inverse(); const gemmi::Op m = HklOperator(to_reference); const gemmi::SpaceGroup before = c.subgroup; if (MoveTwinSubgroupChoice(c, to_reference)) { std::erase_if(merged, [&](const MergedReflection &r) { const auto h = m.apply_to_hkl_without_division({{r.h, r.k, r.l}}); return h[0] % gemmi::Op::DEN != 0 || h[1] % gemmi::Op::DEN != 0 || h[2] % gemmi::Op::DEN != 0; }); merged = ReindexMergedIntoAsu(merged, m, c.subgroup, friedel); cell = CellInBasis(cell, to_reference); } else { c.subgroup = before; } } const gemmi::Op standard = ChooseOutputSetting(cell, c.subgroup); if (!(standard == gemmi::Op::identity())) { const gemmi::SpaceGroup before = c.subgroup; if (MoveTwinSubgroupChoice(c, standard)) { merged = ReindexMergedIntoAsu(merged, HklOperator(standard), c.subgroup, friedel); cell = CellInBasis(cell, standard); } else { c.subgroup = before; } } FrenchWilsonOptions fw_opts; fw_opts.num_threads = config_.nthreads; ApplyFrenchWilson(merged, c.subgroup, fw_opts); t.reason = twin_choice_reason; t.adopted_space_group = adopted.xhm(); t.space_group = c.subgroup.xhm(); t.file_suffix = file_suffix.empty() ? "_twin_" + c.subgroup.short_name() : file_suffix; t.twin_law = c.law.as_hkl().triplet(); t.twin_fraction = fraction.fraction; t.twin_fraction_pairs = fraction.pairs; t.other_subgroups = c.others; logger.Info("Twin hypothesis: {} in {}, twin law {} at a fraction of at least {:.2f} ({} pairs) -> {}.mtz/.cif{}", adopted.xhm(), c.subgroup.xhm(), t.twin_law, fraction.fraction, fraction.pairs, t.file_suffix, file_suffix.empty() ? "" : " where the model chooses it"); return TwinFile{std::move(merged), cell, c.subgroup, c.law, fraction, &twin_merge}; }; const auto prepare_twin_file = [&] { if (!twin_file && twin_merge && twin_choice) { ProcessResult::TwinSubgroupMerge t; twin_file = make_twin_file(*twin_choice, *twin_merge, std::string(), t); if (twin_file) result.twin_subgroup_merge = std::move(t); } if (!model_twin_file && model_twin_merge && model_twin_choice) { ProcessResult::TwinSubgroupMerge t; model_twin_file = make_twin_file(*model_twin_choice, *model_twin_merge, "_mr", t); if (model_twin_file) { t.reason = "the model's group asks for it"; model_twin_report = std::move(t); } } }; // The model's choice of another group, or of the twin hypothesis' subgroup: never the answer's files, // which take from a model at most the enantiomorph and the indexing, but files of their own beside // them, _mr, from the merge it was chosen on, with the model as placed there. struct ModelPriorFile { std::vector merged; UnitCell cell; gemmi::SpaceGroup space_group; std::optional law; TwinLawFraction fraction; std::shared_ptr placed; const RotationScaleMerge::Result *twin_merge = nullptr; // the subgroup merge it is, where it is one }; std::optional model_prior_file; // The coordinates that go with the maps beside the answer's files, kept once the model is done. std::optional placed_model_out; // The twin hypothesis the model is asked about: its own subgroup's merge where it has one, the data's // otherwise - and whether that merge was made only for the model, so is written only where it chooses it. std::optional &model_twin = model_twin_merge ? model_twin_file : twin_file; std::optional &model_twin_entry = model_twin_merge ? model_twin_report : result.twin_subgroup_merge; const bool model_twin_only = model_twin_merge || twin_for_mr_only; // Placed by the rigid body: the model fits against its null in the answer's merge or in the twin // hypothesis' one. Where it is not - or is a prediction, or --mr asked for the search - it is searched // for by molecular replacement below. bool placed_by_rigid_body = false; bool twin_chosen_by_model = false; if (model_run && !model_predicted && !config_.force_mr) { phase("Validating against model"); Step("Model validation"); TimingMark mark(logger, "model validation"); // The ledger's P1 merge (started above) is taken before anything below acts on what the // validation decides: relabelling the outcomes and the group is what it must not see happen. // Every such relabelling reaches the P1 merge afterwards, through merge_to_written. ModelValidationResult validation = model_validation_run.get(); model_validation_log.ReplayInto(logger); take_maps(result.outputs->maps, validation); take_ledger_p1(); // A model that was asked for and could not be used has to say so where anyone will see // it. Without this the run ends successfully with no R-free, no maps and nothing in the // report - indistinguishable from a run that was never given --model at all. result.model_validation = validation; // The ambiguity warning above was issued before the model was read; take it back only // where the model actually settled the indexing. if (validation.indexing_decided) result.indexing_ambiguity_operators.clear(); if (validation.indexing_decided && !ambiguity_warning_text.empty()) { std::erase_if(result.warnings, [](const ReportWarning &w) { return w.code == PathologyCode::INDEXING_AMBIGUITY; }); const auto pos = result.merge_statistics_text.find(ambiguity_warning_text); if (pos != std::string::npos) result.merge_statistics_text.erase(pos, ambiguity_warning_text.size()); logger.Info("Indexing ambiguity resolved against the model; warning withdrawn"); } bool indexing_moved_cell = false; if (data_sg.has_value()) { const gemmi::SpaceGroup *adopted = AdoptModelFrame(validation, sm.merged, *data_sg, experiment_.GetScalingSettings().GetMergeFriedel(), logger); if (adopted->number != data_sg->number) { // The written reflections and the report that describes them, but deliberately not // end_msg: the _process.h5 already holds the per-image reflections as they were // integrated, and a file whose recorded group did not match its own reflections // would mis-merge on a later --mode scale. An enantiomorphic pair merges identically // anyway - same Laue class, same absences - so nothing is lost by leaving it. experiment_.SetSpaceGroup(*adopted); result.space_group = *adopted; } // The unmerged export below comes from the integrated observations, not from the merge, // so it follows the same relabeling - otherwise the two files would describe different // indexings of one dataset. The enantiomorph is a label, so only the indexing moves it. // The cell moves with the indices: an alternative indexing that is only a near-symmetry of // the lattice (b within 2% of c, beta within a degree of 90) permutes the axes or flips an // angle, and the written cell has to be the one the written indices are on. const gemmi::Op &op = validation.indexing_op; if (!(op == gemmi::Op::identity())) { const UnitCell before = *result.consensus_cell; reseat_integrated(HklOperator(op)); indexing_moved_cell = !(*result.consensus_cell == before); merge_to_written = merge_to_written.combine(op); determined_to_written = determined_to_written.combine(op); result.setting_operator = IndexTriplet(determined_to_written); } } // A model that fits and is written on other axes than the data - the depositor's setting, // an I-centred cell, an unreduced triclinic one - takes the files into its own setting, // so the reflections and the deposited coordinates can be used together as they are. The // validation is then made again on the relabelled data, so the maps and the placed model // come out on the same axes as the reflections; what it had decided is kept, since the // second one, facing a model already in the data's setting, has nothing left to decide. // The same where the model's indexing moved the cell (a near-symmetry, above): the first // validation placed the model in the cell as it was before. const bool remake = config_.reference_data.empty() && validation.model_fits && (!(validation.change_of_basis_op == gemmi::Op::identity()) || indexing_moved_cell); const bool forecast_held = forecast_kept(validation); if (ahead.run.valid() && !forecast_held) { ahead.write.set_value(false); ahead.run.wait(); } if (remake) { gemmi::Op to_model = validation.change_of_basis_op; to_model.tran = {0, 0, 0}; ModelValidationResult remade; if (forecast_held) { ahead.write.set_value(true); remade = ahead.run.get(); take_maps(result.outputs->maps, remade); relabel_output(to_model.inverse(), "MODEL", &ahead.merged); ahead.log.ReplayInto(logger); } else { if (!(to_model == gemmi::Op::identity())) relabel_output(to_model.inverse(), "MODEL"); const auto sg_now = experiment_.GetGemmiSpaceGroup(); ModelValidationSchedule kept_verdict; kept_verdict.fit_test = false; kept_verdict.gpu_resident_bytes = merge_on_card; remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, *config_.model, logger, sg_now ? &*sg_now : nullptr, /*probe_indexing_ambiguity=*/false, static_cast(config_.nthreads), experiment_.GetWavelength_A(), report_shell_d_min, kept_verdict); take_maps(result.outputs->maps, remade); } if (remade.ok) { KeepModelVerdict(remade, validation); validation = std::move(remade); result.model_validation = validation; } else if (forecast_held) { logger.Warning("Model validation: the validation in the model's setting failed ({}), and the " "first left its maps to it - no maps were made", remade.failure_reason); } } placed_by_rigid_body = validation.ok && validation.model_fits; // The coordinates that go with the maps just written, in the frame the reflection files // beside them carry. Written whenever the validation ran, not only where the rigid-body // step moved the model: it is also re-fractionalized into the data cell and may be // relabelled to the model's enantiomorph, so "did not move" is not "identical to the input // file". A rejected model is scored, placed and mapped like any other, and that is the case // where someone most wants to see the model in the density, so it gets its file too. // Written beside the reflection files below: nothing after this reads it. // The model asked about the twin hypothesis too, where one was written: scored against the // subgroup's merge, plainly and as twinned by its law (the twin target, at a scanned fraction), // against the answer's merge as above. The maps and the placed model come from whichever merge // the model fits better on R-free; the answer's files are the answer's either way - the model // chooses the maps, never the merge (see docs/RUGNUX_REPORT.md). A merge made only for the model to // test is chosen only where the model fits against its null there too; otherwise it is not written. prepare_twin_file(); bool maps_from_twin = false; if (model_twin && validation.ok) { TwinFile &mt = *model_twin; ProcessResult::TwinSubgroupMerge &mt_report = *model_twin_entry; const double answer_r_free = validation.r_free; const auto twin_r_free = [](const ModelValidationResult &v) { return v.twin_target ? std::min(v.r_free, v.r_free_twin) : v.r_free; }; const auto maps_from = [&](const ModelValidationResult &v) { return twin_r_free(v) < answer_r_free && (!model_twin_only || v.model_fits); }; ModelValidationSchedule twin_schedule; twin_schedule.gpu_resident_bytes = merge_on_card; twin_schedule.maps_superseded = [&](const ModelValidationResult &v) { return !maps_from(v); }; phase("Validating against model (twin hypothesis)"); auto twin_validation = ValidateAgainstModel( mt.merged, mt.cell, *config_.model, logger, &mt.space_group, /*probe_indexing_ambiguity=*/config_.reference_data.empty(), static_cast(config_.nthreads), experiment_.GetWavelength_A(), {}, twin_schedule, &mt.law); take_maps(result.outputs->maps, twin_validation); maps_from_twin = twin_validation.ok && maps_from(twin_validation); const bool placed_in_twin = twin_validation.ok && twin_validation.model_fits; placed_by_rigid_body = placed_by_rigid_body || placed_in_twin; // The subgroup file follows its own validation's indexing, as the answer follows its own: // the file, the R-factors and the maps then describe one indexing. if (twin_validation.ok) { const gemmi::Op op = twin_validation.indexing_op; mt.space_group = *AdoptModelFrame(twin_validation, mt.merged, mt.space_group, experiment_.GetScalingSettings().GetMergeFriedel(), logger); if (!(op == gemmi::Op::identity())) { mt.law = op.inverse().combine(mt.law).combine(op); mt.cell = CellInBasis(mt.cell, HklOperator(op)); } mt_report.twin_law = mt.law.as_hkl().triplet(); mt_report.space_group = mt.space_group.xhm(); } logger.Info("Model validation, twin hypothesis: R-free {:.4f} against the answer's merge in {}, " "{:.4f} against the merge in {}{} - the maps and the placed model come from {}", answer_r_free, experiment_.GetSpaceGroupOrP1().xhm(), twin_validation.r_free, mt.space_group.xhm(), twin_validation.twin_target ? fmt::format(" and {:.4f} as twinned at a fraction of {:.2f}", twin_validation.r_free_twin, twin_validation.twin_fraction) : std::string(), maps_from_twin ? "the merge in " + mt.space_group.xhm() : "the answer's merge"); if (maps_from_twin && twin_validation.placed_model) placed_model_out = PlacedModelOutput{twin_validation.placed_model, mt.cell, mt.space_group}; // The model placed there and fitting better there chooses the subgroup: its files, _mr. They // carry the model's own group where it has the subgroup's rotations - the screws and the hand are the // model's, as the coordinates beside them are, unless the anomalous map at its atoms is inverted - and // none of the reflections that group extinguishes. if (maps_from_twin && placed_in_twin) { twin_chosen_by_model = true; gemmi::SpaceGroup label = mt.space_group; if (const gemmi::SpaceGroup *own = twin_validation.placed_model ? twin_validation.placed_model->find_spacegroup() : nullptr; own != nullptr && !twin_validation.anomalous_hands_disagree && SortedRotations(own->operations()) == SortedRotations(mt.space_group.operations())) label = *own; ModelPriorFile f{mt.merged, mt.cell, label, mt.law, mt.fraction, twin_validation.placed_model, mt.merge}; if (label.xhm() != mt.space_group.xhm()) { const gemmi::GroupOps lops = label.operations(); std::erase_if(f.merged, [&](const MergedReflection &r) { return lops.is_systematically_absent({{r.h, r.k, r.l}}); }); } model_prior_file = std::move(f); result.model_prior_file_suffix = "_mr"; } mt_report.maps_from_this_merge = maps_from_twin; mt_report.model_validation = std::move(twin_validation); } if (validation.placed_model && !maps_from_twin) placed_model_out = PlacedModelOutput{validation.placed_model, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}; } prepare_twin_file(); // Molecular replacement: for a prediction, which has no placement of its own; with --mr; and for an // experimental structure the rigid body did not place (or could not use: no crystal cell). The model is // placed, as many copies as the cell holds, in the groups the answer's merge cannot tell apart and in the // twin hypothesis' subgroup. It takes the enantiomorph label into the answer - only past the anomalous // veto - and nothing else, never an indexing: a different group or the subgroup it chooses goes to the // files of its own, _mr, from the merge it was chosen on. It never makes the rigid body's // verdict (MODEL_FIT) either; its own null is what placed it. if (model_run && (model_predicted || config_.force_mr || !placed_by_rigid_body) && !model_cut()) { phase("Molecular replacement"); Step("Molecular replacement"); TimingMark mark(logger, "model prior"); // As before the validation above, which a predicted model or --mr does not run: the engine's card // memory goes back before the search reserves its rigid-body engines on the card. if (!ledger_ahead.valid()) rsm.reset(); const size_t merge_on_card = rsm && rsm->OnGPU() ? rsm->DeviceBytes() : 0; std::vector merges{{"MERGE", &sm.merged, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}}; if (model_twin) merges.push_back({"TWIN_SUBGROUP_MERGE", &model_twin->merged, model_twin->cell, model_twin->space_group, model_twin->law}); // The search started on the geometry pre-pass's merge, where there is one, taken over where it holds on // this one. Not where the twin hypothesis' merge is searched too: that search is this merge's own. std::optional carried; if (early_mr_ && !model_twin) { TimingMark early_mark(logger, "early molecular replacement wait"); try { MolecularReplacementResult early = early_mr_->run.get(); early_mark.Stop(); early_mr_->log.ReplayInto(logger); carried = CarryMolecularReplacement( early, {"geometry pre-pass's merge", &early_mr_->merged, early_mr_->cell, early_mr_->space_group}, merges[0], logger, static_cast(config_.nthreads), merge_on_card); } catch (const std::exception &e) { logger.Warning("Molecular replacement on the geometry pre-pass's merge failed ({}) - searched again", e.what()); } } MolecularReplacementResult mr = carried ? std::move(*carried) : RunMolecularReplacement(*config_.model, merges, logger, static_cast(config_.nthreads), merge_on_card); if (mr.chosen >= 0) { const MRCandidate &c = mr.candidates[mr.chosen]; if (mr.decision == "ENANTIOMORPH") CheckEnantiomorphHand(mr, sm.merged, experiment_.GetSpaceGroupOrP1(), logger, static_cast(config_.nthreads), merge_on_card); if (mr.decision == "ENANTIOMORPH") { // A change of label only (see ValidateAgainstModel): the two groups index identically. experiment_.SetSpaceGroup(c.space_group); result.space_group = c.space_group; if (twin_file && twin_file->space_group.is_enantiomorphic()) { gemmi::GroupOps e = twin_file->space_group.operations(); e.change_basis_forward(twin_file->space_group.change_of_hand_op()); if (const gemmi::SpaceGroup *other = gemmi::find_spacegroup_by_ops(e)) twin_file->space_group = *other; } logger.Info("Molecular replacement: the answer is relabelled {}, the enantiomorph the placed model chose " "(fit {:+.1f} sd over its null, anomalous map at its atoms not inverted, {:.3f} s)", c.space_group.xhm(), mr.hand_check.fit_sigma, mr.hand_check.seconds); } else if (mr.decision == "SPACE_GROUP") { ModelPriorFile f{sm.merged, *result.consensus_cell, c.space_group, std::nullopt, {}, c.placed}; const gemmi::GroupOps gops = c.space_group.operations(); std::erase_if(f.merged, [&](const MergedReflection &r) { return gops.is_systematically_absent({{r.h, r.k, r.l}}); }); model_prior_file = std::move(f); } else if (mr.decision == "TWIN_SUBGROUP" && model_twin) { model_prior_file = ModelPriorFile{model_twin->merged, model_twin->cell, c.space_group, model_twin->law, model_twin->fraction, c.placed, model_twin->merge}; twin_chosen_by_model = true; } result.model_prior_file_suffix = model_prior_file ? "_mr" : ""; // The maps, the R-factors and CC(model, data): the placement validated against the merge it was // chosen on, with nothing left for that validation to decide. The rigid body's verdict stays. const ModelPriorFile *own = model_prior_file ? &*model_prior_file : nullptr; const gemmi::SpaceGroup data_sg = own ? own->space_group : experiment_.GetSpaceGroupOrP1(); const UnitCell cell = own ? own->cell : *result.consensus_cell; ModelValidationSchedule placed_elsewhere; placed_elsewhere.fit_test = false; placed_elsewhere.gpu_resident_bytes = merge_on_card; phase("Validating against model (molecular replacement)"); ModelValidationResult v = ValidateAgainstModel( own ? own->merged : sm.merged, cell, *config_.model, logger, &data_sg, /*probe_indexing_ambiguity=*/false, static_cast(config_.nthreads), experiment_.GetWavelength_A(), report_shell_d_min, placed_elsewhere, nullptr, c.placed.get()); // Its maps go beside the files of the merge it was validated against. take_maps(own ? result.outputs->model_prior_maps : result.outputs->maps, v); v.maps_suffix = result.model_prior_file_suffix; if (v.ok) { if (result.model_validation) { const ModelValidationResult &rb = *result.model_validation; v.fit_tested = rb.fit_tested; v.model_fits = rb.model_fits; v.null_r_work_mean = rb.null_r_work_mean; v.null_r_work_sd = rb.null_r_work_sd; v.r_work_sigma = rb.r_work_sigma; v.null_replicates = rb.null_replicates; } if (own) model_prior_file->placed = v.placed_model; else placed_model_out = PlacedModelOutput{v.placed_model, cell, data_sg}; result.model_validation = std::move(v); } } result.model_prior = std::move(mr); } if (model_run) { result.model_placement = placed_by_rigid_body ? "RIGID_BODY" : result.model_prior && result.model_prior->chosen >= 0 ? "MR" : "NOT_PLACED"; // A model that was asked for and could not be used has to say so where anyone will see it. Without // this the run ends successfully with no R-free, no maps and nothing in the report - indistinguishable // from a run that was never given --model at all. A model that was used and not placed is a result. std::string why = result.model_validation ? result.model_validation->failure_reason : std::string(); if (why.empty() && result.model_prior && !result.model_prior->ok) why = "the model could not be searched for: " + result.model_prior->failure_reason; if (!(result.model_validation && result.model_validation->ok) && !why.empty()) result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED, "Model validation did not run: " + why}); // The twin hypothesis' merge made only for the model to test: written, and reported, only where it chose it. if (model_twin_only && !twin_chosen_by_model) { model_twin.reset(); model_twin_entry.reset(); } result.model_twin_subgroup_merge = model_twin_report; if (model_prior_file) result.model_prior_file_space_group = model_prior_file->space_group.xhm(); result.outputs->placed_model = placed_model_out; } // The unmerged MTZ (kept with the other outputs, see there) is built from here on, beside the // reflection files: the indexing and the group it is written in are final once the model has been // read, and nothing after this changes the integrated observations it reads. It gets its own copy of // the experiment. if (full && write_output && !geometry_prepass && !superseded && result.consensus_cell && config_.export_unmerged) { unmerged_mtz = std::async(std::launch::async, [&outcomes = indexer->GetIntegrationOutcome(), cell = *result.consensus_cell, x = experiment_, &provenance = config_.provenance, nthreads = config_.nthreads] { return UnmergedMtzTable(outcomes, cell, x, true, provenance, nthreads); }); } // Not on the geometry pre-pass, and not on a pass that is going to be re-run. Pass 1 exists to // choose the space group and post-refine the geometry; the pass after it remakes these outputs // seconds later at the refined geometry, and that is the answer anyone reads. The quality guard is // unaffected: it reads the search merge, which is made well above this. if (result.consensus_cell && write_output && config_.write_merged && !geometry_prepass && !superseded) { phase("Reflection files"); Step("Writing files"); TimingMark writing_mark(logger, "reflection outputs"); const ErrorModelReport em_report{ result.error_model_isa > 0 && result.error_model_isa_resolved ? fmt::format("{:.2f}", result.error_model_isa) : "?", result.error_model_isa_asymptotic > 0 && result.error_model_isa_resolved ? fmt::format("{:.2f}", result.error_model_isa_asymptotic) : std::string(), result.error_model_a > 0 ? fmt::format("{:.3f}", result.error_model_a) : std::string(), result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string()}; // The fulls carry the index they were measured at in the scaler's frame; every relabelling // the merged reflections went through since (merge_to_written) takes them to the written one. if (!(merge_to_written == gemmi::Op::identity())) for (auto &f : sm.scaled_fulls) { const gemmi::Op::Miller h = merge_to_written.apply_to_hkl({{f.h, f.k, f.l}}); f.h = h[0]; f.k = h[1]; f.l = h[2]; } result.outputs->merged = MergedOutput{"", MergedMtzTable(sm.merged, *result.consensus_cell, experiment_, config_.provenance), sm.merged, *result.consensus_cell, experiment_, sm.statistics, em_report, result.twinning, std::move(sm.scaled_fulls)}; if (keep_merged_reflections_) result.merged_reflections = sm.merged; // The twin hypothesis beside it (see prepare_twin_file): the reflection files only, with the // law and its fraction in the mmCIF's _pdbx_reflns_twin. if (twin_file) { DiffractionExperiment twin_x = experiment_; twin_x.SetSpaceGroup(twin_file->space_group); TwinningAnalysisResult law; law.twin_law_operator = twin_file->law.as_hkl().triplet(); law.twin_law_fraction = std::isfinite(twin_file->fraction.fraction) ? twin_file->fraction.fraction : 0.5; const ErrorModelReport twin_em{ twin_merge->isa > 0 && twin_merge->isa_resolved ? fmt::format("{:.2f}", twin_merge->isa) : "?", std::string(), twin_merge->error_model_a > 0 ? fmt::format("{:.3f}", twin_merge->error_model_a) : std::string(), twin_merge->error_model_b > 0 ? fmt::format("{:.4e}", twin_merge->error_model_b) : std::string()}; result.outputs->twin = MergedOutput{result.twin_subgroup_merge->file_suffix, MergedMtzTable(twin_file->merged, twin_file->cell, twin_x, config_.provenance), twin_file->merged, twin_file->cell, twin_x, twin_merge->statistics, twin_em, law, {}}; } // The predicted model's choice (see model_prior_file): its reflections, from the merge it was // chosen on and labelled with the group it chose, and the model as it placed it. if (model_prior_file) { const ModelPriorFile &f = *model_prior_file; DiffractionExperiment mr_x = experiment_; mr_x.SetSpaceGroup(f.space_group); TwinningAnalysisResult law; const MergeStatistics *statistics = &sm.statistics; ErrorModelReport mr_em = em_report; if (f.law) { const RotationScaleMerge::Result &tm = *f.twin_merge; law.twin_law_operator = f.law->as_hkl().triplet(); law.twin_law_fraction = std::isfinite(f.fraction.fraction) ? f.fraction.fraction : 0.5; statistics = &tm.statistics; mr_em = ErrorModelReport{ tm.isa > 0 && tm.isa_resolved ? fmt::format("{:.2f}", tm.isa) : "?", std::string(), tm.error_model_a > 0 ? fmt::format("{:.3f}", tm.error_model_a) : std::string(), tm.error_model_b > 0 ? fmt::format("{:.4e}", tm.error_model_b) : std::string()}; } result.outputs->model_prior = MergedOutput{result.model_prior_file_suffix, MergedMtzTable(f.merged, f.cell, mr_x, config_.provenance), f.merged, f.cell, mr_x, *statistics, mr_em, law, {}}; if (f.placed) result.outputs->model_prior_placed_model = PlacedModelOutput{f.placed, f.cell, f.space_group}; } writing_mark.Stop(); // FINALIST LEDGER (--finalist-ledger), report-only and deliberately behind a flag: these // numbers are an instrument, and reading them as a decision before they are calibrated is // the one way this can do harm. // // It is folded from the P1 merge started above, because that is the substrate the whole // space-group audit says is missing and nothing else in the run has: the same observations // at full resolution, production-scaled with the correction surfaces fitted, and folded // under NO symmetry - so every hypothesis can be folded from it on equal terms. It also // carries half-set intensities, which is what lets the random-noise R floor be formed - the // only reference available to the first step out of P1, and the reason this cannot be done // from the stored MTZ afterwards. Say why there is none: a user who gave the flag and got // no table would otherwise have to read this function to find out whether the instrument // declined, or the crystal simply had nothing to report. if (config_.finalist_ledger && !finalist_ledger) logger.Warning("No finalist ledger: it is folded from a P1 merge, and this run makes none ({}).", !is_rotation ? "stills - the ledger is rotation-only for now" : "a fixed centred space group's absences were never predicted, so a P1 " "merge would be missing whole centring classes"); if (finalist_ledger) { TimingMark mark(logger, "finalist ledger"); take_ledger_p1(); // The scaler still holds the observations in the indexing they were merged in, so every // relabelling since (the written setting, the model's indexing) is applied to this merge // too, so it is read on the axes of the merged output and of the cell it is read with. std::vector p1 = std::move(ledger_p1->merged); if (!(merge_to_written == gemmi::Op::identity())) p1 = ReindexMergedIntoAsu(p1, merge_to_written, gemmi::get_spacegroup_p1(), experiment_.GetScalingSettings().GetMergeFriedel()); // Twin-immune zone evidence for each 2-fold the adopted group holds, read on this merge // because it is folded under no symmetry, at full resolution and production-scaled. // Report-only: nothing reads it back. const auto determined = experiment_.GetGemmiSpaceGroup(); if (determined.has_value() && determined->number > 1) { // The pseudo-translation the zone is normalised by is measured on THIS merge, not // taken from `result.tncs`. That one is fitted on the adopted group's merge, and // where the pseudo-translation is not compatible with the adopted symmetry that // merge has already averaged across its own phase classes, so the vector fitted // there is poorly determined - measured on one crystal, the correlation between E^2 // and cos(2 pi h.u) reads 0.04 on the group merge against 0.25 here, with the // optimum elsewhere entirely, and a vector that names the wrong classes normalises // the zone by noise. It is also the basis this merge is indexed on: a change of // basis committed by the search leaves a vector measured before it pointing // somewhere else. const auto p1_tncs = AnalyzeTranslationalNCS( p1, gemmi::UnitCell(*result.consensus_cell), nullptr, 30, 0, config_.nthreads); result.twin_immune_zones = AnalyzeTwinImmuneZones( p1, gemmi::UnitCell(*result.consensus_cell), *determined, TranslationalNCSVectorForLTest(p1_tncs), config_.nthreads, result.space_group_search ? result.space_group_search->twin_fraction_outside : 0.0); if (!result.twin_immune_zones->zones.empty()) logger.Info("{}", TwinImmuneZonesToText(*result.twin_immune_zones)); } SearchSpaceGroupOptions lopt; lopt.nthreads = static_cast(std::max(1, config_.nthreads)); lopt.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel(); lopt.cell = gemmi::UnitCell(*result.consensus_cell); lopt.enumerate_all_settings = true; lopt.enumerate_all_rotation_sets = true; lopt.merge_isa = ledger_p1->isa; // the error model this merge was scaled with const auto led = SearchSpaceGroup(p1, lopt); logger.Info("Finalist ledger on the full-resolution P1 merge ({} reflections):\n{}", p1.size(), FinalistLedgerToText(led)); } } return false; }