// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include "Rugnux.h" #include "../image_analysis/structure_refinement/ModelValidation.h" #include "../image_analysis/indexing/SpindleCuspLoss.h" #include "../image_analysis/bragg_integration/SpotWidth.h" #include "../image_analysis/bragg_integration/SpotFootprint.h" #include "../image_analysis/bragg_integration/BraggStencil.h" // BRAGG_FOOTPRINT_NSIGMA #include "../image_analysis/hot_pixels/HotPixels.h" #include "../image_analysis/SensorAbsorption.h" #include "DiagnosticOutput.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../reader/JFJochHDF5Reader.h" #include "../common/JFJochMath.h" #include "../common/ParallelFor.h" #include "../common/Logger.h" #include "../common/AzimuthalIntegrationMapping.h" #include "../common/AzimuthalIntegrationProfile.h" #include "../common/CUDAWrapper.h" #include "../common/JFJochException.h" #include "../common/time_utc.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/beam_stop/ShadowFinder.h" #include "../image_analysis/IndexAndRefine.h" #include "../image_analysis/indexing/AnalyzeIndexing.h" #include "../image_analysis/indexing/HarmonicContamination.h" #include "../image_analysis/geom_refinement/BeamCenterFFT.h" #include "../image_analysis/geom_refinement/BeamCenterFromBackground.h" #include "../image_analysis/geom_refinement/GeometryRefiner.h" #include "../image_analysis/indexing/IndexerThreadPool.h" #include "../image_analysis/spot_finding/ImageSpotFinderCPU.h" #include "../image_analysis/spot_finding/SpotUtils.h" #include "gemmi/twin.hpp" #include "../image_analysis/azint/AzIntEngineCPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h" #ifdef JFJOCH_USE_CUDA #include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h" #include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h" #include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h" #endif #include "../image_analysis/scale_merge/Merge.h" #include "../image_analysis/scale_merge/RfreeFlags.h" #include "../image_analysis/scale_merge/RotationScaleMerge.h" #include "../image_analysis/scale_merge/ResolutionCutoff.h" #include "../image_analysis/scale_merge/ReindexAmbiguity.h" #include "../image_analysis/scale_merge/ScalingResult.h" #include "../image_analysis/scale_merge/SearchSpaceGroup.h" #include "../image_analysis/geom_refinement/PostRefine.h" #include "../image_analysis/lattice_search/LatticeSearch.h" #include "../image_analysis/lattice_search/LePageLattice.h" #include "../image_analysis/scale_merge/AnisotropyAnalysis.h" #include "../image_analysis/scale_merge/FrenchWilson.h" #include "../image_analysis/scale_merge/TwinningAnalysis.h" #include "../image_analysis/scale_merge/TranslationalNCS.h" #include "../image_analysis/scale_merge/HKLKey.h" #include "../image_analysis/scale_merge/ScaleOnTheFly.h" #include "../image_analysis/scale_merge/StillsPartialityRefine.h" #include "../image_analysis/WriteReflections.h" #include "../image_analysis/bragg_integration/CalcISigma.h" #include "../common/Definitions.h" #include "../common/CorrelationCoefficient.h" #include #include #include "RugnuxPipeline.h" #include "RugnuxFirstPass.h" // The steps of the first pass that may replace the standing lattice: the rescues, asked where it // indexes too little, and the hypotheses, asked of every run. FirstPassRotationIndexing gives their // order. using namespace rugnux_internal; namespace { auto MaxAxis(const RotationIndexerResult &r) { const auto uc = r.lattice.GetUnitCell(); return std::max({uc.a, uc.b, uc.c}); } } // Rotation-axis SIGN rescue. The sign is a convention the input cannot settle: a PILATUS // miniCBF header carries only "Oscillation_axis omega" and no direction at all, and facilities // disagree about the sign of the NXmx vector (measured across ten of them: most want the // stored [-1,0,0], APS 19-ID and SPring-8 BL41XU want the opposite). Nothing downstream can // repair it either - the default spindle fit refines the axis DIRECTION by fractions of a // degree and cannot flip it, that being a discrete change. // // It is, however, decidable from the data: the wrong sign does not index at all. Measured on // one dataset by changing nothing but the sign: 100% of frames indexed one way, 0 of 60 the // other. So try the other sign and keep whichever indexes more frames - the same // validation-frame count the scheme choice above already uses, so no new metric and no new // threshold. Runs before the long-axis rescue: with the sign wrong every candidate lattice is // wrong, and the coarse pass below would spend itself on a mis-signed cloud. // // Only after a poor pass, so a correctly-signed file costs nothing; and spot finding is not auto Rugnux::FirstPassRun::AxisSignRescue(FirstPass best) -> FirstPass { NearTieGate("axis-sign rescue gate", "standing pass", best.score, 0.5 * static_cast(validation.size())); if (!rx.cancelled_ && best.score < 0.5 * static_cast(validation.size())) { if (const auto gon = rx.experiment_.GetGoniometer(); gon.has_value() && gon->IsScanning()) { GoniometerAxis flipped = *gon; flipped.Axis(-gon->GetAxis()); rx.experiment_.Goniometer(flipped); FirstPass alt = PickBest(*indexer_pool, *indexer); if (alt.result.has_value()) NearTieFrames("axis sign", "opposite sign", alt.score, "file's sign", best.score); // More frames is not enough on its own while both counts sit at the floor: 1/60 against // 0/60 is noise, and adopting it puts every rescue below on the wrong sign (measured: a // run whose file sign, kept, went on to index found nothing once flipped on 1/60). So // the flipped lattice must also beat chance on the pooled spots - the test the run puts // its final lattice to - which a sparse pattern at the right sign passes on few frames // (2/60 frames, 89% of the validation spots against 0% by chance). // A file's sign that found no lattice counts as 0 frames: no answer is not a count the // opposite sign can beat with another 0. const bool more_frames = alt.result.has_value() && alt.score > std::max(best.score, 0); const bool alt_real = more_frames && ValidationEvidenceBeatsChance(MeasurePooled(*indexer, *alt.result)); if (alt_real) { logger.Info("Rotation axis sign: validation frames indexed at the file's axis {}, at the " "opposite sign {} - adopting the opposite sign for this run", Frames(best.score), Frames(alt.score)); best = alt; } else { if (more_frames) logger.Info("Rotation axis sign: validation frames indexed at the opposite sign {}, at " "the file's axis {}, but its lattice does not beat chance on the pooled " "spots - the file's sign is kept", Frames(alt.score), Frames(best.score)); rx.experiment_.Goniometer(*gon); // the file was right; put it back } } } return best; } // Long-axis rescue. When the de-novo cell indexes few validation frames, a long, finely-spaced // axis was likely lost: the unconstrained FFT either collapsed it to a short sub-multiple or let // a denser supercell over-fit the accumulated cloud (a small global-orientation error throws the // many high-order reflections off along the fine axis, so the true cell scores worst on the raw // cloud). Recover the true metric with a second search whose FFT unit-cell bound is widened past // the default, then RE-INDEX at full resolution constrained by the cell that comes back, as a // reference (the -C path): the reference filter drops the collapsed/supercell candidates and // refines an accurate global lattice. Only runs after a poor standard pass, so well-indexing // crystals never pay for it and their result is untouched. Returns the re-index where it indexes // more validation frames than the `standing` pass it was asked about. auto Rugnux::FirstPassRun::LongAxisRescue(const FirstPass &standing) -> std::optional { if (rx.cancelled_ || !standing.result.has_value() || standing.score >= 0.5 * static_cast(validation.size())) return std::nullopt; auto coarse_settings = rx.experiment_.GetIndexingSettings(); // The resolution setting only sizes the FFT histogram - it does not filter which spots go // into it - so this is here to pay for the widened bound below, not to make the long axis // easier to pick out. Measured, coarsening does the opposite: the acceptance angle widens // (0.159 -> 0.579 deg) but the true axis falls from rank 1996 to 16361 of 16384 directions. coarse_settings.FFT_HighResolution_A(3.5f); // The widened bound is what the rescue turns on, and it costs nothing: the histogram length is // 4*pi*max_length/high_resolution, so coarsening the resolution to 3.5 A has already // SHRUNK the transform by 1.75x - spending that back on reach leaves the coarse pass at // 1.14x the standard one. Without it the rescue cannot recover what it exists to recover: // a cell whose long axis is past the default 500 A bound is unrepresentable, so the // "recovered" metric would still be a sub-multiple. Only reached after a poor pass, so a // well-indexing crystal never pays. Stills never reach here. constexpr float RESCUE_MAX_UNIT_CELL_A = 1000.0f; if (coarse_settings.GetFFT_MaxUnitCell_A() < RESCUE_MAX_UNIT_CELL_A) coarse_settings.FFT_MaxUnitCell_A(RESCUE_MAX_UNIT_CELL_A); coarse_settings.IndexingThreads(rx.FirstPassIndexerThreads()); coarse_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); IndexerThreadPool coarse_pool(coarse_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); // Coarse first pass: keep the recovered cell with the LONGEST axis directly. Its full- // resolution per-frame validation would be low (the coarse cell is metrically right but // imprecise), so do NOT score it here - it is only the reference for the constrained re-index. const SchemeIndexers cris = FeedSchemes(coarse_pool); RunIndexing(cris); std::optional coarse_ref; for (auto &ri : cris) { auto r = ri->GetLattice(); if (r.has_value() && (!coarse_ref.has_value() || MaxAxis(*r) > MaxAxis(*coarse_ref))) coarse_ref = r; } // A recovered axis at least 30% longer than the standard pass found signals a rescued metric. if (!rx.cancelled_ && coarse_ref.has_value() && MaxAxis(*coarse_ref) > 1.3 * MaxAxis(*standing.result)) { logger.Info("Long-axis rescue: coarse pass recovered a {:.0f} A axis (was {:.0f} A); " "re-indexing constrained by that cell", MaxAxis(*coarse_ref), MaxAxis(*standing.result)); rx.experiment_.SetUnitCell(coarse_ref->lattice.GetUnitCell()); // The re-index must be able to REPRESENT the cell the coarse pass just recovered. The // standard pool was sized at the default bound, so re-using it here would search for a // long axis it cannot hold and hand back the sub-multiple the rescue exists to replace. // CountIndexed forces the lattice and never touches the pool, so the existing // IndexAndRefine is reused; only the FFT needs the wider one. auto wide_settings = rx.experiment_.GetIndexingSettings(); wide_settings.IndexingThreads(rx.FirstPassIndexerThreads()); wide_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); RaiseFFTBoundForKnownCell(wide_settings, rx.experiment_); FirstPass constrained; if (wide_settings.GetFFT_MaxUnitCell_A() > indexing_bound_A) { IndexerThreadPool wide_pool(wide_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); constrained = PickBest(wide_pool, *indexer); } else { constrained = PickBest(*indexer_pool, *indexer); } rx.experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo if (!constrained.result.has_value()) return std::nullopt; // A standing pass that found no lattice counts as 0 frames. bool more = constrained.score > std::max(standing.score, 0); if (standing.result.has_value() && !SameLattice(*constrained.result, *standing.result) && NearTieFrames("long-axis rescue", "re-index", constrained.score, "standing", standing.score) && DecideLate()) { const int late = CountIndexed(*indexer, *constrained.result, late_validation); const int standing_late = CountIndexed(*indexer, *standing.result, late_validation); const bool early = more; more = constrained.score + late > standing.score + standing_late; LogDecidedLate("long-axis rescue", fmt::format("re-index {}/{} of the later frames, standing {}/{}", late, late_validation.size(), standing_late, late_validation.size()), early, more); } if (more) return constrained; } return std::nullopt; } // The background centre, tried as a SECOND FIRST PASS. // // The estimate itself is free - it is read off the projection --detect-beam-stop has already // built - and a first pass re-uses the spots it has already found, so indexing the run a // second time at the measured centre costs about one more first pass. // That buys the comparison an on-failure search structurally cannot make: a centre wrong // ALONG the spindle does not fail. It keeps 96-100 % of frames indexed and quietly returns a // 2x, 3x or sqrt(3) axis harmonic, so no failure trigger ever fires on it. Indexing both // centres and comparing the two ANSWERS is what can see it. // // What must NOT arbitrate the two is the frame count. Acceptance is a fractional-Miller test, // so a cell twice as long has to place every spot twice as accurately to score the same: // measured on real data, two centres 0.12 px apart gave the deposited cell at 99.23 % and a // halved axis at 100.00 %, and the WRONG answer indexed better. A rule of the form "take the // centre that indexes more" would therefore pick wrong in exactly the case this comparison // exists for. So only what needs no arbiter is decided here: // // * the file's centre does not index a majority -> nothing defends it, so try the measured // one (and the axis sign with it, see below). A run that indexes nothing has nothing to // lose; where the measured one is no better, the file's centre is put back; // * the file's centre indexes and the two agree on the lattice -> keep the file's centre and // say they agree. This is the overwhelming majority, and saying it is worth the pass: it // is a free statement that the header is good enough for this crystal; // * they index and disagree -> say so, with both cells and their primitive-volume ratio. // Which of an integer-related pair is real is not decided on the frame counts, for the // reason above; it is decided, in ONE direction only, on the pooled spots - see the // branch itself. Naming the disagreement is useful even where nothing is decided. // // Where the file's centre indexes nothing and the measured one does not either, the axis-sign // question is asked AGAIN at the measured centre before giving up. The two unknowns are // discrete and coupled, and the rescue above asked its question at a centre that may be 73 px // wrong: on the two depositions here whose header is the geometric centre of the detector, // each error hides the other, both runs sit at 0/60 until both are moved, and moving both // takes them to 60/60 on the deposited lattice. Asking the pair costs one more first pass on // a run that has already failed twice. auto Rugnux::FirstPassRun::BeamCentreCheck(FirstPass best, bool &long_axis_asked) -> FirstPass { rx.JoinBeamCenterCapture(); if (rx.cancelled_ || !rx.config_.beam_center_check || !rx.background_center_) return best; // Consumed, not re-read: a two-pass rotation run comes back through here with the geometry // the first pass post-refined, and a pre-scan estimate of the FILE's centre says nothing // about that one. The pre-scan does not re-measure it either - by then the beam-stop shadow // is already in the mask, so the projection is not rebuilt. const BeamCenterEstimate measured = *rx.background_center_; rx.background_center_.reset(); const float header_x = rx.experiment_.GetBeamX_pxl(), header_y = rx.experiment_.GetBeamY_pxl(); const float measured_x = measured.beam_x_pxl; const float measured_y = measured.beam_y_pxl; const float moved = std::hypot(measured_x - header_x, measured_y - header_y); const Centres c{measured, header_x, header_y, measured_x, measured_y, moved}; // No gate on how significant the move is. It is tempting to skip the second pass when the // two centres are within the fit's own noise - it would save it on a quarter of the runs // here - but that is deciding at a threshold before the evidence is in, and the evidence // is cheap. It would also gate away exactly the case worth asking about: measured on real // data, a 0.12 px change of centre is the whole difference between the deposited cell and // a halved axis, and 0.12 px is well inside the sigma of any fit that could gate it. const int majority = static_cast(validation.size()) / 2; TryBeamCenter(measured_x, measured_y); FirstPass alt = IndexAtMeasuredCentre(); const bool header_indexes = best.result.has_value() && best.score > majority; const bool measured_indexes = alt.result.has_value() && alt.score > majority; NearTieGate("beam-centre majority gate", "file centre", best.score, majority); NearTieGate("beam-centre majority gate", "measured centre", alt.score, majority); if (!header_indexes) return CentreWhenFileFails(std::move(best), std::move(alt), c, long_axis_asked); RestoreBeamCenter(header_x, header_y); if (measured_indexes) return CentreWhenBothIndex(std::move(best), alt, c); logger.Info("Beam centre check: the measured centre ({:.2f},{:.2f}) indexes " "{} validation frames against the file's {} - the file's " "centre stands", measured_x, measured_y, Frames(alt.score), Frames(best.score)); return best; } // The first pass at the measured centre: the speculative one started beside the file centre's // (StartCentreSpeculation) where it read what the run has here, otherwise run here. auto Rugnux::FirstPassRun::IndexAtMeasuredCentre() -> FirstPass { FirstPass alt; if (centre_spec) centre_spec->done.get(); if (centre_spec && centre_spec->experiment_key == rx.ExperimentKey() && centre_spec->key == rx.SpotFindingKey(*spot_mapping) && centre_spec->spots_per_image == first_pass_spots_per_image) { // What PickBest would have found and fed here: hand the spots to the run's cache and // its memo as PrefetchSpots does, and score the schemes on the run's own indexer. spot_cache = centre_spec->spots; { std::lock_guard lock(rx.first_pass_spots_->m); auto &entries = rx.first_pass_spots_->entries; if (std::none_of(entries.begin(), entries.end(), [&](const auto &e) { return e.first == centre_spec->key; })) { if (entries.size() == MAX_SPOT_KEYS) entries.erase(entries.begin()); entries.emplace_back(centre_spec->key, centre_spec->spots); } } alt = ScoreSchemes(centre_spec->ris, *indexer); } else alt = PickBest(*indexer_pool, *indexer); centre_spec.reset(); return alt; } // The file's centre does not index a majority. auto Rugnux::FirstPassRun::CentreWhenFileFails(FirstPass best, FirstPass alt, const Centres &c, bool &long_axis_asked) -> FirstPass { const float header_x = c.header_x, header_y = c.header_y; const float measured_x = c.measured_x, measured_y = c.measured_y; const float moved = c.moved; const int majority = static_cast(validation.size()) / 2; bool measured_indexes = alt.result.has_value() && alt.score > majority; // Nothing defends the file's centre: the data do not index at it, and the // background says it is `moved` px away. So try the measured one, and if that // does not index either, ask the AXIS-SIGN question again there. The two unknowns // are discrete and coupled, and the sign rescue above asked its question at a // centre that may itself be wrong - each error hides the other, and only moving // both reveals anything. bool joint = false; const auto gonio = rx.experiment_.GetGoniometer(); if (!measured_indexes && gonio.has_value() && gonio->IsScanning()) { GoniometerAxis flipped = *gonio; flipped.Axis(-gonio->GetAxis()); rx.experiment_.Goniometer(flipped); const FirstPass both = PickBest(*indexer_pool, *indexer); if (both.result.has_value() && both.score > majority) { logger.Warning("Beam centre from the background: the file's centre " "({:.2f},{:.2f}) indexes {} validation frames and the " "measured ({:.2f},{:.2f}) {}, but the measured centre " "WITH the opposite rotation-axis sign indexes {}. The " "file has both wrong, and neither shows while the other " "stands; the run takes the measured centre and the opposite " "sign", header_x, header_y, Frames(best.score), measured_x, measured_y, Frames(alt.score), Frames(both.score)); alt = both; measured_indexes = true; joint = true; } else { rx.experiment_.Goniometer(*gonio); } } // Third coupled unknown: how DEEP the pass reads. A centre 170 px out and a spot // list too deep to index at ANY centre hide each other exactly as the centre and // the axis sign do - the measured centre is right, and it is thrown away because // the full list indexes nothing there either. Measured on one such crystal: the // file's centre indexed 1/60 and the measured one 0/60, so the file's was kept and // the run died; with the ladder's lean seed the measured centre indexes 60/60, and // the run then indexes every frame of the sweep and merges to the deposited // resolution. So ask the ladder here too, before giving the centre up. // // The depth is asked at BOTH centres, and the centre moves only if it is the // centre that paid. A lean rung at the measured centre scored against the whole // spot list at the file's changes two things at once, and the count cannot say // which of them the gain came from - the same reason a trial centre is scored on // its own spots at all. Measured on one crystal: the measured centre's ladder // reached 40/60 on a FIVE-fold axis harmonic while the file's own centre reached // 49/60 on the deposited cell, and the run moved the centre, shipped the // supercell, and skipped the beam-centre walk that would have found the crystal's // own. An exact tie keeps the file's centre: nothing then says the centre is wrong. bool lean = false; bool deeper_at_home = false; if (!measured_indexes) { const FirstPassState before = committed; RestoreBeamCenter(header_x, header_y); const auto home = FirstPassLadder(best.score, 1, majority + 1); const FirstPassState home_state = committed; committed = before; ApplyState(committed); TryBeamCenter(measured_x, measured_y); const auto away = FirstPassLadder(alt.score, 1, majority + 1); const FirstPassState away_state = committed; // Both centres index a majority, and on the same lattice. Then the two counts // are two readings of one hypothesis at two geometries, and which is higher is // the count's noise: measured on one crystal, the same two ladders gave the // measured centre 56/60 against the file's 50/60, and 48/60 against 50/60 once // a few thousand defective pixels were masked - and the run that then kept the // file's centre, 12.6 px out, merged with twice the frames rejected. So the // count decides nothing here: the file's centre stays for this pass, and the // measured one is carried forward to be judged on the merges (RunAllPasses). const bool same_lattice = home && away && home->result->search_result.system == away->result->search_result.system && home->result->search_result.centering == away->result->search_result.centering && std::max(home->vol, away->vol) < 1.02 * std::min(home->vol, away->vol); if (home && away && !same_lattice) NearTieFrames("beam centre (lean ladder)", "file centre", home->score, "measured centre", away->score); if (home && (!away || same_lattice || home->score >= away->score)) { committed = home_state; RestoreBeamCenter(header_x, header_y); ApplyState(committed); best = *home; deeper_at_home = true; if (same_lattice && geometry_prepass) { rx.beam_center_alternative_ = std::array{measured_x, measured_y}; rx.beam_center_alternative_state_ = away_state; logger.Warning("Beam centre check: reading less of each frame, the " "file's centre indexes {} validation frames and the " "measured centre {:.2f} px away {}, on the same " "lattice - the counts cannot say which centre is right, " "so the run carries BOTH centres forward and adopts " "whichever measures more of the crystal", Frames(home->score), moved, Frames(away->score)); } else logger.Warning("Beam centre check: reading less of each frame takes the " "file's centre to {} validation frames against {} at " "the measured centre {:.2f} px away - it is how deep this " "run was reading and not the centre, so the centre stays", Frames(home->score), Frames(away ? away->score : 0), moved); } else if (away) { alt = *away; measured_indexes = true; lean = true; } } if (measured_indexes) { if (lean) logger.Warning("Beam centre from the background: neither centre indexes the " "whole spot list, but at the measured ({:.2f},{:.2f}) the " "lean first pass above indexes {} against the file's " "{} - the beam centre in this file is wrong, and only a " "pass that reads less of each frame could see it", measured_x, measured_y, Frames(alt.score), Frames(best.score)); else if (!joint) logger.Warning("Beam centre from the background: the file's ({:.2f},{:.2f}) " "indexes {} validation frames and the measured " "({:.2f},{:.2f}) indexes {} - the beam centre in this " "file is wrong, and the run adopts the measured one", header_x, header_y, Frames(best.score), measured_x, measured_y, Frames(alt.score)); best = alt; } else if (!deeper_at_home) { best = CentreByPooledSpots(std::move(best), std::move(alt), c, long_axis_asked); } return best; } // Nothing worked. Put the file's centre back, so a run that fails for some // other reason fails at the geometry it was given rather than at a fit that has // just been tested and found no better. // // Unless the frame counts are both at the floor and the POOLED spots do decide: // a crystal whose spots are a minority of every frame indexes 0/60 at any centre, // and then only the pooled excess over chance - each lattice measured against its // own wrong-spindle null, so a larger cell is not credited for fitting more by // chance - says which centre sees it. // // Each centre is judged on the best lattice it gives, so the long-axis rescue // is asked at BOTH before they are compared. The lattices standing here index // no majority anywhere, which is exactly the pass that rescue exists for, and // the two counts of spots on two lattices that are not the crystal's are close // to a coin toss: measured on a 545 A axis, the measured centre (3.6 px out // across the spindle) won it on 16.7% against 15.5%, the rescue then ran // there alone and could not see the long axis, and the run shipped a P1 // sub-cell - while at the file's centre the same rescue indexes 60/60. auto Rugnux::FirstPassRun::CentreByPooledSpots(FirstPass best, FirstPass alt, const Centres &c, bool &long_axis_asked) -> FirstPass { const float header_x = c.header_x, header_y = c.header_y; const float measured_x = c.measured_x, measured_y = c.measured_y; const float moved = c.moved; if (auto rescued = LongAxisRescue(alt)) alt = *rescued; PooledEvidence at_measured, at_header; if (alt.result.has_value()) at_measured = MeasurePooled(*indexer, *alt.result); RestoreBeamCenter(header_x, header_y); if (auto rescued = LongAxisRescue(best)) best = *rescued; if (best.result.has_value()) at_header = MeasurePooled(*indexer, *best.result); long_axis_asked = true; if (alt.result.has_value() && best.result.has_value() && NearTiePooled("beam centre (pooled spots)", "file centre", at_header, "measured centre", at_measured) && DecideLate()) { // Each centre on its own spots, as above. const bool early = ValidationEvidenceBeatsChance(at_measured) && ValidationEvidencePrefers(at_header, at_measured); const std::vector all = AllValidationFrames(); at_header = MeasurePooled(*indexer, *best.result, all); TryBeamCenter(measured_x, measured_y); at_measured = MeasurePooled(*indexer, *alt.result, all); RestoreBeamCenter(header_x, header_y); LogDecidedLate("beam centre (pooled spots)", "the pooled spots counted again on all the frames at both centres", early, ValidationEvidenceBeatsChance(at_measured) && ValidationEvidencePrefers(at_header, at_measured)); } if (alt.result.has_value() && ValidationEvidenceBeatsChance(at_measured) && ValidationEvidencePrefers(at_header, at_measured)) { TryBeamCenter(measured_x, measured_y); logger.Warning("Beam centre from the background: neither centre indexes a " "majority of the validation frames ({} at the measured, " "{} at the file's), but the pooled spots put " "{:.1f}% on the lattice at the measured ({:.2f},{:.2f}) against " "{:.1f}% at the file's (chance {:.1f}% and {:.1f}%) - the run " "adopts the measured centre", Frames(alt.score), Frames(best.score), 100.0 * at_measured.on_lattice / std::max(1, at_measured.spots), measured_x, measured_y, 100.0 * at_header.on_lattice / std::max(1, at_header.spots), 100.0 * at_measured.by_chance / std::max(1, at_measured.spots), 100.0 * at_header.by_chance / std::max(1, at_header.spots)); best = alt; } else { logger.Info("Beam centre check: the file's centre ({} frames) and the " "measured one {:.2f} px away ({}) - the measured centre puts no " "more of the pooled spots on its lattice, so the file's centre is " "kept", Frames(best.score), moved, Frames(alt.score)); } return best; } // The file's centre indexes a majority, and so does the measured one. auto Rugnux::FirstPassRun::CentreWhenBothIndex(FirstPass best, const FirstPass &alt, const Centres &c) -> FirstPass { const BeamCenterEstimate &measured = c.measured; const float measured_x = c.measured_x, measured_y = c.measured_y; const float moved = c.moved; // Primitive volumes, as everywhere else here: two passes can find the same // lattice in different settings, and a centred setting's cell is an exact // integer multiple of its primitive one. const double ratio = alt.vol < best.vol ? best.vol / alt.vol : alt.vol / best.vol; const bool same_class = best.result->search_result.system == alt.result->search_result.system && best.result->search_result.centering == alt.result->search_result.centering; // The primitive volume is what the beam centre decides. A centre error along // the spindle makes the FFT take an axis harmonic, which changes the volume by // an integer factor or by sqrt(3); the Bravais class can differ for a reason // that has nothing to do with the centre, so the two are reported separately. if (ratio < 1.02 && same_class) { rx.Note(fmt::format("beam centre kept: measured {:.1f} px off, same lattice", moved)); logger.Info("Beam centre check: indexing at the measured centre " "({:.2f},{:.2f}) returns the same lattice as the file's - " "{}-centred {}, cells agreeing to {:.2f} % in volume. The {:.2f} px " "difference does not decide this crystal's cell", measured_x, measured_y, alt.result->search_result.centering, gemmi::crystal_system_str(alt.result->search_result.system), 100.0 * (ratio - 1.0), moved); // Unless the two are further apart than this geometry absorbs. A centre // wrong by about one reflection spacing indexes the same cell just as // well, with every reflection one index out over part of the sweep, and // only the merge shows it - so the measured centre is kept as a hypothesis // for RunAllPasses, which tries it if the file's centre merges // inconsistently. if (geometry_prepass && moved > 3.0f * measured.sigma_pxl && moved > BeamCenterNeed_pxl(rx.experiment_)) rx.beam_center_same_lattice_alternative_ = std::array{measured_x, measured_y}; } else if (ratio < 1.02) { // Same cell, different metric symmetry read off it. The cell is not in // question, but the CLASS is not cosmetic either: GeometryRefiner holds // the cell to it for the rest of the run, so a class that is one axis // equality away from the truth is refined, predicted, integrated and // merged with that equality imposed. // // Whether the centre decided it is arithmetic. The equality is held to // LATTICE_SEARCH_DIST_TOLERANCE, relative, on a pair of reduced-cell // axes; a centre wrong by `moved` px moves an axis of length a by // moved*p*a/(D*lambda) relative, the FFT reading that axis off spot // positions the centre displaces. Both axes of the pair carry their own // error, so the centre can open or close a gap of their sum - and where // that reaches the tolerance the higher class is not a fact about the // crystal but a consequence of where the beam was assumed to be. // // Nothing available here can settle it: both centres index, and the frame // count is the one statistic this check already refuses to arbitrate on. // So carry the other centre out of the pass and let the two arms be judged // on their merges (RunAllPasses). const auto reduced = best.result->search_result.primitive_reduced.GetUnitCell(); const double axis_pair = std::max(reduced.a + reduced.b, reduced.b + reduced.c); const double induced = moved * rx.experiment_.GetPixelSize_mm() * axis_pair / (rx.experiment_.GetDetectorDistance_mm() * rx.experiment_.GetWavelength_A()); const bool centre_can_decide = induced >= LATTICE_SEARCH_DIST_TOLERANCE; logger.Info("Beam centre check: indexing at the measured centre " "({:.2f},{:.2f}) returns the same CELL as the file's - the " "primitive volumes agree to {:.2f} % - but a different metric " "symmetry: {}-centred {} from the file's centre against " "{}-centred {} from the measured one", measured_x, measured_y, 100.0 * (ratio - 1.0), best.result->search_result.centering, gemmi::crystal_system_str(best.result->search_result.system), alt.result->search_result.centering, gemmi::crystal_system_str(alt.result->search_result.system)); if (centre_can_decide && geometry_prepass) { rx.beam_center_alternative_ = std::array{measured_x, measured_y}; logger.Warning("Beam centre check: {:.2f} px is worth {:.2f} % on the " "axis pair the promotion compares, against the {:.2f} % " "the metric symmetry is decided to - the difference is " "large enough to have decided it either way, so the run " "carries BOTH centres forward and adopts whichever " "merges better", moved, 100.0 * induced, 100.0 * LATTICE_SEARCH_DIST_TOLERANCE); } else { logger.Info("Beam centre check: {:.2f} px is worth {:.2f} % on the axis " "pair the promotion compares, against the {:.2f} % the " "metric symmetry is decided to - the run keeps the file's " "centre and the symmetry it found", moved, 100.0 * induced, 100.0 * LATTICE_SEARCH_DIST_TOLERANCE); } } else { best = CentresDisagree(std::move(best), alt, c, ratio); } return best; } // The two centres index different lattices. auto Rugnux::FirstPassRun::CentresDisagree(FirstPass best, const FirstPass &alt, const Centres &c, double ratio) -> FirstPass { const float header_x = c.header_x, header_y = c.header_y; const float measured_x = c.measured_x, measured_y = c.measured_y; const auto &hc = best.result->search_result.conventional.GetUnitCell(); const auto &mc = alt.result->search_result.conventional.GetUnitCell(); const double nearest = std::round(ratio); const bool harmonic = (nearest >= 2.0 && std::abs(ratio - nearest) < 0.15) || std::abs(ratio - std::sqrt(3.0)) < 0.15; logger.Warning("Beam centre check: the two centres DISAGREE about the " "lattice. The file's ({:.2f},{:.2f}) gives {}-centred {}, " "{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} (primitive volume " "{:.0f} A^3, {} frames); the measured ({:.2f},{:.2f}) gives " "{}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} " "(primitive volume {:.0f} A^3, {} frames){}", header_x, header_y, best.result->search_result.centering, gemmi::crystal_system_str(best.result->search_result.system), hc.a, hc.b, hc.c, hc.alpha, hc.beta, hc.gamma, best.vol, Frames(best.score), measured_x, measured_y, alt.result->search_result.centering, gemmi::crystal_system_str(alt.result->search_result.system), mc.a, mc.b, mc.c, mc.alpha, mc.beta, mc.gamma, alt.vol, Frames(alt.score), harmonic ? fmt::format(" - the volumes differ by {:.2f}x, so one " "is an axis harmonic of the other and the " "beam centre is what settles which", ratio) : ""); // Deliberately no decision on the frame counts. They are the one statistic // that must not choose: on the measured case the harmonic indexed MORE. // // Decided in either direction on the pooled excess over chance, the same // measurement the failed-pass branch above adopts a centre on. Each lattice is // scored against its own wrong-spindle null, so the larger cell is not credited // for the spots a denser lattice catches by accident, and the smaller one is // not credited for the spots it misses. // // Both directions need it. Where the measured centre holds the LARGER cell, // nothing else in the run repairs it - the second pass refuses a lattice larger // than pass 1's (the supercell guard below). Where it holds the SMALLER one, // the second pass can drop to it only if its own beam refinement happens to // walk the whole centre error, and that rescue is chaotic: measured on one // crystal, a 0.1 % change in the pass-1 distance switched off a 6.5 px walk and // the run kept the doubled axis. const double nearest_n = std::round(ratio); const bool integer_harmonic = nearest_n >= 2.0 && nearest_n <= 4.0 && std::abs(ratio - nearest_n) < 0.15; bool adopted_measured = false; if (integer_harmonic) { const PooledEvidence at_header = MeasurePooled(*indexer, *best.result); TryBeamCenter(measured_x, measured_y); const PooledEvidence at_measured = MeasurePooled(*indexer, *alt.result); adopted_measured = ValidationEvidenceBeatsChance(at_measured) && ValidationEvidencePrefers(at_header, at_measured); NearTiePooled("beam centre (harmonic pair)", "file centre", at_header, "measured centre", at_measured); logger.Info("Beam centre check: pooled validation spots on the lattice " "{:.1f}% at the file's centre and {:.1f}% at the measured " "(chance {:.1f}% and {:.1f}%)", 100.0 * at_header.on_lattice / std::max(1, at_header.spots), 100.0 * at_measured.on_lattice / std::max(1, at_measured.spots), 100.0 * at_header.by_chance / std::max(1, at_header.spots), 100.0 * at_measured.by_chance / std::max(1, at_measured.spots)); if (adopted_measured) { logger.Warning("Beam centre check: the spots are on the {} cell of the " "measured centre, not on the file's - so the {:.2f}x is " "the file's centre error and not the crystal, and the " "run adopts the measured centre", alt.vol > best.vol ? "larger" : "smaller", ratio); best = alt; } else { RestoreBeamCenter(header_x, header_y); } } if (!adopted_measured) logger.Warning("Beam centre check: the run keeps the file's centre. The " "frame counts cannot arbitrate this - a cell twice as long " "must place every spot twice as accurately to score the " "same, so the harmonic is often the one that indexes more. " "Where the beam centre is known, give it with " "--beam-x/--beam-y"); return best; } // Where the first pass indexes no majority of the validation frames, the end wedge is asked too // and judged against the standing pass as any later scheme is. Only then: on a run the schemes // index it would be one more lattice for the axis-harmonic arbiter, which is not decided on // frame counts. Asked once, after the axis-sign, beam-centre and long-axis rescues (below), not // inside every pass they run: a consecutive wedge indexes part of the validation frames at the // WRONG axis sign too (measured: 33/60 where the spread at the right sign indexes 60/60), and // counted in the pass a rescue is gated on, it lifts that pass past the gate and the rescue // that would have indexed every frame is never asked. auto Rugnux::FirstPassRun::AddEndWedge(FirstPass bp, IndexerThreadPool &pool, IndexAndRefine &idx) -> FirstPass { NearTieGate("end-wedge gate", "standing pass", bp.score, static_cast(validation.size()) / 2); if (rx.cancelled_ || bp.score > static_cast(validation.size()) / 2) return bp; SchemeIndexers ris; ris.push_back(FeedScheme(pool, end_wedge[0].second)); RunIndexing({ris[0].get()}); return ScoreSchemes(ris, idx, &end_wedge, std::move(bp)); } // Short-axis hypothesis - the small-molecule case. // // LAST, after the axis-sign, beam-centre and long-axis rescues, deliberately. Those all // run the standard floor and compare their result against `best` on volume ratios; a cell // adopted here is a k-fold sub-cell, so running before them makes every one of those // comparisons read an integer ratio and report an axis harmonic that is not one (measured: // the beam-centre check announced a "3.00x harmonic ... the beam centre is what settles // which" on a pair that differs only by the FLOOR). Here the comparison is like with like, // and this pass sees whatever geometry those rescues settled on. // // fft_min_unit_cell_A (10 A) is a protein constant: shorter than any protein axis and longer // than a small molecule's. It is applied to the FFT's PER-DIRECTION PEAK SEARCH // (FFTIndexerCPU.cpp / FFTIndexerGPU.cu skip every histogram bin below it) and only then to // the assembled cell (FFTIndexer::ReduceAndRefine, PostIndexingRefinement). The first of // those is what does the damage: the transform reports ONE peak per direction, so a // lattice row whose real-space repeat is shorter than the floor is reported at its first // HARMONIC above it, and the cell built out of those harmonics is an exact integer // sublattice of the true one - a real-space supercell. Measured on chemical-crystallography // data: index 2, 3 and 6 exactly, and the one dataset whose true reduced cell clears 10 A is // the one that comes out right. Because the floor is applied to the REDUCED cell it rejects // the lattice and not merely a basis choice, so nothing downstream can re-describe its way // back. // // Lowering the floor by default is not available: it only ever ADDS candidates, and the // Bravais classification the candidate pool feeds is under-determined, so it moves crystals // whose cell it never touched (measured: 2 regressions and 1 improvement over 20 protein // datasets, all three in the metric symmetry rather than the cell). So the low floor runs as // a SECOND hypothesis and the standard pass's answer stands unless the evidence takes it. // // What the evidence can be, before integration: a sublattice's per-spot residuals are // dominated pointwise by the finer lattice it contains, so every margin statistic - squares, // counts, trimmed sums - is biased toward the LARGER cell, and a spurious supercell indexes // every frame its true cell does. Only the SATURATION is informative. Hence the same shape // the scheme ratchet in PickBest already uses: an integer primitive-volume ratio plus a tie // on validation frames, or an outright win, which runs against the bias and so is real. The // statistic that genuinely decides an axis multiple is the occupancy of the reflection class // the multiple adds - AbsenceEvidence in SearchSpaceGroup - and that needs integrated // intensities, so it cannot run here. // Not when a cell was given. RaiseFFTBoundForKnownCell has already lowered the floor for // the standard pass against that cell, so this pass has nothing left to add - and it takes // its settings from rx.experiment_, which never carried the RAISED max bound, so on a cell // longer than the default search it could only ever return a harmonic of what was asked // for, tie on frames the way a sub-multiple always does, and be adopted over it. auto Rugnux::FirstPassRun::ShortAxisHypothesis(FirstPass best) -> FirstPass { if (!rx.cancelled_ && !rx.experiment_.GetUnitCell().has_value()) { std::unique_ptr short_pool; std::vector> sris; if (short_spec && short_spec->key == rx.ExperimentKey() && short_spec->mapping == spot_mapping && short_spec->spots_per_image == first_pass_spots_per_image) { for (auto &f : short_spec->futures) f.get(); short_pool = std::move(short_spec->pool); sris = std::move(short_spec->sris); } else { short_spec.reset(); short_pool = std::make_unique(ShortAxisSettings(rx.experiment_), IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); sris = FeedSchemes(*short_pool); RunIndexing(sris); } for (size_t i = 0; i < sris.size() && !rx.cancelled_; i++) { auto found = sris[i]->GetLattice(); if (!found.has_value()) continue; RotationIndexerResult alt = std::move(*found); const std::string &name = schemes[i].first; // Primitive volumes, for the reason PickBest gives: two searches can find the same // lattice in different settings, and a centred setting's cell is an exact integer // multiple of its primitive one. const double vol = std::abs( alt.lattice.ToPrimitive(alt.search_result.centering).CalcVolume()); if (!(vol > 1.0) || !(best.vol > 1.0)) continue; const double ratio = best.vol / vol; const double nearest = std::round(ratio); // The gate is here rather than after the scoring because scoring is the expensive // half - CountIndexed runs the real per-image path over the validation set, seconds // on a protein - while the volume ratio is free. On every protein measured the low // floor returns the SAME lattice (ratio 1.00-1.01), so this costs them one extra FFT // and nothing else. if (!(vol < best.vol) || nearest < 2.0 || std::abs(ratio - nearest) >= 0.15) { logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': primitive volume " "{:.0f} A^3 against the standing cell's {:.0f} A^3, ratio {:.2f} - " "not an integer sub-cell, keeping the standing cell", IndexingSettings::fft_min_unit_cell_limit_A, name, vol, best.vol, ratio); continue; } // The premise of this pass, checked rather than asserted. The pass exists because a // lattice row shorter than the standard floor is reported at its first HARMONIC // above it, so a cell it may rightfully adopt has a reduced axis BELOW that floor - // and the floor is applied to the reduced cell, which is the one to measure. Where // the alternative has no such axis the floor cannot be what the standard pass // tripped over, and an integer volume ratio is then an ordinary axis multiple: the // question the axis-harmonic arbiter settles just above on the occupancy of the // class the multiple adds. That is intensity-class evidence about the very // reflections in dispute, where the tie below is a frame count this comment already // says a sublattice cannot lose - so where the two disagree the arbiter is right and // this block must not overrule it. Measured on the corpus: the four small-molecule // sets the pass is for all have a reduced axis under 9 A, and the two protein sets // it halved have 35 and 47 A. const auto alt_reduced = alt.lattice.ToPrimitive(alt.search_result.centering) .NiggliReduce().GetUnitCell(); const float alt_shortest = std::min({alt_reduced.a, alt_reduced.b, alt_reduced.c}); const float standard_floor = rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A(); if (alt_shortest >= standard_floor) { logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': a {:.0f}x sub-cell " "(vol {:.0f} A^3), but its shortest reduced axis is {:.1f} A, above " "the {:.1f} A floor the standard pass already searched - the floor " "is not why that cell was missed, so this is an ordinary axis " "multiple and not this pass's to decide; keeping the standing cell", IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol, alt_shortest, standard_floor); continue; } const int alt_score = CountIndexed(*indexer, alt); NearTieFrames("short-axis hypothesis", "sub-cell", alt_score, "standing", best.score); // A pass that indexes nothing is not tied with a pass that indexes nothing. Without // this, 0 against 0 passes the tie band and the volume ratio alone decides, which is // the one thing this block must never do. if (alt_score <= 0 || best.score <= 0) { logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {}/{} validation " "frames against {} - nothing indexes, keeping the standing cell", IndexingSettings::fft_min_unit_cell_limit_A, name, alt_score, static_cast(validation.size()), best.score); continue; } const bool tied = static_cast(alt_score) >= best.score * 0.9f - 0.5f; const bool clearly_more = static_cast(alt_score) > best.score * 1.1f + 0.5f; if (!tied && !clearly_more) { logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {:.0f}x sub-cell " "(vol {:.0f} A^3) indexes {}/{} validation frames against {} - " "keeping the standing cell", IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol, alt_score, static_cast(validation.size()), best.score); continue; } const auto uc = alt.lattice.GetUnitCell(); logger.Warning("Short-axis pass (floor {:.0f} A), scheme '{}': the standing " "cell is a {:.0f}x supercell. Its {:.0f} A^3 primitive volume against " "this pass's {:.0f} A^3, at {}/{} validation frames against {} - " "adopting {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}. The shortest axis " "is below the {:.0f} A default FFT floor, which is why the standard " "pass could only build a multiple of it", IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, best.vol, vol, alt_score, static_cast(validation.size()), best.score, uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A()); best.score = alt_score; best.vol = vol; best.name = name + " (short-axis)"; best.result = std::move(alt); } } return best; } // Sub-lattice hypothesis: the standing cell's seven index-2 sub-lattices, in its own // orientation and geometry, each scored on the validation frames. // // The FFT can only offer the cells its peaks build, and two lattices seen together - a twin // domain or a second crystal whose reciprocal lattice shares an index-2 sub-lattice with // this one's - build the lattice that contains them both: a cell twice the size of either, // which indexes both crystals' spots and so wins every comparison of candidates the FFT // returns. Where the true cell is then never offered, nothing downstream can choose it. // Offering it costs seven frame counts. // // The frame count only nominates: a coarser lattice asks less accuracy of every spot, so a // sub-lattice of the true cell can index as many frames as the cell itself, or a few more // where the cell's own score is depressed. What decides is the comparison this pass already // makes between a cell and its axis harmonic: the share of the validation frames' pooled // spots each puts on itself over and above what a wrong spindle angle puts there, each // against its own null (ValidationEvidencePrefers). A real superstructure keeps its cell on // that count, because the class it adds holds reflections; a cell built out of two crystals // loses it, because it has to place every spot of the one it describes twice as accurately // with an orientation fitted to both. The nominee must also index a majority of the frames, // the bar the false-promotion test above asks of a cell that "works", and no fewer frames // than the standing cell: a standing cell that indexes almost nothing is not this crystal's // lattice, its sub-lattices are not hypotheses about it, and the rescues below are what // such a run needs. auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass { if (!rx.cancelled_ && best.result.has_value() && best.score > 0) { const CrystalLattice prim = best.result->lattice .ToPrimitive(best.result->search_result.centering); const Coord A = prim.Vec0(), B = prim.Vec1(), C = prim.Vec2(); std::optional sub_best; std::string counts; for (int u = 0; u < 2; u++) for (int v = 0; v < 2; v++) for (int w = 0; w < 2; w++) { if (u + v + w == 0 || rx.cancelled_) continue; const Coord t = (A * static_cast(u) + B * static_cast(v) + C * static_cast(w)) / 2.0f; const CrystalLattice sub = (w ? CrystalLattice(A, B, t) : v ? CrystalLattice(A, t, C) : CrystalLattice(t, B, C)).NiggliReduce(); RotationIndexerResult cand = *best.result; cand.lattice = sub; cand.search_result = LatticeSearchResult{}; cand.search_result.primitive_reduced = sub; cand.search_result.conventional = sub; cand.unconstrained.reset(); cand.extra_lattices.clear(); const int score = CountIndexed(*indexer, cand); counts += fmt::format(" ({},{},{}) {}", u, v, w, score); if (!sub_best || score > sub_best->score) sub_best = FirstPass{std::move(cand), score, fmt::format("{} (sub-lattice {}{}{})", best.name, u, v, w), std::abs(sub.CalcVolume())}; } bool adopt = false; std::string verdict = " - none indexes a majority of the frames and as many as the standing cell"; const int majority = static_cast(validation.size()) / 2; bool nominated = sub_best && sub_best->score >= best.score && sub_best->score > majority; // Carried only where the gate is what decides: a sub-lattice that indexes fewer frames than // the standing cell is not nominated whatever the gate says. if (sub_best && NearTieGate("sub-lattice majority gate", "best sub-lattice", sub_best->score, majority) && sub_best->score >= best.score && DecideLate()) { const int late = CountIndexed(*indexer, *sub_best->result, late_validation); const int standing_late = CountIndexed(*indexer, *best.result, late_validation); const bool early = nominated; nominated = sub_best->score + late >= best.score + standing_late && LateOverGate(sub_best->score, late, majority); LogDecidedLate("sub-lattice majority gate", fmt::format("best sub-lattice {}/{} of the later frames, the standing cell {}/{}", late, late_validation.size(), standing_late, late_validation.size()), early, nominated); } if (nominated) { PooledEvidence at_standing = MeasurePooled(*indexer, *best.result); PooledEvidence at_sub = MeasurePooled(*indexer, *sub_best->result); if (NearTiePooled("sub-lattice hypothesis", "standing cell", at_standing, "sub-lattice", at_sub) && DecideLate()) { const bool early = ValidationEvidencePrefers(at_standing, at_sub); const std::vector all = AllValidationFrames(); at_standing = MeasurePooled(*indexer, *best.result, all); at_sub = MeasurePooled(*indexer, *sub_best->result, all); LogDecidedLate("sub-lattice hypothesis", "the pooled spots counted again on all the frames", early, ValidationEvidencePrefers(at_standing, at_sub)); } adopt = ValidationEvidencePrefers(at_standing, at_sub); verdict = fmt::format(" - the best puts {}/{} pooled spots on itself ({} by chance) against " "the standing cell's {}/{} ({}), {}", at_sub.on_lattice, at_sub.spots, at_sub.by_chance, at_standing.on_lattice, at_standing.spots, at_standing.by_chance, adopt ? "adopting it" : "keeping the standing cell"); } logger.Info("Sub-lattice hypothesis: the standing cell ({:.0f} A^3) indexes {} validation " "frames; its index-2 sub-lattices (half-translation u,v,w):{}{}", best.vol, Frames(best.score), counts, verdict); if (adopt) best = std::move(*sub_best); } return best; } // A lattice that indexes almost none of the validation frames is not this crystal's lattice. // Forcing it anyway integrates every image against it; the few partials that survive yield no // fulls in the 3D combine, and the run then dies inside the merge complaining about resolution - // far from the cause. Stop here instead, while the reason is still visible. The floor is one // frame in six, several times below the weakest crystal that still merges. It is checked AFTER // the long-axis rescue, so a metric that rescue recovers is never rejected on its pre-rescue // score. // The second (refined-geometry) pass re-indexes DE NOVO, and the post-refined geometry - fitted // to the lattice pass 1 already found - can tip the blind FFT, or the long-axis rescue above, // onto another lattice: an axis harmonic of a long cell (measured: a tripled ~100 A c-axis at // 7/60), or a related centred cell that indexes fewer frames than pass 1's lattice does at // this very geometry (measured: a C-centred 303 A cell at 41/60 against pass 1's triclinic // cell at 60/60). Pass 1's lattice is therefore carried here as a hypothesis of its own and // the two are judged on the same validation frames: where the re-index found a different // lattice and indexes fewer frames than pass 1's, or too few to be this crystal's lattice at // all, pass 1's whole result is integrated instead. The same lattice found again is kept as // the re-index refined it at this geometry. This is the floor-side companion of the // supercell-collapse guard in RunAllPasses, which only sees a pass-2 lattice that is LARGER. auto Rugnux::FirstPassRun::Pass1Fallback(FirstPass best) -> FirstPass { RotationIndexerResult pass1_lattice = *rx.prepass_result_; // Pass 1's result carries pass 1's goniometer; forcing it whole would put the // uncorrected angles back (the same repair as the supercell re-run in RunAllPasses). if (rx.prepass_rotation_scale_ && pass1_lattice.axis) pass1_lattice.axis = ScaleRotation(*pass1_lattice.axis, *rx.prepass_rotation_scale_); // Pass 1's lattice was FITTED at pass 1's detector distance, and this pass // integrates at the post-refined one. Re-scoring it here is not the same as // re-fitting it: a real-space cell is measured against the distance the spots were // seen at, so carrying it across a distance change scales the whole cell by the // ratio of the two. Measured on a 545 A axis over 225 -> 226 mm: 0.4 %, exactly // 225/226, shipped as the run's answer. Scale it to the distance it will be used // at; the orientation is unaffected. const float d_fit = rx.prepass_result_->geom.GetDetectorDistance_mm(); const float d_use = rx.experiment_.GetDetectorDistance_mm(); const bool rescaled = d_fit > 0 && d_use > 0 && d_fit != d_use; if (rescaled) { const float f = d_use / d_fit; const auto &l = pass1_lattice.lattice; pass1_lattice.lattice = CrystalLattice(l.Vec0() * f, l.Vec1() * f, l.Vec2() * f); } const CrystalLattice pass1_primitive = pass1_lattice.lattice .ToPrimitive(pass1_lattice.search_result.centering).NiggliReduce(); const double pass1_vol = std::abs(pass1_primitive.CalcVolume()); // The same lattice whatever setting or class it is held in - a centred cell and the // triclinic primitive of the same points are one lattice, and the re-index's symmetric // setting must not lose to its own unconstrained primitive on a frame count: the Niggli- // reduced primitive edges agree within 2 % (the battery's lattice-identity test). const auto reduced_edges = [](const CrystalLattice &l) { const auto uc = l.GetUnitCell(); std::array e = {uc.a, uc.b, uc.c}; std::sort(e.begin(), e.end()); return e; }; const auto e1 = reduced_edges(pass1_primitive); const auto e2 = reduced_edges(best.result->lattice .ToPrimitive(best.result->search_result.centering).NiggliReduce()); bool same_lattice = true; for (int i = 0; i < 3; i++) same_lattice = same_lattice && std::abs(e2[i] - e1[i]) <= 0.02 * e1[i]; const int floor = static_cast(validation.size()) / 6; const int pass1_score = (best.score < floor || !same_lattice) ? CountIndexed(*indexer, pass1_lattice) : -1; NearTieGate("pass-1 fallback floor", "re-index", best.score, floor); bool pass1_more = pass1_score > best.score; if (pass1_score >= 0 && !same_lattice && NearTieFrames("pass-1 fallback", "re-index", best.score, "pass-1 lattice", pass1_score) && DecideLate()) { const int late = CountIndexed(*indexer, *best.result, late_validation); const int pass1_late = CountIndexed(*indexer, pass1_lattice, late_validation); const bool early = pass1_more; pass1_more = pass1_score + pass1_late > best.score + late; LogDecidedLate("pass-1 fallback", fmt::format("re-index {}/{} of the later frames, pass-1 lattice {}/{}", late, late_validation.size(), pass1_late, late_validation.size()), early, pass1_more); } if (best.score < floor || pass1_more) { const auto &pc = pass1_lattice.search_result.conventional.GetUnitCell(); const auto &bc = best.result->search_result.conventional.GetUnitCell(); logger.Warning("Two-pass: the refined-geometry re-index indexes {}/{} validation frames " "({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}), pass-1's lattice " "{}/{} at this geometry ({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} " "{:.2f}) - integrating with pass-1's lattice", best.score, static_cast(validation.size()), best.result->search_result.centering, gemmi::crystal_system_str(best.result->search_result.system), bc.a, bc.b, bc.c, bc.alpha, bc.beta, bc.gamma, pass1_score, static_cast(validation.size()), pass1_lattice.search_result.centering, gemmi::crystal_system_str(pass1_lattice.search_result.system), pc.a, pc.b, pc.c, pc.alpha, pc.beta, pc.gamma); if (rescaled) logger.Info("Two-pass: pass-1's cell was fitted at {:.3f} mm and is used at " "{:.3f} mm - scaled by {:.6f} to the distance it is integrated at", d_fit, d_use, d_use / d_fit); best.result = std::move(pass1_lattice); best.score = pass1_score; best.vol = pass1_vol; best.name = "pass-1 lattice (refined-geometry re-index found less)"; } return best; }