leonarski_fandClaude Opus 5.5
97cac203a6
Pure refactor, no behaviour change. The 2460-line function with ~25 lambdas
becomes Rugnux::FirstPassRun (RugnuxFirstPass.h): the state the steps share
(spot cache, validation frames, schemes, committed spot-list state, the
speculative passes) as members, and one member function per step taking
and returning the standing lattice (FirstPass). FirstPassRotationIndexing is
now the ordered list of decisions:
PrefetchCertainFrames, IndexSchemes (+ centre/short-axis speculation),
AxisSignRescue, BeamCentreCheck (IndexAtMeasuredCentre,
CentreWhenFileFails, CentreByPooledSpots, CentreWhenBothIndex,
CentresDisagree), LongAxisRescue, AddEndWedge, ReleaseSpotEngines,
RequireLattice, ShortAxisHypothesis, SubLatticeHypothesis, Pass1Fallback,
last-resort FirstPassLadder, MeasureVerdict, RejectChanceLattice,
AdoptLattice, StartLeftoverCensus, ReportTiltWalk, MeasureSpotBudget,
Pass1SupercellGuard.
Files: RugnuxFirstPass.cpp (order + shared machinery: spots, scoring,
schemes, ladder), RugnuxFirstPassRescues.cpp, RugnuxFirstPassAccept.cpp.
Bodies are moved verbatim (Rugnux members reached through rx.); every log
line is unchanged. Literal duplicates folded: the two MAX_SPOT_KEYS
constants, and the feed-every-scheme / run-every-indexer loops of the
long-axis rescue and the short-axis fallback (now FeedSchemes/RunIndexing,
the same calls in the same order). The leftover census, which outlives the
pass, captures what it needs explicitly instead of [&].
Verified against the rc175 0b2dde214 binary on 16 sets (7kcn, 8v4o, 5nw5,
9rci, 6z9g, 9min, nothing_1/2, 7atg, myob/cytc_x10sa, 8a1a, kdp, insu low
ISa, 6p8j, 9qw8) and -N 8 on 7kcn/8v4o/9rci: p.mtz md5-identical, logs
identical as multisets apart from timings.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
1046 lines
65 KiB
C++
1046 lines
65 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <cstdlib>
|
|
#include "Rugnux.h"
|
|
#include "../image_analysis/structure_refinement/ModelValidation.h"
|
|
#include "../writer/WriteModel.h"
|
|
#include "../image_analysis/indexing/SpindleCuspLoss.h"
|
|
#include "../image_analysis/bragg_integration/SpotWidth.h"
|
|
#include "../image_analysis/bragg_integration/SpotFootprint.h"
|
|
#include "../image_analysis/bragg_integration/BraggStencil.h" // BRAGG_FOOTPRINT_NSIGMA
|
|
#include "../image_analysis/hot_pixels/HotPixels.h"
|
|
#include "../image_analysis/SensorAbsorption.h"
|
|
#include "DiagnosticOutput.h"
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <condition_variable>
|
|
#include <deque>
|
|
#include <fstream>
|
|
#include <cstring>
|
|
#include <functional>
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <numeric>
|
|
#include <map>
|
|
#include <set>
|
|
#include <sstream>
|
|
#include <thread>
|
|
|
|
#include "../reader/JFJochHDF5Reader.h"
|
|
#include "../common/JFJochMath.h"
|
|
#include "../common/ParallelFor.h"
|
|
#include "../common/Logger.h"
|
|
#include "../common/AzimuthalIntegrationMapping.h"
|
|
#include "../common/AzimuthalIntegrationProfile.h"
|
|
#include "../common/CUDAWrapper.h"
|
|
#include "../common/JFJochException.h"
|
|
#include "../common/time_utc.h"
|
|
#include "../writer/FileWriter.h"
|
|
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
|
#include "../image_analysis/beam_stop/ShadowFinder.h"
|
|
#include "../image_analysis/IndexAndRefine.h"
|
|
#include "../image_analysis/indexing/AnalyzeIndexing.h"
|
|
#include "../image_analysis/indexing/HarmonicContamination.h"
|
|
#include "../image_analysis/geom_refinement/BeamCenterFFT.h"
|
|
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
|
|
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
|
|
#include "../image_analysis/indexing/IndexerThreadPool.h"
|
|
#include "../image_analysis/spot_finding/ImageSpotFinderCPU.h"
|
|
#include "../image_analysis/spot_finding/SpotUtils.h"
|
|
#include "gemmi/twin.hpp"
|
|
#include "../image_analysis/azint/AzIntEngineCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
|
|
#ifdef JFJOCH_USE_CUDA
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h"
|
|
#include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h"
|
|
#endif
|
|
#include "../image_analysis/scale_merge/Merge.h"
|
|
#include "../image_analysis/scale_merge/RfreeFlags.h"
|
|
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
|
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
|
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
|
|
#include "../image_analysis/scale_merge/ScalingResult.h"
|
|
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
|
|
#include "../image_analysis/geom_refinement/PostRefine.h"
|
|
#include "../image_analysis/lattice_search/LatticeSearch.h"
|
|
#include "../image_analysis/lattice_search/LePageLattice.h"
|
|
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
|
|
#include "../image_analysis/scale_merge/FrenchWilson.h"
|
|
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
|
|
#include "../image_analysis/scale_merge/TranslationalNCS.h"
|
|
#include "../image_analysis/scale_merge/HKLKey.h"
|
|
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
|
|
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
|
|
#include "../image_analysis/WriteReflections.h"
|
|
#include "../image_analysis/bragg_integration/CalcISigma.h"
|
|
#include "../common/Definitions.h"
|
|
#include "../common/CorrelationCoefficient.h"
|
|
#include <array>
|
|
#include <map>
|
|
#include "RugnuxPipeline.h"
|
|
#include "RugnuxFirstPass.h"
|
|
|
|
// The steps of the first pass that may replace the standing lattice: the rescues, asked where it
|
|
// indexes too little, and the hypotheses, asked of every run. FirstPassRotationIndexing gives their
|
|
// order.
|
|
|
|
using namespace rugnux_internal;
|
|
|
|
namespace {
|
|
auto MaxAxis(const RotationIndexerResult &r) {
|
|
const auto uc = r.lattice.GetUnitCell();
|
|
return std::max({uc.a, uc.b, uc.c});
|
|
}
|
|
}
|
|
|
|
// Rotation-axis SIGN rescue. The sign is a convention the input cannot settle: a PILATUS
|
|
// miniCBF header carries only "Oscillation_axis omega" and no direction at all, and facilities
|
|
// disagree about the sign of the NXmx vector (measured across ten of them: most want the
|
|
// stored [-1,0,0], APS 19-ID and SPring-8 BL41XU want the opposite). Nothing downstream can
|
|
// repair it either - the default spindle fit refines the axis DIRECTION by fractions of a
|
|
// degree and cannot flip it, that being a discrete change.
|
|
//
|
|
// It is, however, decidable from the data: the wrong sign does not index at all. Measured on
|
|
// one dataset by changing nothing but the sign: 100% of frames indexed one way, 0 of 60 the
|
|
// other. So try the other sign and keep whichever indexes more frames - the same
|
|
// validation-frame count the scheme choice above already uses, so no new metric and no new
|
|
// threshold. Runs before the long-axis rescue: with the sign wrong every candidate lattice is
|
|
// wrong, and the coarse pass below would spend itself on a mis-signed cloud.
|
|
//
|
|
// Only after a poor pass, so a correctly-signed file costs nothing; and spot finding is not
|
|
auto Rugnux::FirstPassRun::AxisSignRescue(FirstPass best) -> FirstPass {
|
|
if (!rx.cancelled_ && best.score < 0.5 * static_cast<double>(validation.size())) {
|
|
if (const auto gon = rx.experiment_.GetGoniometer(); gon.has_value() && gon->IsScanning()) {
|
|
GoniometerAxis flipped = *gon;
|
|
flipped.Axis(-gon->GetAxis());
|
|
rx.experiment_.Goniometer(flipped);
|
|
FirstPass alt = PickBest(*indexer_pool, *indexer);
|
|
// More frames is not enough on its own while both counts sit at the floor: 1/60 against
|
|
// 0/60 is noise, and adopting it puts every rescue below on the wrong sign (measured: a
|
|
// run whose file sign, kept, went on to index found nothing once flipped on 1/60). So
|
|
// the flipped lattice must also beat chance on the pooled spots - the test the run puts
|
|
// its final lattice to - which a sparse pattern at the right sign passes on few frames
|
|
// (2/60 frames, 89% of the validation spots against 0% by chance).
|
|
const bool alt_real = alt.result.has_value() && alt.score > best.score
|
|
&& ValidationEvidenceBeatsChance(MeasurePooled(*indexer, *alt.result));
|
|
if (alt_real) {
|
|
logger.Info("Rotation axis sign: the file's axis indexes {}/{} validation frames, "
|
|
"the opposite sign {}/{} - adopting the opposite sign for this run",
|
|
best.score, static_cast<int>(validation.size()),
|
|
alt.score, static_cast<int>(validation.size()));
|
|
best = alt;
|
|
} else {
|
|
if (alt.result.has_value() && alt.score > best.score)
|
|
logger.Info("Rotation axis sign: the opposite sign indexes {}/{} validation frames "
|
|
"against the file's {}/{}, but its lattice does not beat chance on the "
|
|
"pooled spots - the file's sign is kept",
|
|
alt.score, static_cast<int>(validation.size()),
|
|
best.score, static_cast<int>(validation.size()));
|
|
rx.experiment_.Goniometer(*gon); // the file was right; put it back
|
|
}
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Long-axis rescue. When the de-novo cell indexes few validation frames, a long, finely-spaced
|
|
// axis was likely lost: the unconstrained FFT either collapsed it to a short sub-multiple or let
|
|
// a denser supercell over-fit the accumulated cloud (a small global-orientation error throws the
|
|
// many high-order reflections off along the fine axis, so the true cell scores worst on the raw
|
|
// cloud). Recover the true metric with a second search whose FFT unit-cell bound is widened past
|
|
// the default, then RE-INDEX at full resolution constrained by the cell that comes back, as a
|
|
// reference (the -C path): the reference filter drops the collapsed/supercell candidates and
|
|
// refines an accurate global lattice. Only runs after a poor standard pass, so well-indexing
|
|
// crystals never pay for it and their result is untouched. Returns the re-index where it indexes
|
|
// more validation frames than the `standing` pass it was asked about.
|
|
auto Rugnux::FirstPassRun::LongAxisRescue(const FirstPass &standing) -> std::optional<FirstPass> {
|
|
if (rx.cancelled_ || !standing.result.has_value()
|
|
|| standing.score >= 0.5 * static_cast<double>(validation.size()))
|
|
return std::nullopt;
|
|
auto coarse_settings = rx.experiment_.GetIndexingSettings();
|
|
// The resolution setting only sizes the FFT histogram - it does not filter which spots go
|
|
// into it - so this is here to pay for the widened bound below, not to make the long axis
|
|
// easier to pick out. Measured, coarsening does the opposite: the acceptance angle widens
|
|
// (0.159 -> 0.579 deg) but the true axis falls from rank 1996 to 16361 of 16384 directions.
|
|
coarse_settings.FFT_HighResolution_A(3.5f);
|
|
// The widened bound is what the rescue turns on, and it costs nothing: the histogram length is
|
|
// 4*pi*max_length/high_resolution, so coarsening the resolution to 3.5 A has already
|
|
// SHRUNK the transform by 1.75x - spending that back on reach leaves the coarse pass at
|
|
// 1.14x the standard one. Without it the rescue cannot recover what it exists to recover:
|
|
// a cell whose long axis is past the default 500 A bound is unrepresentable, so the
|
|
// "recovered" metric would still be a sub-multiple. Only reached after a poor pass, so a
|
|
// well-indexing crystal never pays. Stills never reach here.
|
|
constexpr float RESCUE_MAX_UNIT_CELL_A = 1000.0f;
|
|
if (coarse_settings.GetFFT_MaxUnitCell_A() < RESCUE_MAX_UNIT_CELL_A)
|
|
coarse_settings.FFT_MaxUnitCell_A(RESCUE_MAX_UNIT_CELL_A);
|
|
coarse_settings.IndexingThreads(2);
|
|
coarse_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS));
|
|
IndexerThreadPool coarse_pool(coarse_settings, IndexerConstruction::OnFirstUse);
|
|
// Coarse first pass: keep the recovered cell with the LONGEST axis directly. Its full-
|
|
// resolution per-frame validation would be low (the coarse cell is metrically right but
|
|
// imprecise), so do NOT score it here - it is only the reference for the constrained re-index.
|
|
const SchemeIndexers cris = FeedSchemes(coarse_pool);
|
|
RunIndexing(cris);
|
|
std::optional<RotationIndexerResult> coarse_ref;
|
|
for (auto &ri : cris) {
|
|
auto r = ri->GetLattice();
|
|
if (r.has_value() && (!coarse_ref.has_value() || MaxAxis(*r) > MaxAxis(*coarse_ref)))
|
|
coarse_ref = r;
|
|
}
|
|
// A recovered axis at least 30% longer than the standard pass found signals a rescued metric.
|
|
if (!rx.cancelled_ && coarse_ref.has_value()
|
|
&& MaxAxis(*coarse_ref) > 1.3 * MaxAxis(*standing.result)) {
|
|
logger.Info("Long-axis rescue: coarse pass recovered a {:.0f} A axis (was {:.0f} A); "
|
|
"re-indexing constrained by that cell", MaxAxis(*coarse_ref), MaxAxis(*standing.result));
|
|
rx.experiment_.SetUnitCell(coarse_ref->lattice.GetUnitCell());
|
|
// The re-index must be able to REPRESENT the cell the coarse pass just recovered. The
|
|
// standard pool was sized at the default bound, so re-using it here would search for a
|
|
// long axis it cannot hold and hand back the sub-multiple the rescue exists to replace.
|
|
// CountIndexed forces the lattice and never touches the pool, so the existing
|
|
// IndexAndRefine is reused; only the FFT needs the wider one.
|
|
auto wide_settings = rx.experiment_.GetIndexingSettings();
|
|
wide_settings.IndexingThreads(2);
|
|
wide_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS));
|
|
RaiseFFTBoundForKnownCell(wide_settings, rx.experiment_);
|
|
FirstPass constrained;
|
|
if (wide_settings.GetFFT_MaxUnitCell_A() > indexing_bound_A) {
|
|
IndexerThreadPool wide_pool(wide_settings, IndexerConstruction::OnFirstUse);
|
|
constrained = PickBest(wide_pool, *indexer);
|
|
} else {
|
|
constrained = PickBest(*indexer_pool, *indexer);
|
|
}
|
|
rx.experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo
|
|
if (constrained.result.has_value() && constrained.score > standing.score)
|
|
return constrained;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
// The background centre, tried as a SECOND FIRST PASS.
|
|
//
|
|
// The estimate itself is free - it is read off the projection --detect-beam-stop has already
|
|
// built - and a first pass re-uses the spots it has already found, so indexing the run a
|
|
// second time at the measured centre costs about one more first pass.
|
|
// That buys the comparison an on-failure search structurally cannot make: a centre wrong
|
|
// ALONG the spindle does not fail. It keeps 96-100 % of frames indexed and quietly returns a
|
|
// 2x, 3x or sqrt(3) axis harmonic, so no failure trigger ever fires on it. Indexing both
|
|
// centres and comparing the two ANSWERS is what can see it.
|
|
//
|
|
// What must NOT arbitrate the two is the frame count. Acceptance is a fractional-Miller test,
|
|
// so a cell twice as long has to place every spot twice as accurately to score the same:
|
|
// measured on real data, two centres 0.12 px apart gave the deposited cell at 99.23 % and a
|
|
// halved axis at 100.00 %, and the WRONG answer indexed better. A rule of the form "take the
|
|
// centre that indexes more" would therefore pick wrong in exactly the case this comparison
|
|
// exists for. So only what needs no arbiter is decided here:
|
|
//
|
|
// * the file's centre does not index a majority -> nothing defends it, so try the measured
|
|
// one (and the axis sign with it, see below). A run that indexes nothing has nothing to
|
|
// lose; where the measured one is no better, the file's centre is put back;
|
|
// * the file's centre indexes and the two agree on the lattice -> keep the file's centre and
|
|
// say they agree. This is the overwhelming majority, and saying it is worth the pass: it
|
|
// is a free statement that the header is good enough for this crystal;
|
|
// * they index and disagree -> say so, with both cells and their primitive-volume ratio.
|
|
// Which of an integer-related pair is real is not decided on the frame counts, for the
|
|
// reason above; it is decided, in ONE direction only, on the pooled spots - see the
|
|
// branch itself. Naming the disagreement is useful even where nothing is decided.
|
|
//
|
|
// Where the file's centre indexes nothing and the measured one does not either, the axis-sign
|
|
// question is asked AGAIN at the measured centre before giving up. The two unknowns are
|
|
// discrete and coupled, and the rescue above asked its question at a centre that may be 73 px
|
|
// wrong: on the two depositions here whose header is the geometric centre of the detector,
|
|
// each error hides the other, both runs sit at 0/60 until both are moved, and moving both
|
|
// takes them to 60/60 on the deposited lattice. Asking the pair costs one more first pass on
|
|
// a run that has already failed twice.
|
|
auto Rugnux::FirstPassRun::BeamCentreCheck(FirstPass best, bool &long_axis_asked) -> FirstPass {
|
|
rx.JoinBeamCenterCapture();
|
|
if (rx.cancelled_ || !rx.config_.beam_center_check || !rx.background_center_)
|
|
return best;
|
|
// Consumed, not re-read: a two-pass rotation run comes back through here with the geometry
|
|
// the first pass post-refined, and a pre-scan estimate of the FILE's centre says nothing
|
|
// about that one. The pre-scan does not re-measure it either - by then the beam-stop shadow
|
|
// is already in the mask, so the projection is not rebuilt.
|
|
const BeamCenterEstimate measured = *rx.background_center_;
|
|
rx.background_center_.reset();
|
|
const float header_x = rx.experiment_.GetBeamX_pxl(), header_y = rx.experiment_.GetBeamY_pxl();
|
|
const float measured_x = measured.beam_x_pxl;
|
|
const float measured_y = measured.beam_y_pxl;
|
|
const float moved = std::hypot(measured_x - header_x, measured_y - header_y);
|
|
const Centres c{measured, header_x, header_y, measured_x, measured_y, moved};
|
|
// No gate on how significant the move is. It is tempting to skip the second pass when the
|
|
// two centres are within the fit's own noise - it would save it on a quarter of the runs
|
|
// here - but that is deciding at a threshold before the evidence is in, and the evidence
|
|
// is cheap. It would also gate away exactly the case worth asking about: measured on real
|
|
// data, a 0.12 px change of centre is the whole difference between the deposited cell and
|
|
// a halved axis, and 0.12 px is well inside the sigma of any fit that could gate it.
|
|
const int majority = static_cast<int>(validation.size()) / 2;
|
|
TryBeamCenter(measured_x, measured_y);
|
|
FirstPass alt = IndexAtMeasuredCentre();
|
|
const bool header_indexes = best.result.has_value() && best.score > majority;
|
|
const bool measured_indexes = alt.result.has_value() && alt.score > majority;
|
|
if (!header_indexes)
|
|
return CentreWhenFileFails(std::move(best), std::move(alt), c, long_axis_asked);
|
|
RestoreBeamCenter(header_x, header_y);
|
|
if (measured_indexes)
|
|
return CentreWhenBothIndex(std::move(best), alt, c);
|
|
logger.Info("Beam centre check: the measured centre ({:.2f},{:.2f}) indexes "
|
|
"{}/{} validation frames against the file's {}/{} - the file's "
|
|
"centre stands", measured_x, measured_y, alt.score,
|
|
static_cast<int>(validation.size()), best.score,
|
|
static_cast<int>(validation.size()));
|
|
return best;
|
|
}
|
|
|
|
// The first pass at the measured centre: the speculative one started beside the file centre's
|
|
// (StartCentreSpeculation) where it read what the run has here, otherwise run here.
|
|
auto Rugnux::FirstPassRun::IndexAtMeasuredCentre() -> FirstPass {
|
|
FirstPass alt;
|
|
if (centre_spec)
|
|
centre_spec->done.get();
|
|
if (centre_spec && centre_spec->experiment_key == rx.ExperimentKey()
|
|
&& centre_spec->key == rx.SpotFindingKey(*spot_mapping)
|
|
&& centre_spec->spots_per_image == first_pass_spots_per_image) {
|
|
// What PickBest would have found and fed here: hand the spots to the run's cache and
|
|
// its memo as PrefetchSpots does, and score the schemes on the run's own indexer.
|
|
spot_cache = centre_spec->spots;
|
|
{
|
|
std::lock_guard lock(rx.first_pass_spots_->m);
|
|
auto &entries = rx.first_pass_spots_->entries;
|
|
if (std::none_of(entries.begin(), entries.end(),
|
|
[&](const auto &e) { return e.first == centre_spec->key; })) {
|
|
if (entries.size() == MAX_SPOT_KEYS)
|
|
entries.erase(entries.begin());
|
|
entries.emplace_back(centre_spec->key, centre_spec->spots);
|
|
}
|
|
}
|
|
alt = ScoreSchemes(centre_spec->ris, *indexer);
|
|
} else
|
|
alt = PickBest(*indexer_pool, *indexer);
|
|
centre_spec.reset();
|
|
return alt;
|
|
}
|
|
|
|
// The file's centre does not index a majority.
|
|
auto Rugnux::FirstPassRun::CentreWhenFileFails(FirstPass best, FirstPass alt, const Centres &c,
|
|
bool &long_axis_asked) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
const int majority = static_cast<int>(validation.size()) / 2;
|
|
bool measured_indexes = alt.result.has_value() && alt.score > majority;
|
|
// Nothing defends the file's centre: the data do not index at it, and the
|
|
// background says it is `moved` px away. So try the measured one, and if that
|
|
// does not index either, ask the AXIS-SIGN question again there. The two unknowns
|
|
// are discrete and coupled, and the sign rescue above asked its question at a
|
|
// centre that may itself be wrong - each error hides the other, and only moving
|
|
// both reveals anything.
|
|
bool joint = false;
|
|
const auto gonio = rx.experiment_.GetGoniometer();
|
|
if (!measured_indexes && gonio.has_value() && gonio->IsScanning()) {
|
|
GoniometerAxis flipped = *gonio;
|
|
flipped.Axis(-gonio->GetAxis());
|
|
rx.experiment_.Goniometer(flipped);
|
|
const FirstPass both = PickBest(*indexer_pool, *indexer);
|
|
if (both.result.has_value() && both.score > majority) {
|
|
logger.Warning("Beam centre from the background: the file's centre "
|
|
"({:.2f},{:.2f}) indexes {}/{} validation frames and the "
|
|
"measured ({:.2f},{:.2f}) {}/{}, but the measured centre "
|
|
"WITH the opposite rotation-axis sign indexes {}/{}. The "
|
|
"file has both wrong, and neither shows while the other "
|
|
"stands; the run takes the measured centre and the opposite "
|
|
"sign", header_x, header_y, best.score,
|
|
static_cast<int>(validation.size()), measured_x, measured_y,
|
|
alt.score, static_cast<int>(validation.size()), both.score,
|
|
static_cast<int>(validation.size()));
|
|
alt = both;
|
|
measured_indexes = true;
|
|
joint = true;
|
|
} else {
|
|
rx.experiment_.Goniometer(*gonio);
|
|
}
|
|
}
|
|
// Third coupled unknown: how DEEP the pass reads. A centre 170 px out and a spot
|
|
// list too deep to index at ANY centre hide each other exactly as the centre and
|
|
// the axis sign do - the measured centre is right, and it is thrown away because
|
|
// the full list indexes nothing there either. Measured on one such crystal: the
|
|
// file's centre indexed 1/60 and the measured one 0/60, so the file's was kept and
|
|
// the run died; with the ladder's lean seed the measured centre indexes 60/60, and
|
|
// the run then indexes every frame of the sweep and merges to the deposited
|
|
// resolution. So ask the ladder here too, before giving the centre up.
|
|
//
|
|
// The depth is asked at BOTH centres, and the centre moves only if it is the
|
|
// centre that paid. A lean rung at the measured centre scored against the whole
|
|
// spot list at the file's changes two things at once, and the count cannot say
|
|
// which of them the gain came from - the same reason a trial centre is scored on
|
|
// its own spots at all. Measured on one crystal: the measured centre's ladder
|
|
// reached 40/60 on a FIVE-fold axis harmonic while the file's own centre reached
|
|
// 49/60 on the deposited cell, and the run moved the centre, shipped the
|
|
// supercell, and skipped the beam-centre walk that would have found the crystal's
|
|
// own. An exact tie keeps the file's centre: nothing then says the centre is wrong.
|
|
bool lean = false;
|
|
bool deeper_at_home = false;
|
|
if (!measured_indexes) {
|
|
const FirstPassState before = committed;
|
|
RestoreBeamCenter(header_x, header_y);
|
|
const auto home = FirstPassLadder(best.score, 1, majority + 1);
|
|
const FirstPassState home_state = committed;
|
|
committed = before;
|
|
ApplyState(committed);
|
|
TryBeamCenter(measured_x, measured_y);
|
|
const auto away = FirstPassLadder(alt.score, 1, majority + 1);
|
|
const FirstPassState away_state = committed;
|
|
// Both centres index a majority, and on the same lattice. Then the two counts
|
|
// are two readings of one hypothesis at two geometries, and which is higher is
|
|
// the count's noise: measured on one crystal, the same two ladders gave the
|
|
// measured centre 56/60 against the file's 50/60, and 48/60 against 50/60 once
|
|
// a few thousand defective pixels were masked - and the run that then kept the
|
|
// file's centre, 12.6 px out, merged with twice the frames rejected. So the
|
|
// count decides nothing here: the file's centre stays for this pass, and the
|
|
// measured one is carried forward to be judged on the merges (RunAllPasses).
|
|
const bool same_lattice =
|
|
home && away
|
|
&& home->result->search_result.system == away->result->search_result.system
|
|
&& home->result->search_result.centering
|
|
== away->result->search_result.centering
|
|
&& std::max(home->vol, away->vol) < 1.02 * std::min(home->vol, away->vol);
|
|
if (home && (!away || same_lattice || home->score >= away->score)) {
|
|
committed = home_state;
|
|
RestoreBeamCenter(header_x, header_y);
|
|
ApplyState(committed);
|
|
best = *home;
|
|
deeper_at_home = true;
|
|
if (same_lattice && geometry_prepass) {
|
|
rx.beam_center_alternative_ = std::array<float, 2>{measured_x, measured_y};
|
|
rx.beam_center_alternative_state_ = away_state;
|
|
logger.Warning("Beam centre check: reading less of each frame, the "
|
|
"file's centre indexes {}/{} validation frames and the "
|
|
"measured centre {:.2f} px away {}/{}, on the same "
|
|
"lattice - the counts cannot say which centre is right, "
|
|
"so the run carries BOTH centres forward and adopts "
|
|
"whichever measures more of the crystal",
|
|
home->score, static_cast<int>(validation.size()), moved,
|
|
away->score, static_cast<int>(validation.size()));
|
|
} else
|
|
logger.Warning("Beam centre check: reading less of each frame takes the "
|
|
"file's centre to {}/{} validation frames against {}/{} at "
|
|
"the measured centre {:.2f} px away - it is how deep this "
|
|
"run was reading and not the centre, so the centre stays",
|
|
home->score, static_cast<int>(validation.size()),
|
|
away ? away->score : 0,
|
|
static_cast<int>(validation.size()), moved);
|
|
} else if (away) {
|
|
alt = *away;
|
|
measured_indexes = true;
|
|
lean = true;
|
|
}
|
|
}
|
|
if (measured_indexes) {
|
|
if (lean)
|
|
logger.Warning("Beam centre from the background: neither centre indexes the "
|
|
"whole spot list, but at the measured ({:.2f},{:.2f}) the "
|
|
"lean first pass above indexes {}/{} against the file's "
|
|
"{}/{} - the beam centre in this file is wrong, and only a "
|
|
"pass that reads less of each frame could see it",
|
|
measured_x, measured_y, alt.score,
|
|
static_cast<int>(validation.size()), best.score,
|
|
static_cast<int>(validation.size()));
|
|
else if (!joint)
|
|
logger.Warning("Beam centre from the background: the file's ({:.2f},{:.2f}) "
|
|
"indexes {}/{} validation frames and the measured "
|
|
"({:.2f},{:.2f}) indexes {}/{} - the beam centre in this "
|
|
"file is wrong, and the run adopts the measured one",
|
|
header_x, header_y, best.score,
|
|
static_cast<int>(validation.size()), measured_x, measured_y,
|
|
alt.score, static_cast<int>(validation.size()));
|
|
best = alt;
|
|
} else if (!deeper_at_home) {
|
|
best = CentreByPooledSpots(std::move(best), std::move(alt), c, long_axis_asked);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Nothing worked. Put the file's centre back, so a run that fails for some
|
|
// other reason fails at the geometry it was given rather than at a fit that has
|
|
// just been tested and found no better.
|
|
//
|
|
// Unless the frame counts are both at the floor and the POOLED spots do decide:
|
|
// a crystal whose spots are a minority of every frame indexes 0/60 at any centre,
|
|
// and then only the pooled excess over chance - each lattice measured against its
|
|
// own wrong-spindle null, so a larger cell is not credited for fitting more by
|
|
// chance - says which centre sees it.
|
|
//
|
|
// Each centre is judged on the best lattice it gives, so the long-axis rescue
|
|
// is asked at BOTH before they are compared. The lattices standing here index
|
|
// no majority anywhere, which is exactly the pass that rescue exists for, and
|
|
// the two counts of spots on two lattices that are not the crystal's are close
|
|
// to a coin toss: measured on a 545 A axis, the measured centre (3.6 px out
|
|
// across the spindle) won it on 16.7% against 15.5%, the rescue then ran
|
|
// there alone and could not see the long axis, and the run shipped a P1
|
|
// sub-cell - while at the file's centre the same rescue indexes 60/60.
|
|
auto Rugnux::FirstPassRun::CentreByPooledSpots(FirstPass best, FirstPass alt, const Centres &c,
|
|
bool &long_axis_asked) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
if (auto rescued = LongAxisRescue(alt))
|
|
alt = *rescued;
|
|
PooledEvidence at_measured, at_header;
|
|
if (alt.result.has_value())
|
|
at_measured = MeasurePooled(*indexer, *alt.result);
|
|
RestoreBeamCenter(header_x, header_y);
|
|
if (auto rescued = LongAxisRescue(best))
|
|
best = *rescued;
|
|
if (best.result.has_value())
|
|
at_header = MeasurePooled(*indexer, *best.result);
|
|
long_axis_asked = true;
|
|
if (alt.result.has_value() && ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured)) {
|
|
TryBeamCenter(measured_x, measured_y);
|
|
logger.Warning("Beam centre from the background: neither centre indexes a "
|
|
"majority of the validation frames ({}/{} at the measured, "
|
|
"{}/{} at the file's), but the pooled spots put "
|
|
"{:.1f}% on the lattice at the measured ({:.2f},{:.2f}) against "
|
|
"{:.1f}% at the file's (chance {:.1f}% and {:.1f}%) - the run "
|
|
"adopts the measured centre",
|
|
alt.score, static_cast<int>(validation.size()),
|
|
best.score, static_cast<int>(validation.size()),
|
|
100.0 * at_measured.on_lattice / std::max<int64_t>(1, at_measured.spots),
|
|
measured_x, measured_y,
|
|
100.0 * at_header.on_lattice / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.by_chance / std::max<int64_t>(1, at_measured.spots),
|
|
100.0 * at_header.by_chance / std::max<int64_t>(1, at_header.spots));
|
|
best = alt;
|
|
} else {
|
|
logger.Info("Beam centre check: the file's centre ({}/{} frames) and the "
|
|
"measured one {:.2f} px away ({}/{}) - the measured centre puts no "
|
|
"more of the pooled spots on its lattice, so the file's centre is "
|
|
"kept",
|
|
best.score, static_cast<int>(validation.size()), moved, alt.score,
|
|
static_cast<int>(validation.size()));
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// The file's centre indexes a majority, and so does the measured one.
|
|
auto Rugnux::FirstPassRun::CentreWhenBothIndex(FirstPass best, const FirstPass &alt, const Centres &c) -> FirstPass {
|
|
const BeamCenterEstimate &measured = c.measured;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
// Primitive volumes, as everywhere else here: two passes can find the same
|
|
// lattice in different settings, and a centred setting's cell is an exact
|
|
// integer multiple of its primitive one.
|
|
const double ratio = alt.vol < best.vol ? best.vol / alt.vol : alt.vol / best.vol;
|
|
const bool same_class =
|
|
best.result->search_result.system == alt.result->search_result.system
|
|
&& best.result->search_result.centering == alt.result->search_result.centering;
|
|
// The primitive volume is what the beam centre decides. A centre error along
|
|
// the spindle makes the FFT take an axis harmonic, which changes the volume by
|
|
// an integer factor or by sqrt(3); the Bravais class can differ for a reason
|
|
// that has nothing to do with the centre, so the two are reported separately.
|
|
if (ratio < 1.02 && same_class) {
|
|
rx.Note(fmt::format("beam centre kept: measured {:.1f} px off, same lattice", moved));
|
|
logger.Info("Beam centre check: indexing at the measured centre "
|
|
"({:.2f},{:.2f}) returns the same lattice as the file's - "
|
|
"{}-centred {}, cells agreeing to {:.2f} % in volume. The {:.2f} px "
|
|
"difference does not decide this crystal's cell",
|
|
measured_x, measured_y, alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system),
|
|
100.0 * (ratio - 1.0), moved);
|
|
// Unless the two are further apart than this geometry absorbs. A centre
|
|
// wrong by about one reflection spacing indexes the same cell just as
|
|
// well, with every reflection one index out over part of the sweep, and
|
|
// only the merge shows it - so the measured centre is kept as a hypothesis
|
|
// for RunAllPasses, which tries it if the file's centre merges
|
|
// inconsistently.
|
|
if (geometry_prepass && moved > 3.0f * measured.sigma_pxl
|
|
&& moved > BeamCenterNeed_pxl(rx.experiment_))
|
|
rx.beam_center_same_lattice_alternative_ =
|
|
std::array<float, 2>{measured_x, measured_y};
|
|
} else if (ratio < 1.02) {
|
|
// Same cell, different metric symmetry read off it. The cell is not in
|
|
// question, but the CLASS is not cosmetic either: GeometryRefiner holds
|
|
// the cell to it for the rest of the run, so a class that is one axis
|
|
// equality away from the truth is refined, predicted, integrated and
|
|
// merged with that equality imposed.
|
|
//
|
|
// Whether the centre decided it is arithmetic. The equality is held to
|
|
// LATTICE_SEARCH_DIST_TOLERANCE, relative, on a pair of reduced-cell
|
|
// axes; a centre wrong by `moved` px moves an axis of length a by
|
|
// moved*p*a/(D*lambda) relative, the FFT reading that axis off spot
|
|
// positions the centre displaces. Both axes of the pair carry their own
|
|
// error, so the centre can open or close a gap of their sum - and where
|
|
// that reaches the tolerance the higher class is not a fact about the
|
|
// crystal but a consequence of where the beam was assumed to be.
|
|
//
|
|
// Nothing available here can settle it: both centres index, and the frame
|
|
// count is the one statistic this check already refuses to arbitrate on.
|
|
// So carry the other centre out of the pass and let the two arms be judged
|
|
// on their merges (RunAllPasses).
|
|
const auto reduced = best.result->search_result.primitive_reduced.GetUnitCell();
|
|
const double axis_pair = std::max(reduced.a + reduced.b,
|
|
reduced.b + reduced.c);
|
|
const double induced = moved * rx.experiment_.GetPixelSize_mm() * axis_pair
|
|
/ (rx.experiment_.GetDetectorDistance_mm()
|
|
* rx.experiment_.GetWavelength_A());
|
|
const bool centre_can_decide = induced >= LATTICE_SEARCH_DIST_TOLERANCE;
|
|
logger.Info("Beam centre check: indexing at the measured centre "
|
|
"({:.2f},{:.2f}) returns the same CELL as the file's - the "
|
|
"primitive volumes agree to {:.2f} % - but a different metric "
|
|
"symmetry: {}-centred {} from the file's centre against "
|
|
"{}-centred {} from the measured one", measured_x, measured_y,
|
|
100.0 * (ratio - 1.0), best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system));
|
|
if (centre_can_decide && geometry_prepass) {
|
|
rx.beam_center_alternative_ = std::array<float, 2>{measured_x, measured_y};
|
|
logger.Warning("Beam centre check: {:.2f} px is worth {:.2f} % on the "
|
|
"axis pair the promotion compares, against the {:.2f} % "
|
|
"the metric symmetry is decided to - the difference is "
|
|
"large enough to have decided it either way, so the run "
|
|
"carries BOTH centres forward and adopts whichever "
|
|
"merges better", moved, 100.0 * induced,
|
|
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
|
|
} else {
|
|
logger.Info("Beam centre check: {:.2f} px is worth {:.2f} % on the axis "
|
|
"pair the promotion compares, against the {:.2f} % the "
|
|
"metric symmetry is decided to - the run keeps the file's "
|
|
"centre and the symmetry it found", moved, 100.0 * induced,
|
|
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
|
|
}
|
|
} else {
|
|
best = CentresDisagree(std::move(best), alt, c, ratio);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// The two centres index different lattices.
|
|
auto Rugnux::FirstPassRun::CentresDisagree(FirstPass best, const FirstPass &alt, const Centres &c,
|
|
double ratio) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const auto &hc = best.result->search_result.conventional.GetUnitCell();
|
|
const auto &mc = alt.result->search_result.conventional.GetUnitCell();
|
|
const double nearest = std::round(ratio);
|
|
const bool harmonic = (nearest >= 2.0 && std::abs(ratio - nearest) < 0.15)
|
|
|| std::abs(ratio - std::sqrt(3.0)) < 0.15;
|
|
logger.Warning("Beam centre check: the two centres DISAGREE about the "
|
|
"lattice. The file's ({:.2f},{:.2f}) gives {}-centred {}, "
|
|
"{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} (primitive volume "
|
|
"{:.0f} A^3, {}/{} frames); the measured ({:.2f},{:.2f}) gives "
|
|
"{}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} "
|
|
"(primitive volume {:.0f} A^3, {}/{} frames){}",
|
|
header_x, header_y, best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
hc.a, hc.b, hc.c, hc.alpha, hc.beta, hc.gamma, best.vol,
|
|
best.score, static_cast<int>(validation.size()),
|
|
measured_x, measured_y, alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system),
|
|
mc.a, mc.b, mc.c, mc.alpha, mc.beta, mc.gamma, alt.vol,
|
|
alt.score, static_cast<int>(validation.size()),
|
|
harmonic ? fmt::format(" - the volumes differ by {:.2f}x, so one "
|
|
"is an axis harmonic of the other and the "
|
|
"beam centre is what settles which",
|
|
ratio)
|
|
: "");
|
|
// Deliberately no decision on the frame counts. They are the one statistic
|
|
// that must not choose: on the measured case the harmonic indexed MORE.
|
|
//
|
|
// Decided in either direction on the pooled excess over chance, the same
|
|
// measurement the failed-pass branch above adopts a centre on. Each lattice is
|
|
// scored against its own wrong-spindle null, so the larger cell is not credited
|
|
// for the spots a denser lattice catches by accident, and the smaller one is
|
|
// not credited for the spots it misses.
|
|
//
|
|
// Both directions need it. Where the measured centre holds the LARGER cell,
|
|
// nothing else in the run repairs it - the second pass refuses a lattice larger
|
|
// than pass 1's (the supercell guard below). Where it holds the SMALLER one,
|
|
// the second pass can drop to it only if its own beam refinement happens to
|
|
// walk the whole centre error, and that rescue is chaotic: measured on one
|
|
// crystal, a 0.1 % change in the pass-1 distance switched off a 6.5 px walk and
|
|
// the run kept the doubled axis.
|
|
const double nearest_n = std::round(ratio);
|
|
const bool integer_harmonic = nearest_n >= 2.0 && nearest_n <= 4.0
|
|
&& std::abs(ratio - nearest_n) < 0.15;
|
|
bool adopted_measured = false;
|
|
if (integer_harmonic) {
|
|
const PooledEvidence at_header = MeasurePooled(*indexer, *best.result);
|
|
TryBeamCenter(measured_x, measured_y);
|
|
const PooledEvidence at_measured = MeasurePooled(*indexer, *alt.result);
|
|
adopted_measured = ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured);
|
|
logger.Info("Beam centre check: pooled validation spots on the lattice "
|
|
"{:.1f}% at the file's centre and {:.1f}% at the measured "
|
|
"(chance {:.1f}% and {:.1f}%)",
|
|
100.0 * at_header.on_lattice / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.on_lattice
|
|
/ std::max<int64_t>(1, at_measured.spots),
|
|
100.0 * at_header.by_chance / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.by_chance
|
|
/ std::max<int64_t>(1, at_measured.spots));
|
|
if (adopted_measured) {
|
|
logger.Warning("Beam centre check: the spots are on the {} cell of the "
|
|
"measured centre, not on the file's - so the {:.2f}x is "
|
|
"the file's centre error and not the crystal, and the "
|
|
"run adopts the measured centre",
|
|
alt.vol > best.vol ? "larger" : "smaller", ratio);
|
|
best = alt;
|
|
} else {
|
|
RestoreBeamCenter(header_x, header_y);
|
|
}
|
|
}
|
|
if (!adopted_measured)
|
|
logger.Warning("Beam centre check: the run keeps the file's centre. The "
|
|
"frame counts cannot arbitrate this - a cell twice as long "
|
|
"must place every spot twice as accurately to score the "
|
|
"same, so the harmonic is often the one that indexes more. "
|
|
"Where the beam centre is known, give it with "
|
|
"--beam-x/--beam-y");
|
|
return best;
|
|
}
|
|
|
|
// Where the first pass indexes no majority of the validation frames, the end wedge is asked too
|
|
// and judged against the standing pass as any later scheme is. Only then: on a run the schemes
|
|
// index it would be one more lattice for the axis-harmonic arbiter, which is not decided on
|
|
// frame counts. Asked once, after the axis-sign, beam-centre and long-axis rescues (below), not
|
|
// inside every pass they run: a consecutive wedge indexes part of the validation frames at the
|
|
// WRONG axis sign too (measured: 33/60 where the spread at the right sign indexes 60/60), and
|
|
// counted in the pass a rescue is gated on, it lifts that pass past the gate and the rescue
|
|
// that would have indexed every frame is never asked.
|
|
auto Rugnux::FirstPassRun::AddEndWedge(FirstPass bp, IndexerThreadPool &pool, IndexAndRefine &idx) -> FirstPass {
|
|
if (rx.cancelled_ || bp.score > static_cast<int>(validation.size()) / 2)
|
|
return bp;
|
|
SchemeIndexers ris;
|
|
ris.push_back(FeedScheme(pool, end_wedge[0].second));
|
|
RunIndexing({ris[0].get()});
|
|
return ScoreSchemes(ris, idx, &end_wedge, std::move(bp));
|
|
}
|
|
|
|
// Short-axis hypothesis - the small-molecule case.
|
|
//
|
|
// LAST, after the axis-sign, beam-centre and long-axis rescues, deliberately. Those all
|
|
// run the standard floor and compare their result against `best` on volume ratios; a cell
|
|
// adopted here is a k-fold sub-cell, so running before them makes every one of those
|
|
// comparisons read an integer ratio and report an axis harmonic that is not one (measured:
|
|
// the beam-centre check announced a "3.00x harmonic ... the beam centre is what settles
|
|
// which" on a pair that differs only by the FLOOR). Here the comparison is like with like,
|
|
// and this pass sees whatever geometry those rescues settled on.
|
|
//
|
|
// fft_min_unit_cell_A (10 A) is a protein constant: shorter than any protein axis and longer
|
|
// than a small molecule's. It is applied to the FFT's PER-DIRECTION PEAK SEARCH
|
|
// (FFTIndexerCPU.cpp / FFTIndexerGPU.cu skip every histogram bin below it) and only then to
|
|
// the assembled cell (FFTIndexer::ReduceAndRefine, PostIndexingRefinement). The first of
|
|
// those is what does the damage: the transform reports ONE peak per direction, so a
|
|
// lattice row whose real-space repeat is shorter than the floor is reported at its first
|
|
// HARMONIC above it, and the cell built out of those harmonics is an exact integer
|
|
// sublattice of the true one - a real-space supercell. Measured on chemical-crystallography
|
|
// data: index 2, 3 and 6 exactly, and the one dataset whose true reduced cell clears 10 A is
|
|
// the one that comes out right. Because the floor is applied to the REDUCED cell it rejects
|
|
// the lattice and not merely a basis choice, so nothing downstream can re-describe its way
|
|
// back.
|
|
//
|
|
// Lowering the floor by default is not available: it only ever ADDS candidates, and the
|
|
// Bravais classification the candidate pool feeds is under-determined, so it moves crystals
|
|
// whose cell it never touched (measured: 2 regressions and 1 improvement over 20 protein
|
|
// datasets, all three in the metric symmetry rather than the cell). So the low floor runs as
|
|
// a SECOND hypothesis and the standard pass's answer stands unless the evidence takes it.
|
|
//
|
|
// What the evidence can be, before integration: a sublattice's per-spot residuals are
|
|
// dominated pointwise by the finer lattice it contains, so every margin statistic - squares,
|
|
// counts, trimmed sums - is biased toward the LARGER cell, and a spurious supercell indexes
|
|
// every frame its true cell does. Only the SATURATION is informative. Hence the same shape
|
|
// the scheme ratchet in PickBest already uses: an integer primitive-volume ratio plus a tie
|
|
// on validation frames, or an outright win, which runs against the bias and so is real. The
|
|
// statistic that genuinely decides an axis multiple is the occupancy of the reflection class
|
|
// the multiple adds - AbsenceEvidence in SearchSpaceGroup - and that needs integrated
|
|
// intensities, so it cannot run here.
|
|
// Not when a cell was given. RaiseFFTBoundForKnownCell has already lowered the floor for
|
|
// the standard pass against that cell, so this pass has nothing left to add - and it takes
|
|
// its settings from rx.experiment_, which never carried the RAISED max bound, so on a cell
|
|
// longer than the default search it could only ever return a harmonic of what was asked
|
|
// for, tie on frames the way a sub-multiple always does, and be adopted over it.
|
|
auto Rugnux::FirstPassRun::ShortAxisHypothesis(FirstPass best) -> FirstPass {
|
|
if (!rx.cancelled_ && !rx.experiment_.GetUnitCell().has_value()) {
|
|
std::unique_ptr<IndexerThreadPool> short_pool;
|
|
std::vector<std::unique_ptr<RotationIndexer>> sris;
|
|
if (short_spec && short_spec->key == rx.ExperimentKey() && short_spec->mapping == spot_mapping
|
|
&& short_spec->spots_per_image == first_pass_spots_per_image) {
|
|
for (auto &f : short_spec->futures)
|
|
f.get();
|
|
short_pool = std::move(short_spec->pool);
|
|
sris = std::move(short_spec->sris);
|
|
} else {
|
|
short_spec.reset();
|
|
short_pool = std::make_unique<IndexerThreadPool>(ShortAxisSettings(rx.experiment_),
|
|
IndexerConstruction::OnFirstUse);
|
|
sris = FeedSchemes(*short_pool);
|
|
RunIndexing(sris);
|
|
}
|
|
|
|
for (size_t i = 0; i < sris.size() && !rx.cancelled_; i++) {
|
|
auto found = sris[i]->GetLattice();
|
|
if (!found.has_value())
|
|
continue;
|
|
RotationIndexerResult alt = std::move(*found);
|
|
const std::string &name = schemes[i].first;
|
|
// Primitive volumes, for the reason PickBest gives: two searches can find the same
|
|
// lattice in different settings, and a centred setting's cell is an exact integer
|
|
// multiple of its primitive one.
|
|
const double vol = std::abs(
|
|
alt.lattice.ToPrimitive(alt.search_result.centering).CalcVolume());
|
|
if (!(vol > 1.0) || !(best.vol > 1.0))
|
|
continue;
|
|
const double ratio = best.vol / vol;
|
|
const double nearest = std::round(ratio);
|
|
// The gate is here rather than after the scoring because scoring is the expensive
|
|
// half - CountIndexed runs the real per-image path over the validation set, seconds
|
|
// on a protein - while the volume ratio is free. On every protein measured the low
|
|
// floor returns the SAME lattice (ratio 1.00-1.01), so this costs them one extra FFT
|
|
// and nothing else.
|
|
if (!(vol < best.vol) || nearest < 2.0 || std::abs(ratio - nearest) >= 0.15) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': primitive volume "
|
|
"{:.0f} A^3 against the standing cell's {:.0f} A^3, ratio {:.2f} - "
|
|
"not an integer sub-cell, keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, vol, best.vol,
|
|
ratio);
|
|
continue;
|
|
}
|
|
|
|
// The premise of this pass, checked rather than asserted. The pass exists because a
|
|
// lattice row shorter than the standard floor is reported at its first HARMONIC
|
|
// above it, so a cell it may rightfully adopt has a reduced axis BELOW that floor -
|
|
// and the floor is applied to the reduced cell, which is the one to measure. Where
|
|
// the alternative has no such axis the floor cannot be what the standard pass
|
|
// tripped over, and an integer volume ratio is then an ordinary axis multiple: the
|
|
// question the axis-harmonic arbiter settles just above on the occupancy of the
|
|
// class the multiple adds. That is intensity-class evidence about the very
|
|
// reflections in dispute, where the tie below is a frame count this comment already
|
|
// says a sublattice cannot lose - so where the two disagree the arbiter is right and
|
|
// this block must not overrule it. Measured on the corpus: the four small-molecule
|
|
// sets the pass is for all have a reduced axis under 9 A, and the two protein sets
|
|
// it halved have 35 and 47 A.
|
|
const auto alt_reduced = alt.lattice.ToPrimitive(alt.search_result.centering)
|
|
.NiggliReduce().GetUnitCell();
|
|
const float alt_shortest = std::min({alt_reduced.a, alt_reduced.b, alt_reduced.c});
|
|
const float standard_floor = rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A();
|
|
if (alt_shortest >= standard_floor) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': a {:.0f}x sub-cell "
|
|
"(vol {:.0f} A^3), but its shortest reduced axis is {:.1f} A, above "
|
|
"the {:.1f} A floor the standard pass already searched - the floor "
|
|
"is not why that cell was missed, so this is an ordinary axis "
|
|
"multiple and not this pass's to decide; keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
|
|
alt_shortest, standard_floor);
|
|
continue;
|
|
}
|
|
|
|
const int alt_score = CountIndexed(*indexer, alt);
|
|
// A pass that indexes nothing is not tied with a pass that indexes nothing. Without
|
|
// this, 0 against 0 passes the tie band and the volume ratio alone decides, which is
|
|
// the one thing this block must never do.
|
|
if (alt_score <= 0 || best.score <= 0) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {}/{} validation "
|
|
"frames against {} - nothing indexes, keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, alt_score,
|
|
static_cast<int>(validation.size()), best.score);
|
|
continue;
|
|
}
|
|
const bool tied = static_cast<float>(alt_score) >= best.score * 0.9f - 0.5f;
|
|
const bool clearly_more = static_cast<float>(alt_score) > best.score * 1.1f + 0.5f;
|
|
if (!tied && !clearly_more) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {:.0f}x sub-cell "
|
|
"(vol {:.0f} A^3) indexes {}/{} validation frames against {} - "
|
|
"keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
|
|
alt_score, static_cast<int>(validation.size()), best.score);
|
|
continue;
|
|
}
|
|
const auto uc = alt.lattice.GetUnitCell();
|
|
logger.Warning("Short-axis pass (floor {:.0f} A), scheme '{}': the standing "
|
|
"cell is a {:.0f}x supercell. Its {:.0f} A^3 primitive volume against "
|
|
"this pass's {:.0f} A^3, at {}/{} validation frames against {} - "
|
|
"adopting {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}. The shortest axis "
|
|
"is below the {:.0f} A default FFT floor, which is why the standard "
|
|
"pass could only build a multiple of it",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, best.vol,
|
|
vol, alt_score, static_cast<int>(validation.size()), best.score,
|
|
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
|
|
rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A());
|
|
best.score = alt_score;
|
|
best.vol = vol;
|
|
best.name = name + " (short-axis)";
|
|
best.result = std::move(alt);
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Sub-lattice hypothesis: the standing cell's seven index-2 sub-lattices, in its own
|
|
// orientation and geometry, each scored on the validation frames.
|
|
//
|
|
// The FFT can only offer the cells its peaks build, and two lattices seen together - a twin
|
|
// domain or a second crystal whose reciprocal lattice shares an index-2 sub-lattice with
|
|
// this one's - build the lattice that contains them both: a cell twice the size of either,
|
|
// which indexes both crystals' spots and so wins every comparison of candidates the FFT
|
|
// returns. Where the true cell is then never offered, nothing downstream can choose it.
|
|
// Offering it costs seven frame counts.
|
|
//
|
|
// The frame count only nominates: a coarser lattice asks less accuracy of every spot, so a
|
|
// sub-lattice of the true cell can index as many frames as the cell itself, or a few more
|
|
// where the cell's own score is depressed. What decides is the comparison this pass already
|
|
// makes between a cell and its axis harmonic: the share of the validation frames' pooled
|
|
// spots each puts on itself over and above what a wrong spindle angle puts there, each
|
|
// against its own null (ValidationEvidencePrefers). A real superstructure keeps its cell on
|
|
// that count, because the class it adds holds reflections; a cell built out of two crystals
|
|
// loses it, because it has to place every spot of the one it describes twice as accurately
|
|
// with an orientation fitted to both. The nominee must also index a majority of the frames,
|
|
// the bar the false-promotion test above asks of a cell that "works", and no fewer frames
|
|
// than the standing cell: a standing cell that indexes almost nothing is not this crystal's
|
|
// lattice, its sub-lattices are not hypotheses about it, and the rescues below are what
|
|
// such a run needs.
|
|
auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass {
|
|
if (!rx.cancelled_ && best.result.has_value() && best.score > 0) {
|
|
const CrystalLattice prim = best.result->lattice
|
|
.ToPrimitive(best.result->search_result.centering);
|
|
const Coord A = prim.Vec0(), B = prim.Vec1(), C = prim.Vec2();
|
|
std::optional<FirstPass> sub_best;
|
|
std::string counts;
|
|
for (int u = 0; u < 2; u++)
|
|
for (int v = 0; v < 2; v++)
|
|
for (int w = 0; w < 2; w++) {
|
|
if (u + v + w == 0 || rx.cancelled_)
|
|
continue;
|
|
const Coord t = (A * static_cast<float>(u) + B * static_cast<float>(v)
|
|
+ C * static_cast<float>(w)) / 2.0f;
|
|
const CrystalLattice sub = (w ? CrystalLattice(A, B, t)
|
|
: v ? CrystalLattice(A, t, C)
|
|
: CrystalLattice(t, B, C)).NiggliReduce();
|
|
RotationIndexerResult cand = *best.result;
|
|
cand.lattice = sub;
|
|
cand.search_result = LatticeSearchResult{};
|
|
cand.search_result.primitive_reduced = sub;
|
|
cand.search_result.conventional = sub;
|
|
cand.unconstrained.reset();
|
|
cand.extra_lattices.clear();
|
|
const int score = CountIndexed(*indexer, cand);
|
|
counts += fmt::format(" ({},{},{}) {}", u, v, w, score);
|
|
if (!sub_best || score > sub_best->score)
|
|
sub_best = FirstPass{std::move(cand), score,
|
|
fmt::format("{} (sub-lattice {}{}{})", best.name, u, v, w),
|
|
std::abs(sub.CalcVolume())};
|
|
}
|
|
bool adopt = false;
|
|
std::string verdict = " - none indexes a majority of the frames and as many as the standing cell";
|
|
if (sub_best && sub_best->score >= best.score
|
|
&& sub_best->score > static_cast<int>(validation.size()) / 2) {
|
|
const PooledEvidence at_standing = MeasurePooled(*indexer, *best.result);
|
|
const PooledEvidence at_sub = MeasurePooled(*indexer, *sub_best->result);
|
|
adopt = ValidationEvidencePrefers(at_standing, at_sub);
|
|
verdict = fmt::format(" - the best puts {}/{} pooled spots on itself ({} by chance) against "
|
|
"the standing cell's {}/{} ({}), {}",
|
|
at_sub.on_lattice, at_sub.spots, at_sub.by_chance,
|
|
at_standing.on_lattice, at_standing.spots, at_standing.by_chance,
|
|
adopt ? "adopting it" : "keeping the standing cell");
|
|
}
|
|
logger.Info("Sub-lattice hypothesis: the standing cell ({:.0f} A^3) indexes {}/{} validation "
|
|
"frames; its index-2 sub-lattices (half-translation u,v,w):{}{}",
|
|
best.vol, best.score, static_cast<int>(validation.size()), counts, verdict);
|
|
if (adopt)
|
|
best = std::move(*sub_best);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// A lattice that indexes almost none of the validation frames is not this crystal's lattice.
|
|
// Forcing it anyway integrates every image against it; the few partials that survive yield no
|
|
// fulls in the 3D combine, and the run then dies inside the merge complaining about resolution -
|
|
// far from the cause. Stop here instead, while the reason is still visible. The floor is one
|
|
// frame in six, several times below the weakest crystal that still merges. It is checked AFTER
|
|
// the long-axis rescue, so a metric that rescue recovers is never rejected on its pre-rescue
|
|
// score.
|
|
// The second (refined-geometry) pass re-indexes DE NOVO, and the post-refined geometry - fitted
|
|
// to the lattice pass 1 already found - can tip the blind FFT, or the long-axis rescue above,
|
|
// onto another lattice: an axis harmonic of a long cell (measured: a tripled ~100 A c-axis at
|
|
// 7/60), or a related centred cell that indexes fewer frames than pass 1's lattice does at
|
|
// this very geometry (measured: a C-centred 303 A cell at 41/60 against pass 1's triclinic
|
|
// cell at 60/60). Pass 1's lattice is therefore carried here as a hypothesis of its own and
|
|
// the two are judged on the same validation frames: where the re-index found a different
|
|
// lattice and indexes fewer frames than pass 1's, or too few to be this crystal's lattice at
|
|
// all, pass 1's whole result is integrated instead. The same lattice found again is kept as
|
|
// the re-index refined it at this geometry. This is the floor-side companion of the
|
|
// supercell-collapse guard in RunAllPasses, which only sees a pass-2 lattice that is LARGER.
|
|
auto Rugnux::FirstPassRun::Pass1Fallback(FirstPass best) -> FirstPass {
|
|
RotationIndexerResult pass1_lattice = *rx.prepass_result_;
|
|
// Pass 1's result carries pass 1's goniometer; forcing it whole would put the
|
|
// uncorrected angles back (the same repair as the supercell re-run in RunAllPasses).
|
|
if (rx.prepass_rotation_scale_ && pass1_lattice.axis)
|
|
pass1_lattice.axis = ScaleRotation(*pass1_lattice.axis, *rx.prepass_rotation_scale_);
|
|
// Pass 1's lattice was FITTED at pass 1's detector distance, and this pass
|
|
// integrates at the post-refined one. Re-scoring it here is not the same as
|
|
// re-fitting it: a real-space cell is measured against the distance the spots were
|
|
// seen at, so carrying it across a distance change scales the whole cell by the
|
|
// ratio of the two. Measured on a 545 A axis over 225 -> 226 mm: 0.4 %, exactly
|
|
// 225/226, shipped as the run's answer. Scale it to the distance it will be used
|
|
// at; the orientation is unaffected.
|
|
const float d_fit = rx.prepass_result_->geom.GetDetectorDistance_mm();
|
|
const float d_use = rx.experiment_.GetDetectorDistance_mm();
|
|
const bool rescaled = d_fit > 0 && d_use > 0 && d_fit != d_use;
|
|
if (rescaled) {
|
|
const float f = d_use / d_fit;
|
|
const auto &l = pass1_lattice.lattice;
|
|
pass1_lattice.lattice = CrystalLattice(l.Vec0() * f, l.Vec1() * f, l.Vec2() * f);
|
|
}
|
|
const CrystalLattice pass1_primitive = pass1_lattice.lattice
|
|
.ToPrimitive(pass1_lattice.search_result.centering).NiggliReduce();
|
|
const double pass1_vol = std::abs(pass1_primitive.CalcVolume());
|
|
// The same lattice whatever setting or class it is held in - a centred cell and the
|
|
// triclinic primitive of the same points are one lattice, and the re-index's symmetric
|
|
// setting must not lose to its own unconstrained primitive on a frame count: the Niggli-
|
|
// reduced primitive edges agree within 2 % (the battery's lattice-identity test).
|
|
const auto reduced_edges = [](const CrystalLattice &l) {
|
|
const auto uc = l.GetUnitCell();
|
|
std::array<double, 3> e = {uc.a, uc.b, uc.c};
|
|
std::sort(e.begin(), e.end());
|
|
return e;
|
|
};
|
|
const auto e1 = reduced_edges(pass1_primitive);
|
|
const auto e2 = reduced_edges(best.result->lattice
|
|
.ToPrimitive(best.result->search_result.centering).NiggliReduce());
|
|
bool same_lattice = true;
|
|
for (int i = 0; i < 3; i++)
|
|
same_lattice = same_lattice && std::abs(e2[i] - e1[i]) <= 0.02 * e1[i];
|
|
const int floor = static_cast<int>(validation.size()) / 6;
|
|
const int pass1_score = (best.score < floor || !same_lattice)
|
|
? CountIndexed(*indexer, pass1_lattice) : -1;
|
|
if (best.score < floor || pass1_score > best.score) {
|
|
const auto &pc = pass1_lattice.search_result.conventional.GetUnitCell();
|
|
const auto &bc = best.result->search_result.conventional.GetUnitCell();
|
|
logger.Warning("Two-pass: the refined-geometry re-index indexes {}/{} validation frames "
|
|
"({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}), pass-1's lattice "
|
|
"{}/{} at this geometry ({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} "
|
|
"{:.2f}) - integrating with pass-1's lattice",
|
|
best.score, static_cast<int>(validation.size()),
|
|
best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
bc.a, bc.b, bc.c, bc.alpha, bc.beta, bc.gamma,
|
|
pass1_score, static_cast<int>(validation.size()),
|
|
pass1_lattice.search_result.centering,
|
|
gemmi::crystal_system_str(pass1_lattice.search_result.system),
|
|
pc.a, pc.b, pc.c, pc.alpha, pc.beta, pc.gamma);
|
|
if (rescaled)
|
|
logger.Info("Two-pass: pass-1's cell was fitted at {:.3f} mm and is used at "
|
|
"{:.3f} mm - scaled by {:.6f} to the distance it is integrated at",
|
|
d_fit, d_use, d_use / d_fit);
|
|
best.result = std::move(pass1_lattice);
|
|
best.score = pass1_score;
|
|
best.vol = pass1_vol;
|
|
best.name = "pass-1 lattice (refined-geometry re-index found less)";
|
|
}
|
|
return best;
|
|
}
|