leonarski_fandClaude Opus 5.5
3befddd649
Several pipeline copies of one run build FFT indexer pools on the card at once - the rotation-scale walk's indexing probes start as the output pass begins its image loop, and the speculative geometry probe runs beside the merge - and each indexer is a cuFFT plan plus 0.6-0.9 GB of buffers. Construction had no admission control: a cufftPlanMany or device allocation that found the card full failed straight through, and since IndexerResult carries the exception that ends the run. With 7.2 GB of the card held by another process, a small-molecule rotation set failed 4 of 4 runs this way although each pass fits on its own. Construction now does what the merge's allocations do: hand back the pool's unused memory, wait for the work beside it to end, and build again. Who waits for whom is fixed so that nothing waits in a circle: - the main line waits for the probes (GPUWorkBeside, as the merge already does); - a probe waits for the main line's image pass (new GPUImagePass), never for a probe, itself included. The image pass waits for no probe, so it ends. The walk's indexing probes now hold a GPUWorkBeside too (they are exactly that work), and a run carries gpu_work_beside_ so its pools know which side they are on. Only when an indexer is built changes, never what it computes; with nothing to wait for it still fails at once. Same run with the 7.2 GB hog: 5/5 complete (also at 7.6 and 8.0 GB), p.hkl identical to the unhogged run; a card that cannot hold the run still fails in 0.5 s. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3
1115 lines
70 KiB
C++
1115 lines
70 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <cstdlib>
|
|
#include "Rugnux.h"
|
|
#include "../image_analysis/structure_refinement/ModelValidation.h"
|
|
#include "../writer/WriteModel.h"
|
|
#include "../image_analysis/indexing/SpindleCuspLoss.h"
|
|
#include "../image_analysis/bragg_integration/SpotWidth.h"
|
|
#include "../image_analysis/bragg_integration/SpotFootprint.h"
|
|
#include "../image_analysis/bragg_integration/BraggStencil.h" // BRAGG_FOOTPRINT_NSIGMA
|
|
#include "../image_analysis/hot_pixels/HotPixels.h"
|
|
#include "../image_analysis/SensorAbsorption.h"
|
|
#include "DiagnosticOutput.h"
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <condition_variable>
|
|
#include <deque>
|
|
#include <fstream>
|
|
#include <cstring>
|
|
#include <functional>
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <numeric>
|
|
#include <map>
|
|
#include <set>
|
|
#include <sstream>
|
|
#include <thread>
|
|
|
|
#include "../reader/JFJochHDF5Reader.h"
|
|
#include "../common/JFJochMath.h"
|
|
#include "../common/ParallelFor.h"
|
|
#include "../common/Logger.h"
|
|
#include "../common/AzimuthalIntegrationMapping.h"
|
|
#include "../common/AzimuthalIntegrationProfile.h"
|
|
#include "../common/CUDAWrapper.h"
|
|
#include "../common/JFJochException.h"
|
|
#include "../common/time_utc.h"
|
|
#include "../writer/FileWriter.h"
|
|
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
|
#include "../image_analysis/beam_stop/ShadowFinder.h"
|
|
#include "../image_analysis/IndexAndRefine.h"
|
|
#include "../image_analysis/indexing/AnalyzeIndexing.h"
|
|
#include "../image_analysis/indexing/HarmonicContamination.h"
|
|
#include "../image_analysis/geom_refinement/BeamCenterFFT.h"
|
|
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
|
|
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
|
|
#include "../image_analysis/indexing/IndexerThreadPool.h"
|
|
#include "../image_analysis/spot_finding/ImageSpotFinderCPU.h"
|
|
#include "../image_analysis/spot_finding/SpotUtils.h"
|
|
#include "gemmi/twin.hpp"
|
|
#include "../image_analysis/azint/AzIntEngineCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
|
|
#ifdef JFJOCH_USE_CUDA
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h"
|
|
#include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h"
|
|
#endif
|
|
#include "../image_analysis/scale_merge/Merge.h"
|
|
#include "../image_analysis/scale_merge/RfreeFlags.h"
|
|
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
|
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
|
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
|
|
#include "../image_analysis/scale_merge/ScalingResult.h"
|
|
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
|
|
#include "../image_analysis/geom_refinement/PostRefine.h"
|
|
#include "../image_analysis/lattice_search/LatticeSearch.h"
|
|
#include "../image_analysis/lattice_search/LePageLattice.h"
|
|
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
|
|
#include "../image_analysis/scale_merge/FrenchWilson.h"
|
|
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
|
|
#include "../image_analysis/scale_merge/TranslationalNCS.h"
|
|
#include "../image_analysis/scale_merge/HKLKey.h"
|
|
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
|
|
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
|
|
#include "../image_analysis/WriteReflections.h"
|
|
#include "../image_analysis/bragg_integration/CalcISigma.h"
|
|
#include "../common/Definitions.h"
|
|
#include "../common/CorrelationCoefficient.h"
|
|
#include <array>
|
|
#include <map>
|
|
#include "RugnuxPipeline.h"
|
|
#include "RugnuxFirstPass.h"
|
|
|
|
// The steps of the first pass that may replace the standing lattice: the rescues, asked where it
|
|
// indexes too little, and the hypotheses, asked of every run. FirstPassRotationIndexing gives their
|
|
// order.
|
|
|
|
using namespace rugnux_internal;
|
|
|
|
namespace {
|
|
auto MaxAxis(const RotationIndexerResult &r) {
|
|
const auto uc = r.lattice.GetUnitCell();
|
|
return std::max({uc.a, uc.b, uc.c});
|
|
}
|
|
}
|
|
|
|
// Rotation-axis SIGN rescue. The sign is a convention the input cannot settle: a PILATUS
|
|
// miniCBF header carries only "Oscillation_axis omega" and no direction at all, and facilities
|
|
// disagree about the sign of the NXmx vector (measured across ten of them: most want the
|
|
// stored [-1,0,0], APS 19-ID and SPring-8 BL41XU want the opposite). Nothing downstream can
|
|
// repair it either - the default spindle fit refines the axis DIRECTION by fractions of a
|
|
// degree and cannot flip it, that being a discrete change.
|
|
//
|
|
// It is, however, decidable from the data: the wrong sign does not index at all. Measured on
|
|
// one dataset by changing nothing but the sign: 100% of frames indexed one way, 0 of 60 the
|
|
// other. So try the other sign and keep whichever indexes more frames - the same
|
|
// validation-frame count the scheme choice above already uses, so no new metric and no new
|
|
// threshold. Runs before the long-axis rescue: with the sign wrong every candidate lattice is
|
|
// wrong, and the coarse pass below would spend itself on a mis-signed cloud.
|
|
//
|
|
// Only after a poor pass, so a correctly-signed file costs nothing; and spot finding is not
|
|
auto Rugnux::FirstPassRun::AxisSignRescue(FirstPass best) -> FirstPass {
|
|
NearTieGate("axis-sign rescue gate", "standing pass", best.score, 0.5 * static_cast<double>(validation.size()));
|
|
if (!rx.cancelled_ && best.score < 0.5 * static_cast<double>(validation.size())) {
|
|
if (const auto gon = rx.experiment_.GetGoniometer(); gon.has_value() && gon->IsScanning()) {
|
|
GoniometerAxis flipped = *gon;
|
|
flipped.Axis(-gon->GetAxis());
|
|
rx.experiment_.Goniometer(flipped);
|
|
FirstPass alt = PickBest(*indexer_pool, *indexer);
|
|
if (alt.result.has_value())
|
|
NearTieFrames("axis sign", "opposite sign", alt.score, "file's sign", best.score);
|
|
// More frames is not enough on its own while both counts sit at the floor: 1/60 against
|
|
// 0/60 is noise, and adopting it puts every rescue below on the wrong sign (measured: a
|
|
// run whose file sign, kept, went on to index found nothing once flipped on 1/60). So
|
|
// the flipped lattice must also beat chance on the pooled spots - the test the run puts
|
|
// its final lattice to - which a sparse pattern at the right sign passes on few frames
|
|
// (2/60 frames, 89% of the validation spots against 0% by chance).
|
|
// A file's sign that found no lattice counts as 0 frames: no answer is not a count the
|
|
// opposite sign can beat with another 0.
|
|
const bool more_frames = alt.result.has_value() && alt.score > std::max(best.score, 0);
|
|
const bool alt_real = more_frames && ValidationEvidenceBeatsChance(MeasurePooled(*indexer, *alt.result));
|
|
if (alt_real) {
|
|
logger.Info("Rotation axis sign: validation frames indexed at the file's axis {}, at the "
|
|
"opposite sign {} - adopting the opposite sign for this run",
|
|
Frames(best.score), Frames(alt.score));
|
|
best = alt;
|
|
} else {
|
|
if (more_frames)
|
|
logger.Info("Rotation axis sign: validation frames indexed at the opposite sign {}, at "
|
|
"the file's axis {}, but its lattice does not beat chance on the pooled "
|
|
"spots - the file's sign is kept",
|
|
Frames(alt.score), Frames(best.score));
|
|
rx.experiment_.Goniometer(*gon); // the file was right; put it back
|
|
}
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Long-axis rescue. When the de-novo cell indexes few validation frames, a long, finely-spaced
|
|
// axis was likely lost: the unconstrained FFT either collapsed it to a short sub-multiple or let
|
|
// a denser supercell over-fit the accumulated cloud (a small global-orientation error throws the
|
|
// many high-order reflections off along the fine axis, so the true cell scores worst on the raw
|
|
// cloud). Recover the true metric with a second search whose FFT unit-cell bound is widened past
|
|
// the default, then RE-INDEX at full resolution constrained by the cell that comes back, as a
|
|
// reference (the -C path): the reference filter drops the collapsed/supercell candidates and
|
|
// refines an accurate global lattice. Only runs after a poor standard pass, so well-indexing
|
|
// crystals never pay for it and their result is untouched. Returns the re-index where it indexes
|
|
// more validation frames than the `standing` pass it was asked about.
|
|
auto Rugnux::FirstPassRun::LongAxisRescue(const FirstPass &standing) -> std::optional<FirstPass> {
|
|
if (rx.cancelled_ || !standing.result.has_value()
|
|
|| standing.score >= 0.5 * static_cast<double>(validation.size()))
|
|
return std::nullopt;
|
|
auto coarse_settings = rx.experiment_.GetIndexingSettings();
|
|
// The resolution setting only sizes the FFT histogram - it does not filter which spots go
|
|
// into it - so this is here to pay for the widened bound below, not to make the long axis
|
|
// easier to pick out. Measured, coarsening does the opposite: the acceptance angle widens
|
|
// (0.159 -> 0.579 deg) but the true axis falls from rank 1996 to 16361 of 16384 directions.
|
|
coarse_settings.FFT_HighResolution_A(3.5f);
|
|
// The widened bound is what the rescue turns on, and it costs nothing: the histogram length is
|
|
// 4*pi*max_length/high_resolution, so coarsening the resolution to 3.5 A has already
|
|
// SHRUNK the transform by 1.75x - spending that back on reach leaves the coarse pass at
|
|
// 1.14x the standard one. Without it the rescue cannot recover what it exists to recover:
|
|
// a cell whose long axis is past the default 500 A bound is unrepresentable, so the
|
|
// "recovered" metric would still be a sub-multiple. Only reached after a poor pass, so a
|
|
// well-indexing crystal never pays. Stills never reach here.
|
|
constexpr float RESCUE_MAX_UNIT_CELL_A = 1000.0f;
|
|
if (coarse_settings.GetFFT_MaxUnitCell_A() < RESCUE_MAX_UNIT_CELL_A)
|
|
coarse_settings.FFT_MaxUnitCell_A(RESCUE_MAX_UNIT_CELL_A);
|
|
coarse_settings.IndexingThreads(2);
|
|
coarse_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS));
|
|
IndexerThreadPool coarse_pool(coarse_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_);
|
|
// Coarse first pass: keep the recovered cell with the LONGEST axis directly. Its full-
|
|
// resolution per-frame validation would be low (the coarse cell is metrically right but
|
|
// imprecise), so do NOT score it here - it is only the reference for the constrained re-index.
|
|
const SchemeIndexers cris = FeedSchemes(coarse_pool);
|
|
RunIndexing(cris);
|
|
std::optional<RotationIndexerResult> coarse_ref;
|
|
for (auto &ri : cris) {
|
|
auto r = ri->GetLattice();
|
|
if (r.has_value() && (!coarse_ref.has_value() || MaxAxis(*r) > MaxAxis(*coarse_ref)))
|
|
coarse_ref = r;
|
|
}
|
|
// A recovered axis at least 30% longer than the standard pass found signals a rescued metric.
|
|
if (!rx.cancelled_ && coarse_ref.has_value()
|
|
&& MaxAxis(*coarse_ref) > 1.3 * MaxAxis(*standing.result)) {
|
|
logger.Info("Long-axis rescue: coarse pass recovered a {:.0f} A axis (was {:.0f} A); "
|
|
"re-indexing constrained by that cell", MaxAxis(*coarse_ref), MaxAxis(*standing.result));
|
|
rx.experiment_.SetUnitCell(coarse_ref->lattice.GetUnitCell());
|
|
// The re-index must be able to REPRESENT the cell the coarse pass just recovered. The
|
|
// standard pool was sized at the default bound, so re-using it here would search for a
|
|
// long axis it cannot hold and hand back the sub-multiple the rescue exists to replace.
|
|
// CountIndexed forces the lattice and never touches the pool, so the existing
|
|
// IndexAndRefine is reused; only the FFT needs the wider one.
|
|
auto wide_settings = rx.experiment_.GetIndexingSettings();
|
|
wide_settings.IndexingThreads(2);
|
|
wide_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS));
|
|
RaiseFFTBoundForKnownCell(wide_settings, rx.experiment_);
|
|
FirstPass constrained;
|
|
if (wide_settings.GetFFT_MaxUnitCell_A() > indexing_bound_A) {
|
|
IndexerThreadPool wide_pool(wide_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_);
|
|
constrained = PickBest(wide_pool, *indexer);
|
|
} else {
|
|
constrained = PickBest(*indexer_pool, *indexer);
|
|
}
|
|
rx.experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo
|
|
if (!constrained.result.has_value())
|
|
return std::nullopt;
|
|
// A standing pass that found no lattice counts as 0 frames.
|
|
bool more = constrained.score > std::max(standing.score, 0);
|
|
if (standing.result.has_value() && !SameLattice(*constrained.result, *standing.result)
|
|
&& NearTieFrames("long-axis rescue", "re-index", constrained.score, "standing", standing.score)
|
|
&& DecideLate()) {
|
|
const int late = CountIndexed(*indexer, *constrained.result, late_validation);
|
|
const int standing_late = CountIndexed(*indexer, *standing.result, late_validation);
|
|
const bool early = more;
|
|
more = constrained.score + late > standing.score + standing_late;
|
|
LogDecidedLate("long-axis rescue", fmt::format("re-index {}/{} of the later frames, standing {}/{}", late,
|
|
late_validation.size(), standing_late,
|
|
late_validation.size()),
|
|
early, more);
|
|
}
|
|
if (more)
|
|
return constrained;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
|
|
// The background centre, tried as a SECOND FIRST PASS.
|
|
//
|
|
// The estimate itself is free - it is read off the projection --detect-beam-stop has already
|
|
// built - and a first pass re-uses the spots it has already found, so indexing the run a
|
|
// second time at the measured centre costs about one more first pass.
|
|
// That buys the comparison an on-failure search structurally cannot make: a centre wrong
|
|
// ALONG the spindle does not fail. It keeps 96-100 % of frames indexed and quietly returns a
|
|
// 2x, 3x or sqrt(3) axis harmonic, so no failure trigger ever fires on it. Indexing both
|
|
// centres and comparing the two ANSWERS is what can see it.
|
|
//
|
|
// What must NOT arbitrate the two is the frame count. Acceptance is a fractional-Miller test,
|
|
// so a cell twice as long has to place every spot twice as accurately to score the same:
|
|
// measured on real data, two centres 0.12 px apart gave the deposited cell at 99.23 % and a
|
|
// halved axis at 100.00 %, and the WRONG answer indexed better. A rule of the form "take the
|
|
// centre that indexes more" would therefore pick wrong in exactly the case this comparison
|
|
// exists for. So only what needs no arbiter is decided here:
|
|
//
|
|
// * the file's centre does not index a majority -> nothing defends it, so try the measured
|
|
// one (and the axis sign with it, see below). A run that indexes nothing has nothing to
|
|
// lose; where the measured one is no better, the file's centre is put back;
|
|
// * the file's centre indexes and the two agree on the lattice -> keep the file's centre and
|
|
// say they agree. This is the overwhelming majority, and saying it is worth the pass: it
|
|
// is a free statement that the header is good enough for this crystal;
|
|
// * they index and disagree -> say so, with both cells and their primitive-volume ratio.
|
|
// Which of an integer-related pair is real is not decided on the frame counts, for the
|
|
// reason above; it is decided, in ONE direction only, on the pooled spots - see the
|
|
// branch itself. Naming the disagreement is useful even where nothing is decided.
|
|
//
|
|
// Where the file's centre indexes nothing and the measured one does not either, the axis-sign
|
|
// question is asked AGAIN at the measured centre before giving up. The two unknowns are
|
|
// discrete and coupled, and the rescue above asked its question at a centre that may be 73 px
|
|
// wrong: on the two depositions here whose header is the geometric centre of the detector,
|
|
// each error hides the other, both runs sit at 0/60 until both are moved, and moving both
|
|
// takes them to 60/60 on the deposited lattice. Asking the pair costs one more first pass on
|
|
// a run that has already failed twice.
|
|
auto Rugnux::FirstPassRun::BeamCentreCheck(FirstPass best, bool &long_axis_asked) -> FirstPass {
|
|
rx.JoinBeamCenterCapture();
|
|
if (rx.cancelled_ || !rx.config_.beam_center_check || !rx.background_center_)
|
|
return best;
|
|
// Consumed, not re-read: a two-pass rotation run comes back through here with the geometry
|
|
// the first pass post-refined, and a pre-scan estimate of the FILE's centre says nothing
|
|
// about that one. The pre-scan does not re-measure it either - by then the beam-stop shadow
|
|
// is already in the mask, so the projection is not rebuilt.
|
|
const BeamCenterEstimate measured = *rx.background_center_;
|
|
rx.background_center_.reset();
|
|
const float header_x = rx.experiment_.GetBeamX_pxl(), header_y = rx.experiment_.GetBeamY_pxl();
|
|
const float measured_x = measured.beam_x_pxl;
|
|
const float measured_y = measured.beam_y_pxl;
|
|
const float moved = std::hypot(measured_x - header_x, measured_y - header_y);
|
|
const Centres c{measured, header_x, header_y, measured_x, measured_y, moved};
|
|
// No gate on how significant the move is. It is tempting to skip the second pass when the
|
|
// two centres are within the fit's own noise - it would save it on a quarter of the runs
|
|
// here - but that is deciding at a threshold before the evidence is in, and the evidence
|
|
// is cheap. It would also gate away exactly the case worth asking about: measured on real
|
|
// data, a 0.12 px change of centre is the whole difference between the deposited cell and
|
|
// a halved axis, and 0.12 px is well inside the sigma of any fit that could gate it.
|
|
const int majority = static_cast<int>(validation.size()) / 2;
|
|
TryBeamCenter(measured_x, measured_y);
|
|
FirstPass alt = IndexAtMeasuredCentre();
|
|
const bool header_indexes = best.result.has_value() && best.score > majority;
|
|
const bool measured_indexes = alt.result.has_value() && alt.score > majority;
|
|
NearTieGate("beam-centre majority gate", "file centre", best.score, majority);
|
|
NearTieGate("beam-centre majority gate", "measured centre", alt.score, majority);
|
|
if (!header_indexes)
|
|
return CentreWhenFileFails(std::move(best), std::move(alt), c, long_axis_asked);
|
|
RestoreBeamCenter(header_x, header_y);
|
|
if (measured_indexes)
|
|
return CentreWhenBothIndex(std::move(best), alt, c);
|
|
logger.Info("Beam centre check: the measured centre ({:.2f},{:.2f}) indexes "
|
|
"{} validation frames against the file's {} - the file's "
|
|
"centre stands", measured_x, measured_y, Frames(alt.score), Frames(best.score));
|
|
return best;
|
|
}
|
|
|
|
// The first pass at the measured centre: the speculative one started beside the file centre's
|
|
// (StartCentreSpeculation) where it read what the run has here, otherwise run here.
|
|
auto Rugnux::FirstPassRun::IndexAtMeasuredCentre() -> FirstPass {
|
|
FirstPass alt;
|
|
if (centre_spec)
|
|
centre_spec->done.get();
|
|
if (centre_spec && centre_spec->experiment_key == rx.ExperimentKey()
|
|
&& centre_spec->key == rx.SpotFindingKey(*spot_mapping)
|
|
&& centre_spec->spots_per_image == first_pass_spots_per_image) {
|
|
// What PickBest would have found and fed here: hand the spots to the run's cache and
|
|
// its memo as PrefetchSpots does, and score the schemes on the run's own indexer.
|
|
spot_cache = centre_spec->spots;
|
|
{
|
|
std::lock_guard lock(rx.first_pass_spots_->m);
|
|
auto &entries = rx.first_pass_spots_->entries;
|
|
if (std::none_of(entries.begin(), entries.end(),
|
|
[&](const auto &e) { return e.first == centre_spec->key; })) {
|
|
if (entries.size() == MAX_SPOT_KEYS)
|
|
entries.erase(entries.begin());
|
|
entries.emplace_back(centre_spec->key, centre_spec->spots);
|
|
}
|
|
}
|
|
alt = ScoreSchemes(centre_spec->ris, *indexer);
|
|
} else
|
|
alt = PickBest(*indexer_pool, *indexer);
|
|
centre_spec.reset();
|
|
return alt;
|
|
}
|
|
|
|
// The file's centre does not index a majority.
|
|
auto Rugnux::FirstPassRun::CentreWhenFileFails(FirstPass best, FirstPass alt, const Centres &c,
|
|
bool &long_axis_asked) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
const int majority = static_cast<int>(validation.size()) / 2;
|
|
bool measured_indexes = alt.result.has_value() && alt.score > majority;
|
|
// Nothing defends the file's centre: the data do not index at it, and the
|
|
// background says it is `moved` px away. So try the measured one, and if that
|
|
// does not index either, ask the AXIS-SIGN question again there. The two unknowns
|
|
// are discrete and coupled, and the sign rescue above asked its question at a
|
|
// centre that may itself be wrong - each error hides the other, and only moving
|
|
// both reveals anything.
|
|
bool joint = false;
|
|
const auto gonio = rx.experiment_.GetGoniometer();
|
|
if (!measured_indexes && gonio.has_value() && gonio->IsScanning()) {
|
|
GoniometerAxis flipped = *gonio;
|
|
flipped.Axis(-gonio->GetAxis());
|
|
rx.experiment_.Goniometer(flipped);
|
|
const FirstPass both = PickBest(*indexer_pool, *indexer);
|
|
if (both.result.has_value() && both.score > majority) {
|
|
logger.Warning("Beam centre from the background: the file's centre "
|
|
"({:.2f},{:.2f}) indexes {} validation frames and the "
|
|
"measured ({:.2f},{:.2f}) {}, but the measured centre "
|
|
"WITH the opposite rotation-axis sign indexes {}. The "
|
|
"file has both wrong, and neither shows while the other "
|
|
"stands; the run takes the measured centre and the opposite "
|
|
"sign", header_x, header_y, Frames(best.score), measured_x, measured_y,
|
|
Frames(alt.score), Frames(both.score));
|
|
alt = both;
|
|
measured_indexes = true;
|
|
joint = true;
|
|
} else {
|
|
rx.experiment_.Goniometer(*gonio);
|
|
}
|
|
}
|
|
// Third coupled unknown: how DEEP the pass reads. A centre 170 px out and a spot
|
|
// list too deep to index at ANY centre hide each other exactly as the centre and
|
|
// the axis sign do - the measured centre is right, and it is thrown away because
|
|
// the full list indexes nothing there either. Measured on one such crystal: the
|
|
// file's centre indexed 1/60 and the measured one 0/60, so the file's was kept and
|
|
// the run died; with the ladder's lean seed the measured centre indexes 60/60, and
|
|
// the run then indexes every frame of the sweep and merges to the deposited
|
|
// resolution. So ask the ladder here too, before giving the centre up.
|
|
//
|
|
// The depth is asked at BOTH centres, and the centre moves only if it is the
|
|
// centre that paid. A lean rung at the measured centre scored against the whole
|
|
// spot list at the file's changes two things at once, and the count cannot say
|
|
// which of them the gain came from - the same reason a trial centre is scored on
|
|
// its own spots at all. Measured on one crystal: the measured centre's ladder
|
|
// reached 40/60 on a FIVE-fold axis harmonic while the file's own centre reached
|
|
// 49/60 on the deposited cell, and the run moved the centre, shipped the
|
|
// supercell, and skipped the beam-centre walk that would have found the crystal's
|
|
// own. An exact tie keeps the file's centre: nothing then says the centre is wrong.
|
|
bool lean = false;
|
|
bool deeper_at_home = false;
|
|
if (!measured_indexes) {
|
|
const FirstPassState before = committed;
|
|
RestoreBeamCenter(header_x, header_y);
|
|
const auto home = FirstPassLadder(best.score, 1, majority + 1);
|
|
const FirstPassState home_state = committed;
|
|
committed = before;
|
|
ApplyState(committed);
|
|
TryBeamCenter(measured_x, measured_y);
|
|
const auto away = FirstPassLadder(alt.score, 1, majority + 1);
|
|
const FirstPassState away_state = committed;
|
|
// Both centres index a majority, and on the same lattice. Then the two counts
|
|
// are two readings of one hypothesis at two geometries, and which is higher is
|
|
// the count's noise: measured on one crystal, the same two ladders gave the
|
|
// measured centre 56/60 against the file's 50/60, and 48/60 against 50/60 once
|
|
// a few thousand defective pixels were masked - and the run that then kept the
|
|
// file's centre, 12.6 px out, merged with twice the frames rejected. So the
|
|
// count decides nothing here: the file's centre stays for this pass, and the
|
|
// measured one is carried forward to be judged on the merges (RunAllPasses).
|
|
const bool same_lattice =
|
|
home && away
|
|
&& home->result->search_result.system == away->result->search_result.system
|
|
&& home->result->search_result.centering
|
|
== away->result->search_result.centering
|
|
&& std::max(home->vol, away->vol) < 1.02 * std::min(home->vol, away->vol);
|
|
if (home && away && !same_lattice)
|
|
NearTieFrames("beam centre (lean ladder)", "file centre", home->score, "measured centre", away->score);
|
|
if (home && (!away || same_lattice || home->score >= away->score)) {
|
|
committed = home_state;
|
|
RestoreBeamCenter(header_x, header_y);
|
|
ApplyState(committed);
|
|
best = *home;
|
|
deeper_at_home = true;
|
|
if (same_lattice && geometry_prepass) {
|
|
rx.beam_center_alternative_ = std::array<float, 2>{measured_x, measured_y};
|
|
rx.beam_center_alternative_state_ = away_state;
|
|
logger.Warning("Beam centre check: reading less of each frame, the "
|
|
"file's centre indexes {} validation frames and the "
|
|
"measured centre {:.2f} px away {}, on the same "
|
|
"lattice - the counts cannot say which centre is right, "
|
|
"so the run carries BOTH centres forward and adopts "
|
|
"whichever measures more of the crystal",
|
|
Frames(home->score), moved,
|
|
Frames(away->score));
|
|
} else
|
|
logger.Warning("Beam centre check: reading less of each frame takes the "
|
|
"file's centre to {} validation frames against {} at "
|
|
"the measured centre {:.2f} px away - it is how deep this "
|
|
"run was reading and not the centre, so the centre stays",
|
|
Frames(home->score),
|
|
Frames(away ? away->score : 0), moved);
|
|
} else if (away) {
|
|
alt = *away;
|
|
measured_indexes = true;
|
|
lean = true;
|
|
}
|
|
}
|
|
if (measured_indexes) {
|
|
if (lean)
|
|
logger.Warning("Beam centre from the background: neither centre indexes the "
|
|
"whole spot list, but at the measured ({:.2f},{:.2f}) the "
|
|
"lean first pass above indexes {} against the file's "
|
|
"{} - the beam centre in this file is wrong, and only a "
|
|
"pass that reads less of each frame could see it",
|
|
measured_x, measured_y, Frames(alt.score), Frames(best.score));
|
|
else if (!joint)
|
|
logger.Warning("Beam centre from the background: the file's ({:.2f},{:.2f}) "
|
|
"indexes {} validation frames and the measured "
|
|
"({:.2f},{:.2f}) indexes {} - the beam centre in this "
|
|
"file is wrong, and the run adopts the measured one",
|
|
header_x, header_y, Frames(best.score), measured_x, measured_y,
|
|
Frames(alt.score));
|
|
best = alt;
|
|
} else if (!deeper_at_home) {
|
|
best = CentreByPooledSpots(std::move(best), std::move(alt), c, long_axis_asked);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Nothing worked. Put the file's centre back, so a run that fails for some
|
|
// other reason fails at the geometry it was given rather than at a fit that has
|
|
// just been tested and found no better.
|
|
//
|
|
// Unless the frame counts are both at the floor and the POOLED spots do decide:
|
|
// a crystal whose spots are a minority of every frame indexes 0/60 at any centre,
|
|
// and then only the pooled excess over chance - each lattice measured against its
|
|
// own wrong-spindle null, so a larger cell is not credited for fitting more by
|
|
// chance - says which centre sees it.
|
|
//
|
|
// Each centre is judged on the best lattice it gives, so the long-axis rescue
|
|
// is asked at BOTH before they are compared. The lattices standing here index
|
|
// no majority anywhere, which is exactly the pass that rescue exists for, and
|
|
// the two counts of spots on two lattices that are not the crystal's are close
|
|
// to a coin toss: measured on a 545 A axis, the measured centre (3.6 px out
|
|
// across the spindle) won it on 16.7% against 15.5%, the rescue then ran
|
|
// there alone and could not see the long axis, and the run shipped a P1
|
|
// sub-cell - while at the file's centre the same rescue indexes 60/60.
|
|
auto Rugnux::FirstPassRun::CentreByPooledSpots(FirstPass best, FirstPass alt, const Centres &c,
|
|
bool &long_axis_asked) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
if (auto rescued = LongAxisRescue(alt))
|
|
alt = *rescued;
|
|
PooledEvidence at_measured, at_header;
|
|
if (alt.result.has_value())
|
|
at_measured = MeasurePooled(*indexer, *alt.result);
|
|
RestoreBeamCenter(header_x, header_y);
|
|
if (auto rescued = LongAxisRescue(best))
|
|
best = *rescued;
|
|
if (best.result.has_value())
|
|
at_header = MeasurePooled(*indexer, *best.result);
|
|
long_axis_asked = true;
|
|
if (alt.result.has_value() && best.result.has_value()
|
|
&& NearTiePooled("beam centre (pooled spots)", "file centre", at_header, "measured centre", at_measured)
|
|
&& DecideLate()) {
|
|
// Each centre on its own spots, as above.
|
|
const bool early = ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured);
|
|
const std::vector<int> all = AllValidationFrames();
|
|
at_header = MeasurePooled(*indexer, *best.result, all);
|
|
TryBeamCenter(measured_x, measured_y);
|
|
at_measured = MeasurePooled(*indexer, *alt.result, all);
|
|
RestoreBeamCenter(header_x, header_y);
|
|
LogDecidedLate("beam centre (pooled spots)", "the pooled spots counted again on all the frames at both centres",
|
|
early, ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured));
|
|
}
|
|
if (alt.result.has_value() && ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured)) {
|
|
TryBeamCenter(measured_x, measured_y);
|
|
logger.Warning("Beam centre from the background: neither centre indexes a "
|
|
"majority of the validation frames ({} at the measured, "
|
|
"{} at the file's), but the pooled spots put "
|
|
"{:.1f}% on the lattice at the measured ({:.2f},{:.2f}) against "
|
|
"{:.1f}% at the file's (chance {:.1f}% and {:.1f}%) - the run "
|
|
"adopts the measured centre",
|
|
Frames(alt.score),
|
|
Frames(best.score),
|
|
100.0 * at_measured.on_lattice / std::max<int64_t>(1, at_measured.spots),
|
|
measured_x, measured_y,
|
|
100.0 * at_header.on_lattice / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.by_chance / std::max<int64_t>(1, at_measured.spots),
|
|
100.0 * at_header.by_chance / std::max<int64_t>(1, at_header.spots));
|
|
best = alt;
|
|
} else {
|
|
logger.Info("Beam centre check: the file's centre ({} frames) and the "
|
|
"measured one {:.2f} px away ({}) - the measured centre puts no "
|
|
"more of the pooled spots on its lattice, so the file's centre is "
|
|
"kept",
|
|
Frames(best.score), moved, Frames(alt.score));
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// The file's centre indexes a majority, and so does the measured one.
|
|
auto Rugnux::FirstPassRun::CentreWhenBothIndex(FirstPass best, const FirstPass &alt, const Centres &c) -> FirstPass {
|
|
const BeamCenterEstimate &measured = c.measured;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const float moved = c.moved;
|
|
// Primitive volumes, as everywhere else here: two passes can find the same
|
|
// lattice in different settings, and a centred setting's cell is an exact
|
|
// integer multiple of its primitive one.
|
|
const double ratio = alt.vol < best.vol ? best.vol / alt.vol : alt.vol / best.vol;
|
|
const bool same_class =
|
|
best.result->search_result.system == alt.result->search_result.system
|
|
&& best.result->search_result.centering == alt.result->search_result.centering;
|
|
// The primitive volume is what the beam centre decides. A centre error along
|
|
// the spindle makes the FFT take an axis harmonic, which changes the volume by
|
|
// an integer factor or by sqrt(3); the Bravais class can differ for a reason
|
|
// that has nothing to do with the centre, so the two are reported separately.
|
|
if (ratio < 1.02 && same_class) {
|
|
rx.Note(fmt::format("beam centre kept: measured {:.1f} px off, same lattice", moved));
|
|
logger.Info("Beam centre check: indexing at the measured centre "
|
|
"({:.2f},{:.2f}) returns the same lattice as the file's - "
|
|
"{}-centred {}, cells agreeing to {:.2f} % in volume. The {:.2f} px "
|
|
"difference does not decide this crystal's cell",
|
|
measured_x, measured_y, alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system),
|
|
100.0 * (ratio - 1.0), moved);
|
|
// Unless the two are further apart than this geometry absorbs. A centre
|
|
// wrong by about one reflection spacing indexes the same cell just as
|
|
// well, with every reflection one index out over part of the sweep, and
|
|
// only the merge shows it - so the measured centre is kept as a hypothesis
|
|
// for RunAllPasses, which tries it if the file's centre merges
|
|
// inconsistently.
|
|
if (geometry_prepass && moved > 3.0f * measured.sigma_pxl
|
|
&& moved > BeamCenterNeed_pxl(rx.experiment_))
|
|
rx.beam_center_same_lattice_alternative_ =
|
|
std::array<float, 2>{measured_x, measured_y};
|
|
} else if (ratio < 1.02) {
|
|
// Same cell, different metric symmetry read off it. The cell is not in
|
|
// question, but the CLASS is not cosmetic either: GeometryRefiner holds
|
|
// the cell to it for the rest of the run, so a class that is one axis
|
|
// equality away from the truth is refined, predicted, integrated and
|
|
// merged with that equality imposed.
|
|
//
|
|
// Whether the centre decided it is arithmetic. The equality is held to
|
|
// LATTICE_SEARCH_DIST_TOLERANCE, relative, on a pair of reduced-cell
|
|
// axes; a centre wrong by `moved` px moves an axis of length a by
|
|
// moved*p*a/(D*lambda) relative, the FFT reading that axis off spot
|
|
// positions the centre displaces. Both axes of the pair carry their own
|
|
// error, so the centre can open or close a gap of their sum - and where
|
|
// that reaches the tolerance the higher class is not a fact about the
|
|
// crystal but a consequence of where the beam was assumed to be.
|
|
//
|
|
// Nothing available here can settle it: both centres index, and the frame
|
|
// count is the one statistic this check already refuses to arbitrate on.
|
|
// So carry the other centre out of the pass and let the two arms be judged
|
|
// on their merges (RunAllPasses).
|
|
const auto reduced = best.result->search_result.primitive_reduced.GetUnitCell();
|
|
const double axis_pair = std::max(reduced.a + reduced.b,
|
|
reduced.b + reduced.c);
|
|
const double induced = moved * rx.experiment_.GetPixelSize_mm() * axis_pair
|
|
/ (rx.experiment_.GetDetectorDistance_mm()
|
|
* rx.experiment_.GetWavelength_A());
|
|
const bool centre_can_decide = induced >= LATTICE_SEARCH_DIST_TOLERANCE;
|
|
logger.Info("Beam centre check: indexing at the measured centre "
|
|
"({:.2f},{:.2f}) returns the same CELL as the file's - the "
|
|
"primitive volumes agree to {:.2f} % - but a different metric "
|
|
"symmetry: {}-centred {} from the file's centre against "
|
|
"{}-centred {} from the measured one", measured_x, measured_y,
|
|
100.0 * (ratio - 1.0), best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system));
|
|
if (centre_can_decide && geometry_prepass) {
|
|
rx.beam_center_alternative_ = std::array<float, 2>{measured_x, measured_y};
|
|
logger.Warning("Beam centre check: {:.2f} px is worth {:.2f} % on the "
|
|
"axis pair the promotion compares, against the {:.2f} % "
|
|
"the metric symmetry is decided to - the difference is "
|
|
"large enough to have decided it either way, so the run "
|
|
"carries BOTH centres forward and adopts whichever "
|
|
"merges better", moved, 100.0 * induced,
|
|
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
|
|
} else {
|
|
logger.Info("Beam centre check: {:.2f} px is worth {:.2f} % on the axis "
|
|
"pair the promotion compares, against the {:.2f} % the "
|
|
"metric symmetry is decided to - the run keeps the file's "
|
|
"centre and the symmetry it found", moved, 100.0 * induced,
|
|
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
|
|
}
|
|
} else {
|
|
best = CentresDisagree(std::move(best), alt, c, ratio);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// The two centres index different lattices.
|
|
auto Rugnux::FirstPassRun::CentresDisagree(FirstPass best, const FirstPass &alt, const Centres &c,
|
|
double ratio) -> FirstPass {
|
|
const float header_x = c.header_x, header_y = c.header_y;
|
|
const float measured_x = c.measured_x, measured_y = c.measured_y;
|
|
const auto &hc = best.result->search_result.conventional.GetUnitCell();
|
|
const auto &mc = alt.result->search_result.conventional.GetUnitCell();
|
|
const double nearest = std::round(ratio);
|
|
const bool harmonic = (nearest >= 2.0 && std::abs(ratio - nearest) < 0.15)
|
|
|| std::abs(ratio - std::sqrt(3.0)) < 0.15;
|
|
logger.Warning("Beam centre check: the two centres DISAGREE about the "
|
|
"lattice. The file's ({:.2f},{:.2f}) gives {}-centred {}, "
|
|
"{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} (primitive volume "
|
|
"{:.0f} A^3, {} frames); the measured ({:.2f},{:.2f}) gives "
|
|
"{}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} "
|
|
"(primitive volume {:.0f} A^3, {} frames){}",
|
|
header_x, header_y, best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
hc.a, hc.b, hc.c, hc.alpha, hc.beta, hc.gamma, best.vol,
|
|
Frames(best.score),
|
|
measured_x, measured_y, alt.result->search_result.centering,
|
|
gemmi::crystal_system_str(alt.result->search_result.system),
|
|
mc.a, mc.b, mc.c, mc.alpha, mc.beta, mc.gamma, alt.vol,
|
|
Frames(alt.score),
|
|
harmonic ? fmt::format(" - the volumes differ by {:.2f}x, so one "
|
|
"is an axis harmonic of the other and the "
|
|
"beam centre is what settles which",
|
|
ratio)
|
|
: "");
|
|
// Deliberately no decision on the frame counts. They are the one statistic
|
|
// that must not choose: on the measured case the harmonic indexed MORE.
|
|
//
|
|
// Decided in either direction on the pooled excess over chance, the same
|
|
// measurement the failed-pass branch above adopts a centre on. Each lattice is
|
|
// scored against its own wrong-spindle null, so the larger cell is not credited
|
|
// for the spots a denser lattice catches by accident, and the smaller one is
|
|
// not credited for the spots it misses.
|
|
//
|
|
// Both directions need it. Where the measured centre holds the LARGER cell,
|
|
// nothing else in the run repairs it - the second pass refuses a lattice larger
|
|
// than pass 1's (the supercell guard below). Where it holds the SMALLER one,
|
|
// the second pass can drop to it only if its own beam refinement happens to
|
|
// walk the whole centre error, and that rescue is chaotic: measured on one
|
|
// crystal, a 0.1 % change in the pass-1 distance switched off a 6.5 px walk and
|
|
// the run kept the doubled axis.
|
|
const double nearest_n = std::round(ratio);
|
|
const bool integer_harmonic = nearest_n >= 2.0 && nearest_n <= 4.0
|
|
&& std::abs(ratio - nearest_n) < 0.15;
|
|
bool adopted_measured = false;
|
|
if (integer_harmonic) {
|
|
const PooledEvidence at_header = MeasurePooled(*indexer, *best.result);
|
|
TryBeamCenter(measured_x, measured_y);
|
|
const PooledEvidence at_measured = MeasurePooled(*indexer, *alt.result);
|
|
adopted_measured = ValidationEvidenceBeatsChance(at_measured)
|
|
&& ValidationEvidencePrefers(at_header, at_measured);
|
|
NearTiePooled("beam centre (harmonic pair)", "file centre", at_header, "measured centre", at_measured);
|
|
logger.Info("Beam centre check: pooled validation spots on the lattice "
|
|
"{:.1f}% at the file's centre and {:.1f}% at the measured "
|
|
"(chance {:.1f}% and {:.1f}%)",
|
|
100.0 * at_header.on_lattice / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.on_lattice
|
|
/ std::max<int64_t>(1, at_measured.spots),
|
|
100.0 * at_header.by_chance / std::max<int64_t>(1, at_header.spots),
|
|
100.0 * at_measured.by_chance
|
|
/ std::max<int64_t>(1, at_measured.spots));
|
|
if (adopted_measured) {
|
|
logger.Warning("Beam centre check: the spots are on the {} cell of the "
|
|
"measured centre, not on the file's - so the {:.2f}x is "
|
|
"the file's centre error and not the crystal, and the "
|
|
"run adopts the measured centre",
|
|
alt.vol > best.vol ? "larger" : "smaller", ratio);
|
|
best = alt;
|
|
} else {
|
|
RestoreBeamCenter(header_x, header_y);
|
|
}
|
|
}
|
|
if (!adopted_measured)
|
|
logger.Warning("Beam centre check: the run keeps the file's centre. The "
|
|
"frame counts cannot arbitrate this - a cell twice as long "
|
|
"must place every spot twice as accurately to score the "
|
|
"same, so the harmonic is often the one that indexes more. "
|
|
"Where the beam centre is known, give it with "
|
|
"--beam-x/--beam-y");
|
|
return best;
|
|
}
|
|
|
|
// Where the first pass indexes no majority of the validation frames, the end wedge is asked too
|
|
// and judged against the standing pass as any later scheme is. Only then: on a run the schemes
|
|
// index it would be one more lattice for the axis-harmonic arbiter, which is not decided on
|
|
// frame counts. Asked once, after the axis-sign, beam-centre and long-axis rescues (below), not
|
|
// inside every pass they run: a consecutive wedge indexes part of the validation frames at the
|
|
// WRONG axis sign too (measured: 33/60 where the spread at the right sign indexes 60/60), and
|
|
// counted in the pass a rescue is gated on, it lifts that pass past the gate and the rescue
|
|
// that would have indexed every frame is never asked.
|
|
auto Rugnux::FirstPassRun::AddEndWedge(FirstPass bp, IndexerThreadPool &pool, IndexAndRefine &idx) -> FirstPass {
|
|
NearTieGate("end-wedge gate", "standing pass", bp.score, static_cast<int>(validation.size()) / 2);
|
|
if (rx.cancelled_ || bp.score > static_cast<int>(validation.size()) / 2)
|
|
return bp;
|
|
SchemeIndexers ris;
|
|
ris.push_back(FeedScheme(pool, end_wedge[0].second));
|
|
RunIndexing({ris[0].get()});
|
|
return ScoreSchemes(ris, idx, &end_wedge, std::move(bp));
|
|
}
|
|
|
|
// Short-axis hypothesis - the small-molecule case.
|
|
//
|
|
// LAST, after the axis-sign, beam-centre and long-axis rescues, deliberately. Those all
|
|
// run the standard floor and compare their result against `best` on volume ratios; a cell
|
|
// adopted here is a k-fold sub-cell, so running before them makes every one of those
|
|
// comparisons read an integer ratio and report an axis harmonic that is not one (measured:
|
|
// the beam-centre check announced a "3.00x harmonic ... the beam centre is what settles
|
|
// which" on a pair that differs only by the FLOOR). Here the comparison is like with like,
|
|
// and this pass sees whatever geometry those rescues settled on.
|
|
//
|
|
// fft_min_unit_cell_A (10 A) is a protein constant: shorter than any protein axis and longer
|
|
// than a small molecule's. It is applied to the FFT's PER-DIRECTION PEAK SEARCH
|
|
// (FFTIndexerCPU.cpp / FFTIndexerGPU.cu skip every histogram bin below it) and only then to
|
|
// the assembled cell (FFTIndexer::ReduceAndRefine, PostIndexingRefinement). The first of
|
|
// those is what does the damage: the transform reports ONE peak per direction, so a
|
|
// lattice row whose real-space repeat is shorter than the floor is reported at its first
|
|
// HARMONIC above it, and the cell built out of those harmonics is an exact integer
|
|
// sublattice of the true one - a real-space supercell. Measured on chemical-crystallography
|
|
// data: index 2, 3 and 6 exactly, and the one dataset whose true reduced cell clears 10 A is
|
|
// the one that comes out right. Because the floor is applied to the REDUCED cell it rejects
|
|
// the lattice and not merely a basis choice, so nothing downstream can re-describe its way
|
|
// back.
|
|
//
|
|
// Lowering the floor by default is not available: it only ever ADDS candidates, and the
|
|
// Bravais classification the candidate pool feeds is under-determined, so it moves crystals
|
|
// whose cell it never touched (measured: 2 regressions and 1 improvement over 20 protein
|
|
// datasets, all three in the metric symmetry rather than the cell). So the low floor runs as
|
|
// a SECOND hypothesis and the standard pass's answer stands unless the evidence takes it.
|
|
//
|
|
// What the evidence can be, before integration: a sublattice's per-spot residuals are
|
|
// dominated pointwise by the finer lattice it contains, so every margin statistic - squares,
|
|
// counts, trimmed sums - is biased toward the LARGER cell, and a spurious supercell indexes
|
|
// every frame its true cell does. Only the SATURATION is informative. Hence the same shape
|
|
// the scheme ratchet in PickBest already uses: an integer primitive-volume ratio plus a tie
|
|
// on validation frames, or an outright win, which runs against the bias and so is real. The
|
|
// statistic that genuinely decides an axis multiple is the occupancy of the reflection class
|
|
// the multiple adds - AbsenceEvidence in SearchSpaceGroup - and that needs integrated
|
|
// intensities, so it cannot run here.
|
|
// Not when a cell was given. RaiseFFTBoundForKnownCell has already lowered the floor for
|
|
// the standard pass against that cell, so this pass has nothing left to add - and it takes
|
|
// its settings from rx.experiment_, which never carried the RAISED max bound, so on a cell
|
|
// longer than the default search it could only ever return a harmonic of what was asked
|
|
// for, tie on frames the way a sub-multiple always does, and be adopted over it.
|
|
auto Rugnux::FirstPassRun::ShortAxisHypothesis(FirstPass best) -> FirstPass {
|
|
if (!rx.cancelled_ && !rx.experiment_.GetUnitCell().has_value()) {
|
|
std::unique_ptr<IndexerThreadPool> short_pool;
|
|
std::vector<std::unique_ptr<RotationIndexer>> sris;
|
|
if (short_spec && short_spec->key == rx.ExperimentKey() && short_spec->mapping == spot_mapping
|
|
&& short_spec->spots_per_image == first_pass_spots_per_image) {
|
|
for (auto &f : short_spec->futures)
|
|
f.get();
|
|
short_pool = std::move(short_spec->pool);
|
|
sris = std::move(short_spec->sris);
|
|
} else {
|
|
short_spec.reset();
|
|
short_pool = std::make_unique<IndexerThreadPool>(ShortAxisSettings(rx.experiment_),
|
|
IndexerConstruction::OnFirstUse, rx.gpu_work_beside_);
|
|
sris = FeedSchemes(*short_pool);
|
|
RunIndexing(sris);
|
|
}
|
|
|
|
for (size_t i = 0; i < sris.size() && !rx.cancelled_; i++) {
|
|
auto found = sris[i]->GetLattice();
|
|
if (!found.has_value())
|
|
continue;
|
|
RotationIndexerResult alt = std::move(*found);
|
|
const std::string &name = schemes[i].first;
|
|
// Primitive volumes, for the reason PickBest gives: two searches can find the same
|
|
// lattice in different settings, and a centred setting's cell is an exact integer
|
|
// multiple of its primitive one.
|
|
const double vol = std::abs(
|
|
alt.lattice.ToPrimitive(alt.search_result.centering).CalcVolume());
|
|
if (!(vol > 1.0) || !(best.vol > 1.0))
|
|
continue;
|
|
const double ratio = best.vol / vol;
|
|
const double nearest = std::round(ratio);
|
|
// The gate is here rather than after the scoring because scoring is the expensive
|
|
// half - CountIndexed runs the real per-image path over the validation set, seconds
|
|
// on a protein - while the volume ratio is free. On every protein measured the low
|
|
// floor returns the SAME lattice (ratio 1.00-1.01), so this costs them one extra FFT
|
|
// and nothing else.
|
|
if (!(vol < best.vol) || nearest < 2.0 || std::abs(ratio - nearest) >= 0.15) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': primitive volume "
|
|
"{:.0f} A^3 against the standing cell's {:.0f} A^3, ratio {:.2f} - "
|
|
"not an integer sub-cell, keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, vol, best.vol,
|
|
ratio);
|
|
continue;
|
|
}
|
|
|
|
// The premise of this pass, checked rather than asserted. The pass exists because a
|
|
// lattice row shorter than the standard floor is reported at its first HARMONIC
|
|
// above it, so a cell it may rightfully adopt has a reduced axis BELOW that floor -
|
|
// and the floor is applied to the reduced cell, which is the one to measure. Where
|
|
// the alternative has no such axis the floor cannot be what the standard pass
|
|
// tripped over, and an integer volume ratio is then an ordinary axis multiple: the
|
|
// question the axis-harmonic arbiter settles just above on the occupancy of the
|
|
// class the multiple adds. That is intensity-class evidence about the very
|
|
// reflections in dispute, where the tie below is a frame count this comment already
|
|
// says a sublattice cannot lose - so where the two disagree the arbiter is right and
|
|
// this block must not overrule it. Measured on the corpus: the four small-molecule
|
|
// sets the pass is for all have a reduced axis under 9 A, and the two protein sets
|
|
// it halved have 35 and 47 A.
|
|
const auto alt_reduced = alt.lattice.ToPrimitive(alt.search_result.centering)
|
|
.NiggliReduce().GetUnitCell();
|
|
const float alt_shortest = std::min({alt_reduced.a, alt_reduced.b, alt_reduced.c});
|
|
const float standard_floor = rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A();
|
|
if (alt_shortest >= standard_floor) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': a {:.0f}x sub-cell "
|
|
"(vol {:.0f} A^3), but its shortest reduced axis is {:.1f} A, above "
|
|
"the {:.1f} A floor the standard pass already searched - the floor "
|
|
"is not why that cell was missed, so this is an ordinary axis "
|
|
"multiple and not this pass's to decide; keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
|
|
alt_shortest, standard_floor);
|
|
continue;
|
|
}
|
|
|
|
const int alt_score = CountIndexed(*indexer, alt);
|
|
NearTieFrames("short-axis hypothesis", "sub-cell", alt_score, "standing", best.score);
|
|
// A pass that indexes nothing is not tied with a pass that indexes nothing. Without
|
|
// this, 0 against 0 passes the tie band and the volume ratio alone decides, which is
|
|
// the one thing this block must never do.
|
|
if (alt_score <= 0 || best.score <= 0) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {}/{} validation "
|
|
"frames against {} - nothing indexes, keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, alt_score,
|
|
static_cast<int>(validation.size()), best.score);
|
|
continue;
|
|
}
|
|
const bool tied = static_cast<float>(alt_score) >= best.score * 0.9f - 0.5f;
|
|
const bool clearly_more = static_cast<float>(alt_score) > best.score * 1.1f + 0.5f;
|
|
if (!tied && !clearly_more) {
|
|
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {:.0f}x sub-cell "
|
|
"(vol {:.0f} A^3) indexes {}/{} validation frames against {} - "
|
|
"keeping the standing cell",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
|
|
alt_score, static_cast<int>(validation.size()), best.score);
|
|
continue;
|
|
}
|
|
const auto uc = alt.lattice.GetUnitCell();
|
|
logger.Warning("Short-axis pass (floor {:.0f} A), scheme '{}': the standing "
|
|
"cell is a {:.0f}x supercell. Its {:.0f} A^3 primitive volume against "
|
|
"this pass's {:.0f} A^3, at {}/{} validation frames against {} - "
|
|
"adopting {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}. The shortest axis "
|
|
"is below the {:.0f} A default FFT floor, which is why the standard "
|
|
"pass could only build a multiple of it",
|
|
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, best.vol,
|
|
vol, alt_score, static_cast<int>(validation.size()), best.score,
|
|
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
|
|
rx.experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A());
|
|
best.score = alt_score;
|
|
best.vol = vol;
|
|
best.name = name + " (short-axis)";
|
|
best.result = std::move(alt);
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// Sub-lattice hypothesis: the standing cell's seven index-2 sub-lattices, in its own
|
|
// orientation and geometry, each scored on the validation frames.
|
|
//
|
|
// The FFT can only offer the cells its peaks build, and two lattices seen together - a twin
|
|
// domain or a second crystal whose reciprocal lattice shares an index-2 sub-lattice with
|
|
// this one's - build the lattice that contains them both: a cell twice the size of either,
|
|
// which indexes both crystals' spots and so wins every comparison of candidates the FFT
|
|
// returns. Where the true cell is then never offered, nothing downstream can choose it.
|
|
// Offering it costs seven frame counts.
|
|
//
|
|
// The frame count only nominates: a coarser lattice asks less accuracy of every spot, so a
|
|
// sub-lattice of the true cell can index as many frames as the cell itself, or a few more
|
|
// where the cell's own score is depressed. What decides is the comparison this pass already
|
|
// makes between a cell and its axis harmonic: the share of the validation frames' pooled
|
|
// spots each puts on itself over and above what a wrong spindle angle puts there, each
|
|
// against its own null (ValidationEvidencePrefers). A real superstructure keeps its cell on
|
|
// that count, because the class it adds holds reflections; a cell built out of two crystals
|
|
// loses it, because it has to place every spot of the one it describes twice as accurately
|
|
// with an orientation fitted to both. The nominee must also index a majority of the frames,
|
|
// the bar the false-promotion test above asks of a cell that "works", and no fewer frames
|
|
// than the standing cell: a standing cell that indexes almost nothing is not this crystal's
|
|
// lattice, its sub-lattices are not hypotheses about it, and the rescues below are what
|
|
// such a run needs.
|
|
auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass {
|
|
if (!rx.cancelled_ && best.result.has_value() && best.score > 0) {
|
|
const CrystalLattice prim = best.result->lattice
|
|
.ToPrimitive(best.result->search_result.centering);
|
|
const Coord A = prim.Vec0(), B = prim.Vec1(), C = prim.Vec2();
|
|
std::optional<FirstPass> sub_best;
|
|
std::string counts;
|
|
for (int u = 0; u < 2; u++)
|
|
for (int v = 0; v < 2; v++)
|
|
for (int w = 0; w < 2; w++) {
|
|
if (u + v + w == 0 || rx.cancelled_)
|
|
continue;
|
|
const Coord t = (A * static_cast<float>(u) + B * static_cast<float>(v)
|
|
+ C * static_cast<float>(w)) / 2.0f;
|
|
const CrystalLattice sub = (w ? CrystalLattice(A, B, t)
|
|
: v ? CrystalLattice(A, t, C)
|
|
: CrystalLattice(t, B, C)).NiggliReduce();
|
|
RotationIndexerResult cand = *best.result;
|
|
cand.lattice = sub;
|
|
cand.search_result = LatticeSearchResult{};
|
|
cand.search_result.primitive_reduced = sub;
|
|
cand.search_result.conventional = sub;
|
|
cand.unconstrained.reset();
|
|
cand.extra_lattices.clear();
|
|
const int score = CountIndexed(*indexer, cand);
|
|
counts += fmt::format(" ({},{},{}) {}", u, v, w, score);
|
|
if (!sub_best || score > sub_best->score)
|
|
sub_best = FirstPass{std::move(cand), score,
|
|
fmt::format("{} (sub-lattice {}{}{})", best.name, u, v, w),
|
|
std::abs(sub.CalcVolume())};
|
|
}
|
|
bool adopt = false;
|
|
std::string verdict = " - none indexes a majority of the frames and as many as the standing cell";
|
|
const int majority = static_cast<int>(validation.size()) / 2;
|
|
bool nominated = sub_best && sub_best->score >= best.score && sub_best->score > majority;
|
|
// Carried only where the gate is what decides: a sub-lattice that indexes fewer frames than
|
|
// the standing cell is not nominated whatever the gate says.
|
|
if (sub_best && NearTieGate("sub-lattice majority gate", "best sub-lattice", sub_best->score, majority)
|
|
&& sub_best->score >= best.score && DecideLate()) {
|
|
const int late = CountIndexed(*indexer, *sub_best->result, late_validation);
|
|
const int standing_late = CountIndexed(*indexer, *best.result, late_validation);
|
|
const bool early = nominated;
|
|
nominated = sub_best->score + late >= best.score + standing_late
|
|
&& LateOverGate(sub_best->score, late, majority);
|
|
LogDecidedLate("sub-lattice majority gate",
|
|
fmt::format("best sub-lattice {}/{} of the later frames, the standing cell {}/{}", late,
|
|
late_validation.size(), standing_late, late_validation.size()),
|
|
early, nominated);
|
|
}
|
|
if (nominated) {
|
|
PooledEvidence at_standing = MeasurePooled(*indexer, *best.result);
|
|
PooledEvidence at_sub = MeasurePooled(*indexer, *sub_best->result);
|
|
if (NearTiePooled("sub-lattice hypothesis", "standing cell", at_standing, "sub-lattice", at_sub)
|
|
&& DecideLate()) {
|
|
const bool early = ValidationEvidencePrefers(at_standing, at_sub);
|
|
const std::vector<int> all = AllValidationFrames();
|
|
at_standing = MeasurePooled(*indexer, *best.result, all);
|
|
at_sub = MeasurePooled(*indexer, *sub_best->result, all);
|
|
LogDecidedLate("sub-lattice hypothesis", "the pooled spots counted again on all the frames",
|
|
early, ValidationEvidencePrefers(at_standing, at_sub));
|
|
}
|
|
adopt = ValidationEvidencePrefers(at_standing, at_sub);
|
|
verdict = fmt::format(" - the best puts {}/{} pooled spots on itself ({} by chance) against "
|
|
"the standing cell's {}/{} ({}), {}",
|
|
at_sub.on_lattice, at_sub.spots, at_sub.by_chance,
|
|
at_standing.on_lattice, at_standing.spots, at_standing.by_chance,
|
|
adopt ? "adopting it" : "keeping the standing cell");
|
|
}
|
|
logger.Info("Sub-lattice hypothesis: the standing cell ({:.0f} A^3) indexes {} validation "
|
|
"frames; its index-2 sub-lattices (half-translation u,v,w):{}{}",
|
|
best.vol, Frames(best.score), counts, verdict);
|
|
if (adopt)
|
|
best = std::move(*sub_best);
|
|
}
|
|
return best;
|
|
}
|
|
|
|
// A lattice that indexes almost none of the validation frames is not this crystal's lattice.
|
|
// Forcing it anyway integrates every image against it; the few partials that survive yield no
|
|
// fulls in the 3D combine, and the run then dies inside the merge complaining about resolution -
|
|
// far from the cause. Stop here instead, while the reason is still visible. The floor is one
|
|
// frame in six, several times below the weakest crystal that still merges. It is checked AFTER
|
|
// the long-axis rescue, so a metric that rescue recovers is never rejected on its pre-rescue
|
|
// score.
|
|
// The second (refined-geometry) pass re-indexes DE NOVO, and the post-refined geometry - fitted
|
|
// to the lattice pass 1 already found - can tip the blind FFT, or the long-axis rescue above,
|
|
// onto another lattice: an axis harmonic of a long cell (measured: a tripled ~100 A c-axis at
|
|
// 7/60), or a related centred cell that indexes fewer frames than pass 1's lattice does at
|
|
// this very geometry (measured: a C-centred 303 A cell at 41/60 against pass 1's triclinic
|
|
// cell at 60/60). Pass 1's lattice is therefore carried here as a hypothesis of its own and
|
|
// the two are judged on the same validation frames: where the re-index found a different
|
|
// lattice and indexes fewer frames than pass 1's, or too few to be this crystal's lattice at
|
|
// all, pass 1's whole result is integrated instead. The same lattice found again is kept as
|
|
// the re-index refined it at this geometry. This is the floor-side companion of the
|
|
// supercell-collapse guard in RunAllPasses, which only sees a pass-2 lattice that is LARGER.
|
|
auto Rugnux::FirstPassRun::Pass1Fallback(FirstPass best) -> FirstPass {
|
|
RotationIndexerResult pass1_lattice = *rx.prepass_result_;
|
|
// Pass 1's result carries pass 1's goniometer; forcing it whole would put the
|
|
// uncorrected angles back (the same repair as the supercell re-run in RunAllPasses).
|
|
if (rx.prepass_rotation_scale_ && pass1_lattice.axis)
|
|
pass1_lattice.axis = ScaleRotation(*pass1_lattice.axis, *rx.prepass_rotation_scale_);
|
|
// Pass 1's lattice was FITTED at pass 1's detector distance, and this pass
|
|
// integrates at the post-refined one. Re-scoring it here is not the same as
|
|
// re-fitting it: a real-space cell is measured against the distance the spots were
|
|
// seen at, so carrying it across a distance change scales the whole cell by the
|
|
// ratio of the two. Measured on a 545 A axis over 225 -> 226 mm: 0.4 %, exactly
|
|
// 225/226, shipped as the run's answer. Scale it to the distance it will be used
|
|
// at; the orientation is unaffected.
|
|
const float d_fit = rx.prepass_result_->geom.GetDetectorDistance_mm();
|
|
const float d_use = rx.experiment_.GetDetectorDistance_mm();
|
|
const bool rescaled = d_fit > 0 && d_use > 0 && d_fit != d_use;
|
|
if (rescaled) {
|
|
const float f = d_use / d_fit;
|
|
const auto &l = pass1_lattice.lattice;
|
|
pass1_lattice.lattice = CrystalLattice(l.Vec0() * f, l.Vec1() * f, l.Vec2() * f);
|
|
}
|
|
const CrystalLattice pass1_primitive = pass1_lattice.lattice
|
|
.ToPrimitive(pass1_lattice.search_result.centering).NiggliReduce();
|
|
const double pass1_vol = std::abs(pass1_primitive.CalcVolume());
|
|
// The same lattice whatever setting or class it is held in - a centred cell and the
|
|
// triclinic primitive of the same points are one lattice, and the re-index's symmetric
|
|
// setting must not lose to its own unconstrained primitive on a frame count: the Niggli-
|
|
// reduced primitive edges agree within 2 % (the battery's lattice-identity test).
|
|
const auto reduced_edges = [](const CrystalLattice &l) {
|
|
const auto uc = l.GetUnitCell();
|
|
std::array<double, 3> e = {uc.a, uc.b, uc.c};
|
|
std::sort(e.begin(), e.end());
|
|
return e;
|
|
};
|
|
const auto e1 = reduced_edges(pass1_primitive);
|
|
const auto e2 = reduced_edges(best.result->lattice
|
|
.ToPrimitive(best.result->search_result.centering).NiggliReduce());
|
|
bool same_lattice = true;
|
|
for (int i = 0; i < 3; i++)
|
|
same_lattice = same_lattice && std::abs(e2[i] - e1[i]) <= 0.02 * e1[i];
|
|
const int floor = static_cast<int>(validation.size()) / 6;
|
|
const int pass1_score = (best.score < floor || !same_lattice)
|
|
? CountIndexed(*indexer, pass1_lattice) : -1;
|
|
NearTieGate("pass-1 fallback floor", "re-index", best.score, floor);
|
|
bool pass1_more = pass1_score > best.score;
|
|
if (pass1_score >= 0 && !same_lattice
|
|
&& NearTieFrames("pass-1 fallback", "re-index", best.score, "pass-1 lattice", pass1_score)
|
|
&& DecideLate()) {
|
|
const int late = CountIndexed(*indexer, *best.result, late_validation);
|
|
const int pass1_late = CountIndexed(*indexer, pass1_lattice, late_validation);
|
|
const bool early = pass1_more;
|
|
pass1_more = pass1_score + pass1_late > best.score + late;
|
|
LogDecidedLate("pass-1 fallback", fmt::format("re-index {}/{} of the later frames, pass-1 lattice {}/{}", late,
|
|
late_validation.size(), pass1_late, late_validation.size()),
|
|
early, pass1_more);
|
|
}
|
|
if (best.score < floor || pass1_more) {
|
|
const auto &pc = pass1_lattice.search_result.conventional.GetUnitCell();
|
|
const auto &bc = best.result->search_result.conventional.GetUnitCell();
|
|
logger.Warning("Two-pass: the refined-geometry re-index indexes {}/{} validation frames "
|
|
"({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}), pass-1's lattice "
|
|
"{}/{} at this geometry ({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} "
|
|
"{:.2f}) - integrating with pass-1's lattice",
|
|
best.score, static_cast<int>(validation.size()),
|
|
best.result->search_result.centering,
|
|
gemmi::crystal_system_str(best.result->search_result.system),
|
|
bc.a, bc.b, bc.c, bc.alpha, bc.beta, bc.gamma,
|
|
pass1_score, static_cast<int>(validation.size()),
|
|
pass1_lattice.search_result.centering,
|
|
gemmi::crystal_system_str(pass1_lattice.search_result.system),
|
|
pc.a, pc.b, pc.c, pc.alpha, pc.beta, pc.gamma);
|
|
if (rescaled)
|
|
logger.Info("Two-pass: pass-1's cell was fitted at {:.3f} mm and is used at "
|
|
"{:.3f} mm - scaled by {:.6f} to the distance it is integrated at",
|
|
d_fit, d_use, d_use / d_fit);
|
|
best.result = std::move(pass1_lattice);
|
|
best.score = pass1_score;
|
|
best.vol = pass1_vol;
|
|
best.name = "pass-1 lattice (refined-geometry re-index found less)";
|
|
}
|
|
return best;
|
|
}
|