mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 79 Wiki Activity
Files
0ef6b3f00bdedfcde1c19bcbbd8fb0df70b4cfc4
Jungfraujoch/rugnux/Rugnux.cpp
T
leonarski_fandClaude Opus 5 6a26591786 rugnux: the indexing-ambiguity warning is withdrawn only where the model settled it 
The INDEXING_AMBIGUITY warning was suppressed whenever --model was given on
rotation data, decided before the model was read. A model that then decided
nothing - not tested, rejected, or a probe winner that did not beat its own null
- still silenced it, so WARNING_COUNT, PATHOLOGY_FLAGS and possibly VERDICT
differed from a run without the model although the written reflections were
identical.

The warning is now issued exactly as without a model, and withdrawn (from the
warnings and from the statistics text) after model validation only where the
indexing probe decided the indexing: the model fits and the winner's R-free
margin beats the random-placement null (ModelValidationResult::indexing_decided,
set where the decision is taken). A reference MTZ, or the model reference on
serial stills with -C and -S, suppresses it up front as before.

Verified bare vs --model on three open-arm sets: merged MTZ data identical in
all three; warnings identical where the model decided nothing (identity probe
without null; no twin law); withdrawn where the probe decided (+33 sigma).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-20 18:45:03 +02:00

6871 lines
469 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <cstdlib>
#include "Rugnux.h"
#include "ModelValidation.h"
#include "WriteModel.h"
#include "SpindleCuspLoss.h"
#include "SpotWidth.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <deque>
#include <functional>
#include <future>
#include <mutex>
#include <numeric>
#include <set>
#include <sstream>
#include <thread>
#include "../reader/JFJochHDF5Reader.h"
#include "../common/JFJochMath.h"
#include "../common/ParallelFor.h"
#include "../common/Logger.h"
#include "../common/AzimuthalIntegrationMapping.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include "../common/CUDAWrapper.h"
#include "../common/JFJochException.h"
#include "../common/time_utc.h"
#include "../writer/FileWriter.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
#include "../image_analysis/beam_stop/ShadowFinder.h"
#include "../image_analysis/IndexAndRefine.h"
#include "../image_analysis/indexing/AnalyzeIndexing.h"
#include "../image_analysis/indexing/HarmonicContamination.h"
#include "../image_analysis/geom_refinement/BeamCenterFFT.h"
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
#include "../image_analysis/geom_refinement/BeamCenterFromSpots.h"
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
#include "../image_analysis/indexing/IndexerThreadPool.h"
#include "../image_analysis/spot_finding/ImageSpotFinderCPU.h"
#include "../image_analysis/spot_finding/SpotUtils.h"
#include "gemmi/twin.hpp"
#include "../image_analysis/azint/AzIntEngineCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/RfreeFlags.h"
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
#include "../image_analysis/scale_merge/ScalingResult.h"
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/geom_refinement/PostRefine.h"
#include "../image_analysis/lattice_search/LatticeSearch.h"
#include "../image_analysis/lattice_search/LePageLattice.h"
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
#include "../image_analysis/scale_merge/TranslationalNCS.h"
#include "../image_analysis/scale_merge/HKLKey.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
#include "../image_analysis/WriteReflections.h"
#include "../image_analysis/bragg_integration/CalcISigma.h"
#include "../common/Definitions.h"
#include "../common/CorrelationCoefficient.h"
#include <array>
#include <map>
double MetricViolation(const UnitCell &uc, const gemmi::SpaceGroup &sg) {
const double a = uc.a, b = uc.b, c = uc.c;
const double d2r = PI / 180.0;
const double ab = a * b * std::cos(uc.gamma * d2r);
const double ac = a * c * std::cos(uc.beta * d2r);
const double bc = b * c * std::cos(uc.alpha * d2r);
const double g[3][3] = {{a * a, ab, ac}, {ab, b * b, bc}, {ac, bc, c * c}};
const double scale = std::max({a * a, b * b, c * c});
if (!(scale > 0.0))
return 0.0;
double worst = 0.0;
for (const gemmi::Op &op : sg.operations()) {
double r[3][3];
for (int i = 0; i < 3; i++)
for (int j = 0; j < 3; j++)
r[i][j] = static_cast<double>(op.rot[i][j]) / gemmi::Op::DEN;
for (int i = 0; i < 3; i++)
for (int j = 0; j < 3; j++) {
double t = 0.0;
for (int k = 0; k < 3; k++)
for (int l = 0; l < 3; l++)
t += r[k][i] * g[k][l] * r[l][j];
worst = std::max(worst, std::fabs(t - g[i][j]) / scale);
}
}
return worst;
}
namespace {
// A CUDA out-of-memory, or a host allocation that failed, says nothing about the image that
// happened to be in flight and everything about the machine - the next image will hit it too.
// Skipping it the way a genuinely undecodable image is skipped drops frames out of the middle of
// the dataset and still exits 0, so the run looks complete while every merged number downstream
// has quietly changed. Fail the run instead, and let the operator free the card and repeat it.
bool IsFatalResourceError(const std::exception &e) {
if (dynamic_cast<const std::bad_alloc *>(&e)) return true;
const auto *jf = dynamic_cast<const JFJochException *>(&e);
return jf != nullptr && (jf->Category() == JFJochExceptionCategory::GPUCUDAError
|| jf->Category() == JFJochExceptionCategory::MemAllocFailed);
}
// How precisely the beam centre has to be placed before the estimate is used at all. An estimate
// that fails it is treated as no estimate: the spot symmetry falls through to the background and
// the background to the header. A fit's own scatter will not serve as the gate - the ways either
// estimator goes badly wrong all leave a small one - so the spot symmetry reports instead how
// far its answer moves when the search is started elsewhere, which does separate them. On the 38
// rotation regression crystals that stays below 0.35 px against errors of at most 0.74; over a
// ring of injected geometries it rises past 1.0 px on 96 % of the answers that are 2 px or more
// out, and on none of the 38 as they are.
constexpr float MAX_BEAM_CENTER_SIGMA_PXL = 1.0f;
// The longest cell axis assumed where no cell is known yet. It has to be LARGE: a long axis makes
// the tolerance below small, and the ceiling is the larger of it and the flat bound above, so a
// large default is the one that changes nothing.
constexpr float UNKNOWN_CELL_A_MAX_A = 200.0f;
// How far the beam centre may be wrong before the FIRST PASS stops indexing. A centre error is
// fixed in the LAB frame, so accumulating a sweep smears every reciprocal-lattice point around a
// circle and the FFT amplitude at an axis of length a is multiplied by J0(2 pi delta p a/(D
// lambda)); this is the displacement at which that has fallen to 0.70, and the first zero is
// 2.09x further out. Over the in-house and non-SLS corpora it runs from 0.6 px to 9 px - a
// fifteen-fold spread, which is why the flat bound above is only ever used as a FLOOR under it.
//
// It is printed, and it is deliberately not a relevance test. It says how wrong the header may
// be; the estimator's sigma says how well the estimator knows its own answer, and gating one on
// the other rejects a centre correct to 0.03 px because its sigma was 0.98. Nor is a move that is
// small against it a move that can be ignored: measured on real data, a 0.12 px change of centre
// - 0.03x of this - is the difference between the deposited cell and a halved axis.
float BeamCenterNeed_pxl(const DiffractionExperiment &experiment) {
const auto cell = experiment.GetUnitCell();
const float a_max = cell ? std::max({cell->a, cell->b, cell->c}) : UNKNOWN_CELL_A_MAX_A;
return 0.183f * experiment.GetDetectorDistance_mm() * experiment.GetWavelength_A()
/ (experiment.GetPixelSize_mm() * a_max);
}
// A detector-plane displacement resolved ACROSS the spindle - the only component the tolerance
// above is about. Along the spindle the error translates the derotated cloud rigidly and the FFT
// amplitude cannot see it at all; across it the peaks smear and the amplitude falls. (The lattice
// fit that follows the FFT is not translation-invariant, so "along the spindle" is free for the
// transform and not for the run - which is why this is reported and not used to decide anything.)
// The detector direction wanted is the one whose lab image is perpendicular to the spindle n:
// (ux e1 + uy e2).n = 0, so u is proportional to (-(e2.n), e1.n).
std::optional<float> AcrossSpindle_pxl(const DiffractionExperiment &experiment, float dx, float dy) {
const auto goniometer = experiment.GetGoniometer();
if (!goniometer || !goniometer->IsScanning())
return {};
const auto geom = experiment.GetDiffractionGeometry();
const float beam_x = geom.GetBeamX_pxl(), beam_y = geom.GetBeamY_pxl();
const Coord n = goniometer->GetAxis().Normalize();
const Coord origin = geom.LabCoord(beam_x, beam_y);
const Coord e1 = geom.LabCoord(beam_x + 1.0f, beam_y) - origin;
const Coord e2 = geom.LabCoord(beam_x, beam_y + 1.0f) - origin;
const float ux = -(e2 * n), uy = e1 * n;
const float length = std::hypot(ux, uy);
if (length < 1e-3f)
return {};
return std::abs(dx * ux + dy * uy) / length;
}
// Images the projection is built from when the beam centre is wanted but the beam-stop pre-pass
// is off; with it on, that pre-pass's image count is used and one projection serves both.
constexpr int BEAM_CENTER_PROJECTION_IMAGES = 60;
// Spots per pre-scan frame kept for the beam centre, the strongest first.
constexpr size_t BEAM_CENTER_SPOTS_PER_IMAGE = 500;
// Frames read for the spot symmetry alone when the first sample could not measure it. Both
// estimators are counting statistics over PAIRS of frames - the Friedel one over frames half a
// turn apart, the second crossing over any two frames a reflection is recorded on - so a sweep
// whose frames carry few spots can only be measured by reading more of them. On the two
// rotation regression crystals that decline, this is what turns the decline into an answer
// 0.04 px from the post-refined centre; the sample the beam stop is built from is left alone.
constexpr int BEAM_CENTER_SPARSE_IMAGES = 400;
// Frames the spot symmetry reads, as a multiple of the beam stop's. Its two estimators are
// counts of matched PAIRS and the pairs grow faster than the frames do, so this is the one knob
// that buys independence from WHICH frames were drawn - and that dependence is the estimator's
// largest remaining failure. Asking the same code for one frame more or less re-draws the whole
// sample: over five draws of 38 sweeps, 60 frames leaves three answers more than 1 px out, none
// of which reports an unusual sigma, and 120 leaves none in 186. It is also what makes the end
// margin below free: at 60 frames excluding the ends moved three centres, at 120 it moves none.
constexpr int BEAM_CENTER_IMAGE_FACTOR = 2;
// Images left out at each end of the sweep. Shutter synchronisation disturbs the first and last
// frame or two of a run, and a partly exposed frame is dark over the whole detector - which is
// exactly the kind of error a projection cannot average away and a background comparison reads
// as signal. The margin is capped at a tenth of the sweep so a short run still has a sample.
constexpr int PRESCAN_END_MARGIN_IMAGES = 5;
// Workers reading the pre-scan sample. Each owns a shard of the beam-stop projection so no two
// threads touch the same accumulator, and a shard costs 20 bytes per pixel - 362 MB on a 16M
// detector - so this is capped well below the worker count of the run proper. The accumulation
// is memory-bound rather than compute-bound, so a handful of workers already saturates it.
constexpr size_t PRESCAN_MAX_WORKERS = 8;
// The spot width is measured on a GROWING share of the pre-scan sample: every eighth frame of
// it, then every fourth, then every second, then all of it - an eighth, a quarter, a half and
// the whole. Each stride is a multiple of the next, so a tier's sample really does contain the
// one before it, and the measurement stops at the first tier whose answer has settled
// (spot_width::WidthSettled, in the loop below). A crystal whose frames carry plenty of isolated
// strong spots is characterised from a quarter of the sample; a sparse one still gets all of it.
//
// The tiers add PRESCAN_MAX_WORKERS-sized steps as well: on a 60 frame sample they are 8, 15, 30
// and 60 frames, i.e. 8 + 7 + 15 + 30 read in the four passes, so no pass leaves most of the
// workers idle waiting for the others to finish theirs. A ladder that starts smaller measures
// fewer frames but costs more wall clock, because a pass of five frames occupies eight workers
// for as long as a pass of eight does.
//
// Only the width work is tiered. Every frame of the sample is still READ once - the frames are
// simply visited in tier order - and the beam-stop projection still sees all of them, so the
// mask and the beam centre are what they were. What the tiers cut is the per-frame work the
// width needs on top of the read: the host decompression, the preprocessing and the spot
// finding, which together are two thirds of the pre-scan's cycles once the projection itself is
// decoding on the GPU. The width measurement proper is under 1 % of them.
//
// Measured over the rotation battery: the radius chosen is the whole-sample answer on 38 of 38
// crystals, from an average of 32 frames of the 60 instead of all of them.
constexpr std::array<int, 4> WIDTH_TIER_STRIDE = {8, 4, 2, 1};
// Pick up to requested_images ordinals spread evenly across [0, images_to_process) for the
// first pass of two-pass rotation indexing.
std::vector<int> select_equally_spaced_image_ordinals(int images_to_process, int requested_images) {
std::vector<int> ret;
if (images_to_process <= 0 || requested_images <= 0)
return ret;
const int n = std::min(images_to_process, requested_images);
if (n == 1) {
ret.push_back(0);
return ret;
}
std::set<int> unique_ordinals;
for (int i = 0; i < n; i++)
unique_ordinals.insert(static_cast<int>(
std::llround(static_cast<double>(i) * static_cast<double>(images_to_process - 1) /
static_cast<double>(n - 1))));
ret.assign(unique_ordinals.begin(), unique_ordinals.end());
return ret;
}
// A sample arranged as pairs of images half a turn apart: half the ordinals spread over the
// part of the sweep that has a partner 180 deg later, and those partners. That is what the
// Friedel mirror needs. It is NOT the plain selection rearranged - even on a full turn the two
// sets are all but disjoint, sharing 2 of 60 ordinals on a 3600 image sweep - so the caller
// keeps them apart and gives each consumer the one it needs. Empty when the sweep is shorter
// than half a turn, which is the caller's signal that the spot symmetry cannot be measured here.
std::vector<int> select_half_turn_paired_ordinals(int images_to_process, int requested_images,
float degrees_per_ordinal) {
if (!(degrees_per_ordinal > 0.0f))
return {};
const int half_turn = static_cast<int>(std::lround(180.0 / degrees_per_ordinal));
if (half_turn < 1 || images_to_process - half_turn < 1)
return {};
std::set<int> ordinals;
for (const int ordinal : select_equally_spaced_image_ordinals(images_to_process - half_turn,
requested_images / 2)) {
ordinals.insert(ordinal);
ordinals.insert(ordinal + half_turn);
}
return {ordinals.begin(), ordinals.end()};
}
// Apply a goniometer rotation SCALE: the stage turned k times the angle the file records. What is
// wrong is the SWEEP, not where it began, so the per-frame increment and the per-frame oscillation
// width both scale by k while the starting angle is left alone - the stage reached that position
// before the sweep started. A constant offset in phi is in any case exactly degenerate with a
// rotation of the crystal orientation about the spindle, which indexing refines away, so anchoring
// the stretched sweep at image 0 costs nothing and keeps the first frame at the angle the file gives
// it. Every angle in the pipeline comes from this object, so scaling it is the whole correction.
GoniometerAxis ScaleRotation(const GoniometerAxis &g, float k) {
GoniometerAxis scaled(g.GetName(), g.GetStart_deg(), g.GetIncrement_deg() * k,
g.GetAxis(), g.GetHelicalStep());
if (const auto wedge = g.GetScreeningWedge())
scaled.ScreeningWedge(*wedge * k);
return scaled;
}
// Writing an image to the process file takes the global HDF5 mutex, which is the same mutex every
// worker thread needs to fetch its next image. The write itself is short, but with a worker per
// hardware thread they spend longer queueing for the lock than the writing takes: on a 16 Mpx
// rotation dataset the per-image loop runs 7.7 s with 8 workers and 13.1 s with 48. Give the file
// to one thread and let the workers post to it, and the lock has one taker again.
//
// A DataMessage does not own its pixels - it points into the reader's buffer - so the raw image is
// parked in the queue beside it. Without that the worker frees the pixels on its next iteration and
// the writer reads whatever landed there.
class ProcessFileWriter {
public:
ProcessFileWriter(FileWriter &writer, size_t capacity)
: writer_(writer), capacity_(capacity),
thread_([this] { Run(); }) {}
// Post one image. Blocks while the queue is full, which is what stops a run whose analysis
// outpaces its writer from holding every image it has ever processed in memory.
void Post(const DataMessage &msg, std::shared_ptr<JFJochReaderRawImage> img) {
std::unique_lock lock(m_);
space_.wait(lock, [this] { return queue_.size() < capacity_ || failed_; });
if (failed_)
return; // the error is rethrown by Finish(); dropping the rest is deliberate
queue_.push_back({msg, std::move(img)});
work_.notify_one();
}
// Drain, join, and rethrow whatever the writer thread hit. Must be called before the
// FileWriter is used for anything else - the end message, Finalize().
void Finish() {
{
std::lock_guard lock(m_);
done_ = true;
}
work_.notify_one();
if (thread_.joinable())
thread_.join();
if (error_)
std::rethrow_exception(error_);
}
~ProcessFileWriter() {
{
std::lock_guard lock(m_);
done_ = true;
failed_ = true; // an unwinding run should not wait for the backlog to be written
}
work_.notify_one();
space_.notify_all();
if (thread_.joinable())
thread_.join();
}
private:
struct Job {
DataMessage msg;
std::shared_ptr<JFJochReaderRawImage> img;
};
void Run() {
while (true) {
Job job;
{
std::unique_lock lock(m_);
work_.wait(lock, [this] { return !queue_.empty() || done_; });
if (queue_.empty())
return;
job = std::move(queue_.front());
queue_.pop_front();
}
space_.notify_one();
try {
writer_.Write(job.msg);
} catch (...) {
std::lock_guard lock(m_);
if (!error_)
error_ = std::current_exception();
failed_ = true;
space_.notify_all();
return;
}
}
}
FileWriter &writer_;
const size_t capacity_;
std::mutex m_;
std::condition_variable work_;
std::condition_variable space_;
std::deque<Job> queue_;
bool done_ = false;
bool failed_ = false;
std::exception_ptr error_;
std::thread thread_;
};
}
// What the file says about the beam centre, before a frame is read. Nothing here measures anything -
// it reads the value that is there and says where it looks like it came from - and it is free, which
// is why it runs on every run whether or not a centre was asked for.
//
// A centre ON A MASKED PIXEL is the one of these worth acting on: a real beam does not sit on a dead
// pixel or in a module gap, so the value is wrong about something. Over the 29 non-SLS datasets whose
// outcome is on record (base rate of a bad outcome 48%) it fires on 4 and 3 of those 4 went wrong,
// against 2 false alarms in 42 in-house masters. What it does NOT promise is that correcting the
// centre repairs the run - on one of the three it does not.
//
// The other two are PROVENANCE, not prediction, and are said rather than warned about. A value equal
// to the geometric detector centre, or a whole number of pixels in both coordinates, was typed rather
// than measured - true of 29 of 42 in-house masters, and those are out by a median 3.5 px against
// 0.7 px for the rest - but a typed value is often close enough that as predictors of harm both sit
// at or below the base rate. They are still worth saying, because the spot-symmetry search reaches
// only about +-55 px from wherever it starts: a placeholder tens of pixels out is unreachable by
// construction, all five of its searches agree, and the small sigma they report says nothing is
// wrong. Only the background estimator recovers that case, and only a human reading this line knows
// to look.
// Proper rotations of a crystal system's holohedry - what a lattice of that class carries, and the
// bound SearchSpaceGroup caps its point-group candidates at.
static size_t HolohedryRotations(gemmi::CrystalSystem system) {
switch (system) {
case gemmi::CrystalSystem::Triclinic: return 1;
case gemmi::CrystalSystem::Monoclinic: return 2;
case gemmi::CrystalSystem::Orthorhombic: return 4;
case gemmi::CrystalSystem::Trigonal: return 6;
case gemmi::CrystalSystem::Tetragonal: return 8;
case gemmi::CrystalSystem::Hexagonal: return 12;
case gemmi::CrystalSystem::Cubic: return 24;
}
return 1;
}
// The lattice METRIC can carry more rotational symmetry than the group the intensities supported.
// That is not by itself a mistake - a pseudo-symmetric metric is ordinary, and only the intensities
// can say whether the extra rotations are real - but it is where the corpus's symmetry under-calls
// live, so the run says it out loud rather than leaving the user to notice. It decides nothing: no
// threshold, no promotion, no demotion.
//
// The metric symmetry is Le Page's, and its obliquity is the continuous measure of how nearly a
// two-fold holds; 1 degree is the middle of the band over which the answer is stable.
// Following Le Page (1982) J. Appl. Cryst. 15, 255-259
static void WarnIfMetricHostsMoreSymmetry(const std::optional<UnitCell> &cell,
const gemmi::SpaceGroup &sg) {
if (!cell)
return;
Logger logger("Rugnux");
const gemmi::UnitCell gc(cell->a, cell->b, cell->c, cell->alpha, cell->beta, cell->gamma);
const size_t metric_rotations =
gemmi::find_lattice_symmetry(gc, sg.centring_type(), LATTICE_MAX_OBLIQUITY_DEG).sym_ops.size();
const size_t group_rotations = sg.operations().sym_ops.size();
if (metric_rotations <= group_rotations)
return;
// The proper rotations of each holohedry, which is what find_lattice_symmetry returns.
const char *metric_system = "a higher";
switch (metric_rotations) {
case 2: metric_system = "monoclinic"; break;
case 4: metric_system = "orthorhombic"; break;
case 6: metric_system = "trigonal"; break;
case 8: metric_system = "tetragonal"; break;
case 12: metric_system = "hexagonal"; break;
case 24: metric_system = "cubic"; break;
default: break;
}
logger.Warning("The cell metric is {} - it admits {} rotations where {} has {}. Whether the extra "
"ones were tested against the intensities depends on the lattice class the search "
"was given: where that class is lower than the metric, they were never enumerated, "
"so their absence here is not evidence against them. This is a note, not a "
"correction. Where the higher symmetry is real, merging in it would raise "
"multiplicity and completeness.",
metric_system, metric_rotations, sg.xhm(), group_rotations);
}
static void LogHeaderBeamCenter(const DiffractionExperiment &experiment, const PixelMask &pixel_mask) {
Logger logger("Rugnux");
const float beam_x = experiment.GetBeamX_pxl();
const float beam_y = experiment.GetBeamY_pxl();
const auto width = static_cast<int>(experiment.GetXPixelsNumConv());
const auto height = static_cast<int>(experiment.GetYPixelsNumConv());
if (std::hypot(beam_x - static_cast<float>(width) / 2.0f,
beam_y - static_cast<float>(height) / 2.0f) < 1.0f)
logger.Info("Beam centre ({:.2f},{:.2f}) is the geometric centre of the detector - a value "
"written rather than measured", beam_x, beam_y);
else if (beam_x == std::round(beam_x) && beam_y == std::round(beam_y))
logger.Info("Beam centre ({:.0f},{:.0f}) is a whole number of pixels in both coordinates - a "
"value written rather than measured", beam_x, beam_y);
const int x = static_cast<int>(std::lround(beam_x));
const int y = static_cast<int>(std::lround(beam_y));
if (x >= 0 && y >= 0 && x < width && y < height
&& pixel_mask.GetMask()[static_cast<size_t>(y) * width + x] != 0)
logger.Warning("Beam centre ({:.2f},{:.2f}) lands on a masked pixel. A real beam does not, so "
"the value in the file is suspect; --estimate-beam-center measures it",
beam_x, beam_y);
}
Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
PixelMask pixel_mask, ProcessConfig config)
: reader_(reader), experiment_(std::move(experiment)),
pixel_mask_(std::move(pixel_mask)), config_(std::move(config)),
user_fixed_sg_(experiment_.GetGemmiSpaceGroup()) {
// Bit 9 describes where THIS run found the beam stop, so a mask read back from a file that
// already carries one starts clear; the user mask (bit 8) is left as it was loaded.
pixel_mask_.ClearBeamStopMask(experiment_);
LogHeaderBeamCenter(experiment_, pixel_mask_);
// A manually asserted stage calibration applies to everything, before anything reads an angle.
if (config_.rotation_scale.has_value())
if (const auto g = experiment_.GetGoniometer()) {
Logger("Rugnux").Info("Goniometer rotation scale {:.5f} applied from the command line",
*config_.rotation_scale);
experiment_.Goniometer(ScaleRotation(*g, *config_.rotation_scale));
}
}
// Report the detector geometry the way XDS states it, so a run can be compared line by line against
// the IDXREF.LP / CORRECT.LP of XDS on the same data. XDS is never given this geometry - the durin
// plugin hands it image data only (plugin_get_header returns dimensions, bytes per pixel, pixel size
// and frame count, nothing else) and XDS refines its own from XDS.INP - which is precisely why
// having ours in the same convention is worth printing.
//
// The two laboratory frames coincide: x along increasing detector column, y along increasing row,
// z along the beam. XDS places a pixel at
// x_lab(i,j) = (i-ORGX)*QX*X_axis + (j-ORGY)*QY*Y_axis + DETECTOR_DISTANCE*(X_axis x Y_axis)
// and DiffractionGeometry::LabCoord is that same expression with X_axis = poni_rot*(1,0,0) and
// Y_axis = poni_rot*(0,1,0). So the tilt goes out as the two axis vectors rather than as angles,
// which is also how XDS reports it after refinement. Taking the axes as differences of LabCoord
// keeps them in step with whatever the geometry currently is, tilt included.
//
// ORGX/ORGY are 1-based - XDS counts pixels from 1 - and are the foot of the perpendicular from the
// crystal rather than the direct beam, which is what our beam centre is too
// (docs/DETECTOR_GEOMETRY.md), so they map across without correction.
//
// ROTATION_AXIS is printed once per pass, and it is not always the vector stored in the file: the
// sign rescue further down adopts the opposite sign when the file's own indexes almost nothing, and
// the following pass then prints what it adopted. On the corpus the first and last line of a run
// disagree in sign on 12 of 115 rotation runs, so the LAST one - the axis the run actually
// integrated at - is the one to read.
//
// That sign is not reliably the one XDS wants - but it is not a constant either, so do NOT negate
// this line on principle. It is paired with the detector axis directions printed just above it and
// has to be read together with them: two rotation datasets whose detector axes are both +x/+y wanted
// the negation, while a third whose axes are negative wanted the sign as printed. The measure is the
// fraction of COLSPOT spots IDXREF can index - 8.9% as printed against 92.6% negated, and 1060/10080
// against 9624/10080; the other way round, 43.9% as printed against 13.5%.
//
// Read the HIGH member of each pair. That statistic has a chance floor which reaches about 15% on a
// dense reciprocal lattice - a null with the frame numbers permuted scored 14.9% against a real
// 15.4% - so a low value means "IDXREF gave up", not a measured near-zero: both wrong-sign runs
// above stopped on INSUFFICIENT PERCENTAGE (< 50%) OF INDEXED REFLECTIONS, one of them after
// refining the right cell and rejecting 9020 spots as too far from their ideal positions, which is
// the signature of a mis-signed spindle. So try both signs and keep the one that indexes; a sign
// carried over from another file is worth nothing.
//
// One of those two runs read a file our own converter had written, whose axis default was the
// negative sign. It therefore says what XDS does with a given sign, which is what this comment is
// about, and nothing about what a foreign file stores.
static void LogXDSGeometry(const DiffractionExperiment &experiment) {
Logger logger("Rugnux");
const auto geom = experiment.GetDiffractionGeometry();
const float beam_x = geom.GetBeamX_pxl();
const float beam_y = geom.GetBeamY_pxl();
const float pixel_mm = geom.GetPixelSize_mm();
const Coord origin = geom.LabCoord(beam_x, beam_y);
const Coord x_axis = ((geom.LabCoord(beam_x + 1.0f, beam_y) - origin) / pixel_mm).Normalize();
const Coord y_axis = ((geom.LabCoord(beam_x, beam_y + 1.0f) - origin) / pixel_mm).Normalize();
logger.Info("XDS convention: ORGX= {:.2f} ORGY= {:.2f} DETECTOR_DISTANCE= {:.4f}",
beam_x + 1.0f, beam_y + 1.0f, geom.GetDetectorDistance_mm());
logger.Info("XDS convention: DIRECTION_OF_DETECTOR_X-AXIS= {:.6f} {:.6f} {:.6f}",
x_axis.x, x_axis.y, x_axis.z);
logger.Info("XDS convention: DIRECTION_OF_DETECTOR_Y-AXIS= {:.6f} {:.6f} {:.6f}",
y_axis.x, y_axis.y, y_axis.z);
logger.Info("XDS convention: INCIDENT_BEAM_DIRECTION= 0 0 1 X-RAY_WAVELENGTH= {:.6f} QX= QY= {:.6f}",
geom.GetWavelength_A(), pixel_mm);
if (const auto goniometer = experiment.GetGoniometer())
logger.Info("XDS convention: ROTATION_AXIS= {:.6f} {:.6f} {:.6f}",
goniometer->GetAxis().x, goniometer->GetAxis().y, goniometer->GetAxis().z);
}
// The FFT recovers a basis vector only up to fft_max_unit_cell_A: FFTIndexer sizes its projected
// histogram from that value and the transform's last usable bin IS that length. So a reference cell
// longer than the bound cannot be represented by the search meant to find it - the run then reports
// "found no lattice", which is strictly worse than not giving the cell at all (measured on a 545 A
// axis: a partial triclinic solution without -C, nothing with it). Raise the bound to cover the cell,
// with the margin DIALS uses when a cell is known (max_cell = 1.3 * longest axis, see
// dials/algorithms/indexing/indexer.py find_max_cell).
//
// The floor at the short end is adjusted here too, for the same reason and against the same cell -
// see the second half of the body.
//
// ROTATION ONLY. Stills fire the FFT once per image across every worker and routinely run with a
// known cell, so widening the transform there would cost the whole serial run for nothing; the
// caller applies this inside its rotation branch.
static void RaiseFFTBoundForKnownCell(IndexingSettings &settings, const DiffractionExperiment &x) {
const auto cell = x.GetUnitCell();
if (!cell.has_value() || !cell->is_finite())
return;
constexpr float KNOWN_CELL_MARGIN = 1.3f;
const float longest = std::max({cell->a, cell->b, cell->c});
// Clamp rather than let the setter throw. A cell that needs more than the search can ever reach
// is a real possibility - the long-axis rescue calls this with whatever its coarse pass recovered,
// and that has been seen to be an implausible harmonic - and a bad candidate must not take the
// whole run down. Searching as far as the indexer goes is the best available answer; if the cell
// really is that long the pass simply fails to index, which the caller already handles.
const float needed = std::min(KNOWN_CELL_MARGIN * longest,
IndexingSettings::fft_max_unit_cell_limit_A);
if (needed > settings.GetFFT_MaxUnitCell_A()) {
settings.FFT_MaxUnitCell_A(needed);
Logger("Rugnux").Info("FFT search bound raised to {:.0f} A to cover the given cell "
"(longest axis {:.1f} A)", needed, longest);
}
// The same argument at the other end of the search. Every candidate whose reduced cell has an
// axis below fft_min_unit_cell_A is discarded, and the 10 A default is shorter than any protein
// axis but longer than a small molecule's: a given cell with an 8 A axis was thrown away before
// it could be compared against the reference it was supposed to match. Lower the floor to reach
// the cell that was asked for, with the reciprocal of the margin used above.
constexpr float KNOWN_CELL_FLOOR_MARGIN = 1.0f / KNOWN_CELL_MARGIN;
const float shortest = std::min({cell->a, cell->b, cell->c});
const float floor_A = std::max(KNOWN_CELL_FLOOR_MARGIN * shortest,
IndexingSettings::fft_min_unit_cell_limit_A);
if (floor_A < settings.GetFFT_MinUnitCell_A()) {
settings.FFT_MinUnitCell_A(floor_A);
Logger("Rugnux").Info("FFT shortest-axis floor lowered to {:.1f} A to cover the given cell "
"(shortest axis {:.1f} A)", floor_A, shortest);
}
}
std::optional<CalibrationResult> Rugnux::RebinAndRefit(const CalibrationResult &first,
int start_image, int end_image,
int images_to_process,
const AzimuthalIntegrationMapping &mapping,
const std::vector<SpotToSave> &spots) {
Logger logger("Rugnux");
const DiffractionGeometry before = experiment_.GetDiffractionGeometry();
// How far a geometry moves a ring ON THE DETECTOR, against the radial width of one profile bin.
// Below one bin, re-binning would put every pixel back where it already is. A beam-centre shift
// moves a ring by its own size; a distance change moves it in proportion to the ring's radius.
const auto [radius_min, radius_max] = ProfileRadiusRange_pxl(mapping, before);
if (!(radius_max > radius_min))
return std::nullopt;
const float bin_pxl = (radius_max - radius_min) / static_cast<float>(mapping.GetQBinCount());
const auto ring_gap_pxl = [&](const DiffractionGeometry &a, const DiffractionGeometry &b) {
return std::hypot(a.GetBeamX_pxl() - b.GetBeamX_pxl(), a.GetBeamY_pxl() - b.GetBeamY_pxl())
+ radius_max * std::abs(a.GetDetectorDistance_mm() - b.GetDetectorDistance_mm())
/ std::max(1e-3f, b.GetDetectorDistance_mm());
};
const auto ring_shift_pxl = [&](const DiffractionGeometry &g) { return ring_gap_pxl(g, before); };
// Which geometries are worth re-binning about. The fitted one is the obvious candidate, but it was
// itself fitted on the profile now believed mis-binned, so where the beam centre is badly out it is
// wrong and re-binning about it only digs deeper - measured, rms 3.75 -> 5.66 on an exposure 200 px
// off. The spots' geometry is the one that does not degrade there, having never read the header. So
// try both where they differ and keep whichever comes back better; they cost a read each, and a
// calibration run is a handful of images.
std::vector<DiffractionGeometry> candidates;
const auto add = [&](const DiffractionGeometry &g) {
if (!(ring_shift_pxl(g) > bin_pxl))
return; // this geometry bins the images the way they are already binned
for (const auto &c : candidates)
if (ring_gap_pxl(g, c) < bin_pxl)
return; // and this one bins them the way another candidate already does
candidates.push_back(g);
};
add(first.geometry);
if (first.spots_geometry)
add(*first.spots_geometry);
if (candidates.empty()) {
logger.Info("Powder calibration: the fitted geometry moves a ring by {:.2f} px, under the "
"{:.2f} px a profile bin spans - re-binning would change nothing",
ring_shift_pxl(first.geometry), bin_pxl);
return std::nullopt;
}
std::optional<CalibrationResult> best;
for (const auto &geometry : candidates) {
if (cancelled_)
break;
logger.Info("Powder calibration: re-integrating binned about beam ({:.1f},{:.1f}) at {:.3f} mm, "
"which moves a ring {:.2f} px against a {:.2f} px bin",
geometry.GetBeamX_pxl(), geometry.GetBeamY_pxl(),
geometry.GetDetectorDistance_mm(), ring_shift_pxl(geometry), bin_pxl);
// The mapping is the only thing the new geometry is needed for. Preprocessing does not depend on
// where the beam is, so it keeps experiment_ and nothing here mutates the run's own geometry.
DiffractionExperiment rebinned_experiment(experiment_);
rebinned_experiment.BeamX_pxl(geometry.GetBeamX_pxl()).BeamY_pxl(geometry.GetBeamY_pxl())
.DetectorDistance_mm(geometry.GetDetectorDistance_mm())
.PoniRot1_rad(geometry.GetPoniRot1_rad())
.PoniRot2_rad(geometry.GetPoniRot2_rad());
const AzimuthalIntegrationMapping rebinned_mapping(rebinned_experiment, pixel_mask_);
std::atomic<int> next_ordinal = 0;
std::mutex merge_m;
AzimuthalIntegrationProfile summed(rebinned_mapping);
const int nthreads = std::max(1, config_.nthreads);
std::vector<std::future<void>> workers;
workers.reserve(nthreads);
for (int t = 0; t < nthreads; ++t) {
workers.emplace_back(std::async(std::launch::async, [&]() {
std::vector<uint8_t> decompression_buffer;
JFJochReaderRawImage img;
ImagePreprocessorCPU preprocessor(experiment_, pixel_mask_);
ImagePreprocessorBuffer buffer(experiment_.GetPixelsNum());
AzIntEngineCPU azint(rebinned_mapping);
AzimuthalIntegrationProfile local(rebinned_mapping);
while (!cancelled_) {
const int ordinal = next_ordinal.fetch_add(1);
const int image_idx = start_image + ordinal * config_.stride;
if (image_idx >= end_image) break;
try {
if (!reader_.ReadRawImage(image_idx, img)) continue;
const uint8_t *image_ptr = img.image.GetUncompressedPtr(decompression_buffer);
preprocessor.Analyze(buffer, image_ptr, img.image.GetMode());
azint.Run(buffer, local);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Error("Failed to re-integrate image {}: {}", image_idx, e.what());
}
}
std::lock_guard lock(merge_m);
summed += local;
}));
}
for (auto &w : workers)
w.get();
if (cancelled_)
return std::nullopt;
// The profile is measured about this geometry, so this is also what the fit starts from and what
// its ring points are recovered against.
try {
auto refit = CalibrateFromProfile(summed.GetResult(), rebinned_mapping, geometry,
config_.calibrant_ring_q,
config_.calibration_refine_tilt, spots);
logger.Info("Powder calibration: that gives {} ring points at rms {:.3f} px",
refit.ring_points, refit.rms_radial_pxl);
// A refit that is not a measurement never beats one that is, whatever its residual. The
// residual of a fit with a parameter pinned at a value nobody measured is not comparable
// with the residual of one where every parameter was fitted - and this pass hands the
// NEXT pass the geometry it pins to, so taking a non-measurement here launders it into
// the following one's input.
if (!best || (refit.converged && !best->converged)
|| (refit.converged == best->converged
&& refit.ring_points * 2 >= best->ring_points
&& refit.rms_radial_pxl < best->rms_radial_pxl))
best = std::move(refit);
} catch (const std::exception &e) {
logger.Warning("Powder calibration: that profile could not be fitted ({})", e.what());
}
}
if (!best)
return std::nullopt;
// Only if it is actually better. Re-binning about a geometry that was itself wrong makes the fit
// worse, not better, and the first pass is then the answer to keep - it is measured on a profile
// binned about a geometry now believed wrong, which is weak, but it is the one the run has.
// Symmetric with the best-of fold above: a fit that is not a measurement never wins on residual,
// and never loses on one either. The one-directional form threw away a CONVERGED re-fit whose rms
// happened not to beat a non-converged first pass - which is the motivating case, where the first
// pass declined the tilt and pinned the header's.
if ((first.converged && !best->converged)
|| (first.converged == best->converged
&& (best->ring_points * 2 < first.ring_points
|| best->rms_radial_pxl >= first.rms_radial_pxl))) {
logger.Info("Powder calibration: the re-binned fit is no better than the first "
"({} points at rms {:.3f} against {} at {:.3f}) - keeping the first pass",
best->ring_points, best->rms_radial_pxl, first.ring_points, first.rms_radial_pxl);
return std::nullopt;
}
// The header the answer is compared against is the run's own, not the pass before it: a reader wants
// to know how far the calibration moved the FILE.
best->header_distance_mm = first.header_distance_mm;
return best;
}
void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, RugnuxObserver *observer) {
Logger logger("Rugnux");
// Two-pass rotation runs this twice. The shadow does not move, and re-detecting would find
// nothing (its pixels are masked by now) and so clear the mask the first pass established;
// the beam centre must not be re-read either, because by then the geometry is post-refined.
const auto &current = pixel_mask_.GetMask();
const bool have_shadow = std::any_of(current.begin(), current.end(),
[](uint32_t v) { return (v & (1u << PixelMask::BeamStopPixelBit)) != 0; });
const bool want_shadow = config_.detect_beam_stop.has_value() && !have_shadow;
const bool want_beam_center = config_.estimate_beam_center && !beam_center_placed_;
// The spot width is a property of the crystal, not of the geometry, so the second pass reuses the
// radius the first one measured rather than re-reading the frames.
//
// It is READ OFF the geometry as it stands, though, and a beam centre this same pass is about to
// correct leaves that geometry provisional: candidates are excluded within MIN_BEAM_DISTANCE_PX
// of the centre and stratified into resolution bands about it, so a placeholder header centre
// both measures a different set of spots and labels them at the wrong resolution - and
// spot_width_measured_ would then stop the corrected pass from ever revisiting it. Where a
// second pass follows, leave the width to that one, which starts from the corrected centre. A
// run with no second pass measures it here: a provisional radius beats none.
const bool second_pass_follows = want_beam_center && config_.rotation_indexing
&& config_.two_pass_rotation && experiment_.IsRotationIndexing();
const bool want_width = config_.adaptive_integration_radius && !spot_width_measured_
&& !second_pass_follows;
if (!want_shadow && !want_beam_center && !want_width)
return;
// The two consumers want different frames and each gets its own set. The shadow is built from
// the even spread over the sweep it has always been built from, so that asking for a beam
// centre cannot move the mask; the spot symmetry needs frames paired half a turn apart, where
// the sweep reaches that far. The two sets barely overlap, so the loop below walks their union
// and every frame is read once.
//
// Both samples are taken from the middle of the sweep, both ends left out: an even spread over
// [0, images_to_process) always picks the very first and the very last image, and those are the
// ones a shutter out of step spoils - the last carries under 90 % of the sample's median counts
// on 12 of 38 regression sweeps and is essentially unexposed on 5.
const auto goniometer = experiment_.GetGoniometer();
const int margin = std::min(PRESCAN_END_MARGIN_IMAGES, images_to_process / 10);
auto shadow_sample = select_equally_spaced_image_ordinals(images_to_process - 2 * margin,
frame_count);
auto spot_sample = want_beam_center && goniometer
? select_half_turn_paired_ordinals(
images_to_process - 2 * margin,
BEAM_CENTER_IMAGE_FACTOR * frame_count,
// Degrees per ORDINAL: the shifted goniometer's increment already
// is that, and rotation data cannot be strided anyway.
std::abs(goniometer->GetIncrement_deg()))
: std::vector<int>();
for (int &ordinal : shadow_sample)
ordinal += margin;
for (int &ordinal : spot_sample)
ordinal += margin;
const bool want_spot_symmetry = !spot_sample.empty();
const std::set<int> shadow_set(shadow_sample.begin(), shadow_sample.end());
const std::set<int> spot_set(spot_sample.begin(), spot_sample.end());
std::set<int> sample = shadow_set;
sample.insert(spot_set.begin(), spot_set.end());
if (sample.empty())
return;
// Say which of the two this is reading for. It runs before the first image of the run proper is
// processed, so without this the job sits at its initial status for the whole pre-scan - which is
// tens of seconds once the beam centre asks for its own frames on top of the projection.
if (observer)
observer->OnPhase(want_shadow ? (want_beam_center ? "Beam stop and beam centre" : "Beam stop")
: (want_beam_center ? "Beam centre" : "Spot width"));
ShadowFinder finder(experiment_, pixel_mask_);
// The spot symmetry and the spot width are read in the same pass as the projection rather than in
// a pre-pass of their own. The image is preprocessed a second time for them: the shadow is
// accumulated from raw counts and the spot finder works on the converted image.
const bool want_spots = want_spot_symmetry || want_width;
std::unique_ptr<AzimuthalIntegrationMapping> prescan_mapping;
if (want_spots)
prescan_mapping = std::make_unique<AzimuthalIntegrationMapping>(experiment_, pixel_mask_);
// Everything reading one frame needs that cannot be shared between threads: its own scratch for
// each of the two decompressions, its own preprocessed image and its own spot finder.
struct PreScanWorker {
std::vector<uint8_t> shadow_buffer;
std::vector<uint8_t> decompression_buffer;
JFJochReaderRawImage raw_image;
std::unique_ptr<ImagePreprocessorCPU> preprocessor;
std::unique_ptr<ImagePreprocessorBuffer> preprocessed;
std::unique_ptr<ImageSpotFinder> spot_finder;
};
const auto make_worker = [&] {
PreScanWorker w;
if (want_spots) {
w.preprocessor = std::make_unique<ImagePreprocessorCPU>(experiment_, pixel_mask_);
w.preprocessed = std::make_unique<ImagePreprocessorBuffer>(experiment_.GetPixelsNum());
if (config_.spot_finding.adaptive_threshold)
w.spot_finder = std::make_unique<AdaptiveSpotFinderCPU>(*prescan_mapping);
else
w.spot_finder = std::make_unique<ImageSpotFinderCPU>(experiment_.GetXPixelsNumConv(),
experiment_.GetYPixelsNumConv());
}
return w;
};
// The spots of one image, and what the two consumers take off them. Taken as a step of its own
// because the sparse-sweep pass below reads images the shadow does not. The frame they belong to
// is stamped by the caller, which is what keeps the pool independent of the order the workers
// finish in.
const auto find_spots = [&](PreScanWorker &w, CompressedImage &image, int image_idx,
bool for_beam_center, std::vector<BeamCenterSpot> &out,
bool for_width, std::vector<spot_width::FluxCurve> &curves,
std::vector<float> &spot_q) {
try {
w.preprocessor->Analyze(*w.preprocessed,
image.GetUncompressedPtr(w.decompression_buffer), image.GetMode());
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Warning("Pre-scan: failed to preprocess image {}: {}", image_idx, e.what());
return;
}
auto spots = w.spot_finder->Run(*w.preprocessed, config_.spot_finding);
// Every frame's WHOLE spot list feeds the powder measurement below - the rings are a property
// of the sample, and the frames read here are already spread over the sweep, which is what
// sharpens them against the crystal's own rows.
{
const auto &g = experiment_.GetDiffractionGeometry();
for (const auto &spot : spots) {
const Coord c = spot.RawCoord();
const float d = g.PxlToRes(c.x, c.y);
if (d > 0.0f && std::isfinite(d))
spot_q.push_back(2 * PI / d);
}
}
// The width goes first: it wants the whole spot list, to know which spots are isolated.
if (for_width)
MeasureSpotFluxCurves(*w.preprocessed, static_cast<int>(experiment_.GetXPixelsNumConv()),
static_cast<int>(experiment_.GetYPixelsNumConv()),
experiment_.GetDiffractionGeometry(), spots, curves);
if (!for_beam_center)
return;
// The strongest of a crowded frame: the symmetry is over-determined either way, and the
// matching is quadratic in the spots of one frame.
if (spots.size() > BEAM_CENTER_SPOTS_PER_IMAGE) {
std::partial_sort(spots.begin(), spots.begin() + BEAM_CENTER_SPOTS_PER_IMAGE, spots.end(),
[](const DiffractionSpot &a, const DiffractionSpot &b) {
return a.Count() > b.Count(); });
spots.resize(BEAM_CENTER_SPOTS_PER_IMAGE);
}
for (const auto &spot : spots) {
const Coord centroid = spot.RawCoord();
out.push_back({centroid.x, centroid.y, static_cast<float>(spot.Count()), 0});
}
};
std::vector<float> frame_angle_deg;
std::vector<float> prescan_spot_q;
std::vector<BeamCenterSpot> beam_center_spots;
std::vector<spot_width::FluxCurve> width_curves;
// Images the width ended up being measured on, for the log: the tiers below stop as soon as the
// answer has settled, so this is a property of the crystal and worth reporting.
size_t width_images = 0;
// Frames that could actually be read. Counted here rather than asked of the shadow finder, which
// is only fed when the shadow is wanted and would report zero for a pass that read every frame
// for the beam centre alone.
std::atomic<size_t> frames_read{0};
// Read the sample on several workers. The reader serialises on the HDF5 lock, but the
// decompression, the projection and the spot finding - which is all of the cost on a large
// detector - run in parallel. Each worker accumulates into a shard of its own, so nothing is
// locked while an image is added, and the per-frame results are stitched together in sample
// order below so the beam centre sees the same input however the workers interleaved.
{
const std::vector<int> ordinals(sample.begin(), sample.end());
std::vector<std::vector<BeamCenterSpot>> spots_of(ordinals.size());
std::vector<std::vector<spot_width::FluxCurve>> curves_of(ordinals.size());
std::vector<std::vector<float>> spot_q_of(ordinals.size());
// A frame joins the pool only if it could be read, as it did when this was a serial loop.
std::vector<char> spot_read(ordinals.size(), 0);
const size_t nworkers = std::min<size_t>(std::max<size_t>(config_.nthreads, 1),
std::min(PRESCAN_MAX_WORKERS, ordinals.size()));
finder.SetShardCount(nworkers);
// The order the frames are visited in, and where each width tier ends in it. The tiers
// stride the PROJECTION's sample rather than the union read below, so asking for a beam
// centre - which brings frames of its own - cannot move which frames the width is measured
// on; the frames only the beam centre wants are appended to the last tier. Without a width
// to measure there is one tier and the sample is read in its own order.
std::vector<size_t> visit;
std::vector<size_t> tier_end;
visit.reserve(ordinals.size());
{
std::vector<size_t> width_pos;
for (size_t i = 0; i < ordinals.size(); i++)
if (shadow_set.contains(ordinals[i])) width_pos.push_back(i);
std::vector<char> taken(ordinals.size(), 0);
const size_t n_tier = want_width ? WIDTH_TIER_STRIDE.size() : 1;
for (size_t tier = 0; tier < n_tier; tier++) {
const size_t stride = want_width ? WIDTH_TIER_STRIDE[tier] : 1;
for (size_t j = 0; j < width_pos.size(); j += stride)
if (!taken[width_pos[j]]) {
taken[width_pos[j]] = 1;
visit.push_back(width_pos[j]);
}
tier_end.push_back(visit.size());
}
for (size_t i = 0; i < ordinals.size(); i++)
if (!taken[i]) visit.push_back(i);
tier_end.back() = visit.size();
}
// Kept across the tiers rather than rebuilt for each. A worker's preprocessed image is a
// buffer the size of the detector, so each is still built on its own worker the first time
// that worker runs and its pages are touched by the thread that reads them.
std::vector<PreScanWorker> workers(nworkers);
std::vector<char> worker_built(nworkers, 0);
bool width_settled = !want_width;
std::optional<float> previous_r80;
size_t phase_begin = 0;
for (const size_t phase_end : tier_end) {
std::atomic<size_t> next{phase_begin};
std::vector<std::future<void>> futures;
futures.reserve(nworkers);
for (size_t t = 0; t < nworkers; t++)
futures.emplace_back(std::async(std::launch::async, [&, t] {
if (!worker_built[t]) {
workers[t] = make_worker();
worker_built[t] = 1;
}
PreScanWorker &w = workers[t];
for (size_t v = next.fetch_add(1); v < phase_end; v = next.fetch_add(1)) {
const size_t i = visit[v];
const int ordinal = ordinals[i];
const int image_idx = start_image + ordinal * config_.stride;
bool read = false;
try {
read = reader_.ReadRawImage(image_idx, w.raw_image);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Warning("Pre-scan: failed to load image {}: {}", image_idx,
e.what());
continue;
}
if (!read) continue;
DataMessage msg{};
msg.image = w.raw_image.image;
msg.number = ordinal;
msg.original_number = image_idx;
frames_read.fetch_add(1, std::memory_order_relaxed);
// Gated on want_shadow, not on the frame set alone: AddImage projects the
// whole image, and with --detect-beam-stop=off (or a shadow already in the
// mask) that is the pass's largest per-frame cost paid for a result nobody
// reads. The set still decides WHICH frames feed it, so the mask is built
// from the frames it always was.
if (want_shadow && shadow_set.contains(ordinal))
finder.AddImage(msg, w.shadow_buffer, t);
// The width is measured on the projection's own even spread over the sweep,
// so asking for a beam centre cannot move it either.
const bool for_beam_center = spot_set.contains(ordinal);
const bool for_width = !width_settled && shadow_set.contains(ordinal);
if (for_beam_center) spot_read[i] = 1;
if (for_beam_center || for_width)
find_spots(w, msg.image, image_idx, for_beam_center, spots_of[i],
for_width, curves_of[i], spot_q_of[i]);
}
}));
for (auto &f : futures) f.get();
phase_begin = phase_end;
// What this tier makes of the width, pooled in sample order so the answer does not
// depend on how the workers interleaved. A tier too sparse to measure settles nothing:
// the test is against what the smaller sample actually said, not against the radius it
// fell back to.
if (width_settled) continue;
width_images = static_cast<size_t>(
std::count_if(visit.begin(), visit.begin() + static_cast<std::ptrdiff_t>(phase_end),
[&](size_t i) { return shadow_set.contains(ordinals[i]); }));
width_curves.clear();
for (const auto &c : curves_of)
width_curves.insert(width_curves.end(), c.begin(), c.end());
const auto r80 = spot_width::R80AtReference(width_curves);
width_settled = r80 && previous_r80 && spot_width::WidthSettled(*r80, *previous_r80);
previous_r80 = r80;
}
for (const auto &q : spot_q_of)
prescan_spot_q.insert(prescan_spot_q.end(), q.begin(), q.end());
// Frame numbering follows the sample order, exactly as the serial read did.
for (size_t i = 0; i < ordinals.size(); i++) {
if (!spot_read[i]) continue;
const int frame = static_cast<int>(frame_angle_deg.size());
// The goniometer has been shifted so that a LOCAL ordinal maps to the angle of the
// original image it came from (see RunPipeline), so the angle is asked for by ordinal,
// as everywhere else.
frame_angle_deg.push_back(goniometer->GetAngle_deg(static_cast<float>(ordinals[i])));
for (auto &spot : spots_of[i]) {
spot.frame = frame;
beam_center_spots.push_back(spot);
}
}
}
// The integration radius from the recorded spot width. r1 is the aperture the integrator learns
// the profile width over and a second moment over a disk of radius a saturates at a^2/4, so at the
// shipped r1 = 4 the learned sigma cannot exceed 2 px and a wider spot is fitted with a profile
// the model cannot represent. r2 follows r1, and r3 is taken so that the r2..r3 background ring
// keeps the area it has at the shipped 4/6/13 - a ring that shrinks with the disk is what makes a
// bare --integration-radius worse than the default it replaces.
if (want_width) {
spot_width_measured_ = true;
const auto r80 = spot_width::R80AtReference(width_curves);
if (!r80) {
logger.Info("Spot width: not measurable on {} spots from {} images, keeping the "
"integration radius at r1={:.1f}", width_curves.size(), width_images,
experiment_.GetBraggIntegrationSettings().GetR1());
} else {
const float r1 = spot_width::R1ForWidth(*r80);
const float r2 = r1 + 2.0f;
const float r3 = std::sqrt(r2 * r2 + 133.0f);
BraggIntegrationSettings bis = experiment_.GetBraggIntegrationSettings();
// Remembered where the APERTURE moved, so the starvation guard in RunAllPasses has
// something to give back and cannot announce a fallback that changes nothing.
//
// The test is on all three radii and not on r1 alone. It is the same test today - r2 and
// r3 are taken from r1 just above, and the adaptive rule does not run at all when the
// radii were given explicitly - so this changes no run. It is written this way because
// what the guard is about is the aperture, and r1 is only a proxy for it. A prototype
// that widened r2 and r3 at high angle while leaving r1 at the shipped 4 took the
// neighbour-starved fraction of a dense low-symmetry crystal from 2.4% to 9.0%, eight
// times the bound this guard exists to enforce, and the guard could not fire: nothing
// had set bragg_before_adaptive_, because r1 had not moved.
if (r1 != bis.GetR1() || r2 != bis.GetR2() || r3 != bis.GetR3())
bragg_before_adaptive_ = bis;
bis.R1(r1).R2(r2).R3(r3);
experiment_.ImportBraggIntegrationSettings(bis);
logger.Info("Spot width: r80 = {:.2f} px at {:.0f} A ({} spots from {} images) => "
"integration radii r1={:.1f} r2={:.1f} r3={:.2f}", *r80, spot_width::D_REF_A,
width_curves.size(), width_images, r1, r2, r3);
}
}
// Powder contamination, measured on every run that finds spots here. A crystalline phase other
// than the crystal - hexagonal ice, a shower of microcrystals, salt out of the cryoprotectant -
// diffracts as rings, and its spots are handed to the indexer and to scaling as if they were this
// crystal's. Only ice can be named in advance; the rest has to be measured, and measuring it costs
// nothing here, the spots having been found for the width and the centre already.
//
// Reported on every run whatever it says. What is DONE about it is decided later and on evidence:
// a first pass that fails indexes again with the rings set aside and keeps whichever answer indexes
// more frames (see RunPipeline). This line is the diagnostic, and it is the one that tells a user
// their crystal was in a powder.
{
const float w = config_.spot_finding.ice_ring_width_Q_recipA;
powder_ = MeasurePowderRings(prescan_spot_q, w);
prescan_spot_quantile_d_A_.clear();
for (const float frac : {0.25f, 0.5f})
if (const auto d = SpotResolutionQuantile(prescan_spot_q, frac))
prescan_spot_quantile_d_A_.push_back(*d);
if (!powder_.rings_q_recipA.empty()) {
logger.Info("Powder rings: {} rings over {} pooled spots hold {:.0f}% of them above the "
"smooth spot density - the coarsest at {:.2f} A, the finest at {:.2f} A{}",
powder_.rings_q_recipA.size(), prescan_spot_q.size(),
100.0f * powder_.spot_fraction,
2 * PI / powder_.rings_q_recipA.front(),
2 * PI / powder_.rings_q_recipA.back(),
powder_.resolved_to_d_A
? fmt::format("; past {:.2f} A the rings have merged and cannot be told "
"from the crystal", *powder_.resolved_to_d_A)
: std::string());
}
}
if (frames_read.load() == 0) {
logger.Warning("Pre-scan: no image could be read. Skipping.");
return;
}
if (want_shadow) {
// Every pixel is compared against the ring it sits on, so the rings have to be drawn about
// the beam. A centre far enough out draws them across the background's own radial fall-off
// instead of along it, and what comes back describes the fall-off rather than the hardware:
// displacing a centre by 50 px on a clean sweep leaves the mask where it was, by 150 px
// masks a tenth of the detector. A centre in a file can be that wrong - one sweep here
// states one 351 px from where the data put it - so the rings are drawn about the centre
// this same projection measures rather than the one the file claims. That fit costs no
// frame of its own, and an unmasked shadow biases it by a few pixels at most, which is
// three orders below what this comparison would notice. The centre the run reports and
// consumes is the one fitted below instead, with the shadow out of the way.
if (const auto ring_center = FindBeamCenterFromBackground(experiment_, pixel_mask_,
finder.GetMeanProjection())) {
finder.BeamCenter(ring_center->beam_x_pxl, ring_center->beam_y_pxl);
logger.Info("Beam stop: comparing each pixel against its ring about the measured "
"centre ({:.2f},{:.2f}) +- {:.2f} px, {:.2f} px from the file's",
ring_center->beam_x_pxl, ring_center->beam_y_pxl, ring_center->sigma_pxl,
std::hypot(ring_center->beam_x_pxl - experiment_.GetBeamX_pxl(),
ring_center->beam_y_pxl - experiment_.GetBeamY_pxl()));
} else {
logger.Info("Beam stop: the background does not place a centre on this run, so the "
"rings are drawn about the file's ({:.2f},{:.2f})",
experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl());
}
const auto shadow = finder.GetMask(config_.nthreads);
const auto shadowed = std::count(shadow.begin(), shadow.end(), 1u);
pixel_mask_.LoadBeamStopMask(experiment_, shadow);
// Not "this beamline has no beam stop": it is also what a comparison drawn about the wrong
// point returns, so an unexpected answer here is worth reading against the centre named
// above first.
if (shadowed == 0)
logger.Warning("Beam stop: no shadow found in {} images. The comparison is made about "
"the centre named above, so this may point at that centre rather than "
"at the experiment",
finder.GetFrameCount());
else
logger.Info("Beam stop shadow: {} pixels ({:.2f}% of the detector) found in {} images",
shadowed, 100.0 * static_cast<double>(shadowed) / static_cast<double>(shadow.size()),
finder.GetFrameCount());
}
// The beam centre is fitted again here, so that the shadow is in the mask by the time it is
// placed and the holder arm - the largest azimuthal asymmetry on the detector - is out of the
// way of the background the centre is read from. This is the answer the run reports and
// consumes; the fit above only chose where to draw the rings, and is thrown away.
// What the scattered background makes the centre, measured on EVERY run and committed on none.
// The fit reads the projection the beam-stop pre-pass has just built and no frame of its own, so
// it costs a fraction of a second on top of a pass that has already run - which is what makes it
// affordable to do always, and doing it always is the point. A header beam centre is the metadata
// field the field jokes about, and nothing in any package says how wrong a given one is: on a run
// that indexes and merges perfectly well this line still reports that the file claims a centre the
// data put 73 px away. The run is not touched by it. What consumes the number is the second first
// pass in RunPipeline, and --estimate-beam-center's fall-through below, which is the same fit.
//
// Only where the projection exists: with --detect-beam-stop=off nothing accumulates it, and
// projecting every frame for this alone is the largest per-frame cost of the pre-scan. On the
// second pass of a two-pass run the shadow is already in the mask, so this does not run again -
// by then the centre has been post-refined and the file's value is no longer what is in it.
if (want_shadow && config_.beam_center_check) {
const auto t0 = std::chrono::steady_clock::now();
BeamCenterFFTResult capture;
background_center_ = FindBeamCenter(experiment_, pixel_mask_, finder.GetMeanProjection(),
config_.nthreads, &capture);
measured_beam_center_ = background_center_;
const auto capture_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0).count();
if (!capture.point.empty())
logger.Info("Beam centre capture: the strongest point-symmetry centre on the WHOLE "
"detector is ({:.1f},{:.1f}), {:.1f} px from the file's; line mirrors put x "
"at {:.1f} and y at {:.1f}; margins {:.1f}%/{:.1f}%/{:.1f}% (point/x/y, "
"under ~3% the surface is flat). One transform set scores every candidate "
"centre, so this costs the same however wrong the file is - {} ms including "
"the walk that refines it",
capture.point[0].beam_x_pxl, capture.point[0].beam_y_pxl,
std::hypot(capture.point[0].beam_x_pxl - experiment_.GetBeamX_pxl(),
capture.point[0].beam_y_pxl - experiment_.GetBeamY_pxl()),
capture.line_x.empty() ? std::numeric_limits<float>::quiet_NaN()
: capture.line_x[0].beam_x_pxl,
capture.line_y.empty() ? std::numeric_limits<float>::quiet_NaN()
: capture.line_y[0].beam_y_pxl,
100.0f * capture.margin_point, 100.0f * capture.margin_line_x,
100.0f * capture.margin_line_y, capture_ms);
const float need = BeamCenterNeed_pxl(experiment_);
if (!background_center_) {
logger.Info("Beam centre check: neither the capture nor the walk places the centre on "
"this run - the file's ({:.2f},{:.2f}) is neither confirmed nor contradicted",
experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl());
} else {
// Against the DIRECT BEAM, not the PONI. Both the FFT capture and the walk read the
// centrosymmetry of the scattered background about the point the beam lands on; the
// geometry's beam_x_pxl is the PONI, which parts from it by distance*tan(rot)/pixel as
// soon as the detector is tilted. Measured on a calibrant at 110/150/200/300 mm the
// capture sits 0.5-1.5 px from the fitted direct beam and 5.0/6.8/8.4/17.9 px from the
// fitted PONI, so comparing it with the PONI reports a disagreement the data do not
// have - on the in-house files, whose header carries a 0.234 deg tilt, 3.3 px of it at
// 120 mm against a stated need of 0.7 px.
const auto [direct_x, direct_y] = experiment_.GetDiffractionGeometry().GetDirectBeam_pxl();
const float dx = background_center_->beam_x_pxl - direct_x;
const float dy = background_center_->beam_y_pxl - direct_y;
const float moved = std::hypot(dx, dy);
const auto across = AcrossSpindle_pxl(experiment_, dx, dy);
// Everything a reader needs to judge the header, on one line: what the file says, what the
// background says, how far apart they are, how well the background knows its own answer,
// and what this particular geometry asks of the centre. The last is what turns a bare pixel
// count into a verdict - 3 px is nothing at 100 mm and a lost lattice at 500 mm.
logger.Info("Beam centre check: the file puts the direct beam at ({:.2f},{:.2f}), the "
"scattered background "
"puts it at ({:.2f},{:.2f}) +- {:.2f} px - a difference of {:.2f} px, against the "
"{:.2f} px this geometry asks the centre to be right to. Measured only; the "
"run keeps the centre in the file",
direct_x, direct_y,
background_center_->beam_x_pxl, background_center_->beam_y_pxl,
background_center_->sigma_pxl, moved, need);
// Three sigma, because the fit is a PRECISION and not an accuracy: on a background with no
// curvature there is nothing to separate shift from amplitude and it follows the noise in
// g', confidently. A move it cannot resolve is not evidence about the header either way.
if (moved <= 3.0f * background_center_->sigma_pxl)
logger.Info("Beam centre check: that is under three times the fit's own sigma, so the "
"file's centre is as good as this measurement can tell - the second first "
"pass below indexes it anyway, because a fraction of a pixel is enough to "
"decide a cell and no sigma small enough to gate on would notice");
else if (across && *across > need)
// Info and not a warning: `need` is where the FFT amplitude at a 200 A axis has fallen
// to 0.70, which is a floor under what matters and not a threshold for failure - it is
// exceeded on two of three ordinary in-house sweeps whose lattice comes out right. What
// is worth a warning is what the second pass below then finds, not this.
logger.Info("Beam centre check: {:.2f} px of the difference is ACROSS the spindle, more "
"than the {:.2f} px this geometry absorbs, and a centre error is fixed in "
"the lab frame - it smears the accumulated reciprocal-space cloud, so the "
"first pass can lose peaks or take an axis harmonic. The second first pass "
"below indexes both centres and reports whether it did", *across, need);
else
logger.Info("Beam centre check: that is a real difference, and {} - the second first "
"pass below indexes both centres and compares what they give",
across ? "it is mostly along the spindle, where nothing announces itself"
: "this is not a rotation sweep, so there is no spindle to resolve it on");
}
}
if (!want_beam_center)
return;
beam_center_placed_ = true;
// The spot symmetry first where the sweep supports it: it is exact geometry rather than a
// property of the scattering, and it measures the sets the background cannot - a flat radial
// profile at long wavelength leaves that fit nothing to bite on. The background answers
// wherever the sweep does not reach half a turn, which is every still and every screening
// wedge, so the two cover between them what neither covers alone.
std::string source = "spot symmetry";
std::optional<BeamCenterEstimate> estimate;
SpindleEstimate spindle;
// The bound an estimate has to meet. One-sided by construction: the flat constant was calibrated
// on a 244 mm / 0.95 A / 0.15 mm geometry and is far too tight for a loose one, so where the
// geometry asks less of the centre the bound is relaxed to what it asks and never below the
// constant. Nothing it accepts today is lost.
const float need = BeamCenterNeed_pxl(experiment_);
const float ceiling = std::max(MAX_BEAM_CENTER_SIGMA_PXL, need);
if (want_spot_symmetry)
estimate = FindBeamCenterFromSpotSymmetry(experiment_, frame_angle_deg, beam_center_spots,
config_.fit_spindle ? &spindle : nullptr);
// Where it does not come out - no answer at all, or one that moves when the search is started
// elsewhere - read more of the sweep and ask again. Both estimators are counts of matched spot
// PAIRS, the Friedel one over frames half a turn apart and the second crossing over any two
// frames one reflection is recorded on, so a sweep whose frames carry few spots, or whose pairs
// match rarely, is measurable only from more of them. More data first and a weaker estimator
// second: this runs before the fall-through, and only what survives it falls through. The extra
// frames go to the spot finder alone, so the beam-stop projection keeps the images it was
// validated on.
if (want_spot_symmetry && (!estimate || estimate->sigma_pxl > ceiling)) {
// What has already been read FOR THE SPOTS, which is not the whole sample: the frames the
// shadow was built from carry none of them, and testing against the union would silently
// skip the ones this pass exists to read.
std::vector<int> extra;
for (int ordinal : select_half_turn_paired_ordinals(
images_to_process - 2 * margin, BEAM_CENTER_SPARSE_IMAGES,
std::abs(goniometer->GetIncrement_deg()))) {
ordinal += margin;
if (!spot_set.contains(ordinal))
extra.push_back(ordinal);
}
if (!extra.empty()) {
logger.Info("Beam centre: the spot symmetry does not come out on {} frames "
"({} spots); reading {} more frames",
frame_angle_deg.size(), beam_center_spots.size(), extra.size());
// Its own phase: this reads several times the frames the pass above did, and is the one
// step of the pre-scan a user is likely to sit through wondering what is happening.
if (observer)
observer->OnPhase("Beam centre (reading more frames)");
// Read on workers as above - this pass is several times the size of the first one - and
// append in `extra` order so the pool does not depend on how they interleaved.
std::vector<std::vector<BeamCenterSpot>> extra_spots(extra.size());
std::vector<char> extra_read(extra.size(), 0);
const size_t nworkers = std::min<size_t>(std::max<size_t>(config_.nthreads, 1),
std::min(PRESCAN_MAX_WORKERS, extra.size()));
std::atomic<size_t> next{0};
std::vector<std::future<void>> futures;
futures.reserve(nworkers);
for (size_t t = 0; t < nworkers; t++)
futures.emplace_back(std::async(std::launch::async, [&] {
PreScanWorker w = make_worker();
for (size_t i = next.fetch_add(1); i < extra.size(); i = next.fetch_add(1)) {
const int image_idx = start_image + extra[i] * config_.stride;
bool read = false;
try {
read = reader_.ReadRawImage(image_idx, w.raw_image);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Warning("Pre-scan: failed to load image {}: {}", image_idx, e.what());
continue;
}
if (read) {
extra_read[i] = 1;
// The extra frames go to the beam centre alone; the width and the powder
// rings have already been measured on the projection's own sample.
std::vector<float> unused_q;
find_spots(w, w.raw_image.image, image_idx, true, extra_spots[i],
false, width_curves, unused_q);
}
}
}));
for (auto &f : futures) f.get();
for (size_t i = 0; i < extra.size(); i++) {
if (!extra_read[i]) continue;
const int frame = static_cast<int>(frame_angle_deg.size());
frame_angle_deg.push_back(goniometer->GetAngle_deg(static_cast<float>(extra[i])));
for (auto &spot : extra_spots[i]) {
spot.frame = frame;
beam_center_spots.push_back(spot);
}
}
estimate = FindBeamCenterFromSpotSymmetry(experiment_, frame_angle_deg,
beam_center_spots,
config_.fit_spindle ? &spindle : nullptr);
}
}
if (config_.fit_spindle && std::isfinite(spindle.azimuth_rad))
logger.Info("Spindle: azimuth about the beam {:.3f} mrad, tip towards it {:.3f} mrad "
"(Friedel vote {:.0f} -> {:.0f})",
1e3 * spindle.azimuth_rad, 1e3 * spindle.tip_rad,
spindle.vote_excess_nominal, spindle.vote_excess);
if (!estimate || estimate->sigma_pxl > ceiling) {
// Say so. Below about 220 deg of sweep the spot symmetry never clears this bound and the
// background answers instead - measured within 0.23 px of the full-sweep centre at every span
// from 120 to 220 deg - so the fall-through is what covers short sweeps, and until this line
// existed nothing in the log said the spot arm had even been tried.
if (want_spot_symmetry && estimate)
logger.Info("Beam centre: the spot symmetry answers ({:.2f},{:.2f}) at sigma {:.2f} px, "
"over the {:.2f} px ceiling - falling through to the background",
estimate->beam_x_pxl, estimate->beam_y_pxl, estimate->sigma_pxl, ceiling);
else if (want_spot_symmetry)
logger.Info("Beam centre: the spot symmetry does not come out on this sweep - falling "
"through to the background");
source = "background";
// The same fit the check above has already done wherever the projection exists - same mask,
// same geometry, same projection - so this asks for it again only where it was not run.
estimate = background_center_
? background_center_
: FindBeamCenter(experiment_, pixel_mask_, finder.GetMeanProjection(),
config_.nthreads);
}
if (!estimate) {
logger.Info("Beam centre: not measurable, keeping ({:.2f},{:.2f})",
experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl());
return;
}
const float dx = estimate->beam_x_pxl - experiment_.GetBeamX_pxl();
const float dy = estimate->beam_y_pxl - experiment_.GetBeamY_pxl();
const float moved = std::hypot(dx, dy);
const auto across = AcrossSpindle_pxl(experiment_, dx, dy);
// Two conditions, and the second is the new one. A move under three times the estimator's own
// sigma is not a measurement of anything, and a centre that is not moved cannot move the
// two-pass loop off its fixed point either. On the 39 rotation regression crystals this adopts
// 38 and keeps the header on one, a 0.74 px move at sigma 0.32.
const bool commit = estimate->sigma_pxl <= ceiling && moved > 3.0f * estimate->sigma_pxl;
const char *verdict = "COMMIT";
if (!commit)
verdict = estimate->sigma_pxl > ceiling ? "reject: sigma over the ceiling (kept header)"
: "reject: the move is under 3 sigma (kept header)";
logger.Info("Beam centre from {}: ({:.2f},{:.2f}) -> ({:.2f},{:.2f}), moved {:.2f} px, "
"sigma {:.2f} px against a {:.2f} px ceiling => {}",
source, experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(),
estimate->beam_x_pxl, estimate->beam_y_pxl, moved, estimate->sigma_pxl,
ceiling, verdict);
if (across)
logger.Info("Beam centre: {:.2f} px of that move is across the spindle, where this geometry "
"asks for {:.2f} px. That is a floor and not a tolerance - a move 0.03x of it has "
"decided a halved axis", *across, need);
if (across && *across > need)
logger.Warning("Beam centre: the file's centre is {:.2f} px out across the spindle, more than "
"the {:.2f} px this geometry absorbs - expect the first pass to lose peaks or "
"to take an axis harmonic", *across, need);
if (commit)
experiment_.BeamX_pxl(estimate->beam_x_pxl).BeamY_pxl(estimate->beam_y_pxl);
}
void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_process,
RugnuxObserver *observer) {
Logger logger("Rugnux");
// Rotation has its own two-pass; this is stills only.
if (experiment_.IsRotationIndexing()) {
logger.Warning("--refine-geometry is stills-only; rotation data uses two-pass indexing. Ignoring.");
return;
}
// The bundle anchors a known cell; without one there is nothing to refine against.
const auto cell = experiment_.GetUnitCell();
if (!cell.has_value()) {
logger.Warning("--refine-geometry needs a reference unit cell (-C / -S / reference MTZ). Skipping.");
return;
}
const int refine_frames = std::max(1, config_.refine_geometry.value());
if (observer)
observer->OnPhase("Geometry refinement (first pass)");
const auto dataset = reader_.GetDataset();
// Index a spread sample large enough to yield ~refine_frames strong frames even at a low hit rate.
const int sample_budget = std::min(images_to_process, std::max(refine_frames * 50, 8000));
const std::vector<int> sample = select_equally_spaced_image_ordinals(images_to_process, sample_budget);
// First-pass engines, built from the current (nominal) geometry: an index-only pass (no
// integration) to obtain each frame's spots + assigned HKL + orientation.
AzimuthalIntegrationMapping mapping(experiment_, pixel_mask_);
IndexerThreadPool pool(experiment_.GetIndexingSettings(), IndexerConstruction::OnFirstUse);
IndexAndRefine indexer(experiment_, &pool, /*retain_outcomes=*/false);
auto pass_settings = config_.spot_finding;
pass_settings.enable = true;
pass_settings.indexing = true;
pass_settings.quick_integration = false;
// A frame is worth bundling only if it indexed this many spots; the bundle then takes the
// refine_frames strongest of those.
constexpr int MIN_STRONG_SPOTS = 20;
// Stop sampling once there are comfortably more strong frames than the bundle will use, instead of
// draining the whole budget. The budget is sized for a low-hit-rate dataset (refine_frames * 50, at
// least 8000), so on data that indexes well it meant re-indexing the ENTIRE run to keep 200 frames -
// 99% of this pass was sampling. Four times the bundle leaves the "strongest N" selection a real
// pool to choose from.
const int strong_target = refine_frames * 4;
// The sample is cut into a FIXED number of interleaved stripes - stripe s is sample positions
// s, s+STRIPES, s+2*STRIPES, ... - and each stripe stops once it has contributed its share of the
// strong frames. Two things follow, both of which the previous shared-cursor-with-a-shared-stop
// arrangement got wrong. Every stripe is spread over the WHOLE run, so stopping early no longer
// means fitting the geometry to the beginning of it. And which frames get examined depends only
// on the data: it is not a race between the workers, and it does not change with -N, because a
// stripe is processed identically whichever worker happens to claim it.
constexpr int STRIPES = 32;
const int stripe_strong_target = std::max(1, (strong_target + STRIPES - 1) / STRIPES);
std::vector<GeomRefineFrame> frames;
std::mutex frames_mutex;
std::atomic<int> next_stripe = 0;
std::atomic<int> examined = 0;
auto worker = [&]() {
pin_gpu(); // round-robin per worker thread; must precede engine construction
// Before the analysis engine, which page-locks these bytes for its uploads and unregisters
// them when it is destroyed: it must not outlive the buffer it was given.
JFJochReaderRawImage img;
MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, indexer,
/*enable_fused_adaptive_gpu=*/true);
AzimuthalIntegrationProfile profile(mapping);
for (int stripe = next_stripe.fetch_add(1); stripe < STRIPES && !cancelled_;
stripe = next_stripe.fetch_add(1)) {
int stripe_strong = 0;
for (int idx = stripe; idx < static_cast<int>(sample.size()) && !cancelled_
&& stripe_strong < stripe_strong_target; idx += STRIPES) {
examined.fetch_add(1, std::memory_order_relaxed);
const int ordinal = sample[idx];
const int image_idx = start_image + ordinal * config_.stride;
bool read = false;
try {
read = reader_.ReadRawImage(image_idx, img);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Warning("Geometry refinement: failed to load image {}: {}", image_idx, e.what());
continue;
}
if (!read) continue;
DataMessage msg{};
msg.image = img.image;
msg.number = ordinal;
msg.original_number = image_idx;
if (dataset->efficiency.size() > image_idx)
msg.image_collection_efficiency = dataset->efficiency[image_idx];
try {
analysis.Analyze(msg, profile, pass_settings);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
continue;
}
if (!msg.indexing_result.value_or(false) || !msg.indexing_lattice.has_value())
continue;
GeomRefineFrame f;
f.lattice = *msg.indexing_lattice;
f.ordinal = ordinal;
for (const auto &s : msg.spots)
if (s.indexed && s.lattice == 0)
f.spots.push_back(GeomRefineSpot{s.x, s.y,
static_cast<int32_t>(s.h), static_cast<int32_t>(s.k), static_cast<int32_t>(s.l)});
if (f.spots.size() >= 6) {
if (static_cast<int>(f.spots.size()) >= MIN_STRONG_SPOTS)
stripe_strong++;
const std::unique_lock ul(frames_mutex);
frames.push_back(std::move(f));
}
}
}
};
std::vector<std::future<void>> futures;
futures.reserve(config_.nthreads);
for (int i = 0; i < config_.nthreads; ++i)
futures.push_back(std::async(std::launch::async, worker));
for (auto &fut : futures)
fut.get();
if (cancelled_)
return;
// Keep the strongest frames (most indexed spots) for the bundle - the true cell indexes many
// spots per frame, and orientation diversity comes for free from independent serial stills.
std::vector<GeomRefineFrame> strong;
for (auto &f : frames)
if (static_cast<int>(f.spots.size()) >= MIN_STRONG_SPOTS)
strong.push_back(std::move(f));
// frames is in the order the workers happened to finish, so the spot count alone does not order
// it: without the ordinal tie-break, equally strong frames would swap places between runs and a
// different bundle would be refined.
std::sort(strong.begin(), strong.end(),
[](const GeomRefineFrame &a, const GeomRefineFrame &b) {
if (a.spots.size() != b.spots.size()) return a.spots.size() > b.spots.size();
return a.ordinal < b.ordinal;
});
if (static_cast<int>(strong.size()) > refine_frames)
strong.resize(refine_frames);
logger.Info("Geometry refinement: indexed {} of {} sampled frames (budget {}), bundling {} strong "
"frames (>= {} spots)",
frames.size(), std::min<int>(examined.load(), static_cast<int>(sample.size())),
sample.size(), strong.size(), MIN_STRONG_SPOTS);
GeometryRefinerSettings gr_settings;
gr_settings.crystal_system = experiment_.GetCrystalSystem();
gr_settings.num_threads = config_.nthreads;
const GeometryRefinerResult r = RefineGlobalGeometry(
experiment_.GetDiffractionGeometry(), *cell, strong, gr_settings);
if (!r.ok) {
logger.Warning("Geometry refinement did not run (too few strong frames / did not converge); "
"keeping the input geometry");
return;
}
logger.Info("Geometry refinement: beam ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, distance {:.4f} -> {:.4f} mm",
experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(), r.beam_x_px, r.beam_y_px,
experiment_.GetDetectorDistance_mm(), r.distance_mm);
logger.Info("Geometry refinement: cell a/b/c {:.3f}/{:.3f}/{:.3f} -> {:.3f}/{:.3f}/{:.3f} A "
"(median residual {:.3f} px, {} frames, {} spots)",
cell->a, cell->b, cell->c, r.cell.a, r.cell.b, r.cell.c,
r.median_residual_px, r.frames_used, r.spots_used);
experiment_.BeamX_pxl(r.beam_x_px).BeamY_pxl(r.beam_y_px).DetectorDistance_mm(r.distance_mm);
experiment_.SetUnitCell(r.cell);
// The beam centre now comes from indexed spots, which the background cannot better.
beam_center_placed_ = true;
}
ProcessResult Rugnux::Run(RugnuxObserver *observer) {
// Each pass times only itself, so the canonical result's processing_time_s is the last pass alone -
// on the rotation two-pass that is under a third of what the run actually took, the pre-scan and the
// first pass being invisible in it. Time the whole thing here, where every pass is inside.
const auto run_start = std::chrono::steady_clock::now();
ProcessResult result = RunAllPasses(observer);
result.total_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - run_start).count();
return result;
}
namespace {
// How much better the alternative beam centre's search merge has to be for the run to move off
// the centre the file gave it (the beam-centre arbitration in RunAllPasses).
constexpr double MAX_CC_HALF_LOSS = 0.05;
// Say so rather than printing a zero that reads as a measurement. ("not compared", not "not
// measured": completeness IS measured and reported; what did not happen is its use in arbitrating
// between the two passes.)
std::string SearchMergeCompletenessText(const ProcessResult &pass1, const ProcessResult &pass2) {
if (!pass1.search_merge_completeness_measured || !pass2.search_merge_completeness_measured)
return "completeness not compared";
return fmt::format("completeness {:.1f}% vs {:.1f}%", pass2.search_merge_completeness,
pass1.search_merge_completeness);
}
// Whether the two passes describe their lattice in the same setting, which is what makes a count
// of reflections at a given Miller index the same count on both sides. A pass that re-indexes at
// the refined geometry can settle on a centred setting of the same lattice the other pass held
// primitive; the cells are then different bases and only the counts taken over all reflections
// survive the change.
bool SameSetting(const ProcessResult &pass1, const ProcessResult &pass2) {
if (pass1.consensus_centering != pass2.consensus_centering
|| !pass1.consensus_cell || !pass2.consensus_cell)
return false;
const UnitCell &c1 = *pass1.consensus_cell, &c2 = *pass2.consensus_cell;
const auto same_length = [](double a, double b) { return std::abs(a - b) <= 0.05 * (a + b) / 2; };
return same_length(c1.a, c2.a) && same_length(c1.b, c2.b) && same_length(c1.c, c2.c);
}
// Pass 2 is normally the better answer, which is why it is the canonical output - but it is not
// guaranteed to be. Returns why the refined pass is worse, in the words the run reports it in, or
// an empty string when it is not.
//
// The comparison is made on each pass's SEARCH merge, not on the final one. Each pass determines
// its own space group, so the two final merges can be in different groups, and a completeness or a
// correlation in different groups are not the same measurement. The search merge is P1 on both
// sides (or the group the user fixed), over the full resolution range and before any correction
// surface is fitted - and it is counted over the centring-allowed reflections on both sides, so
// that a pass which finds a centring the other did not is not judged on the extinguished half it
// merged as noise (see the capture in RunPipeline).
//
// What the two passes are compared on there is the SIGNAL each measured - the reflections it
// merged at I/sigma >= 2 - and not, as this guard decided until it was measured, that merge's
// CC1/2. Two measurements settled it:
// - Against an EXTERNAL arbiter (the site's own XDS merge of the same images, scored over a
// resolution range common to the arms) the count of unique reflections at I/sigma >= 2 agreed
// with which geometry is the more accurate one on 23 of 27 arm-dataset pairs. R_meas, CC1/2
// and ISa - which measure whether a merge agrees with ITSELF - agreed on 13 of 27, a coin flip.
// - This merge's CC1/2 is pooled over the whole range, uncut and uncorrected, so the shells with
// no signal in them dominate it and they are exactly the shells a geometry move disturbs: it
// reads 0.13 on a crystal whose data merge at 0.995, and 0.93 against 0.76 on two runs whose
// merged data agree to 0.7% of R_meas. Deciding on it sent two crystals back to a geometry
// that ships 22% fewer reflections than the one they refused.
//
// Three ways the refined pass can be worse, then: it merges more unique reflections than the cell
// it settled on can hold, it measured decisively less signal, or it lost the axial rows.
//
// The completeness arm only bites when the search merge is in a fixed group (-S). De novo that
// merge is in P1 and does not count its possible reflections at all, so there is no completeness
// and it does not arm. Counting them there was measured and not taken: it costs 5.2 ms, but a
// merge that fills its own asymmetric unit reads 27-43% of the P1 hemisphere over 28 rotation
// crystals, so the 100.5% bound is never approached and no decision changes - it would only put a
// figure into the report that reads as the dataset's completeness and is not one.
//
// Asked twice, from the same numbers: in RunAllPasses, where the answer is acted on, and inside a
// refined pass as soon as its own search merge is in - a pass that already knows it will be thrown
// away leaves its report and its files unwritten (see the superseded flag in RunPipeline).
std::string RefinedPassIsWorse(const ProcessResult &pass1, const ProcessResult &pass2) {
constexpr double MAX_CREDIBLE_COMPLETENESS = 100.5; // rounding headroom, nothing more
// How much of the signal the refined pass has to lose before the run moves back off the
// geometry it measured. This is not a quality bar: it is set above what the two passes differ
// by for reasons that are not the geometry, and they never integrate quite the same way - the
// canonical pass integrates at the measured spot width where the pre-pass integrated at the
// default, and it predicts with the pre-pass's smoothed mosaicity. Measured over ~100 rotation
// crystals, that alone moves the count by -2% to +35%, and on a crystal where the two passes
// ran at the SAME geometry it moved it +10% (and the CC1/2 this guard used to decide on by
// -0.25). Below this, the difference says nothing about the geometry, and a comparison that
// cannot tell keeps the geometry the run measured.
constexpr double DECISIVE_SIGNAL_LOSS = 0.10;
// ... and the loss has to be a loss rather than a dilution. The wider disk pulls weaker
// reflections into the merge and the error model is then refitted over them, so a pass can hold
// MORE reflections and fewer of them at I/sigma >= 2 without having measured less of the
// crystal: measured, one crystal merges 2.1x the reflections in its refined pass, a third of
// the strong ones, and its refined pass is the better answer by every final statistic. Where
// the refined pass holds no more reflections and fewer strong ones, the loss is a loss.
//
// The other arm: the axial rows. A bulk statistic is an average over tens of thousands of
// reflections and the principal axial rows are a few dozen of them, so a pass can measure half
// the row and move it by nothing - yet those are the reflections the systematic absences are
// read off, and losing them costs a screw axis and with it the space group. It happens because
// a reflection on an axial row crosses the Ewald sphere over a narrow range of the sweep, where
// a general reflection is spread over it: a pass that loses a wedge of the sweep loses whole
// axial rows while its bulk statistics improve. Measured over a dozen rotation crystals the two
// passes agree on this count to within 9% and usually exactly; the crystal this arm was written
// for measured 44%.
//
// It is asked only where the two counts are counts of the SAME reflections: the a and b of a
// C-centred setting are the face diagonals of the primitive cell, so "h00" is a different
// direction in the two merges and the two rows are not comparable at all. And it needs pass 1
// to have measured enough of the rows for the difference to mean anything - below eight there
// is no zone a screw could be claimed on either side. A pass that measured decisively more
// signal has said something about the bulk of the data that a few dozen reflections cannot
// answer, and is kept.
constexpr double MIN_AXIAL_ROWS_KEPT = 0.75;
constexpr int64_t MIN_AXIAL_ROWS_TO_COMPARE = 8;
const double strong1 = static_cast<double>(pass1.search_merge_strong_reflections);
const double strong2 = static_cast<double>(pass2.search_merge_strong_reflections);
const int64_t axial1 = pass1.search_merge_axial_reflections;
const int64_t axial2 = pass2.search_merge_axial_reflections;
const bool measured_less = strong1 > 0 && strong2 < (1.0 - DECISIVE_SIGNAL_LOSS) * strong1
&& pass2.search_merge_reflections <= pass1.search_merge_reflections;
const bool measured_more = strong1 > 0 && strong2 > (1.0 + DECISIVE_SIGNAL_LOSS) * strong1;
const bool lost_axial_rows = SameSetting(pass1, pass2)
&& axial1 >= MIN_AXIAL_ROWS_TO_COMPARE
&& axial2 < MIN_AXIAL_ROWS_KEPT * static_cast<double>(axial1)
&& !measured_more;
const bool worse = pass2.lattice_conflicts_with_fixed_sg
|| (pass1.search_merge_completeness_measured
&& pass2.search_merge_completeness_measured
&& pass2.search_merge_completeness > MAX_CREDIBLE_COMPLETENESS)
|| measured_less || lost_axial_rows;
if (!worse)
return {};
return lost_axial_rows && !measured_less
? fmt::format("it holds {} of the {} low-order axial reflections the header-geometry "
"pass measured, and its {} reflections at I/sigma >= 2 against {} do not "
"say it is better", axial2, axial1,
pass2.search_merge_strong_reflections,
pass1.search_merge_strong_reflections)
: fmt::format("{}, {} reflections at I/sigma >= 2 before corrections against {}",
SearchMergeCompletenessText(pass1, pass2),
pass2.search_merge_strong_reflections,
pass1.search_merge_strong_reflections);
}
}
ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) {
// Only the refined passes of the rotation two-pass are judged against a header-geometry pass, and
// only while this holds one; a second Run() on the same object must not inherit the last one's.
quality_guard_pass1_.reset();
// Rotation two-pass geometry post-refinement: the first pass integrates at the header geometry and
// post-refines the detector geometry (distance + beam from the observed spot positions, cell scale + axis
// from phi_obs); the second pass re-indexes and re-integrates with the refined geometry. Only the second
// (refined-geometry) pass writes merged files, under the plain "<prefix>_*" name. The first pass still runs
// under a "<prefix>_01" prefix so that the _process.h5 it writes, when one is asked for, does not collide
// with the second pass's - the ice-ring flags and geometry stored in each are the ones that pass used.
if (config_.mode == ProcessMode::FullAnalysis && config_.rotation_postrefine_geometry
&& experiment_.IsRotationIndexing()) {
Logger logger("Rugnux");
const std::string base_prefix = config_.output_prefix;
const auto gonio_snapshot = experiment_.GetGoniometer();
// What the first pass starts from, so a second first pass at a different beam centre can start
// from the same place (the beam-centre arbitration below).
const DiffractionExperiment experiment_before_first_pass = experiment_;
const int64_t max_spot_count_snapshot = experiment_.GetDatasetSettings().GetMaxSpotCount();
prepass_detector_geometry_.reset();
prepass_rotation_scale_.reset();
prepass_result_.reset();
force_rotation_result_.reset();
bragg_adaptive_.reset();
logger.Info("Rotation two-pass geometry post-refinement: first pass (header geometry) -> {}_01_*",
base_prefix);
// Keep an empty prefix empty (the "compute stats, persist nothing" mode): the "_01" suffix would
// make it non-empty and RunPipeline would then write stray _01_* files despite no output wanted.
config_.output_prefix = base_prefix.empty() ? base_prefix : base_prefix + "_01";
auto pass1 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/true);
pass1.pass_number = 1;
pass1.pass_count = 2;
if (cancelled_) { config_.output_prefix = base_prefix; return pass1; }
// Beam centre as a LATTICE hypothesis, decided on the merge. The first pass indexed both
// centres, found the same cell with a different metric symmetry read off it, and measured that
// the centre disagreement is by itself large enough to have decided that symmetry (see the
// beam-centre check in RunPipeline). The class is handed to GeometryRefiner, which holds the
// cell to it for the rest of the run, so the wrong one costs an axis - refined, predicted,
// integrated and merged with an equality the crystal does not have.
//
// The disagreement is two-sided, so there is no centre to prefer: over the corpus runs that
// reach here the file's centre finds the higher class about as often as the measured one does.
// Nothing before integration separates them either - both index the same frames, and the frame
// count is the one statistic the check already refuses to arbitrate on. So run the first pass
// again at the other centre and keep whichever arm MERGES better, each judged on its own search
// merge: P1, whole range, before any correction surface, which is the merge the two passes are
// already compared in at the end of this function. Because the alternative arm is a whole first
// pass it gets the short-axis pass and the geometry post-refinement of its own, which the
// question needs - on a crystal with an axis under the FFT floor the check runs before that
// pass and compares supercells.
//
// Adopting the alternative moves the run off the geometry it was given, so it has to be
// decisively better rather than merely ahead.
if (!cancelled_ && beam_center_alternative_) {
const auto alt_center = *beam_center_alternative_;
beam_center_alternative_.reset();
logger.Info("Beam centre check: running the first pass again at the measured centre "
"({:.2f},{:.2f}) and judging the two on their merges", alt_center[0],
alt_center[1]);
const DiffractionExperiment file_experiment = experiment_;
const auto file_mosaicity = prepass_mosaicity_;
const auto file_geometry = prepass_detector_geometry_;
const auto file_scale = prepass_rotation_scale_;
const auto file_result = prepass_result_;
experiment_ = experiment_before_first_pass;
// The measured spot width is a property of the crystal and was measured once, before this
// arm; put it back on the experiment so this pass narrows to the pre-pass radius from the
// same place the first arm did and leaves the same widened radii behind for pass 2.
if (bragg_adaptive_)
experiment_.ImportBraggIntegrationSettings(*bragg_adaptive_);
experiment_.BeamX_pxl(alt_center[0]).BeamY_pxl(alt_center[1]);
prepass_mosaicity_.clear();
prepass_detector_geometry_.reset();
prepass_rotation_scale_.reset();
prepass_result_.reset();
ProcessResult alt;
try {
alt = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/true);
} catch (const std::exception &e) {
// A hypothesis that does not survive its own pass is not a failure of the run: the
// file's centre has already produced a complete first pass, and this arm exists only
// to be compared against it.
logger.Warning("Beam centre check: the pass at the measured centre did not complete "
"({}) - the run keeps the file's centre", e.what());
}
alt.pass_number = 1;
alt.pass_count = 2;
const bool alt_merges_better = !cancelled_ && alt.has_merge_statistics
&& pass1.has_merge_statistics
&& alt.search_merge_cc_half
> pass1.search_merge_cc_half + MAX_CC_HALF_LOSS;
if (alt_merges_better) {
const auto cell_text = [](const ProcessResult &r) {
if (!r.consensus_cell)
return std::string("no cell");
const auto &c = *r.consensus_cell;
return fmt::format("{:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}",
c.a, c.b, c.c, c.alpha, c.beta, c.gamma);
};
logger.Warning("Beam centre check: the measured centre merges better - CC1/2 before "
"corrections {:.3f} against {:.3f} on the search merge, on {} against "
"{} - so the run adopts it and the metric symmetry it finds",
alt.search_merge_cc_half, pass1.search_merge_cc_half,
cell_text(alt), cell_text(pass1));
pass1 = std::move(alt);
} else {
logger.Info("Beam centre check: the measured centre does not merge better (CC1/2 "
"before corrections {:.3f} against {:.3f} on the search merge) - the run "
"keeps the file's centre and the symmetry it found",
alt.search_merge_cc_half, pass1.search_merge_cc_half);
experiment_ = file_experiment;
prepass_mosaicity_ = file_mosaicity;
prepass_detector_geometry_ = file_geometry;
prepass_rotation_scale_ = file_scale;
prepass_result_ = file_result;
}
// The arm's own pass asks the same question again at its own centre; the run has already
// decided it, so nothing is to consume a second answer.
beam_center_alternative_.reset();
if (cancelled_) { config_.output_prefix = base_prefix; return pass1; }
}
// Undo the pre-pass goniometer shift (a sub-range or strided run moves start/increment inside
// the pass), but KEEP an axis direction the pass changed. The rotation-axis sign rescue runs
// inside the pass and adopts the opposite axis when the file's own indexes almost nothing;
// restoring the snapshot wholesale threw that away, handing pass 2 the sign that failed and
// then applying pass 1's fitted rotation scale to the axis it was not fitted on. It
// self-healed - pass 2's rescue fires again and a wrong sign indexes almost no frames - but
// only by repeating a whole first pass, and only while "a wrong sign always scores below
// half" holds, which is an invariant nothing states or tests.
if (gonio_snapshot) {
GoniometerAxis restored = *gonio_snapshot;
if (const auto after_pass1 = experiment_.GetGoniometer())
restored.Axis(after_pass1->GetAxis());
experiment_.Goniometer(restored);
}
// A measured spot budget is the pass's own, not the run's: give the second pass the same list to
// measure from, so the two passes cannot ratchet each other down. The canonical pass then reads
// the budget off the refined geometry, where the spots that do lie on the lattice actually do.
experiment_.MaxSpotCount(max_spot_count_snapshot);
// Apply the post-refined detector geometry for the second pass, keeping the header geometry so
// the run can go back to it if the refined pass turns out worse (see the quality guard below).
const std::array<float, 5> header_geometry{experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(),
experiment_.GetDetectorDistance_mm(),
experiment_.GetPoniRot1_rad(),
experiment_.GetPoniRot2_rad()};
if (prepass_detector_geometry_) {
const auto &g = *prepass_detector_geometry_;
experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2])
.PoniRot1_rad(g[3]).PoniRot2_rad(g[4]);
}
// ... and the goniometer rotation scale, on the same measure-then-re-integrate footing: the angles
// in the file are the commanded ones, so a stage that ran fast is a geometry error like any other.
if (prepass_rotation_scale_ && gonio_snapshot) {
experiment_.Goniometer(ScaleRotation(*gonio_snapshot, *prepass_rotation_scale_));
// The pre-pass mosaicity is a width in degrees fitted against the angles the second pass has
// just stopped using, and the override can only ever raise the second pass's own estimate (it
// takes the larger of the two). Carrying it over would hold the second pass at the rocking
// width the uncorrected angles produced - the correction half-applied. Drop it and let the
// second pass fit its own.
prepass_mosaicity_.clear();
logger.Info("Two-pass: goniometer rotation scale {:.5f} will drive the second integration pass "
"(oscillation {:.4f} -> {:.4f} deg per image)", *prepass_rotation_scale_,
gonio_snapshot->GetIncrement_deg(),
gonio_snapshot->GetIncrement_deg() * *prepass_rotation_scale_);
}
// The integration radius the pre-scan measured from the crystal's own spot width, applied now
// that the geometry is fitted. The pre-pass ran at the radius the run started with (see
// bragg_adaptive_), so everything pass 2 consumes from it - the post-refined distance and beam,
// the space group, the smoothed mosaicity - is what the fixed radius produces, and the widened
// disk changes only the intensities of the canonical pass.
if (bragg_adaptive_) {
experiment_.ImportBraggIntegrationSettings(*bragg_adaptive_);
logger.Info("Two-pass: the canonical pass integrates at the measured spot width, "
"r1={:.1f} r2={:.1f} r3={:.2f}", bragg_adaptive_->GetR1(),
bragg_adaptive_->GetR2(), bragg_adaptive_->GetR3());
}
// Let every refined pass from here on ask the quality guard's question of itself, so one that
// is going to be thrown away can skip the report and the files it would have superseded.
quality_guard_pass1_ = pass1;
logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical) -> {}_*",
base_prefix);
config_.output_prefix = base_prefix; // the refined pass is the canonical result (no _02 suffix)
// The canonical pass measures the post-refinement again where the pre-pass moved the geometry
// (it applies nothing), so that the walk below has something to decide on.
postrefine_probe_ = prepass_detector_geometry_.has_value();
auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
// Walk the geometry to the fit's fixed point. The post-refinement is fitted to reflections
// integrated at the very geometry it is correcting, so one fit reaches only as far as those
// reflections see: on a header whose distance is several percent long, the pre-pass absorbs the
// error into an inflated cell, and from that self-consistent pair the fit finds a fraction of it
// and stops. The canonical pass has just re-integrated and re-indexed at what it committed, so
// asking the same fit there is the next step of the same walk - and that re-indexing is also the
// RATIFICATION the old size bound stood in for: a second lattice does not index better at a
// geometry it did not come from, a wrong header does, and a pass that lost the crystal is thrown
// out by the quality guard below, which returns the run to the header geometry.
//
// What starts the walk is a move of more than one step: that is the fit saying the geometry is
// somewhere else entirely rather than a fraction of a percent away, and it leaves every run
// whose fit settles beside its header at the two passes it always had. Once the run IS walking
// it keeps walking while the fit still commits, because the steps do not get smaller in
// proportion to what is left: measured on a header 3.9 % long, the first round took a third of
// the error and each of the next took a third of the rest. The walk is bounded rather than
// stopped at a size, because there is no size that separates the last step of a real walk from
// the first step of a runaway - and it does not run away: started BELOW the optimum the same fit
// turns back up, so it settles around the answer instead of past it.
constexpr int MAX_GEOMETRY_ROUNDS = 4;
std::optional<std::array<float, 5>> adopted_geometry;
int geometry_rounds = 0;
while (!cancelled_ && geometry_rounds < MAX_GEOMETRY_ROUNDS && prepass_detector_geometry_
&& pass2.post_refine && pass2.post_refine->detector_refined
&& (pass2.post_refine->large_move || geometry_rounds > 0)) {
adopted_geometry = *prepass_detector_geometry_;
const auto &g = *adopted_geometry;
logger.Info("Two-pass: the post-refinement at the adopted geometry still moves it "
"(distance {:.3f} -> {:.3f} mm) - re-integrating and re-indexing at what it "
"asks for (round {} of at most {})", pass2.post_refine->distance_before_mm,
pass2.post_refine->distance_after_mm, geometry_rounds + 1,
MAX_GEOMETRY_ROUNDS);
experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2])
.PoniRot1_rad(g[3]).PoniRot2_rad(g[4]);
pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
++geometry_rounds;
}
postrefine_probe_ = false;
// The post-refinement is measured by pass 1 and consumed by pass 2, so carry it onto whichever
// result is returned - it is a result of the run, not of the pass that happened to fit it. Where
// the walk took rounds they are ONE geometry decision, so what the report names is where the
// geometry started and where the run left it, not the first step of the way there.
if (geometry_rounds > 0 && pass1.post_refine && adopted_geometry) {
pass1.post_refine->beam_x_after_px = (*adopted_geometry)[0];
pass1.post_refine->beam_y_after_px = (*adopted_geometry)[1];
pass1.post_refine->distance_after_mm = (*adopted_geometry)[2];
}
pass2.post_refine = pass1.post_refine;
pass2.pass_number = 2 + geometry_rounds;
pass2.pass_count = 2 + geometry_rounds;
if (geometry_rounds > 0)
pass2.pass_decision = fmt::format(
"post-refined geometry adopted over {} rounds: the fit was still walking at each of "
"them, and the detector distance ended at {:.3f} mm against the header's {:.3f}",
geometry_rounds + 1, (*adopted_geometry)[2],
pass1.post_refine ? pass1.post_refine->distance_before_mm : 0.0);
// r2 = r1 + 2 is also the inner edge of the background ring, so widening the signal disk pushes
// that ring out into the neighbours; a reflection whose ring is left with five or fewer clean
// pixels has no background and is dropped whole. On a pattern dense enough in three dimensions
// at once that costs a quarter of the observations - and NOTHING available before integrating
// predicts it: the crystal it happens to has a mid-table predicted spot spacing, wider than
// crystals that survive r1 = 12. What does measure it is the integrator counting its own drops.
// The pass that integrates at the widened radius is now the canonical one, so the measurement
// and the remedy are both here: re-run it at the radius that keeps the data, the same remedy the
// two guards below use and, like them, a pass only on a crystal that was going to be wrong.
if (!cancelled_ && bragg_adaptive_ && bkg_starved_fraction_
&& *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION) {
logger.Info("Two-pass: at the widened integration radius r1={:.1f} the neighbouring "
"reflections left {:.2f}% of the predicted reflections without a background "
"ring (bound {:.2f}%), so this pass is re-run at r1={:.1f} r2={:.1f} r3={:.2f}",
bragg_adaptive_->GetR1(), 100.0 * *bkg_starved_fraction_,
100.0 * spot_width::BKG_STARVED_MAX_FRACTION,
bragg_before_adaptive_->GetR1(), bragg_before_adaptive_->GetR2(),
bragg_before_adaptive_->GetR3());
const double starved = *bkg_starved_fraction_;
const float wide_r1 = bragg_adaptive_->GetR1();
const int passes = pass2.pass_count + 1;
experiment_.ImportBraggIntegrationSettings(*bragg_before_adaptive_);
bragg_adaptive_.reset();
pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
pass2.post_refine = pass1.post_refine;
pass2.pass_number = passes;
pass2.pass_count = passes;
pass2.pass_decision = fmt::format(
"post-refined geometry adopted, re-run at the fixed integration radius: at r1={:.1f} "
"neighbouring reflections left {:.2f}% of the predicted reflections without a "
"background ring", wide_r1, 100.0 * starved);
}
// Supercell-collapse guard for the second pass. The de-novo lattice search can be BISTABLE on a
// periodically-modulated dataset: the tiny post-refine geometry move tips it out of the true-cell
// basin onto a spurious SUPERCELL (a large multiple of pass-1's cell volume), and the space group
// reused from pass-1 is then stamped onto that wrong-shaped cell - catastrophic. Pass 1 (header
// geometry, un-perturbed) found the correct lattice, so re-run the second pass with pass-1's FULL
// indexing result forced (its refined lattice + orientation + rotation axis + geometry - not just the
// cell, which loses the per-frame orientation and integrates worse): the re-run then matches de-novo
// quality but at the refined geometry. Only fires when pass 2 is markedly LARGER than pass 1 - a
// genuine de-novo demotion to a smaller primitive (the pseudo-symmetry case the second pass re-indexes
// de novo to catch) has pass2 <= pass1 and is left untouched.
if (!cancelled_ && prepass_result_ && pass1.consensus_cell && pass2.consensus_cell) {
// PRIMITIVE volumes, like the two guards in RunPipeline: a centred conventional cell is an
// exact integer multiple of its primitive one (C/I 2x, R 3x, F 4x), so two settings of the
// SAME lattice differ by that factor and comparing the conventional cells reads a mere
// change of setting as a supercell - which then forces pass 1's setting and, with it, its
// lower symmetry. Measured: every dataset this guard fired on was at an exact centring
// multiplicity, one of them an F-centred lattice held down to P1 at 3.99x.
const auto primitive_volume = [](const UnitCell &cell, std::optional<char> centering) {
return std::abs(CrystalLattice(cell).ToPrimitive(centering.value_or('P')).CalcVolume());
};
const double v1 = primitive_volume(*pass1.consensus_cell, pass1.consensus_centering);
const double v2 = primitive_volume(*pass2.consensus_cell, pass2.consensus_centering);
if (v1 > 0.0 && v2 > 1.5 * v1) {
logger.Info("Two-pass: second-pass primitive cell volume {:.0f} A^3 is {:.2f}x pass-1's "
"({:.0f} A^3) - a spurious supercell; re-running the second pass with "
"pass-1's result forced",
v2, v2 / v1, v1);
force_rotation_result_ = *prepass_result_;
// Pass 1's result carries pass 1's goniometer, and the per-image path takes its angles
// from there - forcing it whole would put the uncorrected angles back and silently undo
// the rotation scale for the re-run.
if (prepass_rotation_scale_ && force_rotation_result_->axis)
force_rotation_result_->axis =
ScaleRotation(*force_rotation_result_->axis, *prepass_rotation_scale_);
pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
pass2.post_refine = pass1.post_refine;
pass2.pass_number = 3;
pass2.pass_count = 3;
pass2.pass_decision = fmt::format(
"post-refined geometry adopted, re-run with pass-1's lattice forced: the de-novo "
"primitive cell volume {:.0f} A^3 was {:.2f}x pass-1's ({:.0f} A^3), a spurious "
"supercell", v2, v2 / v1, v1);
force_rotation_result_.reset();
}
}
// The refined pass is normally the better answer, which is why it is the canonical output -
// but it is not guaranteed to be, and the guard that decides is RefinedPassIsWorse above. It
// has already been asked once, inside the pass itself, which is how a pass that is about to be
// thrown away knows not to write a report; this is where the answer is acted on.
if (!cancelled_ && pass1.has_merge_statistics && pass2.has_merge_statistics) {
const std::string worse = RefinedPassIsWorse(pass1, pass2);
if (!worse.empty()) {
logger.Warning("Two-pass: the refined pass is worse than the header-geometry pass ({}) "
"- going back to the header geometry. The refined geometry did not help "
"this crystal.", worse);
// The refined pass has already written the canonical files, so re-run at the header
// geometry to replace them - the same remedy the supercell collapse above uses, and it
// only costs a pass on a crystal that was going to be wrong otherwise.
experiment_.BeamX_pxl(header_geometry[0]).BeamY_pxl(header_geometry[1])
.DetectorDistance_mm(header_geometry[2])
.PoniRot1_rad(header_geometry[3]).PoniRot2_rad(header_geometry[4]);
if (prepass_rotation_scale_ && gonio_snapshot) {
// The header rotation angles with it - but not the axis DIRECTION, which is not
// part of the geometry being reverted and may have been corrected by the sign
// rescue (see the restore before pass 2).
GoniometerAxis restored = *gonio_snapshot;
if (const auto current = experiment_.GetGoniometer())
restored.Axis(current->GetAxis());
experiment_.Goniometer(restored);
}
config_.output_prefix = base_prefix;
// This pass is the answer whatever it measures - the guard has had its one chance -
// so it must not conclude from its own search merge that it is going to be re-run.
quality_guard_pass1_.reset();
auto redo = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
redo.post_refine = pass1.post_refine;
redo.pass_number = pass2.pass_count + 1;
redo.pass_count = pass2.pass_count + 1;
redo.pass_decision = fmt::format(
"header geometry re-adopted: the post-refined pass was worse ({})", worse);
return redo;
}
if (pass2.pass_decision.empty())
pass2.pass_decision = fmt::format(
"post-refined geometry adopted ({}, {} reflections at I/sigma >= 2 before "
"corrections against {})", SearchMergeCompletenessText(pass1, pass2),
pass2.search_merge_strong_reflections, pass1.search_merge_strong_reflections);
}
if (pass2.pass_decision.empty())
pass2.pass_decision = (prepass_detector_geometry_ || prepass_rotation_scale_)
? "post-refined geometry adopted"
: "the post-refinement committed no geometry change, so this pass reproduces the first";
return pass2;
}
return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
}
namespace {
// Angle between the crystal's symmetry axes and the spindle. A rotation sweep never records the
// reflections whose reciprocal vector lies close to the spindle - the blind cusp - and normally
// symmetry fills it in from an equivalent elsewhere in reciprocal space. It cannot when a symmetry
// axis IS the spindle: the cusp then maps onto itself, every reflection in it is equivalent only to
// other reflections in it, and the hole stays empty however long the sweep runs. Returns the
// smallest angle in degrees between any proper rotation axis of the space group and the goniometer
// axis, with the order of the axis that achieves it.
std::optional<std::pair<double, int>> ClosestSymmetryAxisToSpindle(const gemmi::SpaceGroup &sg,
const CrystalLattice &lattice,
const Coord &spindle) {
const double spindle_len = spindle.Length();
if (spindle_len < 1e-9)
return std::nullopt;
std::optional<std::pair<double, int>> best;
for (const auto &op : sg.operations().derive_symmorphic().sym_ops) {
if (op.rot == gemmi::Op::identity().rot)
continue;
// Order of the rotation, then project onto its invariant direction by summing its powers:
// (1/n) sum_k W^k annihilates everything except the axis.
const gemmi::Op w{op.rot, {0, 0, 0}, op.notation};
int order = 0;
gemmi::Op cur = gemmi::Op::identity();
std::array<std::array<double, 3>, 3> sum{};
for (int k = 0; k < 6; ++k) {
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
sum[i][j] += static_cast<double>(cur.rot[i][j]) / gemmi::Op::DEN;
cur = cur.combine(w).wrap();
++order;
if (cur.rot == gemmi::Op::identity().rot)
break;
}
if (order < 2)
continue;
// Any non-degenerate column of the projector is the axis in fractional direct coordinates.
for (int c = 0; c < 3; ++c) {
const Coord axis = lattice.Vec0() * static_cast<float>(sum[0][c])
+ lattice.Vec1() * static_cast<float>(sum[1][c])
+ lattice.Vec2() * static_cast<float>(sum[2][c]);
const double len = axis.Length();
if (len < 1e-6 * lattice.Vec0().Length())
continue;
const double cosang = std::abs((axis * spindle) / (len * spindle_len));
const double ang = std::acos(std::min(1.0, cosang)) * 180.0 / PI;
if (!best.has_value() || ang < best->first)
best = std::make_pair(ang, order);
break;
}
}
return best;
}
// === Is the larger of two integer-related cells real, or a spurious axis multiple? ===
//
// The validation-FRAME count cannot answer it: a spurious multiple indexes every frame its true
// sub-cell does, so both reach the same total and the count saturates. The rule that then decided
// the pair was unconditionally against the larger cell, so on a crystal with a real
// pseudo-translation the true cell could not win in any scheme order.
//
// Ask the two candidates the same question at the granularity where it does not saturate: how
// many of the validation frames' SPOTS does each account for? That comparison leans towards the
// smaller cell by construction, and needs no threshold to do so. Acceptance is a fractional-Miller
// test, so multiplying an axis by n multiplies that axis's residual by n: every spot the larger
// cell places on the sub-lattice the smaller cell places n times more accurately, and the spots in
// the tolerance margin the larger cell simply loses. The only thing that can pay for that loss is
// the class of reflections the larger cell ADDS, which is all but empty for a spurious multiple
// and holds the superstructure's satellite rows for a real one. So the larger cell wins the count
// only when the extra periodicity is really there, and the bar it has to clear is set by the
// data rather than by a constant.
//
// What that added class holds is reported alongside, because it is the physics the count is a
// consequence of. Multiplying an axis by n is a CENTRING CONDITION in the larger cell, and the
// index-n sublattices of its Miller lattice WITH A CYCLIC QUOTIENT are the kernels of the linear
// forms f = (u,v,w) mod n with gcd(u,v,w,n) = 1, so enumerate them and report the emptiest: at
// n = 2 and n = 3 that is every index-n sublattice, but at n = 4 the seven whose quotient is
// (Z/2)^2 - a supercell doubling TWO axes - are not of that form, so a real 2a x 2b superstructure
// reads at the chance value here. That is the condition the larger cell would have to satisfy to
// BE its sub-cell. It is reported and NOT thresholded, and
// that is deliberate. Measured on real data the number does not have two populations to separate:
// an absent class reads a few tenths of a percent where nothing but the crystal is in the spot
// list and several percent where the finder also picked up things that are not this crystal's
// reflections, a real superstructure layer reads tens of percent, and a pair whose volumes are in
// an integer ratio without either cell being the other's supercell reads the chance value (n-1)/n
// - so a floor placed anywhere in that range is calibrated on whichever crystals were to hand.
//
// Following the reduction of an axis multiple to a centring condition, and its measurement as an
// absence test, from this project's own lattice-arbiter analysis.
std::vector<std::array<int, 3>> IndexedMillerIndices(const CrystalLattice &latt,
const std::vector<Coord> &cloud, float tol) {
const Coord a = latt.Vec0(), b = latt.Vec1(), c = latt.Vec2();
const float tol_sq = tol * tol;
std::vector<std::array<int, 3>> hkl;
hkl.reserve(cloud.size());
for (const Coord &s : cloud) {
const float fh = a * s, fk = b * s, fl = c * s; // Coord operator* = dot = Miller index
const float h = std::rint(fh), k = std::rint(fk), l = std::rint(fl);
const float dh = fh - h, dk = fk - k, dl = fl - l;
if (dh * dh + dk * dk + dl * dl < tol_sq)
hkl.push_back({static_cast<int>(h), static_cast<int>(k), static_cast<int>(l)});
}
return hkl;
}
struct AddedClassEvidence {
double occupancy = 1.0; // share of the larger cell's indexed spots OFF the sub-lattice
int u = 0, v = 0, w = 0; // the emptiest form, i.e. the condition that would collapse it
};
AddedClassEvidence AddedClassOccupancy(const std::vector<std::array<int, 3>> &hkl, int n) {
AddedClassEvidence out;
if (n < 2 || hkl.empty())
return out;
for (int u = 0; u < n; u++)
for (int v = 0; v < n; v++)
for (int w = 0; w < n; w++) {
if (u == 0 && v == 0 && w == 0)
continue;
// A form sharing a factor with n has a kernel of index < n - it describes a
// different, coarser condition than the one the volume ratio asks about, and
// admitting it could only make the class look emptier than it is.
if (std::gcd(std::gcd(std::gcd(u, v), w), n) != 1)
continue;
size_t off = 0;
for (const auto &m : hkl)
if (((u * m[0] + v * m[1] + w * m[2]) % n + n) % n != 0)
++off;
const double occ = static_cast<double>(off) / static_cast<double>(hkl.size());
if (occ < out.occupancy) {
out.occupancy = occ;
out.u = u; out.v = v; out.w = w;
}
}
return out;
}
}
ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
Logger logger("Rugnux");
ProcessResult result;
// Each pass determines its own space group, so start every one of them from the group the run was
// ASKED for. A pass that determined one leaves it on experiment_, and the pass after it - the
// re-runs in Run(), or a second Run() on the same object - would otherwise inherit that answer and
// skip the search that is the whole point of running again.
experiment_.SetSpaceGroup(user_fixed_sg_);
const auto dataset = reader_.GetDataset();
if (!dataset)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No experiment dataset found in the input file");
if (config_.stride <= 0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image stride must be positive");
const auto total_images_in_file = static_cast<int>(reader_.GetNumberOfImages());
int end_image = config_.end_image;
if (end_image < 0 || end_image > total_images_in_file)
end_image = total_images_in_file;
const int start_image = config_.start_image;
// Round UP: the worker loop below walks ordinals until start + ordinal * stride reaches end_image,
// so a stride that does not divide the range evenly still processes a final partial step. Rounding
// down here would size the per-image arrays one short of the ordinals that index them.
const int images_to_process = (end_image - start_image + config_.stride - 1) / config_.stride;
if (images_to_process <= 0) {
logger.Warning("No images to process (start {}, end {}, stride {}, total {})",
start_image, end_image, config_.stride, total_images_in_file);
return result;
}
const bool full = (config_.mode == ProcessMode::FullAnalysis);
// Powder calibration runs the same per-image engine as the full analysis (spot finding lives inside
// MXAnalysisWithoutFPGA) with indexing switched off in its settings. By rings it needs only the
// azimuthal profile and the azint worker would do - but the spots are worth their half second even
// then, because the circle through them places the beam centre without reference to the header at
// all, which is the one hypothesis that survives a header the profile cannot correct.
const bool calibration = (config_.mode == ProcessMode::Calibration);
const bool calibration_spots = calibration && config_.calibration_method == CalibrationMethod::Spots;
const bool per_image_analysis = full || calibration;
const bool write_files = write_output && !config_.output_prefix.empty();
// Set below the first merge, once the pass can tell that RunAllPasses is going to throw it away and
// run again - see there for what that turns off and why it is knowable at that point.
bool superseded = false;
// Output/runtime invariants. Algorithm settings (indexing, scaling, integration, polarization,
// space group, unit cell, ...) are configured on experiment_ by the caller.
experiment_.BitDepthImage(32).PixelSigned(true);
experiment_.Mode(DetectorMode::Standard);
experiment_.OverwriteExistingFiles(true);
// Offline processing: the output prefix is a local path the operator chose (rugnux -o or the
// viewer's "next to the input file"), so use the trusted setter that permits an absolute path.
experiment_.FilePrefixTrusted(config_.output_prefix.empty() ? "output" : config_.output_prefix);
experiment_.SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
experiment_.ImagesPerTrigger(images_to_process);
experiment_.NumTriggers(1);
if (full)
experiment_.Compression(CompressionAlgorithm::BSHUF_LZ4);
// The pipeline indexes images 0..N-1 within this run; if we process a sub-range/strided
// selection, shift the goniometer so local index i maps to the angle of original image
// start+i*stride (keeping the per-image rotation wedge), otherwise rotation angles would be
// wrong for any start_image != 0.
if (const auto g = experiment_.GetGoniometer();
g.has_value() && (start_image != 0 || config_.stride != 1)) {
const float incr = g->GetIncrement_deg();
GoniometerAxis shifted(g->GetName(),
g->GetStart_deg() + incr * static_cast<float>(start_image),
incr * static_cast<float>(config_.stride),
g->GetAxis(), g->GetHelicalStep());
shifted.ScreeningWedge(g->GetScreeningWedge().value_or(incr));
experiment_.Goniometer(shifted);
}
// Stills global geometry-refinement first pass: bundle-adjust the shared beam/distance/cell from a
// sample of strongly-indexed frames and apply it to experiment_, so everything below (azimuthal
// mapping, the main index+integrate pass, scaling) uses the refined geometry. Rotation is skipped
// inside (it has its own two-pass).
if (full && config_.refine_geometry.has_value())
RefineStillsGeometry(start_image, end_image, images_to_process, observer);
// After any geometry refinement, so the shadow is found about the beam centre actually used,
// and before the azimuthal mapping and the output mask below, which both read pixel_mask_.
if (config_.detect_beam_stop.has_value() || config_.estimate_beam_center
|| config_.adaptive_integration_radius)
PreScan(start_image, images_to_process,
config_.detect_beam_stop.value_or(BEAM_CENTER_PROJECTION_IMAGES), observer);
// The geometry pre-pass integrates at the radius the run started with; the width the pre-scan just
// measured is applied to the canonical pass instead (RunAllPasses, between the two). See
// bragg_adaptive_ for why - in short, the pre-pass keeps only the geometry it fits to the observed
// reflection positions, and those positions move with the signal disk.
if (geometry_prepass && bragg_before_adaptive_) {
bragg_adaptive_ = experiment_.GetBraggIntegrationSettings();
experiment_.ImportBraggIntegrationSettings(*bragg_before_adaptive_);
}
AzimuthalIntegrationMapping mapping(experiment_, pixel_mask_);
JFJochReceiverPlots plots;
plots.Setup(experiment_, mapping);
// Output file (NXmxIntegrated master that links back to the original images).
StartMessage start_message;
experiment_.FillMessage(start_message);
start_message.arm_date = dataset->arm_date;
start_message.az_int_bin_to_q = mapping.GetBinToQ();
start_message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta();
start_message.az_int_q_bin_count = mapping.GetQBinCount();
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
if (mapping.GetAzimuthalBinCount() > 1)
start_message.az_int_bin_to_phi = mapping.GetBinToPhi();
if (full) {
start_message.rois = experiment_.ROI().ExportMetadata();
start_message.max_spot_count = experiment_.GetMaxSpotCount();
}
start_message.master_suffix = "process";
start_message.file_format = FileWriterFormat::NXmxIntegrated;
start_message.write_master_file = true;
start_message.write_images = false;
// The process file LINKS to the original images rather than writing its own, which only an HDF5
// input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs. So
// ask for the links only when the source is HDF5, and otherwise write the images into the file -
// the per-image analysis is the point of it either way, and without pictures nothing downstream
// could show the frame a reflection came from.
if (auto *hdf5 = dynamic_cast<JFJochHDF5Reader *>(&reader_)) {
// With the stride: image i of the process file is source image start_image + i * stride, so
// the linked pictures line up with the per-image analysis written beside them.
start_message.hdf5_source_data =
hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride);
// Everything describing the pixels must describe those linked files. experiment_ cannot: it
// is pinned to signed 32-bit, the container the reader hands images out in, which would type
// the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every
// value on read. Only consumers going through the virtual view ever saw it - our own reader
// resolves the mapping and opens the source itself.
const auto stored = hdf5->GetStoredPixelFormat();
start_message.bit_depth_image = stored.bit_depth;
// Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
// the field is present, and on a signed image those land inside the trusted range.
start_message.bit_depth_readout.reset();
if (!stored.is_signed)
start_message.bit_depth_readout = stored.bit_depth;
start_message.pixel_signed = stored.is_signed;
start_message.error_value = stored.is_signed
? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN
: (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX
start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
} else {
start_message.write_images = true; // no file to link to; the process file carries them
}
// The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
// but the process file reads them. Copy them where a writer is actually built rather than here:
// the default merging path writes no process file at all.
auto fill_writer_maps = [&] {
start_message.pixel_mask["default"] = pixel_mask_.GetMask(experiment_);
if (full && !experiment_.ROI().empty())
start_message.roi_map = experiment_.ExportROIMap();
};
std::unique_ptr<FileWriter> writer;
std::unique_ptr<ProcessFileWriter> writer_queue;
// Whether anything is in place to break the indexing ambiguity, for the warning further down: a
// reference MTZ, or (serial stills) the reference computed from a model just below.
bool ambiguity_reference = !config_.reference_data.empty();
if (write_files && config_.write_process_h5) {
fill_writer_maps();
writer = std::make_unique<FileWriter>(start_message, /*check_overwrite_at_start=*/true,
/*trusted_path=*/true);
// Deep enough that a worker never waits for the writer in the normal case, shallow enough that
// the backlog is bounded by the worker count rather than by the length of the run.
writer_queue = std::make_unique<ProcessFileWriter>(*writer, 4 * std::max(1, config_.nthreads));
}
const char *mode_name = full ? "full analysis"
: calibration ? "powder calibration" : "azimuthal integration";
logger.Info("Processing {} images (range {}-{}, stride {}) using {} threads [{}]",
images_to_process, start_image, end_image, config_.stride, config_.nthreads, mode_name);
LogXDSGeometry(experiment_);
if (observer)
observer->OnPhase(full ? "Full analysis" : calibration ? "Powder calibration" : "Azimuthal integration");
// Full-analysis shared engines.
std::unique_ptr<IndexerThreadPool> indexer_pool;
std::unique_ptr<IndexAndRefine> indexer;
// The FFT search bound indexer_pool was built with. The long-axis rescue needs it to tell whether a
// cell it recovered still fits, since a pool's transform is sized once, at construction.
float indexing_bound_A = 0.0f;
if (per_image_analysis) {
auto indexing_settings = experiment_.GetIndexingSettings();
// Two-pass rotation fires the FFT indexer only twice - the two first-pass schemes, run in
// parallel (the main per-image loop then uses the forced lattice, no FFT). Size the pool to
// those two so a rotation run doesn't pay to initialise (cuFFT plan + GPU alloc) indexers it
// never uses. Stills fire the FFT once per image across all workers, so keep the default there.
if (config_.rotation_indexing && config_.two_pass_rotation && experiment_.IsRotationIndexing()) {
indexing_settings.IndexingThreads(2);
// Two indexer threads leave the rest of the machine idle for the whole first pass, which is
// a third of the run. Split each scheme's candidate-cell refinement over -N/2 threads so the
// two together use -N. The split is by candidate cell, so the result does not depend on it.
// Clamped to what IndexingSettings accepts - a big host would otherwise ask for more than
// the setting's limit and throw before a single image is read.
indexing_settings.RefineThreads(std::clamp(config_.nthreads / 2, 1, MAX_REFINE_THREADS));
RaiseFFTBoundForKnownCell(indexing_settings, experiment_);
}
indexing_bound_A = indexing_settings.GetFFT_MaxUnitCell_A();
indexer_pool = std::make_unique<IndexerThreadPool>(indexing_settings, IndexerConstruction::OnFirstUse);
indexer = std::make_unique<IndexAndRefine>(experiment_, indexer_pool.get());
// With no per-image file to write, nothing reads the message's reflection list.
if (!writer_queue)
indexer->KeepReflectionsInMessage(false);
// The reference that breaks the per-image indexing ambiguity: a reference MTZ where there is
// one, otherwise intensities computed from the model, which carry the same information. Serial
// stills need it here or not at all - each crystal is indexed in its own hand, so by the time
// there is a merge for --model to fit, the hands have already been averaged together. Rotation
// data have one lattice and one hand, so their ambiguity is settled at the merge instead.
std::vector<MergedReflection> model_reference;
if (!config_.reference_data.empty()) {
indexer->ReferenceIntensities(config_.reference_data);
} else if (!config_.model_path.empty() && !experiment_.IsRotationIndexing()
&& experiment_.GetUnitCell().has_value() && experiment_.GetGemmiSpaceGroup().has_value()) {
const auto d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A()
.value_or(experiment_.GetDetectorMaxResolution_A());
model_reference = ModelReferenceIntensities(
config_.model_path, experiment_.GetUnitCell(),
&experiment_.GetSpaceGroupOrP1(), d_min, logger);
if (!model_reference.empty()) {
indexer->ReferenceIntensities(model_reference);
ambiguity_reference = true;
}
}
// Second pass of the rotation two-pass: predict with the smoothed mosaicity fitted in the pre-pass.
if (!geometry_prepass && !prepass_mosaicity_.empty())
indexer->SetPredictionMosaicityOverride(prepass_mosaicity_);
// The geometry pre-pass fits the detector geometry to spot POSITIONS and throws its intensities
// away, so a fixed group's centring absences cost it half its events for nothing. A no-op when
// no group is fixed - prediction is already in P there - so the de-novo path is untouched.
//
// The full pass predicts them too, so a fixed CENTRED group integrates the same reflections a
// de-novo run would. Both merges then do the right thing on their own: the run's own merge
// drops the centring absences again (Merge.cpp `IsSystematicallyAbsent`, against the group it
// is merging in), and the P1 cross-check keeps them because P1 has none. What it buys is not
// only a complete cross-check - an unpredicted node is an unclaimed patch of detector, so a
// neighbouring reflection sweeps those pixels into its background ring and over-subtracts
// (see the reflection mask in BraggIntegrationEngineCPU). A fixed group used to cost real
// intensity accuracy for that reason.
indexer->PredictAllCentringNodes(true);
}
const auto start_time = std::chrono::steady_clock::now();
// First pass of two-pass rotation indexing (full analysis only).
if (full && force_rotation_result_.has_value()) {
// Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
// search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
// integrates markedly worse.
indexer->ForceRotationIndexerResult(*force_rotation_result_);
logger.Info("Rotation indexer result forced from pass 1 (lattice + geometry) - skipping first pass");
} else if (full && config_.forced_rotation_lattice.has_value()) {
indexer->ForceRotationIndexerLattice(*config_.forced_rotation_lattice);
logger.Info("Rotation indexer lattice forced externally - skipping first pass");
} else if (full && config_.rotation_indexing && config_.two_pass_rotation) {
if (observer)
observer->OnPhase("Rotation indexing (first pass)");
// Mid-exposure goniometer angle for an image ordinal (matches the per-image path).
auto rot_angle = [&](int ordinal) -> std::optional<float> {
if (const auto g = experiment_.GetGoniometer())
return g->GetAngle_deg(static_cast<float>(ordinal)) + g->GetWedge_deg() / 2.0f;
return std::nullopt;
};
// Spots for a first-pass image ordinal, cached (a frame is reused across schemes and the
// validation set; re-finding is expensive on the --redo-rotation-spots path). The workers below
// hand their spots back and this thread puts them in, so the map itself is still only ever
// touched from here and needs no locking.
std::map<int, std::vector<SpotToSave>> spot_cache;
// The azimuthal mapping the first pass finds its spots against. It is `mapping` until a
// beam-centre hypothesis moves the centre, and the hypothesis's own mapping while it stands -
// the mapping is where the resolution of every pixel and the finder's radial background bins
// come from, and both move with the centre.
const AzimuthalIntegrationMapping *spot_mapping = &mapping;
// The spots of one frame, found with the caller's own analysis engine. A frame's spots are a
// pure function of that frame and the settings - the engine carries nothing from one image to
// the next, which is why the main image loop can give one to each of its workers - so it does
// not matter which engine, or which thread, finds them.
const auto find_spots = [&](MXAnalysisWithoutFPGA &analysis, AzimuthalIntegrationProfile &profile,
JFJochReaderRawImage &img, int ordinal) {
const int image_idx = start_image + ordinal * config_.stride;
std::vector<SpotToSave> spots;
try {
if (reader_.ReadRawImage(image_idx, img)) {
DataMessage m{};
m.number = ordinal;
m.original_number = image_idx;
auto first_pass = config_.spot_finding;
first_pass.indexing = false;
first_pass.quick_integration = false;
m.image = img.image;
if (dataset->efficiency.size() > image_idx)
m.image_collection_efficiency = dataset->efficiency[image_idx];
analysis.Analyze(m, profile, first_pass);
spots = std::move(m.spots);
}
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Warning("First-pass spot read failed for image {}: {}", image_idx, e.what());
}
return spots;
};
// Find the spots of every ordinal in the list that is not cached yet, on several workers, and
// put them in the cache.
//
// The cache ends up holding exactly what a serial loop would have put there: the results are
// collected into a vector indexed by position and inserted afterwards, so neither the spots of
// a frame nor the set of frames in the cache depends on how the workers interleaved.
//
// Worker t keeps its analysis engine for the whole first pass. Building one is a CUDA stream
// and a full set of device buffers - measured at 0.13 s against 4 ms for a frame - and the
// "wedge" scheme only discovers how far into the sweep it has to read as the frames arrive,
// so this is called several times and rebuilding them each time cost a quarter of the pass.
// The engine belongs to a card, not to a thread, so a later call only has to make that card
// current again; workers take a card by index so an engine always meets the one it was
// built on.
const int spot_workers = get_gpu_count() > 0
? std::min(config_.nthreads, std::max(8, 4 * get_gpu_count()))
: config_.nthreads;
struct SpotEngine {
// Before the analysis engine, which page-locks these bytes for its uploads and
// unregisters them when it is destroyed: it must not outlive the buffer it was given.
JFJochReaderRawImage raw_image;
std::unique_ptr<MXAnalysisWithoutFPGA> analysis;
std::unique_ptr<AzimuthalIntegrationProfile> profile;
};
std::vector<SpotEngine> engines(std::max(spot_workers, 1));
const auto use_gpu_of_worker = [](size_t t) {
if (const int32_t gpus = get_gpu_count(); gpus > 0)
set_gpu(static_cast<int32_t>(t) % gpus);
};
int prefetch_batches = 0;
double prefetch_time_s = 0.0;
const auto prefetch_spots = [&](const std::vector<int> &ordinals) {
std::vector<int> wanted;
for (const int ordinal : ordinals)
if (!spot_cache.contains(ordinal))
wanted.push_back(ordinal);
if (wanted.empty())
return;
const auto prefetch_start = std::chrono::steady_clock::now();
++prefetch_batches;
std::vector<std::vector<SpotToSave>> found(wanted.size());
std::atomic<size_t> next{0};
std::vector<std::future<void>> futures;
const size_t nworkers = std::min<size_t>(engines.size(), wanted.size());
futures.reserve(nworkers);
for (size_t t = 0; t < nworkers; t++)
futures.emplace_back(std::async(std::launch::async, [&, t] {
use_gpu_of_worker(t); // must precede engine construction
SpotEngine &e = engines[t];
if (!e.analysis) {
e.analysis = std::make_unique<MXAnalysisWithoutFPGA>(
experiment_, *spot_mapping, pixel_mask_, *indexer,
/*enable_fused_adaptive_gpu=*/true);
e.profile = std::make_unique<AzimuthalIntegrationProfile>(*spot_mapping);
}
for (size_t i = next.fetch_add(1); i < wanted.size(); i = next.fetch_add(1))
found[i] = find_spots(*e.analysis, *e.profile, e.raw_image, wanted[i]);
}));
for (auto &f : futures)
f.get();
for (size_t i = 0; i < wanted.size(); i++)
spot_cache.emplace(wanted[i], std::move(found[i]));
prefetch_time_s += std::chrono::duration<double>(
std::chrono::steady_clock::now() - prefetch_start).count();
};
// Release the engines on workers of their own, so a stream and its device buffers are freed
// with the card they were made on current.
const auto release_spot_engines = [&] {
std::vector<std::future<void>> futures;
futures.reserve(engines.size());
for (size_t t = 0; t < engines.size(); t++)
futures.emplace_back(std::async(std::launch::async, [&, t] {
use_gpu_of_worker(t);
engines[t] = SpotEngine{};
}));
for (auto &f : futures)
f.get();
};
// Evaluating a beam-centre hypothesis means re-finding the spots at that centre. The spot
// POSITIONS do not move with the centre - which is why the cache is keyed by image alone - but
// the spot LIST does: every spot's d, the ice-ring flag, the resolution mask the finder
// applies, the high-resolution gap cut and the strongest-N budget are all radial, and the
// adaptive finder's ring background is binned about the centre. Scoring a trial centre against
// spots found at the file's put the ladder in a world the run never enters: measured, a centre
// scored 42/60 inside the ladder and 5/60 when that same value was given on the command line.
//
// A trial therefore gets its own azimuthal mapping, its own engines and its own cache, and the
// starting centre's spots are PARKED rather than thrown away - a hypothesis that is not adopted
// has to leave the rest of the run exactly as it found it.
std::unique_ptr<AzimuthalIntegrationMapping> trial_mapping;
std::map<int, std::vector<SpotToSave>> home_spot_cache;
const auto try_beam_center = [&](float x, float y) {
if (trial_mapping)
spot_cache.clear();
else
home_spot_cache.swap(spot_cache);
experiment_.BeamX_pxl(x).BeamY_pxl(y);
release_spot_engines();
trial_mapping = std::make_unique<AzimuthalIntegrationMapping>(experiment_, pixel_mask_);
spot_mapping = trial_mapping.get();
};
const auto restore_beam_center = [&](float x, float y) {
if (!trial_mapping)
return;
experiment_.BeamX_pxl(x).BeamY_pxl(y);
release_spot_engines();
trial_mapping.reset();
spot_mapping = &mapping;
spot_cache = std::move(home_spot_cache);
home_spot_cache.clear();
};
// How many validation frames a candidate global lattice actually indexes per image, scored on
// the *real* per-image path (force the candidate, then run the same indexing+refinement the
// main loop uses). A cell that merely fits the accumulated first-pass cloud (a spurious
// sub-lattice from an under-sampled scheme) indexes few frames, while the true cell indexes
// many - this is the discriminator that lets the best scheme win.
const std::vector<int> validation = select_equally_spaced_image_ordinals(
images_to_process, std::min(images_to_process, 60));
const float index_tol = experiment_.GetIndexingSettings().GetTolerance();
// The validation frames' spots in reciprocal space, under a candidate's own refined geometry
// and axis. The same construction the rotation indexer makes on its accumulated frames, but
// over the validation set and over each frame's WHOLE spot list - the arbiter counts spots and
// wants the deep sample, where the first pass wants a bounded one. A superstructure layer is
// faintest exactly where the first pass's accumulation cap cuts.
auto validation_cloud = [&](const RotationIndexerResult &r) {
std::vector<Coord> cloud;
if (!r.axis)
return cloud;
for (const int ordinal : validation) {
const auto it = spot_cache.find(ordinal);
if (it == spot_cache.end())
continue;
const float ang = rot_angle(ordinal).value_or(
r.axis->GetAngle_deg(static_cast<float>(ordinal)) + r.axis->GetWedge_deg() / 2.0f);
const auto rot = r.axis->GetTransformationAngle(ang);
for (const auto &sp : it->second)
cloud.emplace_back(rot * sp.ReciprocalCoord(r.geom));
}
return cloud;
};
auto validation_settings = config_.spot_finding;
validation_settings.indexing = true;
validation_settings.quick_integration = false;
auto count_indexed = [&](IndexAndRefine &idx, const RotationIndexerResult &r) -> int {
idx.ForceRotationIndexerResult(r);
// Fill the spot cache first. It is a plain map filled on demand, and the loop below runs the
// frames in parallel, where a lookup racing an insert is not something the map survives.
// After the first candidate this costs nothing.
prefetch_spots(validation);
// Each frame's verdict is its own - the score is just how many of them index - so they run
// together. Refining one frame's geometry is a few Ceres solves and no GPU, and it is the
// same call the main image loop already makes from every one of its workers on this same
// IndexAndRefine, writing only unit_cells[] under its own mutex. With the candidate forced,
// GetLattice() returns it and the branch that would advance the indexer's own state is never
// reached. The offline solver stops on an iteration count rather than a clock
// (IndexAndRefine::RefineGeometryIfNeeded), so a loaded machine cannot change a verdict.
std::atomic<int> count = 0;
ParallelFor(static_cast<int>(validation.size()),
std::min<size_t>(validation.size(), config_.nthreads), [&](int i) {
DataMessage m{};
m.number = validation[i];
m.spots = spot_cache.at(validation[i]);
if (idx.IndexFrameOnly(m, validation_settings))
++count;
});
return count.load();
};
// The run's acceptance test counts SPOTS, not frames. Summing each validation frame's spots
// before any threshold lets a four-spot frame contribute four spots of evidence instead of an
// automatic "no", and removes the spots-per-frame denominator that makes a frame count measure
// how sparse the pattern is rather than how good the lattice is: with 4 spots on an average
// frame no lattice, the true one included, can clear a bar of six indexed spots per frame.
//
// `shift` displaces each frame's spots onto another validation frame's spindle angle. The
// lattice, the spots, the geometry and the per-frame orientation polish are all unchanged and
// only the claim under test is removed - that these spots are at these angles - so what a
// displaced run still puts on the lattice is what chance and the polish give for free. That is
// the bar the real count is measured against, and it is measured rather than chosen, so no
// spots-per-frame number survives anywhere in the test.
auto pooled_on_lattice = [&](IndexAndRefine &idx, const RotationIndexerResult &r,
int shift) -> int64_t {
idx.ForceRotationIndexerResult(r);
prefetch_spots(validation);
std::atomic<int64_t> on_lattice = 0;
ParallelFor(static_cast<int>(validation.size()),
std::min<size_t>(validation.size(), config_.nthreads), [&](int i) {
DataMessage m{};
m.number = validation[(i + shift) % validation.size()];
m.spots = spot_cache.at(validation[i]);
on_lattice += idx.CountSpotsOnLattice(m, validation_settings);
});
return on_lattice.load();
};
// The pooled evidence for one candidate lattice: the validation frames' non-ice spots, how many
// of them lie on the candidate, and how many a wrong spindle angle still puts there.
struct PooledEvidence {
int64_t spots = 0;
int64_t on_lattice = 0;
int64_t by_chance = 0;
};
// Displacements are a fixed fraction of the sweep, so the null is a function of the data alone
// and the same file gives the same verdict every time. Sevenths: no crystallographic rotation
// is a seventh of a turn, so a displacement cannot coincide with a symmetry about the spindle
// and hand the null the signal's own score. The median over five of them bounds what one
// unlucky displacement can do.
auto pooled_evidence = [&](IndexAndRefine &idx, const RotationIndexerResult &r) {
PooledEvidence e;
e.on_lattice = pooled_on_lattice(idx, r, 0);
const bool index_ice_rings = experiment_.GetIndexingSettings().GetIndexIceRings();
for (const int ordinal : validation)
for (const auto &sp : spot_cache.at(ordinal))
if (index_ice_rings || !sp.ice_ring)
e.spots++;
const int n = static_cast<int>(validation.size());
std::vector<int64_t> chance;
for (int k = 1; k <= 5; k++)
chance.push_back(pooled_on_lattice(idx, r, std::max(1, k * n / 7)));
std::sort(chance.begin(), chance.end());
e.by_chance = chance[chance.size() / 2];
return e;
};
// This crystal's lattice is the one that puts materially more spots on itself than a wrong
// angle does. The excess is counted against the binomial noise of the null's own rate over the
// same spots, so the bar scales with the evidence in hand and carries no absolute count; the
// significance is the one-false-positive-in-a-thousand bar the spot-budget measurement is
// already set at. A null that indexed nothing is still credited with one spot of noise, as the
// next displacement could have found one.
auto beats_chance = [](const PooledEvidence &e) {
const double p = static_cast<double>(e.by_chance) / static_cast<double>(std::max<int64_t>(1, e.spots));
const double sigma = std::sqrt(std::max(1.0, static_cast<double>(e.spots) * p * (1.0 - p)));
return static_cast<double>(e.on_lattice - e.by_chance) > SPOT_BUDGET_SIGNIFICANCE_Z * sigma;
};
// How deep into an image's intensity-ordered spot list this lattice is still being seen - the
// measured spot budget. Same frames, same per-image path as count_indexed above, but scoring the
// spots rather than the frames: each frame's spots are tallied by rank and the run keeps the
// ranks the tally supports (AddSpotBudgetEvidence / SpotBudgetFromEvidence). A budget deeper than
// that admits detections that are not this crystal's reflections, and the frame gate then counts
// them against the frame - which is how a larger budget can integrate FEWER images.
auto measure_budget = [&](IndexAndRefine &idx, const RotationIndexerResult &r) -> int64_t {
idx.ForceRotationIndexerResult(r);
prefetch_spots(validation);
std::mutex evidence_mutex;
std::vector<int64_t> indexed(experiment_.GetMaxSpotCount(), 0);
std::vector<int64_t> counted(indexed.size(), 0);
const bool index_ice_rings = experiment_.GetIndexingSettings().GetIndexIceRings();
ParallelFor(static_cast<int>(validation.size()),
std::min<size_t>(validation.size(), config_.nthreads), [&](int i) {
DataMessage m{};
m.number = validation[i];
m.spots = spot_cache.at(validation[i]);
if (!idx.IndexFrameOnly(m, validation_settings))
return;
std::unique_lock ul(evidence_mutex);
AddSpotBudgetEvidence(m.spots, index_ice_rings, indexed, counted);
});
return SpotBudgetFromEvidence(indexed, counted);
};
// Feed one first-pass scheme (a set of image ordinals) into its own rotation indexer, ready to
// be indexed. Spots are pulled from the cache here; the FFT + refinement runs separately
// (RunIndexing) so the two schemes' indexing can overlap.
//
// The feed itself stays in frame order and stops where it always did, but it keeps the cache a
// chunk of frames ahead of itself so those frames' spots are found on several workers. A chunk
// rather than the whole list because the "wedge" scheme stops on the accumulation cap, and how
// far into the sweep that is only becomes known as the frames arrive.
constexpr int SPOT_PREFETCH_CHUNK = 64;
size_t first_pass_spots_per_image = 0; // 0 = the indexer's own default
auto feed_scheme = [&](IndexerThreadPool &pool, const std::vector<int> &ordinals) {
auto ri = std::make_unique<RotationIndexer>(experiment_, pool);
ri->MaxSpotsPerImage(first_pass_spots_per_image);
for (size_t i = 0; i < ordinals.size(); i++) {
if (cancelled_ || ri->AccumulationFull())
break;
if (!spot_cache.contains(ordinals[i]))
prefetch_spots({ordinals.begin() + i,
ordinals.begin() + std::min(ordinals.size(), i + SPOT_PREFETCH_CHUNK)});
ri->ProcessImage(ordinals[i], spot_cache.at(ordinals[i]), rot_angle(ordinals[i]));
}
return ri;
};
// Sampling schemes, tried in order of preference. We index each and keep the one that indexes
// the most frames; the schemes cover complementary failure modes and no single one is best for
// every crystal:
// - "spread": images spread over the whole rotation - full angular range, coarse stride; the
// long-standing default, best for typical crystals.
// - "wedge": consecutive frames from the clean start - native stride keeps each reflection's
// rocking curve continuous across frames, so the FFT can trace a long axis whose fine
// reciprocal spacing the coarse spread cannot resolve (e.g. an F-cubic cell viewed down
// [111]); bounded by the accumulation cap to the least radiation-damaged early wedge.
std::vector<int> all_consecutive(images_to_process);
std::iota(all_consecutive.begin(), all_consecutive.end(), 0);
const std::vector<int> spread_ordinals = select_equally_spaced_image_ordinals(
images_to_process, config_.rotation_indexing_image_count);
const std::vector<std::pair<std::string, std::vector<int>>> schemes = {
{"spread", spread_ordinals},
{"wedge", all_consecutive},
};
// indexer_error: what the indexer threw, if it threw. An empty result with no error is a
// verdict on the crystal; an empty result with one is a verdict on the machine.
struct FirstPass { std::optional<RotationIndexerResult> result; int score = -1; std::string name; double vol = 0.0;
std::optional<std::string> indexer_error; };
// Feed both schemes (single-threaded spot cache), run their RunIndexing() concurrently, then keep
// the lattice that indexes the most validation frames on the real per-image path.
auto pick_best = [&](IndexerThreadPool &pool, IndexAndRefine &idx) -> FirstPass {
std::vector<std::unique_ptr<RotationIndexer>> ris;
ris.reserve(schemes.size());
for (const auto &[name, ordinals] : schemes) {
if (cancelled_)
break;
ris.push_back(feed_scheme(pool, ordinals));
}
std::vector<std::future<void>> index_futures;
index_futures.reserve(ris.size());
for (auto &ri : ris) {
RotationIndexer *rp = ri.get();
index_futures.push_back(std::async(std::launch::async, [rp] { rp->RunIndexing(); }));
}
for (auto &f : index_futures)
f.get();
FirstPass bp;
for (size_t i = 0; i < ris.size(); i++) {
if (cancelled_)
break;
if (!bp.indexer_error)
bp.indexer_error = ris[i]->GetIndexerError();
auto found = ris[i]->GetLattice();
if (!found.has_value())
continue;
RotationIndexerResult result = std::move(*found);
int score = count_indexed(idx, result);
const std::string &name = schemes[i].first;
logger.Info("First-pass scheme '{}': indexes {}/{} validation frames", name, score,
static_cast<int>(validation.size()));
// Metric symmetry that costs nearly every frame is not this crystal's symmetry. The
// Bravais class is decided on the UNREFINED FFT candidate against a fixed angular
// tolerance, so a lattice that is pseudo-symmetric to a few tenths of a degree gets
// promoted a class too far; the constraint then snaps a real angle to the ideal one,
// which throws nearly every reflection of every frame out of tolerance (measured: 2/60
// frames for a monoclinic lattice promoted to C-orthorhombic, 39/60 for its primitive
// cell). The indexer's own guard compares the two on the accumulated first-pass cloud,
// where they differ by less than a factor 2 - far too little to act on.
// The bar is a clear majority, not a margin: the unconstrained refinement holds NO cell
// parameter fixed, so it can only index at least as many frames as the constrained one,
// and on genuine symmetry it does index a few more (measured: 47->54 frames on a real
// I-centred orthorhombic crystal, 49->58 on an F-cubic one - a 10% margin demotes both).
// Only a constrained cell that fails outright while its unconstrained cell works is
// evidence of a false promotion, so demand exactly that. It is the same "fails to index
// half the frames" test the long-axis rescue below uses, and it leaves genuine symmetry
// alone; the space group is in any case decided from the intensities later.
const int majority = static_cast<int>(validation.size()) / 2;
if (result.unconstrained && score < majority) {
RotationIndexerResult alt = *result.unconstrained;
const int alt_score = count_indexed(idx, alt);
if (alt_score > majority) {
logger.Info("Scheme '{}': {}-centred {} indexes {}/{} frames but its unconstrained "
"cell indexes {}/{} - the metric symmetry is a false promotion, dropping it",
name, result.search_result.centering,
gemmi::crystal_system_str(result.search_result.system), score,
static_cast<int>(validation.size()), alt_score,
static_cast<int>(validation.size()));
result = std::move(alt);
score = alt_score;
}
}
// Compare PRIMITIVE volumes: two schemes can find the same lattice in different
// settings, and a centred setting's cell is an exact integer multiple of its primitive
// one - a rhombohedral lattice in hexagonal axes is exactly 3x its primitive
// rhombohedral cell. Comparing the centred volumes makes the supercell test below fire
// on that pair and "demote" a perfectly good setting to a threefold-smaller merge, which
// is enough to change the space group the search then picks.
const double vol = std::abs(
result.lattice.ToPrimitive(result.search_result.centering).CalcVolume());
// A later scheme wins if it indexes clearly more frames (>10%).
const bool clearly_more = static_cast<float>(score) > bp.score * 1.1f + 0.5f;
// Integer-supercell tie-break. The validation-frame count saturates - a spurious axis
// multiple (2x/3x...) indexes every frame its true cell does, so both schemes reach the
// same frame total and the count alone cannot tell them apart; the default then keeps
// whichever ran first. When the two schemes tie on frames but their cell volumes are
// related by an integer factor >=2, which of the pair is real is decided on the SPOTS by
// the arbiter below - regardless of scheme order, and in either direction. A near-integer
// ratio separates a real axis multiplication from a centering coincidence. Volumes are
// primitive (see above), so a pure setting difference is a ratio of 1 and never fires.
bool integer_subcell = false;
bool harmonic_settled = false;
double harmonic_ratio = 0.0;
if (bp.result.has_value() && vol > 1.0 && bp.vol > 1.0) {
const bool tied = static_cast<float>(score) >= bp.score * 0.9f - 0.5f;
const double ratio = (vol < bp.vol) ? bp.vol / vol : vol / bp.vol;
const double nearest = std::round(ratio);
// Bounded at 4. The window is absolute in a unit integer spacing, so a ratio
// between two UNRELATED lattices passes it about a third of the time whatever n
// is, and at n = 124 the pair is not a cell and its harmonic at all. Measured
// over the corpus: all 36 firings at n <= 4 read 1.1 to 51.3 points BELOW the
// chance occupancy (n-1)/n, so an index-n sub-lattice really exists in each; all
// 8 above it read within 7.8 points OF chance, so none does. The largest n at
// which a real sub-lattice was ever seen is 3.
const bool integer_multiple = nearest >= 2.0 && nearest <= 4.0
&& std::abs(ratio - nearest) < 0.15;
// Ask the data which of the pair is right instead of assuming the larger is
// spurious: which cell accounts for more of the validation frames' spots (see
// IndexedMillerIndices)? The measurement is made on their FULL spot lists, which
// reach far deeper into each frame's intensity distribution than the first pass's
// own accumulation cap - and a superstructure layer is faintest exactly where that
// cap cuts.
if (tied && !clearly_more && integer_multiple) {
const bool cand_is_larger = vol > bp.vol;
const RotationIndexerResult &larger = cand_is_larger ? result : *bp.result;
const RotationIndexerResult &smaller = cand_is_larger ? *bp.result : result;
const auto hkl = IndexedMillerIndices(larger.lattice, validation_cloud(larger),
index_tol);
const size_t n_smaller = IndexedMillerIndices(smaller.lattice,
validation_cloud(smaller),
index_tol).size();
const auto ev = AddedClassOccupancy(hkl, static_cast<int>(nearest));
const bool larger_is_real = hkl.size() > n_smaller;
logger.Info("Axis-harmonic arbiter: '{}' ({:.0f} A^3) vs '{}' ({:.0f} A^3), {:.2f}x - "
"the larger cell accounts for {} validation spots against the smaller "
"cell's {}, and {:.1f}% of them lie off its ({},{},{}) sub-lattice, so "
"the class it adds is {}",
name, vol, bp.name, bp.vol, ratio, hkl.size(), n_smaller,
100.0 * ev.occupancy, ev.u, ev.v, ev.w,
larger_is_real ? "occupied enough to pay for the longer axis - the "
"larger cell is real"
: "not occupied enough to pay for the longer axis - the "
"larger cell is a multiple");
// Adopt the candidate exactly when its role matches the verdict.
integer_subcell = (cand_is_larger == larger_is_real);
// Only a win for the larger cell settles the pair. Declining it is a fall back
// to the default, and the warning below - which names the beam centre - is then
// still the most useful thing the run can say. It is the right diagnosis in
// particular for the failure this arbiter cannot see: a centre error along the
// spindle translates the derotated cloud rigidly by HALF a lattice spacing, and
// a half-offset lattice is indexable only on a doubled axis. Such a cell fits no
// index-n sublattice at all, so its added class reads the chance value (n-1)/n -
// the largest the occupancy can be - and it accounts for far fewer spots than
// the true cell, which is what this test sees it by.
harmonic_settled = larger_is_real;
}
// sqrt(3) as well as an integer: a hexagonal supercell that lengthens one axis by
// sqrt(3) is the other harmonic seen on real data and its volume ratio is not a
// whole number. This is a REPORT and not a decision - it is deliberately taken
// over the whole pair, `clearly_more` included, because the harmonic is often the
// one that indexes more frames (see below).
if (integer_multiple || std::abs(ratio - std::sqrt(3.0)) < 0.15)
harmonic_ratio = ratio;
}
// Two candidates whose primitive volumes differ by a small integer or by sqrt(3) are
// one lattice and its axis harmonic, and what decides between them is the BEAM CENTRE.
// A centre error along the spindle translates the derotated cloud rigidly, which the
// FFT amplitude cannot see at all, so the peaks stay sharp and the indexer commits -
// with confidence - to a sub-multiple. Measured on real data: a 0.12 px change of
// centre, 0.03x of what the J0 law asks across the spindle, is the whole difference
// between the deposited cell and a halved axis, and the halved one is the answer that
// indexed MORE frames (100.00 % against 99.23 %). So the frame count cannot arbitrate
// this pair, and until this line nothing in the run said what the pair meant.
// ... unless the arbiter above established the larger cell on the data, which is
// exactly the question this warning says the run cannot answer.
if (harmonic_ratio > 0.0 && !harmonic_settled)
logger.Warning("Scheme '{}' (primitive cell volume {:.0f} A^3) and '{}' ({:.0f} A^3) "
"differ by {:.2f}x - one is an axis harmonic of the other. Which of "
"the two is right is settled by the beam centre to a fraction of a "
"pixel, and the harmonic is often the one that indexes MORE frames, "
"so the frame counts above cannot decide it. Check the beam centre "
"(--estimate-beam-center)", name, vol, bp.name, bp.vol, harmonic_ratio);
if (!bp.result.has_value() || clearly_more || integer_subcell) {
if (integer_subcell && vol < bp.vol)
logger.Info("Scheme '{}' cell (vol {:.0f}) is a {:.0f}x sub-cell of '{}' (vol {:.0f}) at "
"equal frame count - adopting the smaller true cell", name, vol,
std::round(bp.vol / vol), bp.name, bp.vol);
else if (integer_subcell)
logger.Info("Scheme '{}' cell (vol {:.0f}) is a {:.0f}x supercell of '{}' (vol {:.0f}) at "
"equal frame count and accounts for more of the spots - adopting the "
"larger true cell", name, vol, std::round(vol / bp.vol), bp.name, bp.vol);
bp.score = score;
bp.vol = vol;
bp.name = name;
bp.result = std::move(result);
}
}
return bp;
};
// Powder-ring rescue - and, with it, the lean seed the rotation first pass never had.
//
// The pre-scan has already measured this sample's powder rings (see PreScan): a crystalline
// phase other than the crystal, whose reflections the indexer is otherwise handed as if they
// were this crystal's. Hexagonal ice is the only such phase whose rings can be named in
// advance, and the spot finder flags it from that fixed list; everything else - a shower of
// microcrystals, a salt out of the cryoprotectant, a phase nobody expected - has to be
// measured, and until it is, every one of its spots goes into the FFT. Measured on one such
// crystal: 876 spots a frame against the 80 a clean crystal on the same beamline gives, 70 %
// of them on rings, no scheme indexing a single validation frame, and the FFT taking a 10-30 A
// cell out of the shells.
//
// Setting the rings aside is half of it. The other half is HOW DEEP into each frame's spot
// list the first pass reads. It has always read 200 spots an image; the per-frame indexer, on
// the same problem, escalates 30 -> 80 -> the whole list and keeps the seed a lattice explains
// the largest fraction of (IndexAndRefine::DetermineLatticeAndSymmetry), because on a flooded
// frame a lean high-quality seed finds the lattice where the full list derails the search. The
// rotation pass gets the same escalation here. It is the general statement of what a
// resolution cut would do on this one crystal - the contaminant and the noise sit at the deep
// end of the intensity-ordered list wherever in resolution they happen to lie - and it needs
// nothing measured about where they are.
//
// It runs LAST: after every other rescue and after the pass-1 fallback above, on a run that
// would otherwise stop with no lattice at all. Measured over 100 datasets: placed earlier it
// pre-empted the rescues that already work - a crystal the beam-centre ladder fixes at 59/60
// and one the pass-1 fallback fixes at 60/60 both came out WORSE, because this ladder had
// already replaced the standing pass those rescues start from. A last resort has to be last.
//
// The two compose: FilterSpotsByCount has already ordered each image's spots non-ring first
// and strongest first, so a lean cap takes the crystal's strongest reflections and nothing
// else. Decided late, the way the axis sign above is: index each rung, keep the one that
// indexes the most validation frames. A run that already indexes never reaches here, and one
// that does cannot lose by trying - where the rings are the crystal's own rows, or where the
// crystal needs its deep spots to index at all, the pass that already ran stands.
// Called from two places, because the two questions it answers are coupled: a centre that is
// wrong and a spot list that is too deep each hide the other, exactly as the centre and the
// axis sign do above. `score_before` is what the caller's standing pass indexes and `min_gain`
// is what a rung must beat it by; nothing is changed and the settings are put back when no
// rung does.
// What the run is CURRENTLY reading each frame at. It starts at what the file asked for, and
// only an ADOPTED rung moves it. Every rung the ladder tries is put back to this, so a rung
// tried and rejected leaves nothing behind - and a later pass that finds no rung restores what
// an earlier pass adopted rather than resetting the run to the defaults.
const auto first_pass_file_d_min = config_.spot_finding.high_resolution_limit;
struct FirstPassState { bool rings = false; size_t seed = 0; std::optional<float> d_min; };
FirstPassState committed{false, 0, first_pass_file_d_min};
const auto apply_state = [&](const FirstPassState &st) {
const std::vector<float> r = st.rings ? powder_.rings_q_recipA : std::vector<float>{};
config_.spot_finding.measured_ring_q_recipA = r;
validation_settings.measured_ring_q_recipA = r;
config_.spot_finding.high_resolution_limit = st.d_min;
validation_settings.high_resolution_limit = st.d_min;
first_pass_spots_per_image = st.seed;
spot_cache.clear(); // the per-image spot budget is spent again under these settings
};
// min_score is an ABSOLUTE bar beside the gain. A caller that takes a rung only when it
// indexes a majority must not be handed a state the ladder committed on a lesser rung:
// measured, the ladder adopted at 29/60, the caller rejected it against a majority of 30, and
// nothing put the settings back - the beam-centre walk that followed then scored every
// hypothesis on the leaked lean spot lists.
const auto first_pass_ladder =
[&](int score_before, int min_gain, int min_score) -> std::optional<FirstPass> {
if (cancelled_)
return std::nullopt;
// The seeds, leanest first. The same ladder the per-frame indexer uses, one rung deeper
// because a rotation cloud pools many frames and can afford more.
// The last rung is EVERY spot of each frame: SIZE_MAX, not 0 - RotationIndexer reads 0 as
// "use my default", which is 200, so a 0 here made the deepest rung a duplicate of the
// 200 one and the log line "the whole spot list" a lie.
constexpr std::array<size_t, 4> SEEDS = {30, 80, 200, std::numeric_limits<size_t>::max()};
struct Rung { bool rings; size_t seed; std::optional<float> d_min; };
// How much of each frame a rung let the first pass see: more spots an image, and further
// in resolution, is more evidence. Rings set aside is not part of it - that is a
// measurement of the sample, not a depth.
const auto Deeper = [](const Rung &a, const Rung &b) {
if (a.seed != b.seed) return a.seed > b.seed;
const float da = a.d_min.value_or(0.0f), db = b.d_min.value_or(0.0f);
return da < db; // a smaller d_min reaches further; 0 means the detector edge
};
// The file's OWN limit, captured before any rung committed anything. Reading it out of
// config_ here would read back the previous rung's floor once a pass has adopted one, and
// the restore path would then "put back" that instead of what the file asked for.
const auto file_limit = first_pass_file_d_min;
std::vector<Rung> rungs;
// Where a floor is worth trying at all - the rings merged somewhere inside the measured
// range, so past that point nothing can be told from the crystal and a pass that reads
// there is reading the contaminant.
std::vector<std::optional<float>> floors = {file_limit};
for (const float d : prescan_spot_quantile_d_A_)
if (!file_limit || d > *file_limit) floors.emplace_back(d);
// Floors outer, seeds inner, so rungs that share the finder-facing settings (rings,
// d_min) sit adjacent: the seed is an indexer-side cap over the SAME found spots
// (MaxSpotsPerImage over the cached list), so the loop below spot-finds once per floor
// instead of once per rung. Adoption is decided over all rungs at once, so the visit
// order carries no meaning.
for (const auto &d_min : floors)
for (const size_t seed : SEEDS) {
if (!powder_.rings_q_recipA.empty())
rungs.push_back({true, seed, d_min});
if (powder_.rings_q_recipA.empty() && seed != SEEDS.back())
rungs.push_back({false, seed, d_min});
}
// Every rung's score, so the choice below can be made on all of them at once.
std::vector<int> rung_score(rungs.size(), -1);
std::vector<FirstPass> rung_pass(rungs.size());
int won = -1;
for (size_t i = 0; i < rungs.size() && !cancelled_; i++) {
// A new floor (or ring handling) changes what the finder returns, so the spot lists
// are spent and found again; a new seed alone does not - it caps how much of the
// SAME lists the indexer reads - and re-finding 60 validation frames per rung was
// most of what a hopeless run's minutes were spent on.
if (i == 0 || rungs[i].rings != rungs[i - 1].rings || rungs[i].d_min != rungs[i - 1].d_min)
apply_state({rungs[i].rings, rungs[i].seed, rungs[i].d_min});
else
first_pass_spots_per_image = rungs[i].seed;
FirstPass alt = pick_best(*indexer_pool, *indexer);
logger.Info(" first-pass rung: {} spots/image, rings {}, stop at {} -> {}/{} frames{}",
rungs[i].seed == std::numeric_limits<size_t>::max()
? std::string("all") : std::to_string(rungs[i].seed),
rungs[i].rings ? "aside" : "kept",
rungs[i].d_min ? fmt::format("{:.2f} A", *rungs[i].d_min)
: std::string("the detector edge"),
alt.result.has_value() ? alt.score : -1,
static_cast<int>(validation.size()),
alt.result.has_value()
? fmt::format(", {}{} {:.1f} {:.1f} {:.1f} {:.1f} {:.1f} {:.1f}",
alt.result->search_result.centering,
gemmi::crystal_system_str(alt.result->search_result.system),
alt.result->search_result.conventional.GetUnitCell().a,
alt.result->search_result.conventional.GetUnitCell().b,
alt.result->search_result.conventional.GetUnitCell().c,
alt.result->search_result.conventional.GetUnitCell().alpha,
alt.result->search_result.conventional.GetUnitCell().beta,
alt.result->search_result.conventional.GetUnitCell().gamma)
: std::string());
// A rung has to WIN by the count the run itself keeps a lattice on - a sixth of the
// validation frames, the bar below - not merely come out ahead. Three frames out of
// sixty is the count's noise, and adopting on it would hand the rescues that follow
// (the beam centre, the long axis) a spot list cut for no reason. Measured: a marginal
// crystal that indexed 26/60 went to 29/60 on a rung and must not take it, where the
// crystals this exists for go from 0/60 to 24 and to 32/60.
if (alt.result.has_value()) {
rung_score[i] = alt.score;
rung_pass[i] = std::move(alt);
}
}
// The rung to keep. A rung has to WIN by the count the run itself keeps a lattice on - a
// sixth of the validation frames, the bar below - not merely come out ahead: three frames
// out of sixty is the count's noise, and adopting on it would hand the rescues that follow
// (the beam centre, the long axis) a spot list cut for no reason. Measured: a marginal
// crystal that indexed 26/60 went to 29/60 on a rung and must not take it.
//
// Among the rungs that do win it is the highest count that is kept, and an exact tie goes
// to the rung that saw the MOST of each frame. Ties are common - on a pattern where the
// contaminant sits past a resolution, every seed behind that resolution returns the same
// lattice and the same count - and where they happen the rung that read deepest rests on
// the most evidence. Only an exact tie, not a band: the count is what tells a lattice that
// works from one that does not, and a rung three frames behind is behind.
for (size_t i = 0; i < rungs.size(); i++) {
if (rung_score[i] < 0 || rung_score[i] - score_before < min_gain
|| rung_score[i] < min_score)
continue;
if (won < 0 || rung_score[i] > rung_score[won]
|| (rung_score[i] == rung_score[won] && Deeper(rungs[i], rungs[won])))
won = static_cast<int>(i);
}
// ...and it must not be an axis SUB-MULTIPLE of a cell another rung found. The frame
// count the choice above is made on cannot arbitrate an axis harmonic - a cell twice as
// long has to place every spot twice as accurately to score the same, which the scheme
// comparison above says in the same words - and here the two biases compound, because
// the leanest and shallowest rung is both the one that scores highest on a harmonic and
// the one a smaller cell is easiest on. Measured on one crystal: the "30 spots/image,
// stop at 4.91 A" rung indexed 48/60 on a HALVED c axis while the deeper rungs found 2x,
// 3x and 4x of the true one; the ladder took the halved cell, and the run ended in a
// different Bravais class on a different cell from the one it reaches with no ladder at
// all - the ladder had pre-empted the beam-centre walk that finds the crystal's own.
//
// So put the winner through the same axis-harmonic arbiter the scheme comparison uses:
// against every rung whose primitive volume is a near-integer multiple of the winner's,
// which cell accounts for more of the validation frames' SPOTS. A rung that loses that
// question is a sub-multiple, and the ladder then adopts NOTHING. It is a last resort,
// and a last resort that cannot tell an axis from its multiple has not rescued anything -
// promoting the rival instead is no better, since on the measured crystal the rival is a
// multiple of the true axis in its own right.
//
// Judged on the spot list the run STANDS on rather than on either rung's cut: the two
// cells must be counted against the same spots, and the deepest list is where the class
// an axis multiple adds is visible at all (see IndexedMillerIndices).
if (won >= 0 && rung_pass[won].vol > 1.0) {
apply_state(committed);
prefetch_spots(validation);
const double won_vol = rung_pass[won].vol;
const size_t won_spots = IndexedMillerIndices(rung_pass[won].result->lattice,
validation_cloud(*rung_pass[won].result),
index_tol).size();
for (size_t i = 0; i < rungs.size() && won >= 0; i++) {
// Only a rung the caller would have taken can demote one: the winner is chosen
// among those, and a rung that indexes too little to be adopted is too little
// evidence to overturn one either.
if (static_cast<int>(i) == won || rung_score[i] < 0
|| rung_score[i] - score_before < min_gain || rung_score[i] < min_score)
continue;
const double ratio = rung_pass[i].vol / won_vol;
const double nearest = std::round(ratio);
// Only a LARGER rival can demote, and only in the window the arbiter is
// calibrated over (see the scheme comparison: every firing at n <= 4 read below
// the chance occupancy, none above it did).
if (nearest < 2.0 || nearest > 4.0 || std::abs(ratio - nearest) >= 0.15)
continue;
const auto hkl = IndexedMillerIndices(rung_pass[i].result->lattice,
validation_cloud(*rung_pass[i].result),
index_tol);
const auto ev = AddedClassOccupancy(hkl, static_cast<int>(nearest));
logger.Info("First-pass ladder: the winning rung's cell ({:.0f} A^3) and the "
"rung at {}/{} frames ({:.0f} A^3) differ by {:.2f}x - one is an "
"axis harmonic of the other. The larger accounts for {} validation "
"spots against the winner's {}, and {:.1f}% of them lie off its "
"({},{},{}) sub-lattice",
won_vol, rung_score[i], static_cast<int>(validation.size()),
rung_pass[i].vol, ratio, hkl.size(), won_spots,
100.0 * ev.occupancy, ev.u, ev.v, ev.w);
if (hkl.size() > won_spots) {
logger.Warning("First-pass ladder: the rung that indexes the most frames does "
"so on a {:.0f}x sub-multiple of an axis another rung found, "
"which the frame count cannot see - the ladder adopts nothing "
"and the run keeps the pass it already stands on",
nearest);
won = -1;
}
}
}
if (won >= 0) {
committed = {rungs[won].rings, rungs[won].seed, rungs[won].d_min};
apply_state(committed);
logger.Info("First-pass seed: {}{}{} takes it from {}/{} validation frames to {}/{}",
rungs[won].seed == std::numeric_limits<size_t>::max()
? std::string("the whole spot list")
: fmt::format("the strongest {} spots an image", rungs[won].seed),
rungs[won].rings
? fmt::format(", with this sample's {} measured powder rings set aside "
"for the whole run", powder_.rings_q_recipA.size())
: std::string(),
rungs[won].d_min && rungs[won].d_min != file_limit
? fmt::format(", stopping spot finding at {:.2f} A", *rungs[won].d_min)
: std::string(),
score_before, static_cast<int>(validation.size()),
rung_pass[won].score, static_cast<int>(validation.size()));
return rung_pass[won];
}
apply_state(committed); // back to what this run stands on, not to the defaults
return std::nullopt;
};
// Find the spots of everything the first pass is certain to want in one go - both schemes'
// opening frames and the validation set. A batch of workers costs an analysis engine each,
// which is far more than a frame is to analyse, so one batch for the lot beats one per scheme.
{
std::vector<int> upfront = spread_ordinals;
upfront.insert(upfront.end(), validation.begin(), validation.end());
for (int i = 0; i < std::min(SPOT_PREFETCH_CHUNK, images_to_process); i++)
upfront.push_back(i);
std::sort(upfront.begin(), upfront.end());
upfront.erase(std::unique(upfront.begin(), upfront.end()), upfront.end());
prefetch_spots(upfront);
}
FirstPass best = pick_best(*indexer_pool, *indexer);
// Rotation-axis SIGN rescue. The sign is a convention the input cannot settle: a PILATUS
// miniCBF header carries only "Oscillation_axis omega" and no direction at all, and facilities
// disagree about the sign of the NXmx vector (measured across ten of them: most want the
// stored [-1,0,0], APS 19-ID and SPring-8 BL41XU want the opposite). Nothing downstream can
// repair it either - the default spindle fit refines the axis DIRECTION by fractions of a
// degree and cannot flip it, that being a discrete change.
//
// It is, however, decidable from the data: the wrong sign does not index at all. Measured on
// one dataset by changing nothing but the sign: 100% of frames indexed one way, 0 of 60 the
// other. So try the other sign and keep whichever indexes more frames - the same
// validation-frame count the scheme choice above already uses, so no new metric and no new
// threshold. Runs before the long-axis rescue: with the sign wrong every candidate lattice is
// wrong, and the coarse pass below would spend itself on a mis-signed cloud.
//
// Only after a poor pass, so a correctly-signed file costs nothing; and spot finding is not
// repeated, the cache being keyed by image and the spots not depending on the axis.
if (!cancelled_ && best.score < 0.5 * static_cast<double>(validation.size())) {
if (const auto gon = experiment_.GetGoniometer(); gon.has_value() && gon->IsScanning()) {
GoniometerAxis flipped = *gon;
flipped.Axis(-gon->GetAxis());
experiment_.Goniometer(flipped);
FirstPass alt = pick_best(*indexer_pool, *indexer);
if (alt.result.has_value() && alt.score > best.score) {
logger.Info("Rotation axis sign: the file's axis indexes {}/{} validation frames, "
"the opposite sign {}/{} - adopting the opposite sign for this run",
best.score, static_cast<int>(validation.size()),
alt.score, static_cast<int>(validation.size()));
best = alt;
} else {
experiment_.Goniometer(*gon); // the file was right; put it back
}
}
}
// The background centre, tried as a SECOND FIRST PASS.
//
// The estimate itself is free - it is read off the projection --detect-beam-stop has already
// built - and a first pass re-uses the spots it has already found, so indexing the run a
// second time at the measured centre costs about what one rung of the ladder below costs.
// That buys the comparison the on-failure ladder structurally cannot make: a centre wrong
// ALONG the spindle does not fail. It keeps 96-100 % of frames indexed and quietly returns a
// 2x, 3x or sqrt(3) axis harmonic, so no failure trigger ever fires on it. Indexing both
// centres and comparing the two ANSWERS is what can see it.
//
// What must NOT arbitrate the two is the frame count. Acceptance is a fractional-Miller test,
// so a cell twice as long has to place every spot twice as accurately to score the same:
// measured on real data, two centres 0.12 px apart gave the deposited cell at 99.23 % and a
// halved axis at 100.00 %, and the WRONG answer indexed better. A rule of the form "take the
// centre that indexes more" would therefore pick wrong in exactly the case this comparison
// exists for. So only what needs no arbiter is decided here:
//
// * the file's centre does not index a majority -> nothing defends it, so try the measured
// one (and the axis sign with it, see below). A run that indexes nothing has nothing to
// lose; where the measured one is no better, the file's centre is put back;
// * the file's centre indexes and the two agree on the lattice -> keep the file's centre and
// say they agree. This is the overwhelming majority, and saying it is worth the pass: it
// is a free statement that the header is good enough for this crystal;
// * they index and disagree -> say so, with both cells and their primitive-volume ratio,
// and change nothing. Naming a disagreement is useful; resolving it needs an arbiter
// nobody has.
//
// Where the file's centre indexes nothing and the measured one does not either, the axis-sign
// question is asked AGAIN at the measured centre before giving up. The two unknowns are
// discrete and coupled, and the rescue above asked its question at a centre that may be 73 px
// wrong: on the two depositions here whose header is the geometric centre of the detector,
// each error hides the other, both runs sit at 0/60 until both are moved, and moving both
// takes them to 60/60 on the deposited lattice. Asking the pair costs one more first pass on
// a run that has already failed twice.
if (!cancelled_ && config_.beam_center_check && background_center_) {
// Consumed, not re-read: a two-pass rotation run comes back through here with the geometry
// the first pass post-refined, and a pre-scan estimate of the FILE's centre says nothing
// about that one. The pre-scan does not re-measure it either - by then the beam-stop shadow
// is already in the mask, so the projection is not rebuilt.
const BeamCenterEstimate measured = *background_center_;
background_center_.reset();
const float header_x = experiment_.GetBeamX_pxl(), header_y = experiment_.GetBeamY_pxl();
const float measured_x = measured.beam_x_pxl;
const float measured_y = measured.beam_y_pxl;
const float moved = std::hypot(measured_x - header_x, measured_y - header_y);
// No gate on how significant the move is. It is tempting to skip the second pass when the
// two centres are within the fit's own noise - it would save it on a quarter of the runs
// here - but that is deciding at a threshold before the evidence is in, and the evidence
// is cheap. It would also gate away exactly the case worth asking about: measured on real
// data, a 0.12 px change of centre is the whole difference between the deposited cell and
// a halved axis, and 0.12 px is well inside the sigma of any fit that could gate it.
{
const int majority = static_cast<int>(validation.size()) / 2;
try_beam_center(measured_x, measured_y);
FirstPass alt = pick_best(*indexer_pool, *indexer);
const bool header_indexes = best.result.has_value() && best.score > majority;
bool measured_indexes = alt.result.has_value() && alt.score > majority;
if (!header_indexes) {
// Nothing defends the file's centre: the data do not index at it, and the
// background says it is `moved` px away. So try the measured one, and if that
// does not index either, ask the AXIS-SIGN question again there. The two unknowns
// are discrete and coupled, and the sign rescue above asked its question at a
// centre that may itself be wrong - each error hides the other, and only moving
// both reveals anything.
bool joint = false;
const auto gonio = experiment_.GetGoniometer();
if (!measured_indexes && gonio.has_value() && gonio->IsScanning()) {
GoniometerAxis flipped = *gonio;
flipped.Axis(-gonio->GetAxis());
experiment_.Goniometer(flipped);
const FirstPass both = pick_best(*indexer_pool, *indexer);
if (both.result.has_value() && both.score > majority) {
logger.Warning("Beam centre from the background: the file's centre "
"({:.2f},{:.2f}) indexes {}/{} validation frames and the "
"measured ({:.2f},{:.2f}) {}/{}, but the measured centre "
"WITH the opposite rotation-axis sign indexes {}/{}. The "
"file has both wrong, and neither shows while the other "
"stands; the run takes the measured centre and the opposite "
"sign", header_x, header_y, best.score,
static_cast<int>(validation.size()), measured_x, measured_y,
alt.score, static_cast<int>(validation.size()), both.score,
static_cast<int>(validation.size()));
alt = both;
measured_indexes = true;
joint = true;
} else {
experiment_.Goniometer(*gonio);
}
}
// Third coupled unknown: how DEEP the pass reads. A centre 170 px out and a spot
// list too deep to index at ANY centre hide each other exactly as the centre and
// the axis sign do - the measured centre is right, and it is thrown away because
// the full list indexes nothing there either. Measured on one such crystal: the
// file's centre indexed 1/60 and the measured one 0/60, so the file's was kept and
// the run died; with the ladder's lean seed the measured centre indexes 60/60, and
// the run then indexes every frame of the sweep and merges to the deposited
// resolution. So ask the ladder here too, before giving the centre up.
//
// The depth is asked at BOTH centres, and the centre moves only if it is the
// centre that paid. A lean rung at the measured centre scored against the whole
// spot list at the file's changes two things at once, and the count cannot say
// which of them the gain came from - the same reason a trial centre is scored on
// its own spots at all. Measured on one crystal: the measured centre's ladder
// reached 40/60 on a FIVE-fold axis harmonic while the file's own centre reached
// 49/60 on the deposited cell, and the run moved the centre, shipped the
// supercell, and skipped the beam-centre walk that would have found the crystal's
// own. An exact tie keeps the file's centre: nothing then says the centre is wrong.
bool lean = false;
bool deeper_at_home = false;
if (!measured_indexes) {
const FirstPassState before = committed;
restore_beam_center(header_x, header_y);
const auto home = first_pass_ladder(best.score, 1, majority + 1);
const FirstPassState home_state = committed;
committed = before;
apply_state(committed);
try_beam_center(measured_x, measured_y);
const auto away = first_pass_ladder(alt.score, 1, majority + 1);
if (home && (!away || home->score >= away->score)) {
committed = home_state;
restore_beam_center(header_x, header_y);
apply_state(committed);
best = *home;
deeper_at_home = true;
logger.Warning("Beam centre check: reading less of each frame takes the "
"file's centre to {}/{} validation frames against {}/{} at "
"the measured centre {:.2f} px away - it is how deep this "
"run was reading and not the centre, so the centre stays",
home->score, static_cast<int>(validation.size()),
away ? away->score : 0,
static_cast<int>(validation.size()), moved);
} else if (away) {
alt = *away;
measured_indexes = true;
lean = true;
}
}
if (measured_indexes) {
if (lean)
logger.Warning("Beam centre from the background: neither centre indexes the "
"whole spot list, but at the measured ({:.2f},{:.2f}) the "
"lean first pass above indexes {}/{} against the file's "
"{}/{} - the beam centre in this file is wrong, and only a "
"pass that reads less of each frame could see it",
measured_x, measured_y, alt.score,
static_cast<int>(validation.size()), best.score,
static_cast<int>(validation.size()));
else if (!joint)
logger.Warning("Beam centre from the background: the file's ({:.2f},{:.2f}) "
"indexes {}/{} validation frames and the measured "
"({:.2f},{:.2f}) indexes {}/{} - the beam centre in this "
"file is wrong, and the run adopts the measured one",
header_x, header_y, best.score,
static_cast<int>(validation.size()), measured_x, measured_y,
alt.score, static_cast<int>(validation.size()));
best = alt;
} else if (!deeper_at_home) {
// Nothing worked. Put the file's centre back, so the ladder below searches
// around the value the run started with rather than around a fit that has just
// been tested and found no better - and so a run that fails for some other
// reason fails at the geometry it was given.
restore_beam_center(header_x, header_y);
logger.Info("Beam centre check: neither the file's centre ({}/{} frames) nor "
"the measured one {:.2f} px away ({}/{}) indexes this run - the "
"file's centre is kept and this is not a beam-centre failure",
best.score, static_cast<int>(validation.size()), moved, alt.score,
static_cast<int>(validation.size()));
}
} else {
restore_beam_center(header_x, header_y);
if (measured_indexes) {
// Primitive volumes, as everywhere else here: two passes can find the same
// lattice in different settings, and a centred setting's cell is an exact
// integer multiple of its primitive one.
const double ratio = alt.vol < best.vol ? best.vol / alt.vol : alt.vol / best.vol;
const auto &hc = best.result->search_result.conventional.GetUnitCell();
const auto &mc = alt.result->search_result.conventional.GetUnitCell();
const bool same_class =
best.result->search_result.system == alt.result->search_result.system
&& best.result->search_result.centering == alt.result->search_result.centering;
// The primitive volume is what the beam centre decides. A centre error along
// the spindle makes the FFT take an axis harmonic, which changes the volume by
// an integer factor or by sqrt(3); the Bravais class can differ for a reason
// that has nothing to do with the centre, so the two are reported separately.
if (ratio < 1.02 && same_class) {
logger.Info("Beam centre check: indexing at the measured centre "
"({:.2f},{:.2f}) returns the same lattice as the file's - "
"{}-centred {}, cells agreeing to {:.2f} % in volume. The {:.2f} px "
"difference does not decide this crystal's cell",
measured_x, measured_y, alt.result->search_result.centering,
gemmi::crystal_system_str(alt.result->search_result.system),
100.0 * (ratio - 1.0), moved);
} else if (ratio < 1.02) {
// Same cell, different metric symmetry read off it. The cell is not in
// question, but the CLASS is not cosmetic either: GeometryRefiner holds
// the cell to it for the rest of the run, so a class that is one axis
// equality away from the truth is refined, predicted, integrated and
// merged with that equality imposed.
//
// Whether the centre decided it is arithmetic. The equality is held to
// LATTICE_SEARCH_DIST_TOLERANCE, relative, on a pair of reduced-cell
// axes; a centre wrong by `moved` px moves an axis of length a by
// moved*p*a/(D*lambda) relative, the FFT reading that axis off spot
// positions the centre displaces. Both axes of the pair carry their own
// error, so the centre can open or close a gap of their sum - and where
// that reaches the tolerance the higher class is not a fact about the
// crystal but a consequence of where the beam was assumed to be.
//
// Nothing available here can settle it: both centres index, and the frame
// count is the one statistic this check already refuses to arbitrate on.
// So carry the other centre out of the pass and let the two arms be judged
// on their merges (RunAllPasses).
const auto reduced = best.result->search_result.primitive_reduced.GetUnitCell();
const double axis_pair = std::max(reduced.a + reduced.b,
reduced.b + reduced.c);
const double induced = moved * experiment_.GetPixelSize_mm() * axis_pair
/ (experiment_.GetDetectorDistance_mm()
* experiment_.GetWavelength_A());
const bool centre_can_decide = induced >= LATTICE_SEARCH_DIST_TOLERANCE;
logger.Info("Beam centre check: indexing at the measured centre "
"({:.2f},{:.2f}) returns the same CELL as the file's - the "
"primitive volumes agree to {:.2f} % - but a different metric "
"symmetry: {}-centred {} from the file's centre against "
"{}-centred {} from the measured one", measured_x, measured_y,
100.0 * (ratio - 1.0), best.result->search_result.centering,
gemmi::crystal_system_str(best.result->search_result.system),
alt.result->search_result.centering,
gemmi::crystal_system_str(alt.result->search_result.system));
if (centre_can_decide && geometry_prepass) {
beam_center_alternative_ = std::array<float, 2>{measured_x, measured_y};
logger.Warning("Beam centre check: {:.2f} px is worth {:.2f} % on the "
"axis pair the promotion compares, against the {:.2f} % "
"the metric symmetry is decided to - the difference is "
"large enough to have decided it either way, so the run "
"carries BOTH centres forward and adopts whichever "
"merges better", moved, 100.0 * induced,
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
} else {
logger.Info("Beam centre check: {:.2f} px is worth {:.2f} % on the axis "
"pair the promotion compares, against the {:.2f} % the "
"metric symmetry is decided to - the run keeps the file's "
"centre and the symmetry it found", moved, 100.0 * induced,
100.0 * LATTICE_SEARCH_DIST_TOLERANCE);
}
} else {
const double nearest = std::round(ratio);
const bool harmonic = (nearest >= 2.0 && std::abs(ratio - nearest) < 0.15)
|| std::abs(ratio - std::sqrt(3.0)) < 0.15;
logger.Warning("Beam centre check: the two centres DISAGREE about the "
"lattice. The file's ({:.2f},{:.2f}) gives {}-centred {}, "
"{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} (primitive volume "
"{:.0f} A^3, {}/{} frames); the measured ({:.2f},{:.2f}) gives "
"{}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} "
"(primitive volume {:.0f} A^3, {}/{} frames){}",
header_x, header_y, best.result->search_result.centering,
gemmi::crystal_system_str(best.result->search_result.system),
hc.a, hc.b, hc.c, hc.alpha, hc.beta, hc.gamma, best.vol,
best.score, static_cast<int>(validation.size()),
measured_x, measured_y, alt.result->search_result.centering,
gemmi::crystal_system_str(alt.result->search_result.system),
mc.a, mc.b, mc.c, mc.alpha, mc.beta, mc.gamma, alt.vol,
alt.score, static_cast<int>(validation.size()),
harmonic ? fmt::format(" - the volumes differ by {:.2f}x, so one "
"is an axis harmonic of the other and the "
"beam centre is what settles which",
ratio)
: "");
// Deliberately no decision. The frame counts above are the one statistic
// that must not choose: on the measured case the harmonic indexed MORE.
logger.Warning("Beam centre check: the run keeps the file's centre. The "
"frame counts cannot arbitrate this - a cell twice as long "
"must place every spot twice as accurately to score the "
"same, so the harmonic is often the one that indexes more. "
"Re-run with --estimate-beam-center to measure the centre "
"from the spot symmetry, which is exact geometry");
}
} else {
logger.Info("Beam centre check: the measured centre ({:.2f},{:.2f}) indexes "
"{}/{} validation frames against the file's {}/{} - the file's "
"centre stands", measured_x, measured_y, alt.score,
static_cast<int>(validation.size()), best.score,
static_cast<int>(validation.size()));
}
}
}
}
// Beam centre as an INDEXING HYPOTHESIS. A beam-centre error is not repairable downstream: it
// is fixed in the LAB frame, so accumulating a sweep smears every reciprocal-lattice point
// around a circle and the FFT amplitude at an axis of length a is multiplied by
// J0(2 pi delta p a/(D lambda)). Past the first zero the true axis is gone and its harmonic
// wins, which is what a first-pass axis doubling usually is. It is decidable from the data on
// exactly the count the scheme choice and the axis-sign rescue already use: the right centre
// indexes and the wrong one does not.
//
// BOTH detector directions are searched, and that is the one thing to keep. The J0 law is
// about the FFT AMPLITUDE, which is translation-invariant, so a centre error along the spindle
// is free for the transform - but the step after it, fitting a lattice whose origin is the
// beam, is not, and a rigidly shifted lattice is fitted by a finer one. Measured by injection
// on two crystals: across the spindle the cell stays right and the indexed fraction collapses
// (99 % to 25 %), while along it the run keeps 96-100 % indexed and quietly adopts a 2x, 3x or
// sqrt(3) supercell. The direction that announces itself is not the dangerous one.
//
// Only after a poor pass, so a correct header costs nothing. Every rung re-finds its spots at
// its own centre (try_beam_center): the positions do not move with the centre but the list
// does, and a rung scored on the file centre's spot list is not a rung the run can adopt.
if (!cancelled_ && !beam_center_searched_ && config_.beam_center_search_pxl > 0
&& best.score < 0.5 * static_cast<double>(validation.size())) {
beam_center_searched_ = true;
const float beam_x = experiment_.GetBeamX_pxl(), beam_y = experiment_.GetBeamY_pxl();
const int reach = config_.beam_center_search_pxl;
// The background centre is measured off the whole projection rather than off a lattice, so
// where it is significant it BOUNDS where the beam can be, and a ladder that reaches
// `reach` px cannot arrive at a centre further away than that: every rung it can try is one
// the measurement has already excluded. The check above indexes the measured centre once
// and is right not to gate on significance for that - one pass is cheap. This ladder is
// 4*reach passes, so the same evidence is worth acting on here. Measured on a deposition
// whose header centre lands on a masked pixel: 170 px away at 0.34 px sigma, 48 rungs, and
// nothing adopted.
const bool centre_past_reach =
measured_beam_center_.has_value()
&& std::hypot(measured_beam_center_->beam_x_pxl - beam_x,
measured_beam_center_->beam_y_pxl - beam_y)
> std::max(3.0f * measured_beam_center_->sigma_pxl, static_cast<float>(reach));
if (centre_past_reach)
logger.Info("Beam-centre hypothesis: the background places the centre {:.2f} px away "
"(+- {:.2f} px), past the {} px this ladder reaches - no centre it can try "
"is the beam, so it is not walked",
std::hypot(measured_beam_center_->beam_x_pxl - beam_x,
measured_beam_center_->beam_y_pxl - beam_y),
measured_beam_center_->sigma_pxl, reach);
logger.Info("Beam-centre hypothesis: {}/{} validation frames at ({:.2f},{:.2f}); trying "
"the centre +-{} px along each detector axis, a pixel at a time",
best.score, static_cast<int>(validation.size()), beam_x, beam_y, reach);
// Adopt only a hypothesis that INDEXES - the same majority the guards above test - rather
// than whichever rung scores best. Taking any improvement was tried and is wrong: on a
// dataset where no centre works the ladder wandered to the far end of its range on 10/60
// against 5/60 and reported a 30 px move, which is the count's noise floor.
const int adopt_bar = static_cast<int>(validation.size()) / 2;
bool adopted = false;
float won_x = beam_x, won_y = beam_y;
// The count is the only thing the ladder steers on, so a ladder that has not seen a THIRD
// of its own bar anywhere in two complete rings has nothing to steer by and is walking the
// noise floor. Measured over the beam-centre walks on the open battery: after two rings the
// two runs that went on to adopt stood at 17/60 and 59/60, and the three that adopted
// nothing stood at 4/60 or below - the two populations do not overlap.
int seen = best.score;
for (int k = 1; k <= reach && !cancelled_ && !adopted && !centre_past_reach; k++) {
for (const auto &[dx, dy]: {std::pair<int, int>{-k, 0}, {k, 0}, {0, -k}, {0, k}}) {
const float trial_x = beam_x + static_cast<float>(dx);
const float trial_y = beam_y + static_cast<float>(dy);
try_beam_center(trial_x, trial_y);
FirstPass alt = pick_best(*indexer_pool, *indexer);
logger.Info("Beam-centre hypothesis ({:.2f},{:.2f}): {}/{} frames", trial_x,
trial_y, alt.score, static_cast<int>(validation.size()));
seen = std::max(seen, alt.score);
// A rung is not adopted on the count alone. A centre error along the spindle
// translates the derotated cloud rigidly, which the FFT amplitude cannot see, so
// the indexer commits to an axis HARMONIC of the true cell - and a cell twice as
// long indexes MORE frames, not fewer, so the count rewards exactly the answer a
// centre error produces. The same test the scheme comparison already makes:
// primitive volumes differing by a small integer or by sqrt(3). Measured with the
// rungs scored honestly, a rung 1 px away indexed 56/60 on a 4x supercell of the
// lattice the starting centre had already found at 4/60.
const double ratio = best.vol > 0.0 && alt.vol > best.vol ? alt.vol / best.vol : 0.0;
const double nearest = std::round(ratio);
const bool harmonic = (nearest >= 2.0 && std::abs(ratio - nearest) < 0.15)
|| std::abs(ratio - std::sqrt(3.0)) < 0.15;
if (harmonic) {
logger.Info("Beam-centre hypothesis ({:.2f},{:.2f}): its cell is {:.2f}x the "
"volume of the one the starting centre found, so it is an axis "
"harmonic and the frame count cannot arbitrate it - not adopted",
trial_x, trial_y, ratio);
continue;
}
if (alt.result.has_value() && alt.score > adopt_bar && alt.score > best.score) {
best = alt;
won_x = trial_x;
won_y = trial_y;
adopted = true;
break;
}
}
if (k == 2 && !adopted && seen < adopt_bar / 3) {
logger.Info("Beam-centre hypothesis: the best of two rings is {}/{} against the "
"{}/{} a centre has to index to be adopted - the ladder is at the "
"count's noise floor and is stopped here", seen,
static_cast<int>(validation.size()), adopt_bar + 1,
static_cast<int>(validation.size()));
break;
}
}
if (!adopted)
restore_beam_center(won_x, won_y);
if (adopted)
logger.Info("Beam centre from indexing: ({:.2f},{:.2f}) -> ({:.2f},{:.2f}), moved "
"{:.2f} px, {}/{} validation frames", beam_x, beam_y, won_x, won_y,
std::hypot(won_x - beam_x, won_y - beam_y), best.score,
static_cast<int>(validation.size()));
}
// Long-axis rescue. When the de-novo cell indexes few validation frames, a long, finely-spaced
// axis was likely lost: the unconstrained FFT either collapsed it to a short sub-multiple or let
// a denser supercell over-fit the accumulated cloud (a small global-orientation error throws the
// many high-order reflections off along the fine axis, so the true cell scores worst on the raw
// cloud). Recover the true metric with a second search whose FFT unit-cell bound is widened past
// the default, then RE-INDEX at full resolution constrained by the cell that comes back, as a
// reference (the -C path): the reference filter drops the collapsed/supercell candidates and
// refines an accurate global lattice. Only runs after a poor standard pass, so well-indexing
// crystals never pay for it and their result is untouched.
auto max_axis = [](const RotationIndexerResult &r) {
const auto uc = r.lattice.GetUnitCell();
return std::max({uc.a, uc.b, uc.c});
};
if (!cancelled_ && best.result.has_value()
&& best.score < 0.5 * static_cast<double>(validation.size())) {
auto coarse_settings = experiment_.GetIndexingSettings();
// The resolution setting only sizes the FFT histogram - it does not filter which spots go
// into it - so this is here to pay for the widened bound below, not to make the long axis
// easier to pick out. Measured, coarsening does the opposite: the acceptance angle widens
// (0.159 -> 0.579 deg) but the true axis falls from rank 1996 to 16361 of 16384 directions.
coarse_settings.FFT_HighResolution_A(3.5f);
// The widened bound is what the rescue turns on, and it costs nothing: the histogram length is
// 4*pi*max_length/high_resolution, so coarsening the resolution to 3.5 A has already
// SHRUNK the transform by 1.75x - spending that back on reach leaves the coarse pass at
// 1.14x the standard one. Without it the rescue cannot recover what it exists to recover:
// a cell whose long axis is past the default 500 A bound is unrepresentable, so the
// "recovered" metric would still be a sub-multiple. Only reached after a poor pass, so a
// well-indexing crystal never pays. Stills never reach here.
constexpr float RESCUE_MAX_UNIT_CELL_A = 1000.0f;
if (coarse_settings.GetFFT_MaxUnitCell_A() < RESCUE_MAX_UNIT_CELL_A)
coarse_settings.FFT_MaxUnitCell_A(RESCUE_MAX_UNIT_CELL_A);
coarse_settings.IndexingThreads(2);
coarse_settings.RefineThreads(std::clamp(config_.nthreads / 2, 1, MAX_REFINE_THREADS));
IndexerThreadPool coarse_pool(coarse_settings, IndexerConstruction::OnFirstUse);
// Coarse first pass: keep the recovered cell with the LONGEST axis directly. Its full-
// resolution per-frame validation would be low (the coarse cell is metrically right but
// imprecise), so do NOT score it here - it is only the reference for the constrained re-index.
std::vector<std::unique_ptr<RotationIndexer>> cris;
cris.reserve(schemes.size());
for (const auto &[name, ordinals] : schemes) {
if (cancelled_) break;
cris.push_back(feed_scheme(coarse_pool, ordinals));
}
std::vector<std::future<void>> cfut;
cfut.reserve(cris.size());
for (auto &ri : cris) {
RotationIndexer *rp = ri.get();
cfut.push_back(std::async(std::launch::async, [rp] { rp->RunIndexing(); }));
}
for (auto &f : cfut) f.get();
std::optional<RotationIndexerResult> coarse_ref;
for (auto &ri : cris) {
auto r = ri->GetLattice();
if (r.has_value() && (!coarse_ref.has_value() || max_axis(*r) > max_axis(*coarse_ref)))
coarse_ref = r;
}
// A recovered axis at least 30% longer than the standard pass found signals a rescued metric.
if (!cancelled_ && coarse_ref.has_value()
&& max_axis(*coarse_ref) > 1.3 * max_axis(*best.result)) {
logger.Info("Long-axis rescue: coarse pass recovered a {:.0f} A axis (was {:.0f} A); "
"re-indexing constrained by that cell", max_axis(*coarse_ref), max_axis(*best.result));
experiment_.SetUnitCell(coarse_ref->lattice.GetUnitCell());
// The re-index must be able to REPRESENT the cell the coarse pass just recovered. The
// standard pool was sized at the default bound, so re-using it here would search for a
// long axis it cannot hold and hand back the sub-multiple the rescue exists to replace.
// count_indexed forces the lattice and never touches the pool, so the existing
// IndexAndRefine is reused; only the FFT needs the wider one.
auto wide_settings = experiment_.GetIndexingSettings();
wide_settings.IndexingThreads(2);
wide_settings.RefineThreads(std::clamp(config_.nthreads / 2, 1, MAX_REFINE_THREADS));
RaiseFFTBoundForKnownCell(wide_settings, experiment_);
FirstPass constrained;
if (wide_settings.GetFFT_MaxUnitCell_A() > indexing_bound_A) {
IndexerThreadPool wide_pool(wide_settings, IndexerConstruction::OnFirstUse);
constrained = pick_best(wide_pool, *indexer);
} else {
constrained = pick_best(*indexer_pool, *indexer);
}
experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo
if (constrained.result.has_value() && constrained.score > best.score)
best = constrained;
}
}
release_spot_engines(); // no more frames to read; give the cards their memory back
if (!cancelled_) {
// The two ways the first pass can end without a usable lattice that are about the DATA.
// They are one situation from the user's side - no crystal was found in these frames - so
// both name the cause and then give the same remedies.
constexpr const char *next_steps =
" Nothing was integrated. Check the beam centre (--beam-x / --beam-y), raise "
"--max-spots if the frames are well populated with spots, and use --mode azint to "
"write the per-image analysis for inspection.";
if (!best.result.has_value()) {
// An indexer that FAILED is a third outcome, and none of those remedies fit it: it
// never examined the frames, so nothing about the data or the processing options is
// implicated and changing them cannot help. Name the failure instead.
if (best.indexer_error)
throw JFJochException(JFJochExceptionCategory::IndexerError,
fmt::format("Rotation indexing could not run: {}. Nothing was "
"integrated. The indexer never examined the frames, "
"so this is a failure of the machine and not of the "
"data - free the resource it needs (a GPU shared with "
"another process is the usual cause) and run again.",
*best.indexer_error));
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
std::string("Two-pass rotation indexing found no lattice.") + next_steps);
}
// Short-axis hypothesis - the small-molecule case.
//
// LAST, after the axis-sign, beam-centre and long-axis rescues, deliberately. Those all
// run the standard floor and compare their result against `best` on volume ratios; a cell
// adopted here is a k-fold sub-cell, so running before them makes every one of those
// comparisons read an integer ratio and report an axis harmonic that is not one (measured:
// the beam-centre check announced a "3.00x harmonic ... the beam centre is what settles
// which" on a pair that differs only by the FLOOR). Here the comparison is like with like,
// and this pass sees whatever geometry those rescues settled on.
//
// fft_min_unit_cell_A (10 A) is a protein constant: shorter than any protein axis and longer
// than a small molecule's. It is applied to the FFT's PER-DIRECTION PEAK SEARCH
// (FFTIndexerCPU.cpp / FFTIndexerGPU.cu skip every histogram bin below it) and only then to
// the assembled cell (FFTIndexer::ReduceAndRefine, PostIndexingRefinement). The first of
// those is what does the damage: the transform reports ONE peak per direction, so a
// lattice row whose real-space repeat is shorter than the floor is reported at its first
// HARMONIC above it, and the cell built out of those harmonics is an exact integer
// sublattice of the true one - a real-space supercell. Measured on chemical-crystallography
// data: index 2, 3 and 6 exactly, and the one dataset whose true reduced cell clears 10 A is
// the one that comes out right. Because the floor is applied to the REDUCED cell it rejects
// the lattice and not merely a basis choice, so nothing downstream can re-describe its way
// back.
//
// Lowering the floor by default is not available: it only ever ADDS candidates, and the
// Bravais classification the candidate pool feeds is under-determined, so it moves crystals
// whose cell it never touched (measured: 2 regressions and 1 improvement over 20 protein
// datasets, all three in the metric symmetry rather than the cell). So the low floor runs as
// a SECOND hypothesis and the standard pass's answer stands unless the evidence takes it.
//
// What the evidence can be, before integration: a sublattice's per-spot residuals are
// dominated pointwise by the finer lattice it contains, so every margin statistic - squares,
// counts, trimmed sums - is biased toward the LARGER cell, and a spurious supercell indexes
// every frame its true cell does. Only the SATURATION is informative. Hence the same shape
// the scheme ratchet in pick_best already uses: an integer primitive-volume ratio plus a tie
// on validation frames, or an outright win, which runs against the bias and so is real. The
// statistic that genuinely decides an axis multiple is the occupancy of the reflection class
// the multiple adds - AbsenceEvidence in SearchSpaceGroup - and that needs integrated
// intensities, so it cannot run here.
// Not when a cell was given. RaiseFFTBoundForKnownCell has already lowered the floor for
// the standard pass against that cell, so this pass has nothing left to add - and it takes
// its settings from experiment_, which never carried the RAISED max bound, so on a cell
// longer than the default search it could only ever return a harmonic of what was asked
// for, tie on frames the way a sub-multiple always does, and be adopted over it.
if (!cancelled_ && !experiment_.GetUnitCell().has_value()) {
auto short_settings = experiment_.GetIndexingSettings();
short_settings.FFT_MinUnitCell_A(IndexingSettings::fft_min_unit_cell_limit_A);
short_settings.IndexingThreads(2);
short_settings.RefineThreads(std::clamp(config_.nthreads / 2, 1, MAX_REFINE_THREADS));
IndexerThreadPool short_pool(short_settings, IndexerConstruction::OnFirstUse);
std::vector<std::unique_ptr<RotationIndexer>> sris;
sris.reserve(schemes.size());
for (const auto &[name, ordinals] : schemes) {
if (cancelled_)
break;
sris.push_back(feed_scheme(short_pool, ordinals));
}
std::vector<std::future<void>> short_futures;
short_futures.reserve(sris.size());
for (auto &ri : sris) {
RotationIndexer *rp = ri.get();
short_futures.push_back(std::async(std::launch::async, [rp] { rp->RunIndexing(); }));
}
for (auto &f : short_futures)
f.get();
for (size_t i = 0; i < sris.size() && !cancelled_; i++) {
auto found = sris[i]->GetLattice();
if (!found.has_value())
continue;
RotationIndexerResult alt = std::move(*found);
const std::string &name = schemes[i].first;
// Primitive volumes, for the reason pick_best gives: two searches can find the same
// lattice in different settings, and a centred setting's cell is an exact integer
// multiple of its primitive one.
const double vol = std::abs(
alt.lattice.ToPrimitive(alt.search_result.centering).CalcVolume());
if (!(vol > 1.0) || !(best.vol > 1.0))
continue;
const double ratio = best.vol / vol;
const double nearest = std::round(ratio);
// The gate is here rather than after the scoring because scoring is the expensive
// half - count_indexed runs the real per-image path over the validation set, seconds
// on a protein - while the volume ratio is free. On every protein measured the low
// floor returns the SAME lattice (ratio 1.00-1.01), so this costs them one extra FFT
// and nothing else.
if (!(vol < best.vol) || nearest < 2.0 || std::abs(ratio - nearest) >= 0.15) {
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': primitive volume "
"{:.0f} A^3 against the standing cell's {:.0f} A^3, ratio {:.2f} - "
"not an integer sub-cell, keeping the standing cell",
IndexingSettings::fft_min_unit_cell_limit_A, name, vol, best.vol,
ratio);
continue;
}
// The premise of this pass, checked rather than asserted. The pass exists because a
// lattice row shorter than the standard floor is reported at its first HARMONIC
// above it, so a cell it may rightfully adopt has a reduced axis BELOW that floor -
// and the floor is applied to the reduced cell, which is the one to measure. Where
// the alternative has no such axis the floor cannot be what the standard pass
// tripped over, and an integer volume ratio is then an ordinary axis multiple: the
// question the axis-harmonic arbiter settles just above on the occupancy of the
// class the multiple adds. That is intensity-class evidence about the very
// reflections in dispute, where the tie below is a frame count this comment already
// says a sublattice cannot lose - so where the two disagree the arbiter is right and
// this block must not overrule it. Measured on the corpus: the four small-molecule
// sets the pass is for all have a reduced axis under 9 A, and the two protein sets
// it halved have 35 and 47 A.
const auto alt_reduced = alt.lattice.ToPrimitive(alt.search_result.centering)
.NiggliReduce().GetUnitCell();
const float alt_shortest = std::min({alt_reduced.a, alt_reduced.b, alt_reduced.c});
const float standard_floor = experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A();
if (alt_shortest >= standard_floor) {
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': a {:.0f}x sub-cell "
"(vol {:.0f} A^3), but its shortest reduced axis is {:.1f} A, above "
"the {:.1f} A floor the standard pass already searched - the floor "
"is not why that cell was missed, so this is an ordinary axis "
"multiple and not this pass's to decide; keeping the standing cell",
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
alt_shortest, standard_floor);
continue;
}
const int alt_score = count_indexed(*indexer, alt);
// A pass that indexes nothing is not tied with a pass that indexes nothing. Without
// this, 0 against 0 passes the tie band and the volume ratio alone decides, which is
// the one thing this block must never do.
if (alt_score <= 0 || best.score <= 0) {
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {}/{} validation "
"frames against {} - nothing indexes, keeping the standing cell",
IndexingSettings::fft_min_unit_cell_limit_A, name, alt_score,
static_cast<int>(validation.size()), best.score);
continue;
}
const bool tied = static_cast<float>(alt_score) >= best.score * 0.9f - 0.5f;
const bool clearly_more = static_cast<float>(alt_score) > best.score * 1.1f + 0.5f;
if (!tied && !clearly_more) {
logger.Info("Short-axis pass (floor {:.0f} A), scheme '{}': {:.0f}x sub-cell "
"(vol {:.0f} A^3) indexes {}/{} validation frames against {} - "
"keeping the standing cell",
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, vol,
alt_score, static_cast<int>(validation.size()), best.score);
continue;
}
const auto uc = alt.lattice.GetUnitCell();
logger.Warning("Short-axis pass (floor {:.0f} A), scheme '{}': the standing "
"cell is a {:.0f}x supercell. Its {:.0f} A^3 primitive volume against "
"this pass's {:.0f} A^3, at {}/{} validation frames against {} - "
"adopting {:.3f} {:.3f} {:.3f} {:.2f} {:.2f} {:.2f}. The shortest axis "
"is below the {:.0f} A default FFT floor, which is why the standard "
"pass could only build a multiple of it",
IndexingSettings::fft_min_unit_cell_limit_A, name, nearest, best.vol,
vol, alt_score, static_cast<int>(validation.size()), best.score,
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
experiment_.GetIndexingSettings().GetFFT_MinUnitCell_A());
best.score = alt_score;
best.vol = vol;
best.name = name + " (short-axis)";
best.result = std::move(alt);
}
}
// A lattice that indexes almost none of the validation frames is not this crystal's lattice.
// Forcing it anyway integrates every image against it; the few partials that survive yield no
// fulls in the 3D combine, and the run then dies inside the merge complaining about resolution -
// far from the cause. Stop here instead, while the reason is still visible. The floor is one
// frame in six, several times below the weakest crystal that still merges. It is checked AFTER
// the long-axis rescue, so a metric that rescue recovers is never rejected on its pre-rescue
// score.
if (best.score < static_cast<int>(validation.size()) / 6) {
// The second (refined-geometry) pass re-indexes DE NOVO, and the post-refined geometry -
// fitted to the lattice pass 1 already found - can tip the blind FFT onto an axis harmonic
// of a long cell that indexes too few frames to clear this floor (measured: a tripled
// ~100 A c-axis at 7/60). Pass 1 found and integrated the true cell at the header geometry,
// so prefer its whole result rather than abort the run on a re-index only the refined
// geometry made fail. This is the floor-side companion of the supercell-collapse guard
// below: that one only reaches a pass-2 lattice that DID clear the floor, whereas here the
// harmonic fell BELOW it and the throw would run before the guard could ever see it.
if (!geometry_prepass && prepass_result_.has_value()) {
const auto &pc = prepass_result_->search_result.conventional.GetUnitCell();
const auto &bc = best.result->search_result.conventional.GetUnitCell();
logger.Warning("Two-pass: the refined-geometry re-index indexes only {}/{} validation "
"frames ({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}) - too "
"few for this crystal's lattice; integrating with pass-1's lattice "
"instead ({}-centred {}, {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f})",
best.score, static_cast<int>(validation.size()),
best.result->search_result.centering,
gemmi::crystal_system_str(best.result->search_result.system),
bc.a, bc.b, bc.c, bc.alpha, bc.beta, bc.gamma,
prepass_result_->search_result.centering,
gemmi::crystal_system_str(prepass_result_->search_result.system),
pc.a, pc.b, pc.c, pc.alpha, pc.beta, pc.gamma);
best.result = *prepass_result_;
// Pass 1's lattice was FITTED at pass 1's detector distance, and this pass
// integrates at the post-refined one. Re-scoring it here is not the same as
// re-fitting it: a real-space cell is measured against the distance the spots were
// seen at, so carrying it across a distance change scales the whole cell by the
// ratio of the two. Measured on a 545 A axis over 225 -> 226 mm: 0.4 %, exactly
// 225/226, shipped as the run's answer. Scale it to the distance it will be used
// at; the orientation is unaffected.
const float d_fit = prepass_result_->geom.GetDetectorDistance_mm();
const float d_use = experiment_.GetDetectorDistance_mm();
if (d_fit > 0 && d_use > 0 && d_fit != d_use) {
const float f = d_use / d_fit;
const auto &l = best.result->lattice;
best.result->lattice = CrystalLattice(l.Vec0() * f, l.Vec1() * f, l.Vec2() * f);
logger.Info("Two-pass: pass-1's cell was fitted at {:.3f} mm and is used at "
"{:.3f} mm - scaled by {:.6f} to the distance it is integrated at",
d_fit, d_use, f);
}
best.score = count_indexed(*indexer, *best.result); // re-score at the refined geometry
best.vol = std::abs(best.result->lattice
.ToPrimitive(best.result->search_result.centering).CalcVolume());
best.name = "pass-1 lattice (refined-geometry re-index too sparse)";
}
}
// The last resort, and ONLY on a run that has no usable lattice: the pass-1 fallback above
// has had its turn, and below this the run stops with an error. A run that indexes must
// not pay for it - measured, an unguarded call spent 41-76 % of the wall clock on a
// healthy dataset evaluating rungs that arithmetically could not be adopted (one crystal
// 41 s guarded against 175 s unguarded, same answer).
if (best.score < static_cast<int>(validation.size()) / 6) {
if (auto rescued = first_pass_ladder(best.score,
static_cast<int>(validation.size()) / 6, 0))
best = *rescued;
}
// Whether the run has a lattice at all is decided here, on the pooled spots rather than on
// the frame count the rescues above are steered by. A frame count asks each frame to clear
// a bar of its own and then counts the frames that did, which measures the spots per frame
// as much as the lattice; the pooled fraction asks the whole validation set at once and is
// judged against what the same lattice scores at a wrong spindle angle, so a sparse pattern
// is refused only when its spots genuinely do not know about the lattice.
const PooledEvidence evidence = pooled_evidence(*indexer, *best.result);
const double pooled = static_cast<double>(evidence.on_lattice)
/ static_cast<double>(std::max<int64_t>(1, evidence.spots));
const double pooled_chance = static_cast<double>(evidence.by_chance)
/ static_cast<double>(std::max<int64_t>(1, evidence.spots));
if (!beats_chance(evidence)) {
{
// Name the cell and Bravais class that was rejected. The commonest cause is a metric
// symmetry promoted one class too far - the constrained cell then misses every
// reflection by the small angle the constraint snapped away - and without the cell in
// the message there is nothing to see that from.
const auto &c = best.result->search_result.conventional.GetUnitCell();
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
fmt::format("Two-pass rotation indexing found only a lattice that "
"takes {:.1f}% of the validation spots against the "
"{:.1f}% a wrong spindle angle takes - it is not this "
"crystal's lattice. It was {}-centred {}, "
"{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}.{}",
100.0 * pooled, 100.0 * pooled_chance,
best.result->search_result.centering,
gemmi::crystal_system_str(best.result->search_result.system),
c.a, c.b, c.c, c.alpha, c.beta, c.gamma, next_steps));
}
}
indexer->ForceRotationIndexerResult(*best.result);
logger.Info("First-pass spot finding: {} frames in {} batches, {:.2f} s",
spot_cache.size(), prefetch_batches, prefetch_time_s);
logger.Info("Two-pass rotation indexing found lattice (scheme '{}': {}/{} validation frames, "
"{}/{} validation spots = {:.1f}% against {:.1f}% at a wrong spindle angle)",
best.name, best.score, static_cast<int>(validation.size()),
evidence.on_lattice, evidence.spots, 100.0 * pooled, 100.0 * pooled_chance);
// The spots in hand were already cut to the budget in force, so the measurement can only
// shorten a budget, never lengthen one - which is why RunAllPasses hands the second pass the
// list the first one started from rather than the list the first one settled on.
if (config_.measure_spot_budget) {
const int64_t budget = measure_budget(*indexer, *best.result);
// The measurement can honestly land under MIN_SPOT_COUNT, a floor written for a
// stills FIT, where fewer spots cannot pin a lattice down. Here the lattice is
// already in hand and merely tested - but the validation gate still needs
// ViableCellMinSpots indexed spots per frame (AnalyzeIndexing), so a shorter list
// could never index anything: keep the floor's worth of spots rather than die on
// the setter. A budget that shallow is a finding in its own right - the lattice and
// the file's geometry explain only the brightest few detections (measured on a
// header detector distance 20% off, compensated by a uniformly scaled cell that
// only matches near the beam centre) - so it is warned about, not just adopted.
const int64_t adopted = std::max<int64_t>(budget, MIN_SPOT_COUNT);
if (budget > 0 && adopted < experiment_.GetMaxSpotCount()) {
if (adopted != budget)
logger.Warning("Spot budget: only the strongest {} spots per image lie on "
"the lattice (measured on {} frames) - fewer than the {} the "
"frame gate can act on, so {} are kept. A budget this shallow "
"usually means the geometry in the file (detector distance, "
"beam centre) or the lattice is wrong, not that the crystal "
"is weak", budget, static_cast<int>(validation.size()),
MIN_SPOT_COUNT, adopted);
else
logger.Info("Spot budget: keeping the strongest {} spots per image "
"(measured on {} frames; was {})", budget,
static_cast<int>(validation.size()), experiment_.GetMaxSpotCount());
experiment_.MaxSpotCount(adopted);
}
}
// Second pass: compare the de-novo lattice with pass 1's HERE, before integrating every image
// with it. A markedly larger cell is the bistable supercell collapse, which was already
// rejected in Run() - but only after the whole pass had run, so it cost an entire extra pass
// on a dataset that ended up on pass-1's lattice anyway. Take pass-1's result now instead:
// the same remedy, one pass earlier.
if (!geometry_prepass && prepass_result_.has_value()) {
// Primitive volumes, like the scheme comparison above: a centred conventional cell is an
// exact integer multiple of its primitive one, so two settings of the SAME lattice differ
// by that factor and comparing the conventional cells reads a mere change of setting as a
// supercell - which then forces pass 1's setting and, with it, its lower symmetry.
const double v1 = std::abs(prepass_result_->lattice
.ToPrimitive(prepass_result_->search_result.centering).CalcVolume());
const double v2 = std::abs(best.result->lattice
.ToPrimitive(best.result->search_result.centering).CalcVolume());
const bool supercell = v1 > 1.0 && v2 > 1.5 * v1;
// Dropping to the primitive sub-cell of pass-1's centred lattice is the same
// disagreement seen from the other side, and the volumes above cannot see it: the two
// have the SAME primitive metric. Integrating there never predicts the reflections the
// centring implies, and the pass then merges in P1 (measured on a C-centred monoclinic
// crystal). Only this direction - a centring pass 2 finds where pass 1 had none is the
// refined geometry doing its job, and the intensity test in the search confirms it.
const bool lost_centring = prepass_result_->search_result.centering != 'P'
&& best.result->search_result.centering == 'P';
if (supercell || lost_centring) {
logger.Info("Two-pass: that lattice ({}-centred, volume {:.0f} A^3) disagrees with "
"pass 1 ({}-centred, {:.0f} A^3) - integrating with pass-1's lattice instead",
best.result->search_result.centering, v2,
prepass_result_->search_result.centering, v1);
indexer->ForceRotationIndexerResult(*prepass_result_);
}
}
}
}
// Main per-image loop, spread over N worker threads pulling from a shared counter. HDF5 reads
// are serialized by the global hdf5_mutex; the analysis runs in parallel.
std::atomic<int> next_ordinal = 0;
std::atomic<int> finished_count = 0;
// Images are written at their own ordinal, so a frame that fails leaves a HOLE rather than
// shifting everything after it up. The extent of the written datasets is therefore the highest
// ordinal reached, not the number that succeeded - counting successes makes the file one image
// short for every failure, and the image it loses is the LAST one, not the one that failed.
std::atomic<int> max_written_ordinal = -1;
auto note_written = [&max_written_ordinal](int ordinal) {
int prev = max_written_ordinal.load(std::memory_order_relaxed);
while (prev < ordinal
&& !max_written_ordinal.compare_exchange_weak(prev, ordinal, std::memory_order_relaxed)) {
}
};
std::atomic<uint64_t> total_uncompressed_bytes = 0;
// What the integrator did to the reflections it was handed, summed over the workers as each one
// finishes (BraggIntegrationCounts). Reading an engine's counts synchronises its stream, so it is
// done once per worker at the end rather than per image.
std::atomic<uint64_t> bragg_predicted = 0, bragg_starved = 0, bragg_starved_nb = 0,
bragg_fallback = 0;
// Calibration by spots: the pooled spot list of the whole run. Ring clustering is O(n^2) in it and
// the Hough circle centre O(n^3) in its first few hundred, so each image contributes a fair share of
// a fixed budget instead of everything it found - a powder ring is over-determined either way.
constexpr size_t CALIBRATION_MAX_SPOTS = 20000;
const size_t calibration_spots_per_image =
std::max<size_t>(1, CALIBRATION_MAX_SPOTS / static_cast<size_t>(images_to_process));
std::mutex calibration_spot_m;
std::vector<SpotToSave> calibration_spot_list;
// Higher-order beam contamination, counted on the spots the lattice did not take. Each worker
// keeps its own tally and adds it in when it is done, so the total is a sum of counts and does
// not depend on how the images were shared out.
std::mutex harmonic_m;
HarmonicEvidence harmonic_evidence;
auto azint_worker = [&]() {
std::vector<uint8_t> decompression_buffer;
std::shared_ptr<JFJochReaderRawImage> img;
ImagePreprocessorCPU preprocessor(experiment_, pixel_mask_);
ImagePreprocessorBuffer buffer(experiment_.GetPixelsNum());
AzIntEngineCPU azint(mapping);
AzimuthalIntegrationProfile profile(mapping);
while (!cancelled_) {
const int ordinal = next_ordinal.fetch_add(1);
const int image_idx = start_image + ordinal * config_.stride;
if (image_idx >= end_image) break;
// Reuse the raw image frame to frame - a fresh 4-byte 16 Mpx frame is 72 MB of
// untouched pages the decoder faults in one by one - except while the writer thread
// still holds the previous one's pixels.
if (!img || img.use_count() > 1)
img = std::make_shared<JFJochReaderRawImage>();
bool read = false;
try {
read = reader_.ReadRawImage(image_idx, *img);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Error("Failed to load image {}: {}", image_idx, e.what());
continue;
}
if (!read) continue;
DataMessage msg{};
msg.image = img->image;
msg.number = ordinal;
msg.original_number = image_idx;
if (dataset->efficiency.size() > image_idx)
msg.image_collection_efficiency = dataset->efficiency[image_idx];
total_uncompressed_bytes += msg.image.GetUncompressedSize();
const auto t0 = std::chrono::steady_clock::now();
try {
const uint8_t *image_ptr = msg.image.GetUncompressedPtr(decompression_buffer);
preprocessor.Analyze(buffer, image_ptr, msg.image.GetMode());
azint.Run(buffer, profile);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Error("Error integrating image {}: {}", image_idx, e.what());
continue;
}
msg.azint_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now() - t0).count();
msg.processing_time_s = msg.azint_time_s;
msg.az_int_profile = profile.GetResult();
msg.az_int_profile_count = profile.GetPixelCount();
msg.az_int_profile_std = profile.GetStd();
msg.bkg_estimate = profile.GetBkgEstimate(mapping.Settings());
msg.ice_ring_score = profile.GetIceRingScore(mapping.Settings(),
config_.spot_finding.ice_ring_width_Q_recipA);
msg.run_number = experiment_.GetRunNumber();
msg.run_name = experiment_.GetRunName();
plots.Add(msg, profile);
if (writer_queue) writer_queue->Post(msg, img);
note_written(ordinal);
if (observer) observer->OnImageProcessed(msg);
const int done = finished_count.fetch_add(1) + 1;
if (observer) observer->OnProgress(done, images_to_process);
}
};
auto full_worker = [&]() {
pin_gpu(); // round-robin per worker thread; must precede engine construction
// Before the analysis engine, which page-locks these bytes for its uploads and unregisters
// them when it is destroyed: it must not outlive the buffer it was given.
std::shared_ptr<JFJochReaderRawImage> img;
MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, *indexer,
/*enable_fused_adaptive_gpu=*/true);
AzimuthalIntegrationProfile profile(mapping);
HarmonicEvidence worker_harmonic;
while (!cancelled_) {
const int ordinal = next_ordinal.fetch_add(1);
const int image_idx = start_image + ordinal * config_.stride;
if (image_idx >= end_image) break;
// Reuse the raw image frame to frame - a fresh 4-byte 16 Mpx frame is 72 MB of
// untouched pages the decoder faults in one by one - except while the writer thread
// still holds the previous one's pixels.
if (!img || img.use_count() > 1)
img = std::make_shared<JFJochReaderRawImage>();
bool read = false;
try {
read = reader_.ReadRawImage(image_idx, *img);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Error("Failed to load image {}: {}", image_idx, e.what());
continue;
}
if (!read) continue;
DataMessage msg{};
msg.image = img->image;
msg.number = ordinal;
msg.original_number = image_idx;
if (dataset->efficiency.size() > image_idx)
msg.image_collection_efficiency = dataset->efficiency[image_idx];
total_uncompressed_bytes += msg.image.GetUncompressedSize();
const auto t0 = std::chrono::steady_clock::now();
try {
analysis.Analyze(msg, profile, config_.spot_finding);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
logger.Error("Error analyzing image {}: {}", image_idx, e.what());
continue;
}
msg.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now() - t0).count();
msg.run_number = experiment_.GetRunNumber();
msg.run_name = experiment_.GetRunName();
if (calibration) {
std::lock_guard lock(calibration_spot_m);
for (size_t i = 0; i < msg.spots.size() && i < calibration_spots_per_image; ++i)
calibration_spot_list.push_back(msg.spots[i]);
}
AddHarmonicEvidence(msg.spots, experiment_.GetDiffractionGeometry(), worker_harmonic);
plots.Add(msg, profile);
if (writer_queue) writer_queue->Post(msg, img);
note_written(ordinal);
if (observer) observer->OnImageProcessed(msg);
const int done = finished_count.fetch_add(1) + 1;
if (observer) observer->OnProgress(done, images_to_process);
}
{
std::lock_guard lock(harmonic_m);
harmonic_evidence.Add(worker_harmonic);
}
const auto bragg = analysis.BraggCounts();
bragg_predicted += bragg.predicted;
bragg_starved += bragg.bkg_starved;
bragg_starved_nb += bragg.bkg_starved_by_neighbour;
bragg_fallback += bragg.profile_fallback;
};
if (observer)
observer->OnPhase("Processing images");
std::function<void()> worker = per_image_analysis ? std::function<void()>(full_worker)
: std::function<void()>(azint_worker);
// How many workers the loop actually wants. Every one of them submits its own kernels to a card,
// and a card runs out of room to accept them long before it runs out of work: measured on two
// GPUs, the loop's own time falls to sixteen workers and then rises again, so forty-eight is
// slower than sixteen on a 16 Mpx set. Sixteen is where that measurement put the minimum, and a
// later one on a single card agrees - eight leaves the loop waiting on the queue rather than the
// card. The cap is per card because that is what the queue depth belongs to, and it only applies
// when -N was left at its default - an explicit -N is a deliberate instruction and is obeyed,
// which is what a previous attempt at this got wrong.
// Only the full-analysis worker drives a card; the azimuthal one preprocesses and integrates on
// the CPU and wants every thread it can have, so the cap must not reach it.
const int gpus = get_gpu_count();
const int image_workers = (per_image_analysis && config_.nthreads_auto && gpus > 0)
? std::min(config_.nthreads, std::max(16, 16 * gpus))
: config_.nthreads;
if (image_workers < config_.nthreads)
logger.Info("Per-image loop: {} of {} threads ({} GPUs) - past what a card can take queue "
"work from, more workers make it slower, not faster; pass -N to override",
image_workers, config_.nthreads, gpus);
std::vector<std::future<void> > futures;
futures.reserve(image_workers);
const auto image_loop_start = std::chrono::steady_clock::now();
for (int i = 0; i < image_workers; ++i)
futures.push_back(std::async(std::launch::async, worker));
for (auto &f: futures)
f.get();
// Everything the workers posted has to be on disk before the end message is written and the file
// finalized, and a write that failed - out of space, above all - has to surface here rather than
// leave a truncated file behind.
if (writer_queue)
writer_queue->Finish();
// Wall time of the per-image loop alone. The per-stage means below cover only what runs inside it,
// so without this there is nothing to compare them against and no way to see how much of a run is
// spent outside it - on the first-pass indexing and on scaling/merging.
result.image_loop_threads = image_workers;
result.image_loop_time_s = std::chrono::duration<double>(
std::chrono::steady_clock::now() - image_loop_start).count();
result.cancelled = cancelled_;
result.images_processed = finished_count.load();
result.harmonic = HarmonicFromEvidence(harmonic_evidence);
result.powder = powder_;
result.powder_excluded_from_indexing = !config_.spot_finding.measured_ring_q_recipA.empty();
if (result.powder_excluded_from_indexing)
result.powder_indexing_d_min_A = config_.spot_finding.high_resolution_limit;
if (result.harmonic.detected)
logger.Warning("Higher-order beam contamination: {:.0f}% of the {} unindexed spots this run "
"could not put on the lattice are on it at one third the wavelength (the same "
"test off a harmonic finds {:.1f}%). They sit at {:.1f} A apparent resolution",
result.harmonic.fraction, result.harmonic.tested,
result.harmonic.null_fraction, result.harmonic.mean_d_A);
// What the stencil cost, as the integrator measured it: the reflections dropped whole for a
// background ring left below six clean pixels, and - of those - the ones the NEIGHBOURS took
// rather than the detector. The second is what a wider r1 can change, and it is what the two-pass
// guard reads; the first is reported beside it because the difference is the detector's own floor.
if (bragg_predicted > 0) {
const auto frac = [&](uint64_t n) {
return 100.0 * static_cast<double>(n) / static_cast<double>(bragg_predicted);
};
bkg_starved_fraction_ = static_cast<double>(bragg_starved_nb) / static_cast<double>(bragg_predicted);
logger.Info("Integration at r1={:.1f} r2={:.1f} r3={:.2f}: of {} predicted reflections, "
"{:.3f}% lost their background ring - {:.3f}% to neighbouring reflections and the "
"rest to the detector; {:.3f}% profile-fit fallbacks",
experiment_.GetBraggIntegrationSettings().GetR1(),
experiment_.GetBraggIntegrationSettings().GetR2(),
experiment_.GetBraggIntegrationSettings().GetR3(),
bragg_predicted.load(), frac(bragg_starved), frac(bragg_starved_nb),
frac(bragg_fallback));
}
// Every image failing is a total failure, not a run that produced nothing: it used to be reported
// only as per-image log lines while the process still exited 0 with no output file.
if (!cancelled_ && images_to_process > 0 && result.images_processed == 0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No image could be analyzed - see the per-image errors above");
result.mean_processing_time = plots.GetMeanProcessingTime();
result.indexing_rate = plots.GetIndexingRate();
// Powder calibration: the whole run has now been accumulated - the summed (q x azimuth) profile in
// plots, or the pooled spot list - so fit the detector geometry to it. The caller reports it and
// writes the PONI file; nothing here is stored in the _process.h5.
if (calibration && !cancelled_) {
if (observer)
observer->OnPhase("Powder calibration");
if (calibration_spots) {
logger.Info("Powder calibration from {} pooled spots", calibration_spot_list.size());
result.calibration = CalibrateFromSpots(calibration_spot_list,
experiment_.GetDiffractionGeometry(),
config_.calibrant_ring_q,
config_.calibration_refine_tilt);
} else {
const auto *summed = plots.GetAzIntProfileObject();
if (!summed)
throw JFJochException(JFJochExceptionCategory::CalibrationError,
"No azimuthal profile was accumulated");
result.calibration = CalibrateFromProfile(summed->GetResult(), mapping,
experiment_.GetDiffractionGeometry(),
config_.calibrant_ring_q,
config_.calibration_refine_tilt,
calibration_spot_list);
// Re-bin, and fit again on a profile that was binned about the answer.
//
// Everything above reads a profile built with the geometry the run STARTED from, and a
// calibration is run precisely because that geometry is in doubt. Binning is not something
// the fit can undo: which pixels landed in which (q, phi) bin was decided when the images
// were read. Get the centre wrong and every ring is smeared across its own sectors - past a
// hundred pixels the fit leaves rms 4.8 px even when it is started from the exact answer.
// Get the distance wrong and the radial sampling is simply the wrong scale, which is why a
// run recovering 110 mm from a 250 mm header ended at 1.1 px where the same data from a
// right header gives 0.4.
//
// So integrate the images once more, binned with what was just fitted. A calibration run is
// a handful of images and the fit is worth far more than the second read.
const auto rebinned = RebinAndRefit(*result.calibration, start_image, end_image,
images_to_process, mapping, calibration_spot_list);
if (rebinned)
result.calibration = rebinned;
}
}
// End message (also written to the file).
EndMessage end_msg;
// The EXTENT of the per-image datasets (see note_written), not the success count: those two
// differ by exactly the number of frames that failed, and the writer sizes its virtual datasets
// from this.
end_msg.max_image_number = static_cast<uint64_t>(max_written_ordinal.load() + 1);
end_msg.images_collected_count = result.images_processed;
end_msg.images_sent_to_write_count = result.images_processed;
end_msg.end_date = time_UTC(std::chrono::system_clock::now());
end_msg.run_number = experiment_.GetRunNumber();
end_msg.run_name = experiment_.GetRunName();
end_msg.bkg_estimate = plots.GetBkgEstimate();
end_msg.spindle_blind_fraction = plots.GetSpindleBlindFraction();
result.spot_resolution_estimate_A = plots.GetResolutionEstimate();
end_msg.ice_ring_score = plots.GetIceRingScoreArray();
end_msg.ice_ring_score_mean = plots.GetIceRingScore();
end_msg.az_int_result["dataset"] = plots.GetAzIntProfile();
end_msg.indexing_rate = result.indexing_rate;
if (full && !cancelled_) {
if (const auto rot = indexer->FinalizeRotationIndexing(); rot.has_value()) {
end_msg.rotation_lattice = rot->lattice;
// The refined geometry, not the nominal StartMessage values. Two things about these five
// numbers that are not obvious from their names. The beam centre is the PONI - the foot of
// the perpendicular from the sample - so with a non-zero tilt it is NOT where the direct
// beam lands; that point is D*tan(rot)/pixel away and is GetDirectBeam_pxl(). And the tilt
// is not written here only for the record: IndexAndRefine puts it on outcome.experiment,
// which is the geometry Bragg prediction and integration then run at. It is fitted with the
// DISTANCE FROZEN and nothing checks the covariance, so the split between PONI and tilt is
// not reproducible between passes even though their sum - the direct beam - is. Neither
// number is meaningful on its own: report or feed back the whole set, or none of it.
end_msg.refined_beam_center_x = rot->geom.GetBeamX_pxl();
end_msg.refined_beam_center_y = rot->geom.GetBeamY_pxl();
end_msg.refined_poni_rot1 = rot->geom.GetPoniRot1_rad();
end_msg.refined_poni_rot2 = rot->geom.GetPoniRot2_rad();
end_msg.refined_poni_rot3 = rot->geom.GetPoniRot3_rad();
// rot3 is not carried: a rotation about the beam is an exact null of this experiment
// (measured, smallest eigenvalue -5e-16), so the fit never moves it and reporting it would
// suggest it had been determined.
result.refined_detector_tilt_deg = std::array<double, 2>{
rot->geom.GetPoniRot1_rad() * 180.0 / PI,
rot->geom.GetPoniRot2_rad() * 180.0 / PI};
if (rot->axis)
end_msg.refined_rotation_axis = rot->axis->GetAxis();
end_msg.rotation_lattice_type = LatticeMessage{
.centering = rot->search_result.centering,
.niggli_class = rot->search_result.niggli_class,
.crystal_system = rot->search_result.system
};
result.rotation_lattice_found = true;
// The consensus cell below is this lattice's conventional cell, so this is the centring
// it is written in - the pair the volume comparisons in Run() need.
result.consensus_centering = rot->search_result.centering;
}
result.consensus_cell = indexer->GetConsensusUnitCell();
end_msg.unit_cell = result.consensus_cell;
}
// Scaling and merging (full analysis only).
if (full && !cancelled_ && result.indexing_rate.has_value() && result.indexing_rate > 0
&& (config_.run_scaling || !config_.reference_data.empty())) {
// Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
// bar reflects what is happening instead of freezing on one label.
auto phase = [&](const std::string &p) {
if (observer) observer->OnPhase(p);
};
phase("Scaling and merging");
// Ice-ring handling (--detect-ice-rings): flag reflections sitting on a hexagonal-ice powder
// ring. Their integrated intensity is contaminated by the strong, variable ice background, so
// they drag the per-image scale fit. Flag them here so scaling (the per-image G fit and the
// fulls refit) skips them, while the combine, the merge and the statistics keep them - dropping
// them outright guts low/mid-resolution completeness on crystals that merge them fine.
// The eleven hexagonal-ice bands cover 16-26% of the unique reflections at typical resolutions
// whether or not the crystal has any ice, so handling ice unconditionally taxes clean data for
// nothing. Gate the flagging on the run's own measured ice strength (1 = no ice; the strongest
// ring over the smooth radial background, measured on the peak-excluded profile so Bragg peaks
// cannot pose as ice). The score is logged either way, so a run that gates off still reports why.
// Ice is looked for on TWO channels, because it arrives in two forms. Fine-grained ice is a
// smooth powder ring and shows up in the radial profile (ice_ring_score). Ice in large
// crystallites arrives as discrete reflections instead, leaves the profile flat, and is only
// visible as a pile-up of found SPOTS on the ring positions - measured against the same q width
// of ice-free flanks beside them, so the fall-off of spot density with resolution cancels. On
// the rotation battery the two channels barely overlap: the crystals with smooth ice read
// 2.1-2.4 / ~1.0, the ones whose spots are mostly ice read ~1.1 / 3.8-14.6, and a clean crystal
// reads 1.04 on both. Either channel alone is blind to half the cases.
const auto ice_score = plots.GetIceRingScore();
const auto ice_spot_ratio = plots.GetIceRingSpotRatio();
const float ice_min_score = experiment_.GetScalingSettings().GetIceMinScore();
const float ice_min_spot_ratio = experiment_.GetScalingSettings().GetIceMinSpotRatio();
const bool ice_present = (!ice_score.has_value() || *ice_score >= ice_min_score)
|| (ice_min_spot_ratio > 0.0f && ice_spot_ratio.has_value()
&& *ice_spot_ratio >= ice_min_spot_ratio);
if (experiment_.IsDetectIceRings() && !ice_present) {
logger.Info("Ice-ring handling: measured ice score {:.2f} < {:.2f} and spot ratio {:.2f} < "
"{:.2f}, no ice detected - ice-ring handling skipped entirely",
*ice_score, ice_min_score, ice_spot_ratio.value_or(NAN), ice_min_spot_ratio);
} else if (experiment_.IsDetectIceRings()) {
const float ice_width = config_.spot_finding.ice_ring_width_Q_recipA;
size_t total = 0, flagged = 0;
for (auto &outcome : indexer->GetIntegrationOutcome()) {
for (auto &r : outcome.reflections) {
++total;
r.on_ice_ring = IsOnIceRing(r.d, ice_width);
if (r.on_ice_ring)
++flagged;
}
}
logger.Info("Ice-ring handling: ice score {:.2f} (gate {:.2f}), spot ratio {:.2f} (gate "
"{:.2f}); flagged {} of {} reflections on ice rings (half-width {:.3f} A^-1); "
"excluded from scaling, kept for merging",
ice_score.value_or(NAN), ice_min_score, ice_spot_ratio.value_or(NAN),
ice_min_spot_ratio, flagged, total, ice_width);
}
// Scale the images and merge. Factored so it can run twice: first in P1 to give the
// space-group search a merged dataset, then again in the determined space group so the
// scaling sees symmetry equivalents and the final statistics are in the right symmetry.
// (With a user-fixed space group it simply runs once, already in that symmetry.)
struct ScaleMergeResult {
std::vector<MergedReflection> merged;
MergeStatistics statistics;
// CC1/2 as measured before the correction surfaces (rotation) - the quantity the two-pass
// quality guard in Run() compares the passes on. Equal to the reported CC1/2 wherever no
// surface was fitted, which is every merge on the stills path.
double cc_half_before_corrections = NAN;
// The reference-range table and its ISa (--report-resolution); see ProcessResult.
std::optional<MergeStatistics> reference_statistics;
double reference_isa = 0.0;
};
// The reference path computes each image's G once (per-image scaling against the
// reference); the scaling loop below is skipped, so G is stable across the two passes.
// Smoothing it more than once would compound the correction, so do it only on the first
// pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly.
// Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single
// allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame
// partials, scales the fulls and merges; the whole hot path runs on the GPU when one is present).
// Ingested once here (after ice flagging) and reused across both space-group passes; a fixed
// (forced) mosaicity is handled by the recompute. The decay + absorption correction surfaces run as
// post-scale-fulls stages (on by default, see ScalingSettings::CorrectionSurfaces); external-
// reference scaling, the stills -B (per-image B-factor) and wedge refinement are unsupported here.
// Re-seat the integrated reflections into a different setting of the SAME lattice, for the two
// arms below. The change of basis is read straight off the two lattices: P[i][j] =
// conv_real[i] . indexed_reciprocal[j] is the coefficient of the indexed cell's j-th axis in the
// new cell's i-th axis, and indices transform with the axes, so it is also the matrix that takes
// hkl across. Rounding it to integers - and declining when the residual is large - keeps it exact
// even though the refined cell is not exactly LatticeSearch's Niggli cell.
//
// The per-image reflections and lattices are already in the _process.h5 by now - they go to file
// as each image is processed, long before the group is known - so re-seating them here leaves the
// stored ones in the OLD setting beside a cell in the new one. Accumulate the change of basis
// (composed, in case more than one fires) and hand it to the writer, so the file says which setting
// its per-image data are in and a reader can bring them across.
gemmi::Mat33 stored_to_final; // identity until something below re-seats
const auto record_reindex = [&](const gemmi::Mat33 &m) {
stored_to_final = m.multiply(stored_to_final);
std::array<int32_t, 9> v{};
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j)
v[3 * i + j] = static_cast<int32_t>(std::lround(stored_to_final.a[i][j]));
end_msg.reindex_matrix = v;
};
const auto reindex_into = [&](const LatticeSearchResult &cand) {
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (reindex_res >= 0.1)
return false;
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(reindex);
for (auto &r : io.reflections) {
const double h = r.h, k = r.k, l = r.l;
r.h = static_cast<int32_t>(std::lround(reindex.a[0][0]*h + reindex.a[0][1]*k + reindex.a[0][2]*l));
r.k = static_cast<int32_t>(std::lround(reindex.a[1][0]*h + reindex.a[1][1]*k + reindex.a[1][2]*l));
r.l = static_cast<int32_t>(std::lround(reindex.a[2][0]*h + reindex.a[2][1]*k + reindex.a[2][2]*l));
}
}
result.consensus_cell = cand.conventional.GetUnitCell();
result.consensus_centering = cand.centering;
end_msg.unit_cell = result.consensus_cell;
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reindex);
end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system };
record_reindex(reindex);
return true;
};
// A space group the USER fixed carries its own Bravais lattice, and that is the setting its
// reflections have to be indexed in - but the indexer answers to the metric, not to the group, and
// LatticeSearch hands back the MOST symmetric setting the metric supports. A tetragonal-P lattice
// therefore never comes back C-centred orthorhombic even though the same lattice has that setting,
// and -S C222 on one merges with the C absence rule applied in the wrong frame: measured, half the
// observations thrown away (2.14 M -> 1.08 M) and ISa 28.3 -> 24.6. Ask the character table for the
// group's own class instead and re-seat the reflections into it.
//
// Two ways an indexed lattice can fail to carry the fixed group's Bravais lattice, and one
// re-seating for both. The CENTRING can differ, as above. Or the centring can agree while the
// METRIC does not - the axes permuted, a=37.909 b=78.031 c=77.594 declared P 41 21 2, which
// needs a=b with the 4-fold along c. That one is invisible to a centring test and merges
// anyway, in a frame the reflections are not in: measured, --mode scale -S P41212 on such a
// run reports COMPLETENESS= 195.3.
// A fixed group of lower symmetry than the lattice but with the same centring (-S P21 on an
// orthorhombic-P lattice, say) satisfies both tests where it stands, merges correctly, and is
// left alone. So is a group the RUN determined - it was determined FROM this lattice, and the
// arm above and the check below are its handling; a de-novo run must behave exactly as it did.
if (user_fixed_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_;
const char indexed_centering = end_msg.rotation_lattice_type->centering;
const double violation = end_msg.unit_cell ? MetricViolation(*end_msg.unit_cell, fixed_sg) : 0.0;
if (indexed_centering != fixed_sg.centring_type() || violation > MAX_METRIC_VIOLATION) {
// A trigonal-P group sits on a hexagonal-P lattice; every other system names its own.
const auto want_system = fixed_sg.crystal_system() == gemmi::CrystalSystem::Trigonal
&& fixed_sg.centring_type() == 'P'
? gemmi::CrystalSystem::Hexagonal : fixed_sg.crystal_system();
// Search the PRIMITIVE cell. rotation_lattice is the conventional one and its centring
// is carried beside it, but the character table is reached through a Niggli reduction
// that takes whatever it is handed to be primitive - so a centred conventional cell
// reduces as a lattice with the wrong point set. This arm is entered exactly when the
// centrings differ, which includes every centred indexed lattice, so the distinction is
// not academic: a C-centred orthorhombic conventional cell read as primitive is itself a
// perfectly good oP metric, and -S P212121 on one would match, reindex by the identity
// and merge in a group whose absence rule deletes half the reflections that exist.
// The other lattice arithmetic in this file goes through ToPrimitive for the same reason.
const auto cand = LatticeSearchForClass(
end_msg.rotation_lattice->ToPrimitive(indexed_centering), want_system,
fixed_sg.centring_type());
if (cand && reindex_into(*cand)) {
const auto &uc = *result.consensus_cell;
logger.Info("Reindexed the indexed {}-centred lattice (metric violation {:.3f} against "
"the group's own operators) into the {}-centred setting the fixed space "
"group {} needs: a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}", indexed_centering, violation, cand->centering,
fixed_sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma);
}
}
}
// The lattice a pass indexed has to carry the Bravais lattice of the group it merges in. Only a
// group the USER fixed can fail that - one the pass determined for itself was determined FROM
// this lattice - and the re-seating above has already had its chance to find a setting that
// does. What comes out otherwise is not a merely suboptimal answer: the absence rule is applied
// across a frame the reflections are not in, and the statistics stop being arithmetic (measured:
// 173.5% complete on a triclinic-P lattice merged in C2, and an undefined R_meas on an F-centred
// cubic one merged in a trigonal-P group).
//
// The two-pass second pass is flagged and not adopted rather than refused: its lattice comes from
// a de-novo re-index at the post-refined geometry, and the first pass - whose lattice did carry
// the group - is still there to go back to. A pass with nothing to fall back on refuses instead;
// the group is the user's assertion, and quietly determining a different one would answer a
// question that was not asked. Name the cell that WAS indexed so the user can act on it.
if (user_fixed_sg_ && end_msg.rotation_lattice_type.has_value() && end_msg.unit_cell.has_value()) {
const gemmi::SpaceGroup &fixed_sg = *user_fixed_sg_;
const bool centring_wrong =
end_msg.rotation_lattice_type->centering != fixed_sg.centring_type();
// The same two tests as the re-seating above, asked again now that it has had its chance.
const double violation = MetricViolation(*end_msg.unit_cell, fixed_sg);
if (centring_wrong || violation > MAX_METRIC_VIOLATION) {
const auto &uc = *end_msg.unit_cell;
const std::string why = centring_wrong
? fmt::format("its lattice is {}-centred and this crystal indexes as {}-centred",
fixed_sg.centring_type(), end_msg.rotation_lattice_type->centering)
: fmt::format("this crystal's cell is not one the group's own operators leave "
"invariant (metric violation {:.3f}, tolerance {:.3f}) - the symmetry "
"axes do not run along the axes the group puts them on",
violation, MAX_METRIC_VIOLATION);
if (prepass_result_) {
logger.Warning("Two-pass: the second pass indexed a lattice the fixed space group {} "
"cannot describe - {} - so this pass will not be adopted",
fixed_sg.xhm(), why);
result.lattice_conflicts_with_fixed_sg = true;
} else {
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, fmt::format(
"The space group {} was fixed for this run, but {}: the crystal indexes as a "
"{}-centred {} lattice (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}), and no setting of that lattice carries the group's. Merging in {} "
"would apply its symmetry to reflections that are not in its frame, so nothing it "
"reported would describe this crystal - a completeness above 100% is the usual "
"symptom. Re-run without a fixed space group to have it determined from the data, "
"or fix one whose lattice this crystal has.",
fixed_sg.xhm(), why, end_msg.rotation_lattice_type->centering,
gemmi::crystal_system_str(end_msg.rotation_lattice_type->crystal_system),
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, fixed_sg.xhm()));
}
}
}
const auto &rot_ss = experiment_.GetScalingSettings();
const bool is_rotation = experiment_.IsRotationIndexing(); // rotation indexing -> rotation scaling/merge
std::optional<RotationScaleMerge> rsm;
if (is_rotation) {
if (experiment_.GetRefineRotationWedgeInScaling()
|| rot_ss.GetRotationWedgeForScaling().has_value())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Rotation scaling/merging (RotationScaleMerge) does not support "
"wedge refinement");
// A reference MTZ is allowed for rotation: it fixes the space group / cell (on the CLI) and
// resolves the indexing ambiguity (below), but is NOT used to scale - the rotation merge stays
// self-consistent.
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
// The geometry pre-pass reads the per-image reflections exactly twice more: they were just
// ingested above, and the post-refinement at the end of the pass reads a narrow
// (hkl, frame, I, sigma, observed position) view of them. Nothing else in the pre-pass
// touches them - its reflections are never written, and the scaling engine reads back only
// per-frame fields - so take the post-refine's view NOW and hand each image's vector back as
// it is read: on a fine-sliced long axis the vectors are gigabytes, held to the end of the
// pass for those 32 bytes a partial. A run with a reference MTZ keeps them: resolving the
// indexing ambiguity re-reads them (and re-ingests).
std::optional<PostRefineObservations> prepass_postrefine_obs;
if (geometry_prepass && rsm && config_.reference_data.empty())
prepass_postrefine_obs = GatherPostRefineObservations(
indexer->GetIntegrationOutcome(), config_.nthreads,
/*release_reflections=*/true, logger);
auto scale_and_merge = [&](const std::string &label, bool for_search) -> ScaleMergeResult {
if (rsm) {
phase("Scale/combine/merge (" + label + ")");
// The geometry pre-pass merges only to choose the space group and to give the quality
// guard something to judge the second pass against; its reflections are never written,
// so the parts of the merge that only fill in an output file are skipped there.
auto r = rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
/*measure_cc_before_corrections=*/true);
result.error_model_isa = r.isa;
result.error_model_isa_asymptotic = r.isa_asymptotic;
if (r.resolution_fit_A) result.resolution_fit_A = r.resolution_fit_A;
result.error_model_a = r.error_model_a;
result.error_model_b = r.error_model_b;
return ScaleMergeResult{std::move(r.merged), std::move(r.statistics),
r.cc_half_before_corrections,
std::move(r.reference_statistics), r.reference_isa};
}
// Stills (rotation goes through RotationScaleMerge above): self-scale each image against the
// running merge with ScaleOnTheFly (fixed partiality), then merge directly. This runs even
// with an external reference: the reference is used only to break the per-image indexing
// ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor (scaling each
// image against a foreign dataset injects cross-dataset systematics and is a worse reference
// than the data's own merge). One pass: the per-image scale G is the exact one-pass solution,
// so iterating would only rebuild the reference from the freshly-scaled (noisy) data and
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
{
phase("Scaling images (" + label + ")");
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome());
indexer->ScaleAllImages(merge_result);
}
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
// orientation tilt against the running merge and recompute each reflection's partiality + scale
// correction (no re-integration). Never for the P1 search pass.
if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) {
phase("Partiality post-refine (" + label + ")");
StillsPartialityRefine refiner(experiment_);
const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads);
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
// The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally
// when the image has no prior CC - which is exactly the state the guard above leaves. So
// the guard was protecting nothing: measured on a jet dataset, of 367 images it left
// unscaled only 8 were still unscaled in the output, and 53 reached the merge at or below
// a fiftieth of the run median, the worst at a 4525th. Whatever produced the final scale,
// it is the final scale that has to be credible.
ScaleOnTheFly::RejectCollapsedScales(indexer->GetIntegrationOutcome());
}
const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
phase("Merging");
MergeOnTheFly merge_engine(experiment_);
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
// intensities and the error model, so the space-group search sees clean data; the final
// in-symmetry merge keeps them so completeness is not lost.
merge_engine.ExcludeIceRings(for_search);
if (result.consensus_cell.has_value())
merge_engine.ReferenceCell(*result.consensus_cell);
// Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells)
// most indexed crystals do not correlate with the true structure; --min-image-cc drops the
// crystals whose per-image CC to the reference is below the limit, so the merge keeps only
// the coherent (real) ones. Set before the error model so the model, the merge and the
// reported statistics are all fitted over the same set of images. The P1 search pass keeps
// every image: its job is to find the symmetry, not to produce final intensities.
merge_engine.FilterByImageCC(!for_search
&& experiment_.GetScalingSettings().GetMinCCForImage() > 0.0);
merge_engine.RefineErrorModel(merge_input);
if (merge_engine.ErrorModelActive())
{
// XDS convention, so the pair reads directly against a CORRECT.LP; the stills ISa is
// already the whole-range 1/b, there being no asymptote tier on this path.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
}
for (size_t i = 0; i < merge_input.size(); ++i)
merge_engine.AddImage(merge_input[i], static_cast<int64_t>(i));
ScaleMergeResult out;
out.merged = merge_engine.ExportReflections();
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
// range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A
// manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the
// space-group search still sees the full range. (Rotation is cut inside RotationScaleMerge.)
const auto &cut_ss = experiment_.GetScalingSettings();
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A);
phase("Computing statistics");
out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min);
// The same kept reflections binned over the reference range (--report-resolution); the
// stills error model is fitted over the whole range already, so no ISa of its own.
if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search)
out.reference_statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data,
std::nullopt, range);
{ // stills: no asymptote tier, and its ISa already IS the whole-range 1/b
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
result.error_model_isa = em.isa;
result.error_model_a = em.a;
result.error_model_b = em.b;
}
out.cc_half_before_corrections = out.statistics.overall.cc_half; // no surfaces on this path
logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label);
return out;
};
// First pass: P1 when searching, or directly the user-fixed space group.
const auto initial_sg = experiment_.GetGemmiSpaceGroup();
auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value());
// The two-pass quality guard judges the second pass against the first on THIS merge, which both
// passes make in the same terms - P1 (or the group the user fixed), full range, no correction
// surfaces - where their final merges need not even be in the same group. See RunAllPasses.
{
const auto &o = sm.statistics.overall;
result.search_merge_completeness_measured = o.possible_unique_reflections > 0;
result.search_merge_completeness = result.search_merge_completeness_measured
? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0;
result.search_merge_cc_half = sm.cc_half_before_corrections;
// ... and the reflections it holds on the principal axial rows, which the guard needs
// separately because CC1/2 cannot express them (see RunAllPasses). Only the low-order
// part is counted: a screw is read off the strong start of its row - the evidence gate
// wants four absences in one zone, and on a 70 A axis that is already at 7 A - while the
// high-order axial reflections are as numerous as they are uninformative and would dilute
// exactly the difference this number exists to show.
//
// Both counts are taken over the reflections this pass's own lattice centring allows, and
// that is what makes them the same measurement on the two sides. Each pass merges on its
// OWN conventional cell, so a pass that finds a centring the other did not carries the
// centring-extinguished half (two thirds, R; three quarters, F) of that cell's reflections
// as measured noise: measured, two crystals compare merges holding 2.0x the reflections of
// the other, with different axial rows, for no reason but the setting. What both settings
// describe is the primitive lattice, and these are its reflections.
const auto centring_allows = [c = result.consensus_centering.value_or('P')]
(const MergedReflection &r) {
switch (c) { // rhombohedral is described obverse everywhere in this code
case 'A': return (r.k + r.l) % 2 == 0;
case 'B': return (r.h + r.l) % 2 == 0;
case 'C': return (r.h + r.k) % 2 == 0;
case 'I': return (r.h + r.k + r.l) % 2 == 0;
case 'F': return (r.h + r.k) % 2 == 0 && (r.h + r.l) % 2 == 0;
case 'R': return (-r.h + r.k + r.l) % 3 == 0;
default: return true;
}
};
constexpr float AXIAL_ROW_LOW_ORDER_D_A = 4.0f;
result.search_merge_axial_reflections = std::count_if(
sm.merged.begin(), sm.merged.end(), [&](const MergedReflection &r) {
return ((r.h == 0) + (r.k == 0) + (r.l == 0)) == 2
&& r.d > AXIAL_ROW_LOW_ORDER_D_A && centring_allows(r);
});
// The signal the pass measured, and the size of the merge it measured it in, which is
// what the guard decides on.
constexpr float STRONG_I_OVER_SIGMA = 2.0f;
result.search_merge_strong_reflections = std::count_if(
sm.merged.begin(), sm.merged.end(), [&](const MergedReflection &r) {
return r.sigma > 0 && r.I / r.sigma >= STRONG_I_OVER_SIGMA && centring_allows(r);
});
result.search_merge_reflections = std::count_if(sm.merged.begin(), sm.merged.end(),
centring_allows);
}
// Rotation two-pass geometry pre-pass: this first scale/merge has now fitted a frame-order-smoothed
// mosaicity and written it back onto the per-frame outcomes. Capture it for the second pass's Bragg
// prediction (so prediction uses the smoothed value, not a fresh per-image estimate) and post-refine +
// apply the detector geometry from this integration. That is all the pre-pass is for: everything
// the second pass consumes from it is fitted here, and the space group is left to the second pass
// (see the search gate below).
// postrefine_probe_ brings the post-refinement into a CANONICAL pass as well, where the
// pre-pass moved the geometry: the fit it committed was made on reflections integrated at the
// geometry BEFORE the move, and whether it has finished walking can only be asked of
// reflections integrated at the geometry after it. The probe measures - it applies nothing and
// leaves the smoothed mosaicity and the pre-pass lattice alone, which stay pass 1's for every
// pass that consumes them; RunAllPasses decides what to do with the answer.
if ((geometry_prepass || postrefine_probe_) && rsm && experiment_.IsRotationIndexing()) {
const auto &outcomes = indexer->GetIntegrationOutcome();
if (geometry_prepass) {
prepass_mosaicity_.assign(outcomes.size(), NAN);
for (size_t o = 0; o < outcomes.size(); ++o)
if (outcomes[o].mosaicity_deg) prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg;
}
if (result.consensus_cell.has_value()) {
if (const auto rot = indexer->FinalizeRotationIndexing(); rot && rot->axis) {
if (geometry_prepass)
prepass_result_ = *rot; // full result (lattice+geom+axis), for the supercell fallback
PostRefineSettings prs;
prs.crystal_system = rot->search_result.system;
prs.num_threads = static_cast<int>(config_.nthreads);
prs.refine_geometry = true;
// The centre the pre-scan measured, where the fit was precise enough to be used as
// a centre at all (the same ceiling --estimate-beam-center adopts one on). It does
// not move the geometry - it only tells the bound on step B's beam move where else
// the centre is independently believed to be.
if (measured_beam_center_
&& measured_beam_center_->sigma_pxl <= std::max(MAX_BEAM_CENTER_SIGMA_PXL,
BeamCenterNeed_pxl(experiment_)))
prs.measured_beam_px = std::array<double, 2>{
static_cast<double>(measured_beam_center_->beam_x_pxl),
static_cast<double>(measured_beam_center_->beam_y_pxl)};
// The first pass's own geometry, not the header's. The outcomes being post-refined
// were integrated at it, and step B holds the tilt fixed while it fits the beam and
// the distance - so handing it the header tilt fits a beam that has to absorb a tilt
// error the images do not have, and the answer describes neither geometry.
//
// On the pre-pass the observations were gathered right after Ingest (see there) and
// the scaling engine is finished - nothing after this point reads it on that pass
// (the search and the reference re-merge run only on the canonical one) - so it is
// freed BEFORE the fit: the engine holds about twice the partial payload again,
// and with it gone the fit no longer sits on the pass's memory peak.
if (prepass_postrefine_obs)
rsm.reset();
auto pr_obs = prepass_postrefine_obs
? std::move(*prepass_postrefine_obs)
: GatherPostRefineObservations(indexer->GetIntegrationOutcome(),
config_.nthreads,
/*release_reflections=*/false, logger);
prepass_postrefine_obs.reset();
const auto pr = PostRefineRotationGeometry(std::move(pr_obs), *rot->axis,
rot->geom, rot->lattice, prs, logger);
result.post_refine = pr;
if (pr.ok && pr.detector_refined) {
// Store, don't apply: this pass continues to write its own (header-geometry) output;
// Run() applies the refined geometry before the second pass.
prepass_detector_geometry_ = std::array<float, 5>{
static_cast<float>(pr.beam_x_after_px),
static_cast<float>(pr.beam_y_after_px),
static_cast<float>(pr.distance_after_mm),
rot->geom.GetPoniRot1_rad(),
rot->geom.GetPoniRot2_rad()};
logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, "
"beam {:.2f},{:.2f} px, tilt {:.4f},{:.4f} deg) - {}",
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px,
rot->geom.GetPoniRot1_rad() * 180.0 / PI,
rot->geom.GetPoniRot2_rad() * 180.0 / PI,
geometry_prepass ? "it will drive the second integration pass"
: "what the fit still asks for at the geometry "
"this pass ran at");
} else {
logger.Info("Two-pass: geometry post-refine committed no detector change{}",
geometry_prepass ? " - the second pass reproduces the first" : "");
}
// DECISION POINT for the goniometer rotation scale. It does not ride on pr.ok - the
// scale is its own cross-validated fit and the crystals that have a stage fault are
// exactly the ones whose cell and detector steps do NOT pass, because the angle error
// is what their residual is made of. A calibration fault is rare (36 of 37 rotation
// datasets sit at 1.0000) and applying a 1 % angle correction to a healthy dataset
// would silently damage it, so the asymmetry is deliberate: committed only when the
// fit is both cross-validated and outside the tolerance. A manual --rotation-scale is
// already on the goniometer and is left alone.
// Only the pre-pass fits the rotation scale. It is applied from the goniometer
// the file came with, so a scale measured again on angles that have already been
// corrected once is not a correction that can be applied on top of that one.
if (pr.rotation_scale_suspect && !config_.rotation_scale.has_value() && geometry_prepass)
prepass_rotation_scale_ = static_cast<float>(pr.rotation_scale);
}
}
}
// For a de-novo search, gate the P1 scaling+merge (the pass that feeds the space-group search) at
// <I/sigma> >= 1 over thin resolution shells: noise-dominated high-res shells otherwise mislead
// the determination, where a single artifact (a zinger the box sum integrates whole, or a
// weak-reflection profile-fit runaway) has an astronomical resolution-normalised E and wrecks the
// intensity-correlation / second-moment statistics. The final in-symmetry merge keeps the full
// range (only this P1 search pass is cut); a manual --scaling-high-resolution, if coarser, wins.
// The geometry pre-pass does not determine the space group. Its integration is at the header
// geometry, which is the thing the second pass exists to improve on, and the symmetry evidence
// moves with the geometry: measured on a rotation crystal, 0.33 mm of distance and 0.65 px of
// beam centre took the screw-axis evidence from -19 to +311 nats, where merging the same
// integration in symmetry instead of P1 moved it 2.5. Deciding on the worse integration also
// fails in one direction only - every statistic the search weighs is a ratio of the candidate
// merge's scatter to the subgroup's, so poorly-integrated data inflate them and REFUSE real
// symmetry (corpus-wide the misses run ten under-calls to one over-call). So the search runs on
// the canonical pass, at the refined geometry, and the pre-pass skips it along with the
// in-symmetry re-merge that only exists to serve it.
const bool search_space_group = !geometry_prepass && !initial_sg.has_value();
double d_min_search = 0.0;
if (search_space_group) {
std::vector<std::pair<float, float>> rs; // (d, I/sigma) over the full P1 merge
rs.reserve(sm.merged.size());
for (const auto &m : sm.merged)
if (std::isfinite(m.I) && std::isfinite(m.sigma) && m.sigma > 0.0f &&
std::isfinite(m.d) && m.d > 0.0f)
rs.emplace_back(m.d, m.I / m.sigma);
if (rs.size() >= 400) {
std::sort(rs.begin(), rs.end(),
[](const auto &a, const auto &b) { return a.first > b.first; }); // low -> high res
const int bins = std::clamp(static_cast<int>(rs.size() / 100), 1, 40);
const size_t per = (rs.size() + bins - 1) / static_cast<size_t>(bins);
for (size_t b = 0; b * per < rs.size(); ++b) {
const size_t lo = b * per, hi = std::min(rs.size(), lo + per);
double sum = 0.0;
for (size_t j = lo; j < hi; ++j) sum += rs[j].second;
if (sum / static_cast<double>(hi - lo) < 1.0) {
// Cut the noise-dominated high-res shells. When even the lowest-res shell fails,
// keep that shell alone rather than abandoning the cut. The bound is absolute
// while the merged I/sigma it tests saturates at the merge's own ISa (times the
// square root of the multiplicity), so a merge whose ISa has collapsed towards 1
// can fail it in every shell - and that is exactly the merge the search must not
// be handed whole, since the noise which drove ISa down is what the cut exists to
// remove. Abandoning it feeds back: more noise into the search, a weaker operator
// statistic, and a lost symmetry.
if (b > 0)
d_min_search = rs[lo].first;
else if (per >= 400)
d_min_search = rs[per - 1].first;
break;
}
}
}
const double d_manual = experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0);
if (rsm && d_min_search > d_manual) {
logger.Info("Space-group search: re-scaling the P1 pass at {:.2f} A (<I/sigma> >= 1)", d_min_search);
rsm->SetDMinLimit(d_min_search);
sm = scale_and_merge("P1, space-group search", true);
rsm->SetDMinLimit(d_manual > 0.0 ? std::optional<double>(d_manual) : std::nullopt);
}
}
std::ostringstream stats_text;
// The indexing-ambiguity paragraph as it went into stats_text, so that a model which then
// settles the ambiguity can withdraw it; empty where there was none.
std::string ambiguity_warning_text;
// True when the point group below was chosen BY THE SEARCH (rather than given by the user) and is
// above triclinic - i.e. a promotion was made, which is what makes a later "the Laue class is
// holohedral so there is no twin law" conclusion circular.
bool promoted_point_group = false;
if (search_space_group) {
SearchSpaceGroupOptions sg_opts;
sg_opts.nthreads = static_cast<size_t>(std::max(1, config_.nthreads));
sg_opts.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
sg_opts.d_min_limit_A = std::max<double>(
d_min_search, experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0));
// Constrain the search to subgroups of the lattice (metric) symmetry found by rotation
// indexing. Centering is not constrained here - it is determined from the absences.
if (end_msg.rotation_lattice_type.has_value())
sg_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
// The cell the reflections are indexed on, which is what lets the search offer a candidate
// in a setting other than gemmi's reference one: a setting names the symmetry directions by
// axis, so a candidate the cell cannot host is refused before it is scored.
if (result.consensus_cell.has_value())
sg_opts.cell = gemmi::UnitCell(*result.consensus_cell);
sg_opts.enumerate_all_settings = true;
sg_opts.enumerate_all_rotation_sets = true;
// The error model this merge was scaled with, so the search's "genuinely present" cut can be
// read on the counting sigma instead of on the merged one (SearchSpaceGroupOptions::merge_isa).
// Set per merge, not once: the two arms below are scaled separately and their ISa differ by up
// to a factor of six. 0 (b not measurable) leaves the cut as the caller set it.
sg_opts.merge_isa = result.error_model_isa;
auto sg_search = SearchSpaceGroup(sm.merged, sg_opts);
// The Lorentz-filtered arm's merge, kept past the point where the all-observation one
// replaces it. The metric re-ask below is a search like any other and answers to the same
// two-arm rule: a promotion is refused by the most damning statistic it can be shown, and
// only one of the two merges has to be starved for that to happen.
std::vector<MergedReflection> merged_filtered;
// ...and that arm's OWN ISa. sg_opts.merge_isa is re-set below for the all-observation
// merge, and the two differ by up to a factor of six, so the re-ask has to keep this one
// or it judges the filtered merge's "genuinely present" cut on the other arm's error model.
double merged_filtered_isa = 0.0;
// Second opinion from a merge that keeps only well-measured observations (see
// RotationScaleMerge::search_min_zeta). Where the two disagree the all-observation merge
// decides. The filter earns its place because a reflection that crosses the Ewald sphere
// near-tangentially is measured worst, and a handful of bad measurements can make a real
// operator look like a twin law; but it discards 40-80% of the observations, which starves
// the operator correlations. Both failure directions are real - the filtered arm has been
// seen to lose an operator the full merge confirms, and, at geometries away from the ones
// this comparison was first checked on, to confirm one the full merge refuses. So the two
// are split rather than ranked: whichever arm finds the LARGER point group decides which
// rotations are real, because a promotion is refused by the most damning statistic it can
// be shown and only one arm has to be starved for that to happen, while the ABSENCES are
// always judged on the all-observation merge, which is the only one holding the weak
// reflections a systematic absence is made of. Both searches keep their vetoes, so a merohedral twin
// stays refused in each (verified on a real twin at the three frame ranges where it
// over-promotes: refused in both arms every time).
if (rsm && rsm->GetSearchMinZeta() > 0.0 && !sg_search.point_group_hm.empty()) {
const double zeta = rsm->GetSearchMinZeta();
rsm->SetSearchMinZeta(0.0);
auto sm_all = scale_and_merge("P1, all observations", true);
rsm->SetSearchMinZeta(zeta);
merged_filtered_isa = sg_opts.merge_isa; // the filtered arm's, before it is replaced
sg_opts.merge_isa = result.error_model_isa; // this arm's own error model
const auto alt = SearchSpaceGroup(sm_all.merged, sg_opts);
// The order of the point group each arm confirmed. Read from the search, not from the
// space group it went on to pick: Stage B leaves best_space_group unset when no
// candidate has enough absences to be eligible, which is common on the filtered arm
// precisely because the filter removes the weak reflections the absences live in. An
// arm that confirmed 422 would then have scored 0 and lost to a P2 arm - a demotion,
// reported as "taking the higher symmetry".
const auto order_of = [](const SearchSpaceGroupResult &r) { return r.point_group_order; };
logger.Info("Space-group search: all-observation merge -> {} (order {}), "
"Lorentz-filtered -> {} (order {})",
alt.point_group_hm.empty() ? "?" : alt.point_group_hm, order_of(alt),
sg_search.point_group_hm.empty() ? "?" : sg_search.point_group_hm,
order_of(sg_search));
// The screw/centering determination in particular comes from the merge with all the
// observations, because systematic absences are decided by the WEAK reflections and the
// filter throws most of them away. Preferring the filtered arm on a tie cost four
// crystals their screw axes (P2(1) read as P2) for exactly that reason, with the point
// group and every intensity statistic identical.
// Checked while both arms are still intact - below, the all-observation arm is adopted
// by value and the two become the same result.
// Which point group, by its symmorphic representative rather than by name - gemmi calls
// both P321 and P312 "32", so two arms that disagree about WHICH 2-folds are real read
// as agreeing. By the representative's full symbol rather than its NUMBER, for the same
// reason one level down: P 1 2 1 and P 1 1 2 are both number 3 and put their 2-fold on
// different axes. Empty when no point group was chosen.
const auto pg_of = [](const SearchSpaceGroupResult &r) {
return r.point_group_representative.has_value() ? r.point_group_representative->xhm()
: std::string();
};
if (order_of(alt) > 0 && order_of(alt) == order_of(sg_search)
&& pg_of(alt) != pg_of(sg_search)) {
// Same order, different symmetry: the two merges disagree about WHICH operators are
// real, and neither is higher, so there is nothing to prefer. Say so instead of
// picking silently - the two imply different molecular-replacement searches, and
// trying both is cheap next to processing the data again.
const std::string a = alt.best_space_group.has_value()
? alt.best_space_group->short_name() : alt.point_group_hm;
const std::string b = sg_search.best_space_group.has_value()
? sg_search.best_space_group->short_name() : sg_search.point_group_hm;
const std::string msg = fmt::format(
"Space group is AMBIGUOUS between {} and {} (point groups {} and {}, the same "
"order): the merge of all observations and the merge of only the well-measured "
"ones each support one of them, and neither is the higher symmetry, so the data "
"do not decide. Processing continues in {} (the all-observation choice); "
"molecular replacement in both is what settles it, and a re-run with -S {} "
"forces the other one.",
a, b, alt.point_group_hm, sg_search.point_group_hm, a, b);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
result.warnings.push_back({PathologyCode::SYMMETRY_AMBIGUITY, msg});
}
if (order_of(alt) > order_of(sg_search))
logger.Info("Space-group search: all-observation merge supports {} where the "
"Lorentz-filtered one supports {} - taking the higher symmetry",
alt.point_group_hm, sg_search.point_group_hm);
if (order_of(alt) < order_of(sg_search)) {
// The filtered arm found symmetry the all-observation arm did not. Discarding it
// was costing real point groups: over the corpus the two arms disagree on eight
// crystals and the higher one is right on all eight, because the filter removes
// the near-tangential observations that make a genuine operator look like a twin,
// and a promotion is refused by the most damning statistic it can be shown.
//
// But the two halves of the answer do not come from the same place. The filtered
// merge is deliberately starved, and ABSENCES live in the weak reflections it
// throws away - adopting its result whole recovers the point group and loses the
// screw (a P6_5 crystal comes back P6). So the point group is taken from the arm
// that found it and the absences are re-judged here, on every observation. That
// is what fixed_point_group is for: Stage A still runs and still reports what it
// would have refused, only the choice is overridden.
sg_opts.fixed_point_group = sg_search.point_group_representative;
auto split = SearchSpaceGroup(sm_all.merged, sg_opts);
sg_opts.fixed_point_group.reset();
logger.Info("Space-group search: Lorentz-filtered merge supports {} where the "
"all-observation one supports {} - taking the higher symmetry, and "
"judging its absences on all observations -> {}",
sg_search.point_group_hm, alt.point_group_hm,
split.best_space_group.has_value() ? split.best_space_group->xhm()
: split.point_group_hm);
sg_search = std::move(split);
} else {
sg_search = alt;
}
// The all-observation merge is the one carried forward: it is what the
// pre-promotion twinning numbers below are measured on, and the filtered merge is the
// second opinion reported above.
merged_filtered = std::move(sm.merged);
sm = std::move(sm_all);
}
// Twinning evidence measured BEFORE any promotion, on the P1/subgroup merge the search was
// given. Reported alongside the post-adoption analysis: once a higher Laue class has been
// adopted, its own twinning test can only say "no twin law exists within this class", which
// is circular - the promotion is what a twin would have caused. These numbers are not.
const auto pre_promotion_twinning = AnalyzeTwinning(sm.merged, nullptr);
result.pre_promotion_twinning = pre_promotion_twinning;
if (pre_promotion_twinning.twinning_suspected)
logger.Warning("Twinning indicators BEFORE the space-group decision (subgroup merge): "
"<|L|> = {:.3f}, second moment = {:.3f} (untwinned 0.500 / 2.00) - if the "
"search promoted the point group, treat the promotion with suspicion",
pre_promotion_twinning.mean_abs_l, pre_promotion_twinning.second_moment);
promoted_point_group = !sg_search.point_group_hm.empty() && sg_search.point_group_hm != "1";
if (!sg_search.refused_point_group_hm.empty())
logger.Warning("Higher symmetry {} was confirmed by the operator correlations but refused: "
"{}. Processing in the lower symmetry (the recoverable direction).",
sg_search.refused_point_group_hm, sg_search.refused_reason);
// Miller-index reindex under a change of basis: (h,k,l)_conv = reindex * (h,k,l)_prim.
const auto reindex_hkl = [](auto &r, const gemmi::Mat33 &m) {
const double h = r.h, k = r.k, l = r.l;
r.h = static_cast<int32_t>(std::lround(m.a[0][0] * h + m.a[0][1] * k + m.a[0][2] * l));
r.k = static_cast<int32_t>(std::lround(m.a[1][0] * h + m.a[1][1] * k + m.a[1][2] * l));
r.l = static_cast<int32_t>(std::lround(m.a[2][0] * h + m.a[2][1] * k + m.a[2][2] * l));
};
// Intensity-based centred-lattice test (XDS CORRECT / POINTLESS style). When the indexer
// demoted to a primitive/triclinic cell but the metric is pseudo-symmetric, the point-group
// search above tested no operators (its candidate set is bounded by the indexer's lattice).
// A genuinely centred lattice and a merely pseudo-symmetric triclinic one are indistinguishable
// from spot POSITIONS - the 2-fold is an INTENSITY property - so decide it here, after P1
// integration: reindex the P1-merged data to the metric candidate's conventional setting and
// re-run the search. The I(h) vs I(Rh) correlation confirms a real centred lattice (high CC ->
// commit the centring) and rejects a pseudo-symmetric triclinic (low CC -> stays P1, no reindex).
std::optional<gemmi::Mat33> commit_reindex;
std::optional<UnitCell> commit_cell;
std::optional<CrystalLattice> commit_lattice;
std::optional<LatticeMessage> commit_lattice_type;
// Only a demoted (triclinic) indexing result reaches here with a primitive cell - that is both
// the case the point-group search could not test (a triclinic holohedry offers no operators)
// and the case where LatticeSearch's Niggli reduction is valid (it assumes a primitive input).
const bool searched_trivial = !sg_search.best_space_group || sg_search.best_space_group->number == 1;
const bool demoted_triclinic = end_msg.rotation_lattice_type.has_value()
&& end_msg.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Triclinic;
if (searched_trivial && demoted_triclinic && end_msg.rotation_lattice.has_value()) {
// The metric only generates the candidate; the intensity CC gate below makes the decision
// (a false pseudo-symmetry is rejected by the correlation).
const auto cand = LatticeSearch(*end_msg.rotation_lattice);
// Exact integer reindex from the indexed setting straight to the candidate conventional
// setting: P[i][j] = conv_real[i] . rot_reciprocal[j]. Derived from the two lattices, so it
// holds even if the refined indexed cell is not exactly LatticeSearch's Niggli cell (whose
// primitive_reduced->conventional reindex would otherwise be applied in the wrong setting).
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (cand.system != gemmi::CrystalSystem::Triclinic && reindex_res < 0.1) {
std::vector<MergedReflection> merged_c = sm.merged;
for (auto &m : merged_c)
reindex_hkl(m, reindex);
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand.system;
o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell());
const auto s2 = SearchSpaceGroup(merged_c, o2);
if (s2.best_space_group.has_value() && s2.best_space_group->number > 1) {
// The centring cannot be confirmed from absences here (integrated in the primitive
// cell, so the centring-absent reflections do not exist) - it is metric-determined.
// Among the point-group-equivalent candidates the intensities cannot separate
// (e.g. P2 / P21 / C2), pick the one matching the metric candidate's centring.
gemmi::SpaceGroup chosen = *s2.best_space_group;
if (chosen.centring_type() != cand.centering)
for (const auto &alt : s2.alternatives)
if (alt.centring_type() == cand.centering) { chosen = alt; break; }
// Commit only if the adopted centring matches the metric candidate whose conventional
// cell we reindex into - otherwise the space group and the written cell disagree.
if (chosen.centring_type() == cand.centering) {
sg_search = s2;
// Swap the metric-matching candidate into the answer and leave the one it
// displaced among the alternatives, so the reported "or ..." list stays the
// set of groups the data cannot separate, each named once.
if (chosen.number != s2.best_space_group->number) {
sg_search.alternatives.push_back(*s2.best_space_group);
std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) {
return a.number == chosen.number;
});
}
sg_search.best_space_group = chosen;
commit_reindex = reindex;
commit_cell = cand.conventional.GetUnitCell();
commit_lattice = end_msg.rotation_lattice->Multiply(reindex);
commit_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class,
.crystal_system = cand.system };
// Say so at the point of decision. The point group here is confirmed
// from the intensities, but the centering is not and cannot be - the
// reflections it extinguishes were never measured. (The report warns
// about it too, from the candidate's centering_untested flag, which
// survives into the pass that is written.)
// Only for a CENTRED candidate: a primitive one extinguishes nothing, so
// there is no untested absence class to warn about. Before the enumeration
// was widened this branch could only ever adopt a centred group, so the
// condition was implicit.
if (cand.centering != 'P')
logger.Warning("The {} centering of {} comes from the lattice METRIC, "
"not from the intensities: the crystal is indexed on "
"the primitive sub-cell, so the reflections a "
"{}-centred lattice extinguishes were never predicted "
"or integrated. The point group ({}) IS confirmed, by "
"the operator correlations.",
cand.centering, chosen.short_name(), cand.centering,
s2.point_group_hm);
}
}
}
}
// A CLASS DECIDED BY ROUNDING. LatticeSearch names the Bravais class by Niggli-reducing the
// indexed cell and looking the reduced cell up in the 44 lattice characters. For a lattice
// whose Buerger cells straddle the Niggli type-I/type-II boundary that lookup is a coin
// flip - an F-centred cubic lattice has reduced forms on both sides, the two sides carry
// different characters, and which side the reduction lands on is decided by the last digits
// of whatever refinement produced the cell rather than by the data. Measured on such a
// lattice: 43% cubic F, 36% tetragonal I, 21% orthorhombic I, and the split is flat over a
// factor of ten in the noise. The class then caps the point-group search
// (SearchSpaceGroupOptions::lattice_system), so from the I-centred sub-cell the cubic
// three-fold is never enumerated and the run reports that nothing was refused - which is
// accurate, because nothing was asked.
//
// Le Page's two-fold search has no such key: it measures each rotation's obliquity on the
// lattice itself, in a primitive basis, and the conventional cell is then built from the
// axes of the rotations it found. Where it carries more rotations than the adopted class's
// holohedry, offer its conventional cell as a SECOND lattice candidate - reindex the merge
// into it and run the same search there, with every gate live. This only ever ASKS. The
// reindex is committed if and only if the search in the new setting confirms a strictly
// higher point group, and the centring the new cell describes, on these intensities; a
// pseudo-symmetric metric fails that and the answer already in hand stands.
//
// Asked whether or not the centred-lattice test above already committed. What that test
// adopts is a lattice class of its own, and a class it settled from a triclinic indexing is
// exactly the kind that can still be short of the metric's - so the question here is not
// answered by its having run. The two cannot disagree: the ask below only ever replaces the
// answer in hand with a STRICTLY higher point group, and it writes all four commit_ values
// together, so the group, the cell and the lattice type stay one answer.
if (end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
// The merge is indexed on the lattice the indexer named, whatever has been committed
// since, so the metric is measured on the primitive form of THAT one.
const CrystalLattice primitive =
end_msg.rotation_lattice->ToPrimitive(end_msg.rotation_lattice_type->centering);
// The lattice class a search has already been given - the indexer's, or the one the
// centred-lattice test above committed. It bounds what any search so far could
// ENUMERATE, which is what says whether the metric's own class is a new question.
const gemmi::CrystalSystem asked_system = commit_lattice_type
? commit_lattice_type->crystal_system
: end_msg.rotation_lattice_type->crystal_system;
const size_t asked_rotations = HolohedryRotations(asked_system);
const auto metric = LePageLattice(primitive);
// Ask the intensities about ONE lattice the metric offers: reindex the merge into its
// conventional setting and run the same search there, with every gate of that search
// live. It commits if and only if the search confirms a strictly higher point group and
// the centring the new cell describes. Called at most twice - the metric's own cell,
// then the sub-lattice below - and only the ask that adopts sets commit_reindex, so the
// second never runs against an answer the first already took.
const auto ask_metric_lattice = [&](const LePageResult &cand, bool sub_lattice) {
// Exact integer reindex from the indexed setting to the candidate's conventional
// one, P[i][j] = conv_real[i] . rot_reciprocal[j], as in the centred-lattice test
// above.
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(),
cand.conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
gemmi::Mat33 reindex;
double reindex_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reindex.a[i][j] = std::round(v);
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
}
if (reindex_res >= 0.1)
return;
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand.system;
o2.cell = gemmi::UnitCell(cand.conventional.GetUnitCell());
const auto reindexed = [&](const std::vector<MergedReflection> &in) {
std::vector<MergedReflection> out = in;
for (auto &m : out)
reindex_hkl(m, reindex);
return out;
};
const auto merged_all = reindexed(sm.merged);
auto s2 = SearchSpaceGroup(merged_all, o2);
// The same two-arm rule the search above follows: the Lorentz-filtered merge can
// confirm an operator the all-observation one refuses, because the filter removes
// the near-tangential measurements that make a genuine operator look like a twin.
// Where it finds the larger point group, the absences are still judged on all
// observations, with the point group pinned.
if (!merged_filtered.empty()) {
SearchSpaceGroupOptions of = o2;
of.merge_isa = merged_filtered_isa; // this arm's own error model
const auto filtered = SearchSpaceGroup(reindexed(merged_filtered), of);
if (filtered.point_group_order > s2.point_group_order) {
o2.fixed_point_group = filtered.point_group_representative;
s2 = SearchSpaceGroup(merged_all, o2);
o2.fixed_point_group.reset();
}
}
// The centring cannot be read from the absences in this setting: reindexing
// into the metric candidate's conventional cell leaves its centring-absent class
// holding no data at all, so among the point-group-equivalent candidates the
// intensities cannot separate, the tie-break takes the lowest number, and that is
// the primitive one. Pick the candidate matching the metric's own centring out of
// the alternatives first, exactly as the centred-lattice test above does -
// otherwise a promotion the intensities DID confirm is refused for naming the
// same lattice in its equivalent primitive setting.
std::optional<gemmi::SpaceGroup> chosen = s2.best_space_group;
if (chosen.has_value() && chosen->centring_type() != cand.centering)
for (const auto &alt : s2.alternatives)
if (alt.centring_type() == cand.centering) { chosen = alt; break; }
const bool point_group_higher = s2.best_space_group.has_value()
&& s2.point_group_order > sg_search.point_group_order;
const bool higher = point_group_higher
&& chosen->centring_type() == cand.centering;
const auto &uc = cand.conventional.GetUnitCell();
// The two asks are told apart by what they were asked against: the class a search
// has already been given, or the rotations the intensities have actually confirmed.
const std::string preamble =
"The cell metric carries " + std::to_string(metric->n_operators) +
(sub_lattice
? " rotations where the intensities have confirmed " +
std::to_string(sg_search.point_group_order) +
", so the search was asked again on a SUB-lattice of the metric - the "
"largest one whose two-folds these intensities confirm one by one, carrying " +
std::to_string(cand.n_operators) + " of those rotations"
: " rotations where the " + std::string(gemmi::crystal_system_str(asked_system)) +
" lattice in hand has " + std::to_string(asked_rotations) +
", so the search was asked again on the metric's own cell");
logger.Info("{} (a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f}, centring {}, worst obliquity {:.3f} deg): {} (order "
"{} against {}){}",
preamble,
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma, cand.centering,
cand.max_obliquity_deg,
s2.best_space_group.has_value() ? s2.best_space_group->xhm() : "nothing",
s2.point_group_order, sg_search.point_group_order,
higher ? " - adopted"
: (point_group_higher
? " - keeping the group already found; the point group IS "
"higher, but no candidate of it carries the centring the "
"metric cell names, which this setting cannot test"
: (s2.refused_reason.empty()
? " - the intensities do not support it, keeping the "
"group already found"
: " - keeping the group already found; " + s2.refused_reason)));
if (higher) {
sg_search = s2;
// Leave the candidate it displaced among the alternatives, so the reported
// "or ..." list stays the set of groups the data cannot separate.
if (chosen->number != s2.best_space_group->number) {
sg_search.alternatives.push_back(*s2.best_space_group);
std::erase_if(sg_search.alternatives, [&](const gemmi::SpaceGroup &a) {
return a.number == chosen->number;
});
}
sg_search.best_space_group = chosen;
commit_reindex = reindex;
commit_cell = cand.conventional.GetUnitCell();
commit_lattice = end_msg.rotation_lattice->Multiply(reindex);
commit_lattice_type = LatticeMessage{ .centering = cand.centering,
.niggli_class = end_msg.rotation_lattice_type->niggli_class,
.crystal_system = cand.system };
}
};
if (metric) {
// The metric's own class, where no search has been given it yet. Where one has -
// the centred-lattice test above reindexes into a metric candidate's cell and runs
// the full search there - asking it again would only re-run the same question.
if (static_cast<size_t>(metric->n_operators) > asked_rotations)
ask_metric_lattice(*metric, false);
// That question is all-or-nothing, and it is asked at the TOP of the metric
// hierarchy: where the truth is an INTERMEDIATE class the answer is correctly "no",
// and the class in between is never asked about, because it is not expressible in
// the metric cell's basis. Measured on a pseudo-hexagonal monoclinic indexing whose
// metric group is hexagonal: three of its twelve operators close into 222 and
// correlate at 0.60 against 0.02 for the rest, and the conventional cell of that
// orthorhombic sub-lattice is the C-centred one the crystal is actually described on.
//
// So ask once more, on the sub-lattice the intensities themselves pick out: score
// every two-fold the metric offers against the P1 merge and keep only those that
// correlate, then let Le Page close what survives into a lattice. Nothing here
// decides a space group - it decides which lattice gets ASKED about, and the ask is
// the same call with the same gates as above. The number to beat here is what the
// intensities have CONFIRMED (the adopted point group's order), not what some
// search was allowed to enumerate: an enumeration that reached the metric's class
// and refused it says nothing about the classes below it.
if (static_cast<size_t>(metric->n_operators) > sg_search.point_group_order) {
// The two-folds are offered in the basis of the lattice handed to LePageLattice,
// which is the PRIMITIVE one; sm.merged is indexed on the lattice the run
// adopted. With a_prim[i] = sum_j P[i][j] a_idx[j] (CrystalLattice::Multiply
// combines basis rows) Miller indices map h_prim = P h_idx, so an operator's
// Miller-index matrix carries back as M_idx = P^-1 M_prim P. P is the identity
// for a primitive adopted lattice.
gemmi::Mat33 to_primitive;
if (end_msg.rotation_lattice_type->centering != 'P')
to_primitive = gemmi::rot_as_mat33(
gemmi::centred_to_primitive(end_msg.rotation_lattice_type->centering)).transpose();
const gemmi::Mat33 from_primitive = to_primitive.inverse();
// An operator counts as real on the same bound Stage A calls one present at -
// SearchSpaceGroupOptions::min_operator_cc, applied by OperatorCorrelation over
// the same reflection population, so nothing new is calibrated here.
const auto keep_operator = [&](const gemmi::Mat33 &m_primitive) {
const gemmi::Mat33 m_indexed =
from_primitive.multiply(m_primitive).multiply(to_primitive);
const auto score = OperatorCorrelation(sm.merged, m_indexed, sg_opts);
return score.has_value() && score->present;
};
const auto sub = LePageLattice(primitive, LATTICE_MAX_OBLIQUITY_DEG, keep_operator);
if (sub && static_cast<size_t>(sub->n_operators) > sg_search.point_group_order
&& sub->n_operators < metric->n_operators)
ask_metric_lattice(*sub, true);
}
}
}
// A DOUBLED DESCRIPTION of a primitive lattice. gemmi states every setting as a change of
// basis from its reference one, and where that change has determinant < 1 the setting names
// a cell of 1/det times the reference volume. In the monoclinic family exactly two settings
// have it - C 1 1 2 and C 1 1 2_1 - and the reason is physical: with unique axis c the
// C-centring vector (1/2,1/2,0) has no component along c, lies wholly in the oblique a-b
// plane and reduces away, which is why the genuine c-unique centred settings are A/B/I 1 1 2.
// (The centred settings of P1, and the C/F settings of the tetragonal groups, are the same
// thing.) Such a group is never a distinct Bravais centring; it is a primitive lattice
// written on twice the cell.
//
// It wins because the candidate table is scored ON the doubled cell, where half the
// reflection positions are not reciprocal-lattice points at all and are absent BY
// CONSTRUCTION, so the centred candidate collects a centring evidence that is tautological.
// Refusing to enumerate those settings was measured and is worse, not better: the
// volume-preserving alternative in the same table keeps the by-construction absences as real
// observations and merges no better than P1. What the centred candidate gets RIGHT is
// discarding them.
//
// So take the win as a SIGNAL rather than as the answer. Halve the cell, put the lattice back
// in the primitive setting of the group's own crystal system, carry the reflections across -
// the by-construction absences map to non-integral indices and are dropped, which is exactly
// what the centred candidate was doing, so nothing that was merging is lost - and re-run the
// search there. The enumeration is untouched, so no candidate that JUDGES another moves and
// no chi2 reference changes; the doubled setting still wins its own table, and only the cell
// its answer is expressed on is corrected.
if (!commit_reindex && sg_search.best_space_group.has_value()
&& sg_search.best_space_group->number > 1
&& end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()) {
const gemmi::SpaceGroup won = *sg_search.best_space_group;
constexpr double den3 = static_cast<double>(gemmi::Op::DEN) * gemmi::Op::DEN * gemmi::Op::DEN;
const double det = won.basisop().det_rot() / den3;
const int multiple = static_cast<int>(std::lround(1.0 / det)); // 2 for C 1 1 2_1, 4 for F 1
if (det < 0.99 && multiple >= 2 && multiple <= 4) {
// Reduce, then classify. The character table is reached through a Niggli reduction
// that takes whatever it is handed to be primitive, so the centred conventional cell
// has to be brought to its primitive one FIRST - asked on the doubled cell the same
// call answers with the same volume, merely permuted. Ask for the group's own system
// in 'P': the metric of a reducible lattice supports the doubled centred description
// too (that is how it got here), and an unrestricted search would hand it straight
// back.
const auto cand = LatticeSearchForClass(
end_msg.rotation_lattice->ToPrimitive(won.centring_type()),
won.crystal_system(), 'P');
// Change of basis from the doubled conventional setting straight to the reduced one,
// P[i][j] = conv_real[i] . doubled_reciprocal[j], as everywhere else here - except
// that its entries are multiples of 1/multiple rather than integers, which is the
// whole point of it.
gemmi::Mat33 reduce;
double reduce_res = 1.0;
if (cand) {
const Coord cv[3] = {cand->conventional.Vec0(), cand->conventional.Vec1(),
cand->conventional.Vec2()};
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
end_msg.rotation_lattice->Cstar()};
reduce_res = 0.0;
for (int i = 0; i < 3; ++i)
for (int j = 0; j < 3; ++j) {
const double v = cv[i] * rs[j];
reduce.a[i][j] = std::round(v * multiple) / multiple;
reduce_res = std::max(reduce_res, std::fabs(v - reduce.a[i][j]));
}
}
// hkl across the same change of basis. A reflection whose image is not integral is
// not a reciprocal-lattice point of the reduced cell - it is one of the positions the
// doubled description invented - and it is dropped rather than reindexed.
const auto reduce_hkl = [&reduce](auto &r) {
const double h = r.h, k = r.k, l = r.l;
const double n[3] = {reduce.a[0][0]*h + reduce.a[0][1]*k + reduce.a[0][2]*l,
reduce.a[1][0]*h + reduce.a[1][1]*k + reduce.a[1][2]*l,
reduce.a[2][0]*h + reduce.a[2][1]*k + reduce.a[2][2]*l};
for (const double x : n)
if (std::fabs(x - std::round(x)) > 1e-3)
return false;
r.h = static_cast<int32_t>(std::lround(n[0]));
r.k = static_cast<int32_t>(std::lround(n[1]));
r.l = static_cast<int32_t>(std::lround(n[2]));
return true;
};
if (!cand || reduce_res >= 0.05) {
logger.Warning("{} describes a cell of {}x the volume its reference setting does - "
"the lattice is a doubled description of a primitive one - but it "
"could not be reduced ({}). Keeping the doubled cell.",
won.short_name(), multiple,
cand ? "the change of basis is not exact" : "no primitive character fits");
} else {
std::vector<MergedReflection> merged_p;
merged_p.reserve(sm.merged.size());
for (auto m : sm.merged)
if (reduce_hkl(m))
merged_p.push_back(m);
SearchSpaceGroupOptions o2 = sg_opts;
o2.lattice_system = cand->system;
o2.cell = gemmi::UnitCell(cand->conventional.GetUnitCell());
const auto s2 = SearchSpaceGroup(merged_p, o2);
// Only ever a re-expression: the reduced cell has to support the same symmetry
// the doubled one found, in the primitive setting the reduction produced.
// Anything less and the doubled answer stands - it is the one that merges.
const bool holds = s2.best_space_group.has_value()
&& s2.point_group_order >= sg_search.point_group_order
&& s2.best_space_group->centring_type() == cand->centering;
if (!holds) {
logger.Warning("{} describes a cell of {}x the volume its reference setting "
"does, but the reduced primitive cell does not carry the same "
"symmetry ({}, order {} against {}). Keeping the doubled cell.",
won.short_name(), multiple,
s2.best_space_group ? s2.best_space_group->short_name() : "nothing",
s2.point_group_order, sg_search.point_group_order);
} else {
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(reduce);
std::vector<Reflection> kept;
kept.reserve(io.reflections.size());
for (auto r : io.reflections)
if (reduce_hkl(r))
kept.push_back(r);
io.reflections = std::move(kept);
}
const size_t dropped = sm.merged.size() - merged_p.size();
result.consensus_cell = cand->conventional.GetUnitCell();
result.consensus_centering = cand->centering;
end_msg.unit_cell = result.consensus_cell;
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reduce);
end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand->centering,
.niggli_class = cand->niggli_class, .crystal_system = cand->system };
sg_search = s2;
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
const auto &uc = *result.consensus_cell;
logger.Info("{} names a cell of {}x the volume of its reference setting: the "
"lattice is a doubled description of a primitive one, and {} of "
"{} merged reflections were positions the doubled cell invented. "
"Reduced to a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} "
"gamma={:.2f} and searched again there: {}",
won.short_name(), multiple, dropped, sm.merged.size(),
uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
s2.best_space_group->short_name());
// The reduction halves the cell, so its change of basis is not integral and
// end_msg.reindex_matrix (int32) cannot carry it. The per-image reflections
// already written to the _process.h5 therefore stay in the doubled setting.
logger.Warning("The per-image reflections in the process file are in the "
"doubled setting the run integrated in; the cell, the merged "
"reflections and the space group are in the reduced one. The "
"change of basis between them is not integral, so it is not "
"recorded in the file.");
}
}
}
}
// Adopt the determined space group and re-scale/merge in it, so scaling uses symmetry
// equivalents and the statistics come out in the right symmetry. P1 stands when nothing
// is determined - and it has to stand as a MERGE of its own, not as whatever `sm` happens
// to hold: everything merged up to here was a search merge, which drops the ice rings and
// (on the Lorentz-filtered arm) most of the observations with them, is never resolution-cut
// and fits no correction surfaces, because its intensities exist only to be searched.
//
// The P1 fallback is a belt, not a brace. SearchSpaceGroup cannot hand back an unset group
// for a merge that has any reflections in it: finding no symmetry at all leaves it on the
// identity point group, whose representative IS P1, and it returns that; finding a point
// group but no screw or centering leaves the primitive symmorphic candidate, which has no
// predicted absences to be inconsistent with and is therefore always eligible; and an empty
// merge never reaches the search, because both merge engines throw on the resolution range
// before returning one. So the fallback buys nothing today - what it buys is that the
// promise above is kept HERE, rather than resting on eligibility gates in another file that
// a later change could tighten without anyone noticing what else was leaning on them.
{
const gemmi::SpaceGroup sg = sg_search.best_space_group.value_or(gemmi::get_spacegroup_p1());
// xhm(), not short_name(): the search can now return a non-reference setting, and
// short_name() cannot tell P 1 2 1 from P 1 1 2.
logger.Info("Adopting space group {} (number {})", sg.xhm(), sg.number);
// A GLIDE plane was confirmed, so the adopted group is not a Sohncke one. Say both:
// a crystal built from L-amino acids is chiral and cannot have a glide plane, so this
// is a statement that the sample is not a protein - and a reader who knows it IS one
// has to be able to see the Sohncke group the same run would otherwise have reported,
// without processing the images again. Both, always, is what the two fields are for.
if (sg_search.glide_space_group.has_value()) {
std::string zones;
for (const auto &z : sg_search.glide_zones)
zones += fmt::format("{}{} ({} absences at {:.2g} of the rest of their plane)",
zones.empty() ? "" : ", ", z.label, z.n_absent, z.mean_u);
const std::string msg = fmt::format(
"Glide plane(s) confirmed from zonal absences - {}. The group is {}, which is "
"NOT a Sohncke group: a crystal of chiral molecules (any protein) cannot have "
"a glide plane, so this says the sample is not one. The best Sohncke group for "
"the same data is {} - re-run with -S {} to process in it.",
zones, sg.xhm(),
sg_search.sohncke_space_group.has_value()
? sg_search.sohncke_space_group->xhm() : std::string("none"),
sg_search.sohncke_space_group.has_value()
? sg_search.sohncke_space_group->xhm() : std::string("P 1"));
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
}
// The group has to be able to DESCRIBE the cell it is adopted on, and where the
// promotion was made HERE it cannot. A two-fold the intensities confirm is one the spot
// positions never offered - the Bravais class is decided on the unrefined indexing
// candidate, and the free refinement can walk into a class afterwards - so no fit under
// this group's constraint has ever been run, and what goes to the report, the master
// file and the MTZ is the free fit's metric re-expressed in the group's setting: a
// C 1 2 1 whose alpha is 88.5, which no monoclinic refinement could have produced. Ask
// the spots what cell the constraint supports; where they refuse it, report the metric
// the group fixes rather than one it cannot describe. A lattice already refined under
// this class leaves only float noise behind and is left alone, which is what the bound
// tests - so a de-novo run whose indexing found the class behaves exactly as it did.
{
std::optional<CrystalLattice> &adopted =
commit_reindex ? commit_lattice : end_msg.rotation_lattice;
// The band between the two bounds is where the cell is merely UNCONSTRAINED. Above
// the upper one it is the wrong cell for this group - the axes permuted, the fold
// running along an axis the group does not put it on - and nothing here can repair
// that: a constrained refit has no cell to find and the metric projection would
// manufacture one, turning a visible mismatch into a plausible-looking lie. That
// case is refused where the group comes in (MAX_METRIC_VIOLATION), and is left
// exactly as it is here.
const double violation = adopted.has_value()
? MetricViolation(adopted->GetUnitCell(), sg) : 0.0;
if (adopted.has_value() && violation > MAX_CONSTRAINED_METRIC_NOISE
&& violation <= MAX_METRIC_VIOLATION) {
const UnitCell before = adopted->GetUnitCell();
const auto refit = indexer->RefineRotationLatticeConstrained(*adopted,
sg.crystal_system());
// A refit is usable only when the constraint it was written in is the one this
// group needs: XtalResidual parameterises monoclinic with the unique axis along
// b, and the search can adopt a group that puts it elsewhere (C 1 1 2), whose
// metric such a fit does not satisfy. Ask the group, rather than enumerate the
// settings - a fit made in the right constraint lands at float noise.
const bool fits_group = refit.has_value()
&& MetricViolation(refit->lattice.GetUnitCell(), sg)
<= MAX_CONSTRAINED_METRIC_NOISE;
// ...and the spots have to keep it. Half is the bar the indexer's own
// pseudo-symmetry guard uses for this very comparison: a constrained cell that
// indexes half of what the free one did is not describing these images, and the
// metric and the intensities are then saying different things - which is a
// geometry error to go and find, not a cell to report.
const bool spots_keep_it = refit.has_value()
&& refit->indexed_fraction > 0.5f * refit->indexed_fraction_before;
if (fits_group && spots_keep_it) {
*adopted = refit->lattice;
const UnitCell uc = adopted->GetUnitCell();
logger.Info("{} was adopted on a cell no fit under it had produced (the "
"promotion was made after integration, on the freely refined "
"metric). Re-refined under the constraint: a={:.3f} b={:.3f} "
"c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, from a={:.3f} "
"b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f} - it "
"indexes {:.1f} % of the accumulated spots against {:.1f} % for "
"the free cell.",
sg.xhm(), uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma,
before.a, before.b, before.c, before.alpha, before.beta,
before.gamma, 100.0 * refit->indexed_fraction,
100.0 * refit->indexed_fraction_before);
} else {
*adopted = SymmetrizeMetric(*adopted, sg);
const UnitCell uc = adopted->GetUnitCell();
const std::string msg = fmt::format(
"The cell {} was adopted on is one that group cannot describe - "
"a={:.3f} b={:.3f} c={:.3f} alpha={:.2f} beta={:.2f} gamma={:.2f}, "
"refined FREE, because the symmetry was confirmed from the intensities "
"after integration and no constrained fit had produced this lattice. {} "
"The reported cell is the nearest metric the group fixes (alpha={:.2f} "
"beta={:.2f} gamma={:.2f}); the reflections are unchanged. A deviation "
"this size is not refinement noise - the geometry the images were "
"integrated at is the place to look.",
sg.xhm(), before.a, before.b, before.c, before.alpha, before.beta,
before.gamma,
refit.has_value()
? (fits_group
? fmt::format("Re-refining under the constraint indexes {:.1f} % "
"of the accumulated spots against {:.1f} % free, so "
"it was refused.",
100.0 * refit->indexed_fraction,
100.0 * refit->indexed_fraction_before)
: "The constrained refinement is not written in this group's "
"setting, so it could not be asked.")
: "There was no rotation spot cloud to re-refine against.",
uc.alpha, uc.beta, uc.gamma);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
}
// Carry the corrected lattice into the cell that is reported and merged on.
// Where a reindex is pending the block below does that and re-ingests; where
// there is none, this is the only place it happens.
if (commit_reindex) {
commit_cell = adopted->GetUnitCell();
} else {
result.consensus_cell = adopted->GetUnitCell();
end_msg.unit_cell = result.consensus_cell;
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(),
result.consensus_cell, static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
}
}
}
// A reindex is committed only when the intensity re-test confirmed a higher symmetry in a
// centred setting: bring the integrated reflections + cell into that setting and re-ingest
// the rotation merge, so the final scaling/merging folds the equivalents correctly.
if (commit_reindex) {
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(*commit_reindex);
for (auto &r : io.reflections)
reindex_hkl(r, *commit_reindex);
}
result.consensus_cell = commit_cell;
result.consensus_centering = commit_lattice_type->centering;
end_msg.unit_cell = commit_cell;
// Keep the lattice metadata (hence the master-file UB matrix and indexed-lattice
// vectors) in the same conventional setting as the cell, reflections and space group.
end_msg.rotation_lattice = commit_lattice;
end_msg.rotation_lattice_type = commit_lattice_type;
record_reindex(*commit_reindex);
if (rsm) {
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
}
}
experiment_.SetSpaceGroup(sg);
end_msg.space_group_name = sg.xhm();
end_msg.space_group_number = static_cast<uint64_t>(sg.number);
result.space_group = sg;
phase("Re-scaling in space group " + sg.short_name());
sm = scale_and_merge(sg.short_name(), false);
// A promotion the operator correlations CONFIRMED and Stage A then refused, put to
// the merge. Every gate that can refuse one is a ratio against another operator or
// against the noise floor, and both references are properties of how the crystal was
// mounted rather than of its symmetry: a 2-fold within a degree of the spindle
// records its mates on the same detector pixel half a turn later, carries no
// geometry-dependent systematic at all, and by being that clean makes every other
// operator look bad; and the best-agreeing operator anywhere can be one of the
// operators under test, which collapses the ratio on worse data and raises it on
// better. So ask the data the question those ratios stand in for - merge under the
// refused group and see whether the reflections it folds together agree.
//
// R_meas is multiplicity-corrected, so a merge that averaged non-equivalent
// reflections has to inflate it, and the error model is refitted per merge, so ISa
// says whether the extra multiplicity was bought with systematic disagreement. Over
// six refused promotions the two signs separate completely: the genuine ones leave
// both flat or better, the false ones inflate R_meas by 21-99% and drop ISa. CC1/2
// and <I/sigma> do NOT separate them - both RISE on a false promotion too, which
// gains multiplicity either way - so the rule is read on R_meas and ISa and nothing
// else.
//
// Both arms are merged over the SAME resolution range, the one the adopted group's
// own merge chose: left to pick their own cutoffs the two are not the same
// measurement and the separation muddies to a couple of percent with the sign lost.
// Rotation only - it is the rotation merge that can be re-cut between merges.
if (rsm && sg_search.refused_point_group_representative.has_value()) {
// Which group the comparison arm merges in. The point group is the refused one -
// passed as its representative, because gemmi calls both P321 and P312 "32" - and
// the absence classes are the adopted group's, because Stage B is asked for the
// group on the adopted merge, where those classes are already empty. That is what
// makes the two arms differ by the added ROTATIONS alone.
SearchSpaceGroupOptions hi_opts = sg_opts;
hi_opts.fixed_point_group = sg_search.refused_point_group_representative;
hi_opts.merge_isa = result.error_model_isa;
if (result.consensus_cell.has_value())
hi_opts.cell = gemmi::UnitCell(*result.consensus_cell);
if (end_msg.rotation_lattice_type.has_value())
hi_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
const auto hi_search = SearchSpaceGroup(sm.merged, hi_opts);
// ...and its CENTRING is taken rather than read. The adopted group has already
// removed its centring-absent class from this merge, so that class holds no data,
// every candidate of the point group is equally consistent with it, and the
// tie-break takes the lowest-numbered - which is the primitive one. Merging in
// that would hand the higher arm the centring-absent reflections as if they were
// data and lose the comparison on noise. Same move the metric-lattice ask makes.
std::optional<gemmi::SpaceGroup> hi_sg;
if (hi_search.best_space_group.has_value()
&& hi_search.best_space_group->centring_type() == sg.centring_type())
hi_sg = hi_search.best_space_group;
else
for (const auto &alt : hi_search.alternatives)
if (alt.centring_type() == sg.centring_type()) { hi_sg = alt; break; }
if (hi_sg.has_value()) {
// scale_and_merge reports the error model of whatever it merged, and the two
// comparison merges are not the run's answer, so it goes back if they lose.
const double em_isa = result.error_model_isa;
const double em_isa_asymptotic = result.error_model_isa_asymptotic;
const double em_a = result.error_model_a;
const double em_b = result.error_model_b;
const auto res_fit = result.resolution_fit_A;
const auto manual_d_min =
experiment_.GetScalingSettings().GetHighResolutionLimit_A();
phase("Merging under the refused point group "
+ sg_search.refused_point_group_hm);
rsm->SetDMinLimit(sm.statistics.overall.d_min);
const auto low = scale_and_merge(sg.short_name() + ", pinned", false);
const double low_isa = result.error_model_isa;
experiment_.SetSpaceGroup(*hi_sg);
const auto high = scale_and_merge(hi_sg->short_name() + ", pinned", false);
const double high_isa = result.error_model_isa;
rsm->SetDMinLimit(manual_d_min);
const double low_r = low.statistics.overall.r_meas;
const double high_r = high.statistics.overall.r_meas;
const auto mult = [](const MergeStatisticsShell &o) {
return o.unique_reflections > 0
? static_cast<double>(o.total_observations) / o.unique_reflections
: 0.0;
};
const std::string measured = fmt::format(
"R_meas {:.4f} -> {:.4f}, ISa {:.2f} -> {:.2f}, multiplicity {:.2f} "
"-> {:.2f}, both merged to {:.2f} A",
low_r, high_r, low_isa, high_isa, mult(low.statistics.overall),
mult(high.statistics.overall), sm.statistics.overall.d_min);
// Neither statistic may get worse. A NaN on either side answers "worse",
// which is the recoverable direction: the run keeps the group it already has.
if (high_r <= low_r && high_isa >= low_isa) {
const std::string msg = fmt::format(
"Point group {} was confirmed by the operator correlations and "
"then refused ({}), so both groups were merged and compared: {}. "
"Merging under it does not degrade the data, which folding "
"non-equivalent reflections together cannot manage, so {} is "
"adopted.",
sg_search.refused_point_group_hm, sg_search.refused_reason,
measured, hi_sg->xhm());
logger.Info("{}", msg);
stats_text << " " << msg << "\n\n";
sg_search = hi_search;
// Leave the candidate it displaced among the alternatives, so the
// reported "or ..." list stays the set of groups the data cannot
// separate, each named once - as the metric-lattice ask does.
if (hi_search.best_space_group->number != hi_sg->number) {
sg_search.alternatives.push_back(*hi_search.best_space_group);
std::erase_if(sg_search.alternatives,
[&](const gemmi::SpaceGroup &a) {
return a.number == hi_sg->number;
});
}
sg_search.best_space_group = *hi_sg;
result.space_group = *hi_sg;
end_msg.space_group_name = hi_sg->xhm();
end_msg.space_group_number = static_cast<uint64_t>(hi_sg->number);
promoted_point_group = true;
phase("Re-scaling in space group " + hi_sg->short_name());
sm = scale_and_merge(hi_sg->short_name(), false);
} else {
logger.Info("Point group {} was refused ({}), and merging under it "
"agrees: {}. Keeping {}.", sg_search.refused_point_group_hm,
sg_search.refused_reason, measured, sg.xhm());
sg_search.refused_reason +=
", and merging under it makes the data worse (" + measured + ")";
experiment_.SetSpaceGroup(sg);
result.error_model_isa = em_isa;
result.error_model_isa_asymptotic = em_isa_asymptotic;
result.error_model_a = em_a;
result.error_model_b = em_b;
result.resolution_fit_A = res_fit;
}
}
}
// After the arbitration, so a promoted run is not told that the metric hosts more
// symmetry than a group it no longer has.
WarnIfMetricHostsMoreSymmetry(result.consensus_cell,
result.space_group.value_or(gemmi::get_spacegroup_p1()));
}
result.space_group_search = sg_search;
} else {
// A space group was fixed by the user; surface it so the viewer/CLI can still show it.
result.space_group = experiment_.GetGemmiSpaceGroup();
// ...and record it, exactly as the search arm above does with the group it determined, or
// the _process.h5 comes out with a cell but no group and re-merging it defaults to P1.
if (const auto &sg = experiment_.GetGemmiSpaceGroup(); sg.has_value()) {
end_msg.space_group_name = sg->xhm();
end_msg.space_group_number = static_cast<uint64_t>(sg->number);
}
}
// Reference-based indexing-ambiguity resolution (rotation). When a reference MTZ is supplied and
// the crystal has an indexing ambiguity (merohedral: lattice symmetry higher than the Laue group),
// pick the reindexing that best correlates with the reference intensities and re-merge in it. The
// twin-law reindex is metric-preserving, so only the hkl labels change (the cell is unchanged).
// Stills resolve the ambiguity per image with ReindexAmbiguityResolver, not here.
if (rsm && !config_.reference_data.empty() && result.consensus_cell
&& experiment_.GetSpaceGroupNumber().has_value()) {
const gemmi::SpaceGroup &sg = experiment_.GetSpaceGroupOrP1();
const auto choice = ChooseReindex(
sm.merged, *result.consensus_cell, sg,
[&](const std::vector<MergedReflection> &m) {
return ReferenceIntensityCC(m, config_.reference_data, sg);
});
if (!choice.is_identity) {
logger.Info("Reference: resolved indexing ambiguity by reindexing to match the reference "
"(CC {:.3f}, vs {:.3f} unchanged)", choice.score, choice.identity_score);
for (auto &io : indexer->GetIntegrationOutcome())
for (auto &r : io.reflections) {
const gemmi::Op::Miller h = choice.op.apply_to_hkl({{r.h, r.k, r.l}});
r.h = h[0]; r.k = h[1]; r.l = h[2];
}
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
static_cast<int>(config_.scaling_iter),
config_.nthreads, logger, config_.observation_dump_path);
rsm->Ingest();
phase("Re-merging in the reference indexing");
sm = scale_and_merge(experiment_.GetGemmiSpaceGroup()->short_name(), false);
}
}
// A pass RunAllPasses is about to throw away and run again. Its report is never printed and
// its files are overwritten seconds later, so everything below that only fills those in would
// be spent on an answer nobody reads. Every trigger of a re-run is settled by here - the
// integrator's background-starvation count (measured at integration), the fixed-group lattice
// conflict (at indexing) and the search merge the quality guard reads (at the first merge) - so
// a pass can tell that it is being superseded, which RunAllPasses cannot until it returns.
// Only the refined passes ask: quality_guard_pass1_ holds the header-geometry pass they are
// judged against and is set only while they run. The supercell guard is deliberately not asked
// here - it compares this pass's cell against pass 1's rather than reading this pass's own
// numbers - so a pass it re-runs still writes a report first.
//
// Nothing below this point is read by a later pass: the analyses are report-only (see each),
// the files are rewritten by the pass that supersedes this one, and the two places that touch
// experiment_ - the model frame and the P1 merge - are undone before the pass ends, the first
// by the SetSpaceGroup at the top of RunPipeline and the second on the spot.
superseded =
!cancelled_ && quality_guard_pass1_ && quality_guard_pass1_->has_merge_statistics
&& ((bragg_adaptive_ && bkg_starved_fraction_
&& *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION)
|| !RefinedPassIsWorse(*quality_guard_pass1_, result).empty());
if (superseded)
logger.Info("This pass is going to be re-run, so it makes no report and writes no files.");
const auto &twin_sg_opt = experiment_.GetGemmiSpaceGroup();
const gemmi::SpaceGroup *twin_sg = twin_sg_opt ? &*twin_sg_opt : nullptr;
// Not on the geometry pre-pass, nor on a superseded one: the analysis goes into that pass's
// statistics text and its written reflections, and neither survives the run. The promotion flag
// below is a different thing - it is what the SEARCH did, the second pass reads it, and it is
// set either way.
if (!geometry_prepass && !superseded) {
// Translational pseudo-symmetry first: its verdict is an input to the twinning analysis,
// which must not read its own L-test when a non-half-integer pseudo-translation has biased
// it. Reporting only - no reflection and no merge depends on the outcome.
if (result.consensus_cell)
result.tncs = AnalyzeTranslationalNCS(sm.merged, gemmi::UnitCell(*result.consensus_cell),
twin_sg);
if (result.tncs.detected)
logger.Warning("Translational pseudo-symmetry: Patterson off-origin peak {:.1f}% of the "
"origin at ({:.3f}, {:.3f}, {:.3f}), {:.1f} A - declare it to the "
"molecular-replacement program",
result.tncs.peak_percent, result.tncs.vector_frac[0],
result.tncs.vector_frac[1], result.tncs.vector_frac[2],
result.tncs.vector_length_A);
result.twinning = AnalyzeTwinning(sm.merged, twin_sg, 20,
TranslationalNCSVectorForLTest(result.tncs));
// Set on the struct the text below is rendered from. AnalyzeTwinning returns a fresh result
// with this flag clear, so setting it only afterwards left the statistics text asserting
// "No twinning: the Laue class is holohedral" as a fact while the report, rendering the same
// struct once the flag was set, said the opposite - two twin verdicts from one run, with the
// flat one landing on exactly the case the flag exists for. (Assigning it before the call
// would not do: the call replaces the whole struct.)
result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group;
if (result.tncs.measurable)
stats_text << TranslationalNCSToText(result.tncs) << "\n";
stats_text << TwinningAnalysisToText(result.twinning) << "\n";
}
// Mark the conclusion as non-authoritative when the Laue class was reached by a promotion the
// search itself made, so the text cannot claim "no twin law exists" on its own say-so.
result.twinning.laue_class_was_chosen_by_promotion = promoted_point_group;
// How the crystal sat on the spindle. Reported always on rotation data (it is a property of
// the mounting, which the user can change). The angle to the nearest symmetry axis is kept
// as a descriptive key, but the WARNING is decided by the exact orbit computation: the old
// any-proper-axis-within-15-deg rule was wrong in both directions - an aligned in-plane
// 2-fold of a dihedral group is repaired by the principal axis, a cubic group is never
// severe in any orientation, and a lone diad PERPENDICULAR to the spindle is severe while
// no axis is anywhere near it.
if (experiment_.IsRotationIndexing() && twin_sg && end_msg.rotation_lattice.has_value()) {
if (const auto gonio = experiment_.GetGoniometer()) {
const auto closest = ClosestSymmetryAxisToSpindle(*twin_sg, *end_msg.rotation_lattice,
gonio->GetAxis());
if (closest.has_value()) {
const auto [angle, order] = *closest;
result.spindle_symmetry_axis_deg = angle;
result.spindle_symmetry_axis_order = order;
stats_text << "Closest symmetry axis to the spindle: " << order << "-fold at "
<< std::fixed << std::setprecision(1) << angle << " deg\n";
}
// The measured resolution of THIS run sets the cone: past merging the question is
// what these data lost, not what the detector could have reached.
double d_min = 0;
for (const auto &r : sm.merged)
if (std::isfinite(r.d) && r.d > 0 && (d_min == 0 || r.d < d_min))
d_min = r.d;
if (const auto loss = SpindleUnrepairedFraction(
*twin_sg, *end_msg.rotation_lattice, gonio->GetAxis(),
experiment_.GetWavelength_A(), d_min)) {
result.spindle_lost_unique_fraction = loss->lost_unique_fraction;
end_msg.spindle_lost_unique_fraction =
static_cast<float>(loss->lost_unique_fraction);
stats_text << "Unique reflections the mounting made unmeasurable: "
<< std::fixed << std::setprecision(2)
<< 100.0 * loss->lost_unique_fraction << "% (of the "
<< std::setprecision(2) << 100.0 * (1.0 - std::cos(loss->theta_max_deg * PI / 180.0))
<< "% a sweep's blind cone holds at " << std::setprecision(2)
<< d_min << " A)\n\n";
// Warn when the point group recovers less than half the cone - the same half
// that fixes the per-image trigger threshold. The loss is a coherent cap about
// the spindle direction, which costs a map more than the same percentage lost
// at random.
if (loss->cone_fraction >= 0.5) {
const std::string msg = fmt::format(
"The mounting makes {:.1f}% of unique reflections unmeasurable: the "
"measured point group cannot map that part of the sweep's blind cone "
"onto measured territory, however long the sweep runs. The loss is a "
"coherent cap about the spindle direction rather than a scatter, so "
"overall completeness may still look reasonable while the gap sits at "
"the spindle direction. A second sweep about a different axis, or "
"re-mounting, recovers it.",
100.0 * loss->lost_unique_fraction);
logger.Warning("{}", msg);
stats_text << " !! " << msg << "\n\n";
result.warnings.push_back({PathologyCode::SPINDLE_CAP, msg});
}
} else {
stats_text << "\n";
}
}
}
// Indexing-ambiguity (alternative-indexing) advisory. When the lattice metric symmetry exceeds
// the Laue symmetry the crystal can be validly indexed in several hands related by twin-law
// operators. For an obvious merohedral case (P3/P4/P6...) users expect this; but a PSEUDO-merohedral
// metric (e.g. a C2 crystal whose beta makes it pseudo-F-orthorhombic) is easy to miss, so surface it. The
// reindex operator is applied directly to (h,k,l), so its triplet reads as an h,k,l transform.
if (result.consensus_cell && twin_sg != nullptr) {
const auto twin_ops = ReindexAmbiguityOperators(*result.consensus_cell, *twin_sg, 2.0);
if (!twin_ops.empty()) {
std::string laws;
for (const auto &op : twin_ops) {
std::string t = op.triplet();
std::replace(t.begin(), t.end(), 'x', 'h');
std::replace(t.begin(), t.end(), 'y', 'k');
std::replace(t.begin(), t.end(), 'z', 'l');
laws += (laws.empty() ? "" : " ; ") + t;
}
// A model can resolve it too, but whether it does is only known once the model has
// been fitted (validation step below), so the warning stands here exactly as in a run
// without a model, and is withdrawn there if the model's indexing probe decided it.
// On serial stills with -C and -S the model is already in `ambiguity_reference`.
if (!ambiguity_reference) {
const std::string msg = fmt::format(
"Indexing ambiguity: this cell / space group admits alternative indexing "
"(reindex operator(s): {}). {} are indexed in one hand at random, and rugnux can only "
"break this against an external reference. WITHOUT one the merge mixes the hands and "
"CC1/2 is degraded; a reference MTZ (-z) or a model ({}) resolves it.",
laws, experiment_.IsRotationIndexing() ? "Lattices" : "Serial-stills crystals",
experiment_.IsRotationIndexing() ? "--model" : "--model, which needs -C and -S here");
logger.Warning("{}", msg);
ambiguity_warning_text = " !! " + msg + "\n\n";
stats_text << ambiguity_warning_text;
result.warnings.push_back({PathologyCode::INDEXING_AMBIGUITY, msg});
} else {
logger.Info("Indexing ambiguity present (reindex operator(s): {}); resolved "
"against the supplied reference.", laws);
}
}
}
// Dataset-wide Wilson B-factor estimate (like XDS's WILSON LINE B). Diagnostic only - it is not
// fed back into scaling; it just lands in the printed statistics, the mmCIF, and the log, none
// of which the geometry pre-pass or a superseded pass produces.
if (!geometry_prepass && !superseded) {
const GlobalWilsonB wilson = CalcGlobalWilsonB(sm.merged);
sm.statistics.wilson_b = wilson.b;
sm.statistics.wilson_b_correlation = wilson.correlation;
if (std::isfinite(wilson.b) && wilson.b > 0.0)
logger.Info("Wilson B-factor estimate: {:.2f} A^2 (correlation {:.3f}, {} shells)",
wilson.b, wilson.correlation, wilson.n_shells);
// Diffraction anisotropy. Report-only, like the twinning and Wilson analyses above: it
// describes the data and corrects nothing. The tensor and its resolution signature come from
// the merged intensities; the error bar the verdict is gated on has to come from the
// unmerged observations, because a merge has exact Laue symmetry by construction and the
// tensor directions the symmetry forbids - the only place a dataset measures its own
// systematic error - are identically zero in it.
if (result.consensus_cell) {
AnisotropyRunInfo aniso_run;
if (sm.statistics.sweep_quality.measured && sm.statistics.sweep_quality.sweep_deg > 0.0f)
aniso_run.observed_rotation_deg = sm.statistics.sweep_quality.sweep_deg;
aniso_run.dose_term_in_scale_model =
experiment_.GetScalingSettings().GetCorrectionSurfaces();
aniso_run.radiation_damage_relative_b = sm.statistics.radiation_damage_delta_b;
sm.statistics.anisotropy = AnalyzeAnisotropy(
sm.merged,
ScaledObservations(indexer->GetIntegrationOutcome(),
experiment_.IsRotationIndexing(), twin_sg,
experiment_.GetGoniometer()
? experiment_.GetGoniometer()->GetWedge_deg() : 0.0f),
*result.consensus_cell, twin_sg, aniso_run);
stats_text << AnisotropyToText(sm.statistics.anisotropy) << "\n";
}
}
stats_text << sm.statistics;
result.merge_statistics_text = stats_text.str();
result.has_merge_statistics = true;
result.merge_statistics = sm.statistics;
result.reference_merge_statistics = sm.reference_statistics;
result.reference_isa = sm.reference_isa;
result.cc_half_before_corrections = sm.cc_half_before_corrections;
// Per-image form of the sweep-quality ranges, for the _process.h5: one code per image, 0 where
// the image is in no flagged range, plus the vocabulary the codes index. Only filled when the
// diagnostic ran, so absent datasets mean "not looked for" rather than "all clean". The
// disposition rides alongside as a second per-image array: what the reason SAW and what became
// of the frame are different statements, and a consumer needs both.
if (sm.statistics.sweep_quality.measured) {
const auto &sq = sm.statistics.sweep_quality;
end_msg.sweep_quality.assign(end_msg.max_image_number, 0);
for (const auto &r : sq.ranges)
for (int64_t i = std::max<int64_t>(0, r.first_image);
i <= r.last_image && i < static_cast<int64_t>(end_msg.sweep_quality.size()); ++i)
end_msg.sweep_quality[i] = static_cast<uint8_t>(r.reason) + 1;
for (int r = 0; r <= static_cast<int>(SweepQualityReason::InconsistentWithMerge); ++r)
end_msg.sweep_quality_reasons.emplace_back(
SweepQualityReasonCode(static_cast<SweepQualityReason>(r)));
end_msg.frame_disposition.assign(
sq.frame_disposition.begin(),
sq.frame_disposition.begin()
+ std::min<int64_t>(end_msg.max_image_number, sq.frame_disposition.size()));
for (int d = 0; d <= static_cast<int>(FrameDisposition::Rejected); ++d)
end_msg.frame_disposition_codes.emplace_back(
FrameDispositionCode(static_cast<FrameDisposition>(d)));
}
{
// Stride rather than take the head: the merged list is ordered by hkl, so the first N
// reflections are one corner of reciprocal space and would not show the intensity range.
constexpr size_t MAX_POINTS = 4000;
const size_t n = sm.merged.size();
const size_t step = std::max<size_t>(1, n / MAX_POINTS);
result.merged_i_sigma.reserve(std::min(n, MAX_POINTS + 1));
for (size_t i = 0; i < n; i += step)
if (std::isfinite(sm.merged[i].I) && std::isfinite(sm.merged[i].sigma)
&& sm.merged[i].sigma > 0.0f)
result.merged_i_sigma.emplace_back(sm.merged[i].I, sm.merged[i].sigma);
}
result.has_reference = !config_.reference_data.empty();
// Inherit the campaign's shared R-free test set from the reference MTZ (overriding the
// per-hkl hash that the merge assigned), so every dataset flags the same free reflections.
if (config_.reference_has_free_flags && !config_.reference_data.empty() && !sm.merged.empty()) {
const size_t matched = ApplyReferenceFreeFlags(sm.merged, experiment_.GetSpaceGroupOrP1(),
config_.reference_data);
logger.Info("R-free flags: inherited the reference test set ({} of {} merged reflections matched)",
matched, sm.merged.size());
}
// Radiation-damage report (rotation): the per-image scale CC-to-merge and mosaicity across the
// sweep (the per-image scaling the rotation merge already fits, binned by frame = dose). A
// per-image CC that falls, and/or a mosaicity that rises, with frame number is the classic
// radiation-damage signature - a data-quality-vs-dose read complementary to the fitted decay
// correction. The full per-image table is written to <prefix>_scaling.txt for detail. Both this
// and the sweep-quality report below are text, so a superseded pass makes neither.
if (experiment_.IsRotationIndexing() && !superseded) {
const auto &outs = indexer->GetIntegrationOutcome();
const int nf = static_cast<int>(outs.size());
constexpr int nb = 10;
std::array<double, nb> cc_sum{}, mos_sum{};
std::array<int, nb> cc_n{}, mos_n{};
for (int f = 0; f < nf; ++f) {
const int b = nf > 1 ? std::min(nb - 1, f * nb / nf) : 0;
if (outs[f].image_scale_cc && std::isfinite(*outs[f].image_scale_cc)) {
cc_sum[b] += *outs[f].image_scale_cc; cc_n[b]++;
}
if (outs[f].mosaicity_deg && std::isfinite(*outs[f].mosaicity_deg)) {
mos_sum[b] += *outs[f].mosaicity_deg; mos_n[b]++;
}
}
int tot = 0;
for (int n : cc_n) tot += n;
if (tot >= 20) {
std::ostringstream os;
os << fmt::format("Radiation-damage report (per-image scale over {} frames):\n", nf);
os << " dose <CC to merge> <mosaicity deg>\n";
for (int b = 0; b < nb; ++b) {
if (cc_n[b] == 0 && mos_n[b] == 0) continue;
os << fmt::format(" {:3d}-{:3d}% {:>7} {:>7}\n", b * 10, (b + 1) * 10,
cc_n[b] ? fmt::format("{:.3f}", cc_sum[b] / cc_n[b]) : std::string("-"),
mos_n[b] ? fmt::format("{:.3f}", mos_sum[b] / mos_n[b]) : std::string("-"));
}
if (cc_n[0] > 0 && cc_n[nb - 1] > 0) {
const double cc0 = cc_sum[0] / cc_n[0], cc9 = cc_sum[nb - 1] / cc_n[nb - 1];
os << fmt::format(" => per-image CC to merge {:.3f} (first 10%) -> {:.3f} (last 10%){}",
cc0, cc9,
cc9 < cc0 - 0.05 ? " (falling: radiation damage / crystal decay)" : "");
}
// Relative B-factor across the sweep (measured before any decay correction, against the
// low-dose start): the resolution-dependent complement to the CC/mosaicity read above.
// Radiation damage fades the high-resolution intensity, so only a POSITIVE change is dose.
// A curve no straight line describes gets no first->last number at all, and a batch whose
// data could not measure one prints "-" rather than a value.
if (!sm.statistics.radiation_damage_b_batch.empty()) {
if (std::isfinite(sm.statistics.radiation_damage_delta_b))
os << fmt::format("\n => relative B-factor change over run = {:+.2f} A^2 (first->last){}",
sm.statistics.radiation_damage_delta_b,
sm.statistics.radiation_damage_delta_b > 5.0
? " (significant dose-dependent scaling: radiation damage)" : "");
else
os << "\n => relative B-factor: the per-batch curve below is not a trend, so no "
"first->last number describes it - dose does not come back, so whatever moved "
"here was not dose; see the sweep-quality report";
os << fmt::format("\n per-batch relative-B (A^2, {:.0f} deg/batch):",
sm.statistics.radiation_damage_batch_deg);
for (float bb : sm.statistics.radiation_damage_b_batch)
os << (std::isfinite(bb) ? fmt::format(" {:.1f}", bb) : std::string(" -"));
}
logger.Info("{}", os.str());
result.radiation_damage_text = os.str();
}
// Sweep-quality report: the stretches of the sweep over which the crystal delivered much
// less than the rest of the run, what each one looks like, what became of it and what
// keeping it costs the merged intensities. Frame numbers are processed-image ordinals,
// inclusive, as in <prefix>_image.dat.
const auto &sq = sm.statistics.sweep_quality;
if (sq.measured) {
const int n_sweep = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected;
auto pct = [&](int n) { return n_sweep > 0 ? 100.0 * n / n_sweep : 0.0; };
std::ostringstream os;
os << fmt::format("Sweep quality over {:.0f} deg (per-image scale with the incident flux, "
"which varied {:.2f}x, already divided out):\n",
sq.sweep_deg, sq.flux_peak_to_trough);
os << fmt::format(" {} frames: {} merged ({:.1f}%), {} downgraded ({:.1f}%), "
"{} rejected ({:.1f}%, {:.1f} deg)\n",
n_sweep, sq.frames_merged, pct(sq.frames_merged),
sq.frames_downgraded, pct(sq.frames_downgraded),
sq.frames_rejected, pct(sq.frames_rejected), sq.rejected_deg);
if (sq.ranges.empty()) {
os << " no stretch of the sweep is materially worse than the run";
} else {
os << " frames rotation diagnosis severity scale CC indexed disposition dCC1/2\n";
for (const auto &r : sq.ranges)
os << fmt::format(" {:<17s} {:6.1f} deg {:<16s} {:5.2f} {:5.2f} {:5.2f} {:4.0f}% {:<11s} {}\n",
fmt::format("{}-{}", r.first_image, r.last_image), r.rotation_deg,
SweepQualityReasonText(r.reason), r.severity, r.mean_relative_scale,
r.mean_relative_cc, 100.0 * r.indexed_fraction,
FrameDispositionCode(r.disposition),
std::isfinite(r.delta_cc_half)
? fmt::format("{:+.4f} +/- {:.4f}", r.delta_cc_half,
r.delta_cc_half_se)
: std::string("-"));
os << " => severity is the fraction of the run's typical diffracting power missing "
"over the range; dCC1/2 is what keeping it costs the merged intensities";
if (sq.modulation_peak_to_trough >= 1.05f)
os << fmt::format("\n => once-per-revolution modulation of the per-image scale: "
"{:.1f}x peak to trough", sq.modulation_peak_to_trough);
}
if (!sq.delta_cc_half_batch.empty()) {
os << fmt::format("\n => delta-CC1/2 per {:.0f} deg:", sq.delta_cc_half_batch_deg);
for (float d : sq.delta_cc_half_batch)
os << (std::isfinite(d) ? fmt::format(" {:+.3f}", d) : std::string(" -"));
}
logger.Info("{}", os.str());
}
}
// Model validation runs BEFORE the reflection files are written, because it is what settles the
// frame they are written in: the enantiomorph, which merged intensities cannot choose, and -
// with no reference MTZ - the alternative indexing. Both are relabelings of the same
// measurements, so the statistics computed above are unaffected; what changes is that the file,
// the R-factors and the maps then all describe one indexing instead of two.
if (result.consensus_cell && write_files && !config_.model_path.empty() && !geometry_prepass
&& !superseded) {
phase("Validating against model");
const auto data_sg = experiment_.GetGemmiSpaceGroup();
// With a reference MTZ the merohedral indexing was already resolved against it (rotation
// merge / stills scaling), so trust that; only probe indexing by R-free when model-only.
// The merge's own shell bounds, so CC(model, data) is reported on the shells CC1/2 was.
std::vector<float> report_shell_d_min;
for (const auto &sh : sm.statistics.shells)
report_shell_d_min.push_back(sh.d_min);
const auto validation =
ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
config_.output_prefix, logger,
data_sg ? &*data_sg : nullptr,
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
static_cast<size_t>(config_.nthreads),
experiment_.GetWavelength_A(), report_shell_d_min);
// A model that was asked for and could not be used has to say so where anyone will see
// it. Without this the run ends successfully with no R-free, no maps and nothing in the
// report - indistinguishable from a run that was never given --model at all.
result.model_validation = validation;
if (!validation.failure_reason.empty())
result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED,
"Model validation did not run: " + validation.failure_reason});
// The ambiguity warning above was issued before the model was read; take it back only
// where the model actually settled the indexing.
if (validation.indexing_decided && !ambiguity_warning_text.empty()) {
std::erase_if(result.warnings, [](const ReportWarning &w) {
return w.code == PathologyCode::INDEXING_AMBIGUITY;
});
const auto pos = result.merge_statistics_text.find(ambiguity_warning_text);
if (pos != std::string::npos)
result.merge_statistics_text.erase(pos, ambiguity_warning_text.size());
logger.Info("Indexing ambiguity resolved against the model; warning withdrawn");
}
if (data_sg.has_value()) {
const gemmi::SpaceGroup *adopted =
AdoptModelFrame(validation, sm.merged, *data_sg,
experiment_.GetScalingSettings().GetMergeFriedel(), logger);
if (adopted->number != data_sg->number) {
// The written reflections and the report that describes them, but deliberately not
// end_msg: the _process.h5 already holds the per-image reflections as they were
// integrated, and a file whose recorded group did not match its own reflections
// would mis-merge on a later --mode scale. An enantiomorphic pair merges identically
// anyway - same Laue class, same absences - so nothing is lost by leaving it.
experiment_.SetSpaceGroup(*adopted);
result.space_group = *adopted;
}
// The unmerged export below comes from the integrated observations, not from the merge,
// so it follows the same relabeling - otherwise the two files would describe different
// indexings of one dataset. The enantiomorph is a label, so only the indexing moves it.
const gemmi::Op &op = validation.indexing_op;
if (!(op == gemmi::Op::identity()))
for (auto &io : indexer->GetIntegrationOutcome())
for (auto &r : io.reflections) {
const gemmi::Op::Miller h = op.apply_to_hkl({{r.h, r.k, r.l}});
r.h = h[0]; r.k = h[1]; r.l = h[2];
}
}
// The coordinates that go with the maps just written, in the frame the reflection files
// beside them carry. Written whenever the validation ran, not only where the rigid-body
// step moved the model: it is also re-fractionalized into the data cell and may be
// relabelled to the model's enantiomorph, so "did not move" is not "identical to the input
// file". A rejected model is scored, placed and mapped like any other, and that is the case
// where someone most wants to see the model in the density, so it gets its file too.
if (validation.placed_model)
WritePlacedModel(*validation.placed_model, *result.consensus_cell,
experiment_.GetSpaceGroupOrP1(), config_.output_prefix, logger);
}
// Not on the geometry pre-pass, and not on a pass that is going to be re-run. Pass 1 exists to
// choose the space group and post-refine the geometry; the pass after it remakes these files
// seconds later at the refined geometry, and that is the answer anyone reads. Writing them
// twice costs 0.6 s of a 15 s run - a fifth of the merged mmCIF on a large crystal - for a file
// that is superseded before the run ends. The quality guard is unaffected: it reads the search
// merge, which is made well above this.
if (result.consensus_cell && write_files && config_.write_merged && !geometry_prepass
&& !superseded) {
phase("Writing reflections");
const ErrorModelReport em_report{
result.error_model_isa > 0 ? fmt::format("{:.2f}", result.error_model_isa) : "?",
result.error_model_isa_asymptotic > 0 ? fmt::format("{:.2f}", result.error_model_isa_asymptotic)
: std::string(),
result.error_model_a > 0 ? fmt::format("{:.3f}", result.error_model_a) : std::string(),
result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string()};
WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics,
em_report, result.twinning, config_.output_prefix,
static_cast<size_t>(std::max(1, config_.nthreads)));
// Per-image scaling table (G, B-factor, mosaicity, wedge, CC) for inspection / XDS
// comparison. The offline self-scaling result is otherwise not exposed (process.h5's
// per-image arrays are only filled on the online per-image path). Sourced from the
// partials, which carry the first-pass per-image scale.
ScalingResult(indexer->GetIntegrationOutcome()).SaveToFile(config_.output_prefix);
// P1 cross-check dataset. The group the files above are written in was chosen by the
// search, and if that choice is wrong nothing in them says so - every statistic was
// computed in the group that was assumed, and the only way back is to process the images
// again. Merging the same integration once more in P1 removes that: the user can re-merge
// it in any subgroup, run MR or refinement in it, or hand it to POINTLESS. It is a proper
// merge - correction surfaces fitted, ice rings and near-tangential observations kept -
// not the deliberately degraded one the search itself ran on, and it is the same file
// `rugnux --mode scale -S P1` makes from a _process.h5. Written after the merged output,
// so the answer of the run is already on disk in the group that was determined.
//
// Written on EVERY de-novo run, including one whose search concluded P1 and where the
// file therefore repeats the merged output. Whether a file exists must not depend on
// what the pipeline decided: a script harvesting results would otherwise have to
// reproduce the search's decision to know whether to expect it, and a missing file
// would not distinguish "the run chose P1" from "the run failed".
//
// With a user-fixed space group (-S) the file is written only when prediction still ran
// in P, which is what makes the merge complete. It does whenever the fixed group is
// primitive - most of them - and now on every run, since the full pass predicts every node.
// A fixed CENTRED group without that gate rejects its own centring absences at prediction
// and never integrates them (IndexAndRefine.cpp, `.centering = ...`); a P1 merge of such a
// run would be missing whole centring classes - a misleading P1 dataset, not merely a
// smaller one - so nothing is written and the log says why.
//
// Rotation only, for now. The stills merge re-fits each image's scale and each
// reflection's partiality onto the integration outcomes, and _unmerged.mtz is written
// from those afterwards - so on stills this merge changes a file that is the run's own
// output (measured: same length, different bytes). The fix is to run this below the
// unmerged export rather than here, which needs the merge lambda hoisted out of this
// block; deferred, as rotation is what the online pipeline processes.
const bool p1_integration_complete =
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
if (config_.write_p1_crosscheck && is_rotation && !p1_integration_complete)
logger.Warning("No P1 cross-check dataset written: the fixed space group's centring "
"absences were never predicted, so a P1 merge of this run would be "
"missing whole centring classes. Re-run without -S to get one.");
// The ledger runs off the P1 cross-check merge below, so every reason that merge is not
// built is also a reason there is no ledger. Say which one it was: a user who gave
// --finalist-ledger and got no table would otherwise have to read this function to find out
// whether the instrument declined, or the crystal simply had nothing to report.
if (config_.finalist_ledger
&& !(config_.write_p1_crosscheck && p1_integration_complete && is_rotation))
logger.Warning("No finalist ledger: it is folded from the P1 cross-check merge, and "
"this run builds none ({}).",
!is_rotation ? "stills - the cross-check is rotation-only for now"
: !config_.write_p1_crosscheck ? "--no-p1-crosscheck was given"
: "a fixed centred space group's absences were never predicted, so a P1 "
"merge would be missing whole centring classes");
if (config_.write_p1_crosscheck && p1_integration_complete && is_rotation) {
// The whole GROUP, not its number: the search can adopt a non-reference setting, and
// a number only ever names the reference one - so restoring through the number would
// silently re-seat P 1 1 2(1) as P 1 2(1) 1 for everything written after this block,
// the unmerged MTZ included, leaving two files that describe one dataset in two
// different settings.
const auto determined_group = experiment_.GetGemmiSpaceGroup();
const gemmi::SpaceGroup *determined =
determined_group.has_value() ? &*determined_group : nullptr;
// scale_and_merge reports the error model AND the CC1/2 resolution fit of whatever it
// merged; this merge is not the run's answer, so both go back afterwards. The fit in
// particular is what the report calls the number to quote, and a P1 merge of the same
// observations has n_ops times the unique reflections at a fraction of the
// multiplicity, so its fall-off is not this crystal's.
const double em_isa = result.error_model_isa;
const double em_isa_asymptotic = result.error_model_isa_asymptotic;
const double em_a = result.error_model_a;
const double em_b = result.error_model_b;
const auto res_fit = result.resolution_fit_A;
// Both the merge and the MTZ read the group from the experiment, so it is set for
// the whole of it and restored after.
experiment_.SpaceGroupNumber(1);
const auto p1 = scale_and_merge("P1 cross-check", false);
// FINALIST LEDGER (--finalist-ledger), report-only and deliberately behind a flag: these
// numbers are an instrument, and reading them as a decision before they are calibrated
// is the one way this can do harm.
//
// It runs here because THIS merge is the substrate the whole space-group audit says is
// missing and nothing else in the run has: the same observations at full resolution,
// production-scaled with the correction surfaces fitted, and folded under NO symmetry -
// so every hypothesis can be folded from it on equal terms. It is already being built
// for the P1 cross-check file, so the ledger costs one search over it and no merge at
// all. It also carries half-set intensities, which is what lets the random-noise R
// floor be formed - the only reference available to the first step out of P1, and the
// reason this cannot be done from the stored MTZ afterwards.
if (config_.finalist_ledger) {
SearchSpaceGroupOptions lopt;
lopt.nthreads = static_cast<size_t>(std::max(1, config_.nthreads));
lopt.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
if (result.consensus_cell.has_value())
lopt.cell = gemmi::UnitCell(*result.consensus_cell);
lopt.enumerate_all_settings = true;
lopt.enumerate_all_rotation_sets = true;
lopt.merge_isa = result.error_model_isa;
const auto led = SearchSpaceGroup(p1.merged, lopt);
logger.Info("Finalist ledger on the full-resolution P1 merge ({} reflections):\n{}",
p1.merged.size(), FinalistLedgerToText(led));
}
const std::string path = config_.output_prefix + "_P1.mtz";
WriteMtzReflections(p1.merged, *result.consensus_cell, experiment_, path);
experiment_.SetSpaceGroup(determined_group);
result.error_model_isa = em_isa;
result.error_model_isa_asymptotic = em_isa_asymptotic;
result.error_model_a = em_a;
result.error_model_b = em_b;
result.resolution_fit_A = res_fit;
if (determined != nullptr && determined->number > 1)
logger.Info("P1 cross-check dataset written to {} ({} unique reflections): the "
"same observations merged in P1 instead of {}, so a wrong space group "
"can be recovered from without reprocessing. It is not the result of "
"this run.", path, p1.merged.size(),
determined ? determined->short_name() : "?");
else
logger.Info("P1 cross-check dataset written to {} ({} unique reflections). This "
"run determined P1, so it repeats the merged output; it is written "
"anyway, so the file is there on every run.", path, p1.merged.size());
}
}
}
// Unmerged observations. After the merge, because that is where the space group is settled and
// the file has to be written in it, but from the integrated observations rather than the merged
// ones: the partiality and the per-image scale are left for the reading program, which fits a
// scale model of its own. write_files, not write_output - without an output prefix there is
// nowhere to put it, and the name would come out as a bare "_unmerged.mtz".
if (full && !cancelled_ && write_files && !geometry_prepass && !superseded && result.consensus_cell) {
if (config_.export_unmerged) {
if (observer) observer->OnPhase("Writing unmerged reflections");
const std::string path = config_.output_prefix + "_unmerged.mtz";
WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell,
experiment_, true, path);
logger.Info("Unmerged observations written to {}", path);
}
if (config_.export_unmerged_partials) {
if (observer) observer->OnPhase("Writing unmerged reflections");
const std::string path = config_.output_prefix + "_unmerged_partials.mtz";
WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell,
experiment_, false, path);
logger.Info("Unmerged partials written to {}", path);
}
}
// When merging is on, the _process.h5 is skipped because the merged reflections are the wanted
// output and the per-image file is large. If nothing indexed there are no merged reflections, so
// the run would finish successfully having written no file at all. Write one now: the per-image
// messages went past while there was no writer, but the end message still carries the per-image
// scalars (spot counts, background, resolution estimate) and the dataset metadata - which is what
// anyone asking "why did nothing index" needs, and it is small precisely because nothing indexed.
if (!writer && full && write_files && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) {
logger.Warning("No image indexed, so there are no merged reflections to write - writing "
"{}_process.h5 with the per-image analysis instead", config_.output_prefix);
fill_writer_maps();
writer = std::make_unique<FileWriter>(start_message, /*check_overwrite_at_start=*/true,
/*trusted_path=*/true);
}
if (writer) {
writer->WriteHDF5(end_msg);
writer->Finalize();
result.written_master_path = config_.output_prefix + "_process.h5";
}
if (observer)
observer->OnPhase(cancelled_ ? "Cancelled" : "Done");
// Total wall time for the whole run, including scaling/merging/writing above (not just the
// per-image pass), so the reported rate reflects everything rugnux did.
result.processing_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - start_time).count();
if (result.processing_time_s > 0.0) {
result.frame_rate_hz = static_cast<double>(result.images_processed) / result.processing_time_s;
result.throughput_MBs = static_cast<double>(total_uncompressed_bytes) / (result.processing_time_s * 1e6);
}
logger.Info("{} {} images in {:.2f} s ({:.2f} Hz)", cancelled_ ? "Cancelled after" : "Processed",
result.images_processed, result.processing_time_s, result.frame_rate_hz);
// The geometry this pass integrated at, for the report. Taken from experiment_ rather than from the
// caller's copy: the second pass of the rotation two-pass runs at the post-refined geometry.
result.used_beam_x_pxl = experiment_.GetBeamX_pxl();
result.used_beam_y_pxl = experiment_.GetBeamY_pxl();
result.used_distance_mm = experiment_.GetDetectorDistance_mm();
{
const auto &g = experiment_.GetDiffractionGeometry();
constexpr double DEG = 180.0 / PI;
result.used_detector_tilt_deg = {g.GetPoniRot1_rad() * DEG, g.GetPoniRot2_rad() * DEG,
g.GetPoniRot3_rad() * DEG};
const auto direct = g.GetDirectBeam_pxl();
result.used_direct_beam_x_pxl = direct.first;
result.used_direct_beam_y_pxl = direct.second;
}
return result;
}
Reference in New Issue View Git Blame Copy Permalink