Jungfraujoch/rugnux/Rugnux.cpp

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#include <cstdlib>
#include "Rugnux.h"
#include "../image_analysis/structure_refinement/ModelValidation.h"
#include "../image_analysis/indexing/SpindleCuspLoss.h"
#include "../image_analysis/bragg_integration/SpotWidth.h"
#include "../image_analysis/bragg_integration/SpotFootprint.h"
#include "../image_analysis/bragg_integration/BraggStencil.h"   // BRAGG_FOOTPRINT_NSIGMA
#include "../image_analysis/hot_pixels/HotPixels.h"
#include "../image_analysis/SensorAbsorption.h"
#include "DiagnosticOutput.h"

#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <deque>
#include <cstring>
#include <functional>
#include <future>
#include <mutex>
#include <numeric>
#include <map>
#include <set>
#include <sstream>
#include <thread>

#include "../reader/JFJochHDF5Reader.h"
#include "../reader/JFJochStartMessageReader.h"
#include "../common/HostMemory.h"
#include "../common/JFJochMath.h"
#include "../common/ParallelFor.h"
#include "../common/Logger.h"
#include "../common/AzimuthalIntegrationMapping.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include "../common/CUDAWrapper.h"
#include "../common/JFJochException.h"
#include "../common/time_utc.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
#include "../image_analysis/beam_stop/ShadowFinder.h"
#include "../image_analysis/IndexAndRefine.h"
#include "../image_analysis/indexing/AnalyzeIndexing.h"
#include "../image_analysis/indexing/HarmonicContamination.h"
#include "../image_analysis/geom_refinement/BeamCenterFFT.h"
#include "../image_analysis/geom_refinement/BeamCenterFromBackground.h"
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
#include "../image_analysis/indexing/IndexerThreadPool.h"
#include "../image_analysis/spot_finding/ImageSpotFinderCPU.h"
#include "../image_analysis/spot_finding/SpotUtils.h"
#include "gemmi/twin.hpp"
#include "../image_analysis/azint/AzIntEngineCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
#ifdef JFJOCH_USE_CUDA
#include "../image_analysis/image_preprocessing/ImagePreprocessorGPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h"
#include "../image_analysis/spot_finding/AdaptiveSpotFinderGPU.h"
#endif
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/RfreeFlags.h"
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
#include "../image_analysis/scale_merge/ScalingResult.h"
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/geom_refinement/PostRefine.h"
#include "../image_analysis/lattice_search/LatticeSearch.h"
#include "../image_analysis/lattice_search/LePageLattice.h"
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
#include "../image_analysis/scale_merge/FrenchWilson.h"
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
#include "../image_analysis/scale_merge/TranslationalNCS.h"
#include "../image_analysis/scale_merge/HKLKey.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
#include "../image_analysis/WriteReflections.h"
#include "../image_analysis/bragg_integration/CalcISigma.h"
#include "../common/Definitions.h"
#include "../common/CorrelationCoefficient.h"
#include <array>
#include <map>
#include "RugnuxPipeline.h"

using namespace rugnux_internal;

std::pair<double, double> ValidationEvidenceDifference(const ValidationSpotEvidence &current,
                                                       const ValidationSpotEvidence &candidate) {
    const auto excess = [](const ValidationSpotEvidence &e) {
        return static_cast<double>(e.on_lattice - e.by_chance) / static_cast<double>(std::max<int64_t>(1, e.spots));
    };
    const auto rate_var = [](const ValidationSpotEvidence &e) {
        const double n = static_cast<double>(std::max<int64_t>(1, e.spots));
        const double p = static_cast<double>(e.on_lattice) / n;
        return p * (1.0 - p) / n;
    };
    return {excess(candidate) - excess(current), std::sqrt(rate_var(candidate) + rate_var(current))};
}

bool ValidationEvidencePrefers(const ValidationSpotEvidence &current, const ValidationSpotEvidence &candidate) {
    const auto [difference, sigma] = ValidationEvidenceDifference(current, candidate);
    return difference > SPOT_BUDGET_SIGNIFICANCE_Z * sigma;
}

bool ValidationEvidenceBeatsChance(const ValidationSpotEvidence &e) {
    return ValidationEvidenceBeatsChance(e, std::erfc(SPOT_BUDGET_SIGNIFICANCE_Z / std::sqrt(2.0)));
}

double ValidationEvidenceLog10Tail(const ValidationSpotEvidence &e) {
    const int64_t n = e.reflections + e.reflections_by_chance;
    const double p = 1.0 / (1.0 + VALIDATION_NULL_DISPLACEMENTS);
    // Summed in logs: a real lattice's tail is far below the smallest double.
    std::vector<double> log_terms;
    for (int64_t j = e.reflections; j <= n; j++)
        log_terms.push_back(std::lgamma(n + 1.0) - std::lgamma(j + 1.0) - std::lgamma(n - j + 1.0)
                            + j * std::log(p) + (n - j) * std::log(1.0 - p));
    if (log_terms.empty())
        return 0.0;
    const double top = *std::max_element(log_terms.begin(), log_terms.end());
    double sum = 0.0;
    for (const double t : log_terms)
        sum += std::exp(t - top);
    return (top + std::log(sum)) / std::log(10.0);
}

bool ValidationEvidenceBeatsChance(const ValidationSpotEvidence &e, double significance) {
    return ValidationEvidenceLog10Tail(e) < std::log10(significance);
}

RotationScaleWalk WalkRotationScale(double first_fit,
                                    const std::function<ValidationSpotEvidence(float)> &index_at,
                                    const std::function<std::optional<double>(float)> &refit_at,
                                    int max_rounds, int64_t sweep_images) {
    RotationScaleWalk walk;
    auto next = static_cast<float>(first_fit);
    if (next == walk.scale)
        return walk;
    const auto score = [](const ValidationSpotEvidence &e) {
        return 100.0 * static_cast<double>(e.on_lattice - e.by_chance)
               / static_cast<double>(std::max<int64_t>(1, e.spots));
    };
    walk.evidence = index_at(walk.scale);
    walk.trail = fmt::format("{:.5f}: {:.1f}%", walk.scale, score(walk.evidence));
    double last_step = 0.0;
    for (int round = 0; round < max_rounds && next != walk.scale; ++round) {
        const ValidationSpotEvidence e = index_at(next);
        walk.trail += fmt::format(", {:.5f}: {:.1f}%", next, score(e));
        if (!ValidationEvidencePrefers(walk.evidence, e))
            break;
        const double step = next - walk.scale;
        walk.scale = next;
        walk.evidence = e;
        // Each refit reads a share of the error left, so the steps contract by about the ratio of the
        // last two. Where the step that ratio predicts moves the end of the sweep by less than half a
        // frame, the scale it would reach indexes the frames as this one does, and the refit is not made.
        if (last_step != 0.0 && std::fabs(step * step / last_step) * static_cast<double>(sweep_images) < 0.5) {
            walk.trail += fmt::format(" (next step predicted {:.5f}, under half a frame)", step * step / last_step);
            break;
        }
        last_step = step;
        const auto fit = refit_at(walk.scale);
        if (!fit)
            break;
        next = static_cast<float>(walk.scale * *fit);
    }
    return walk;
}

namespace {
    // Images the pre-scan reads when the beam-stop pre-pass is off and only the spot width is wanted;
    // with it on, that pre-pass's image count is used and one sample serves both.
    constexpr int BEAM_CENTER_PROJECTION_IMAGES = 60;

    // The least rotation the two-pass geometry pre-pass integrates, whatever --prepass-fraction asks.
    constexpr double PREPASS_MIN_ROTATION_DEG = 120.0;

    // About how wide a block of the pre-pass subset is (PrepassSubset). Wide against a reflection's
    // rocking width, so that few of the reflections the subset holds are cut by a block's edges.
    constexpr double PREPASS_SUBSET_BLOCK_DEG = 10.0;
}

namespace rugnux_internal {
    // A CUDA out-of-memory, or a host allocation that failed, says nothing about the image that
    // happened to be in flight and everything about the machine - the next image will hit it too.
    // Skipping it the way a genuinely undecodable image is skipped drops frames out of the middle of
    // the dataset and still exits 0, so the run looks complete while every merged number downstream
    // has quietly changed. Fail the run instead, and let the operator free the card and repeat it.
    bool IsFatalResourceError(const std::exception &e) {
        if (dynamic_cast<const std::bad_alloc *>(&e) || dynamic_cast<const std::length_error *>(&e)) return true;
        // Not a resource, but just as fatal: a memo verification (verify_first_pass_memo) that found a
        // difference must end the run, not pass for a probe that scored nothing.
        if (dynamic_cast<const MemoMismatch *>(&e)) return true;
        // Nor a collection followed from a broker that is gone: the frames it had not reported will never
        // be read, and a run that skipped them would look complete.
        if (dynamic_cast<const LiveCollectionLost *>(&e)) return true;
        const auto *jf = dynamic_cast<const JFJochException *>(&e);
        return jf != nullptr && (jf->Category() == JFJochExceptionCategory::GPUCUDAError
                                 || jf->Category() == JFJochExceptionCategory::MemAllocFailed);
    }

    // The longest cell axis assumed where no cell is known yet. It has to be LARGE: a long axis makes
    // the tolerance below small, and the ceiling is the larger of it and the flat bound above, so a
    // large default is the one that changes nothing.
    constexpr float UNKNOWN_CELL_A_MAX_A = 200.0f;

    // How far the beam centre may be wrong before the FIRST PASS stops indexing. A centre error is
    // fixed in the LAB frame, so accumulating a sweep smears every reciprocal-lattice point around a
    // circle and the FFT amplitude at an axis of length a is multiplied by J0(2 pi delta p a/(D
    // lambda)); this is the displacement at which that has fallen to 0.70, and the first zero is
    // 2.09x further out. Over the in-house and non-SLS corpora it runs from 0.6 px to 9 px - a
    // fifteen-fold spread, which is why the flat bound above is only ever used as a FLOOR under it.
    //
    // It is printed, and it is deliberately not a relevance test. It says how wrong the header may
    // be; the estimator's sigma says how well the estimator knows its own answer, and gating one on
    // the other rejects a centre correct to 0.03 px because its sigma was 0.98. Nor is a move that is
    // small against it a move that can be ignored: measured on real data, a 0.12 px change of centre
    // - 0.03x of this - is the difference between the deposited cell and a halved axis.
    float BeamCenterNeed_pxl(const DiffractionExperiment &experiment) {
        const auto cell = experiment.GetUnitCell();
        const float a_max = cell ? std::max({cell->a, cell->b, cell->c}) : UNKNOWN_CELL_A_MAX_A;
        return 0.183f * experiment.GetDetectorDistance_mm() * experiment.GetWavelength_A()
               / (experiment.GetPixelSize_mm() * a_max);
    }

    // Pick up to requested_images ordinals spread evenly across [0, images_to_process) for the
    // first pass of two-pass rotation indexing.
    std::vector<int> select_equally_spaced_image_ordinals(int images_to_process, int requested_images) {
        std::vector<int> ret;
        if (images_to_process <= 0 || requested_images <= 0)
            return ret;

        const int n = std::min(images_to_process, requested_images);
        if (n == 1) {
            ret.push_back(0);
            return ret;
        }

        std::set<int> unique_ordinals;
        for (int i = 0; i < n; i++)
            unique_ordinals.insert(static_cast<int>(
                std::llround(static_cast<double>(i) * static_cast<double>(images_to_process - 1) /
                             static_cast<double>(n - 1))));

        ret.assign(unique_ordinals.begin(), unique_ordinals.end());
        return ret;
    }

    // Apply a goniometer rotation SCALE: the stage turned k times the angle the file records. What is
    // wrong is the SWEEP, not where it began, so the per-frame increment and the per-frame oscillation
    // width both scale by k while the starting angle is left alone - the stage reached that position
    // before the sweep started. A constant offset in phi is in any case exactly degenerate with a
    // rotation of the crystal orientation about the spindle, which indexing refines away, so anchoring
    // the stretched sweep at image 0 costs nothing and keeps the first frame at the angle the file gives
    // it. Every angle in the pipeline comes from this object, so scaling it is the whole correction.
    GoniometerAxis ScaleRotation(const GoniometerAxis &g, float k) {
        GoniometerAxis scaled(g.GetName(), g.GetStart_deg(), g.GetIncrement_deg() * k,
                              g.GetAxis(), g.GetHelicalStep());
        if (const auto wedge = g.GetScreeningWedge())
            scaled.ScreeningWedge(*wedge * k);
        return scaled;
    }

}

// What the file says about the beam centre, before a frame is read. Nothing here measures anything -
// it reads the value that is there and says where it looks like it came from - and it is free, which
// is why it runs on every run whether or not a centre was asked for.
//
// A centre ON A MASKED PIXEL is the one of these worth acting on: a real beam does not sit on a dead
// pixel or in a module gap, so the value is wrong about something. Over the 29 non-SLS datasets whose
// outcome is on record (base rate of a bad outcome 48%) it fires on 4 and 3 of those 4 went wrong,
// against 2 false alarms in 42 in-house masters. What it does NOT promise is that correcting the
// centre repairs the run - on one of the three it does not.
//
// The other two are PROVENANCE, not prediction, and are said rather than warned about. A value equal
// to the geometric detector centre, or a whole number of pixels in both coordinates, was typed rather
// than measured - true of 29 of 42 in-house masters, and those are out by a median 3.5 px against
// 0.7 px for the rest - but a typed value is often close enough that as predictors of harm both sit
// at or below the base rate. They are still worth saying: only a human reading this line knows to
// look.
static void LogHeaderBeamCenter(const DiffractionExperiment &experiment, const PixelMask &pixel_mask) {
    Logger logger("Rugnux");
    const float beam_x = experiment.GetBeamX_pxl();
    const float beam_y = experiment.GetBeamY_pxl();
    const auto width = static_cast<int>(experiment.GetXPixelsNumConv());
    const auto height = static_cast<int>(experiment.GetYPixelsNumConv());

    if (std::hypot(beam_x - static_cast<float>(width) / 2.0f,
                   beam_y - static_cast<float>(height) / 2.0f) < 1.0f)
        logger.Info("Beam centre ({:.2f},{:.2f}) is the geometric centre of the detector - a value "
                    "written rather than measured", beam_x, beam_y);
    else if (beam_x == std::round(beam_x) && beam_y == std::round(beam_y))
        logger.Info("Beam centre ({:.0f},{:.0f}) is a whole number of pixels in both coordinates - a "
                    "value written rather than measured", beam_x, beam_y);

    const int x = static_cast<int>(std::lround(beam_x));
    const int y = static_cast<int>(std::lround(beam_y));
    if (x >= 0 && y >= 0 && x < width && y < height
        && pixel_mask.GetMask()[static_cast<size_t>(y) * width + x] != 0)
        logger.Warning("Beam centre ({:.2f},{:.2f}) lands on a masked pixel. A real beam does not, so "
                       "the value in the file is suspect; the beam centre check below measures it",
                       beam_x, beam_y);
}

Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
                             PixelMask pixel_mask, ProcessConfig config)
    : reader_(reader), experiment_(std::move(experiment)),
      pixel_mask_(std::move(pixel_mask)), config_(std::move(config)),
      user_fixed_sg_(experiment_.GetGemmiSpaceGroup()), user_cell_(experiment_.GetUnitCell()) {
    // Bit 9 describes where THIS run found the beam stop, so a mask read back from a file that
    // already carries one starts clear; the user mask (bit 8) is left as it was loaded.
    pixel_mask_.ClearBeamStopMask(experiment_);

    LogHeaderBeamCenter(experiment_, pixel_mask_);

    // What the run plans its host memory against, decided once from the machine's total (HostMemory.h):
    // work that would not fit it side by side runs one part after the other, and the large per-worker
    // buffers of the model's steps are held a share of it at a time. Only when things run changes.
    const size_t machine = get_host_total_memory();
    host_budget_ = config_.host_memory_budget_bytes > 0 ? config_.host_memory_budget_bytes : DefaultHostMemoryBudget();
    HostWorkerPool::Instance().SetSize(
            std::max<size_t>(1, static_cast<size_t>(HOST_WORKER_POOL_FRACTION * static_cast<double>(host_budget_))));
    Logger("Rugnux").Info("Host memory budget {:.1f} GB ({}): what does not fit it side by side runs in series",
                          static_cast<double>(host_budget_) / 1e9,
                          config_.host_memory_budget_bytes > 0
                                  ? std::string("set by the caller")
                                  : fmt::format("{:.0f}% of the machine's {:.1f} GB", 100.0 * HOST_MEMORY_BUDGET_FRACTION,
                                                static_cast<double>(machine) / 1e9));

    // Every decision on a rotation run - per-frame scaling, the correction surfaces, the outlier test,
    // the error model, the CC1/2 half-sets and the resolution cut, the frame ledger, the space-group
    // search - is taken with Friedel mates merged, -A or not. Splitting them halves the multiplicity
    // every one of those gates was calibrated at, which is a noisier instrument rather than a more
    // careful one: it flipped symmetry, resolution and frame-rejection calls against the bare run on
    // several sets, none of them through the anomalous signal. It is also what XDS, AIMLESS, POINTLESS
    // and DIALS decide on. The anomalous signal is not lost: the merge always keeps the Bijvoet split,
    // and I(+)/I(-), SigAno and CCanom come from it in either mode. Stills keep -A as it was - their
    // merge has no separate split.
    if (experiment_.IsRotationIndexing() && !experiment_.GetScalingSettings().GetMergeFriedel()) {
        ScalingSettings s = experiment_.GetScalingSettings();
        experiment_.ImportScalingSettings(s.MergeFriedel(true));
        Logger("Rugnux").Info("-A on rotation data: every processing decision is made with Friedel mates "
                              "merged; I(+)/I(-), SigAno and CCanom are reported from the Bijvoet split "
                              "the merge keeps, exactly as without -A");
    }

    // A manually asserted stage calibration applies to everything, before anything reads an angle.
    if (config_.rotation_scale.has_value())
        if (const auto g = experiment_.GetGoniometer()) {
            Logger("Rugnux").Info("Goniometer rotation scale {:.5f} applied from the command line",
                                  *config_.rotation_scale);
            experiment_.Goniometer(ScaleRotation(*g, *config_.rotation_scale));
        }
}

namespace rugnux_internal {
// The FFT recovers a basis vector only up to fft_max_unit_cell_A: FFTIndexer sizes its projected
// histogram from that value and the transform's last usable bin IS that length. So a reference cell
// longer than the bound cannot be represented by the search meant to find it - the run then reports
// "found no lattice", which is strictly worse than not giving the cell at all (measured on a 545 A
// axis: a partial triclinic solution without -C, nothing with it). Raise the bound to cover the cell,
// with the margin DIALS uses when a cell is known (max_cell = 1.3 * longest axis, see
// dials/algorithms/indexing/indexer.py find_max_cell).
//
// The floor at the short end is adjusted here too, for the same reason and against the same cell -
// see the second half of the body.
//
// ROTATION ONLY. Stills fire the FFT once per image across every worker and routinely run with a
// known cell, so widening the transform there would cost the whole serial run for nothing; the
// caller applies this inside its rotation branch.
void RaiseFFTBoundForKnownCell(IndexingSettings &settings, const DiffractionExperiment &x) {
    const auto cell = x.GetUnitCell();
    if (!cell.has_value() || !cell->is_finite())
        return;

    constexpr float KNOWN_CELL_MARGIN = 1.3f;
    const float longest = std::max({cell->a, cell->b, cell->c});
    // Clamp rather than let the setter throw. A cell that needs more than the search can ever reach
    // is a real possibility - the long-axis rescue calls this with whatever its coarse pass recovered,
    // and that has been seen to be an implausible harmonic - and a bad candidate must not take the
    // whole run down. Searching as far as the indexer goes is the best available answer; if the cell
    // really is that long the pass simply fails to index, which the caller already handles.
    const float needed = std::min(KNOWN_CELL_MARGIN * longest,
                                  IndexingSettings::fft_max_unit_cell_limit_A);
    if (needed > settings.GetFFT_MaxUnitCell_A()) {
        settings.FFT_MaxUnitCell_A(needed);
        Logger("Rugnux").Info("FFT search bound raised to {:.0f} A to cover the given cell "
                              "(longest axis {:.1f} A)", needed, longest);
    }

    // The same argument at the other end of the search. Every candidate whose reduced cell has an
    // axis below fft_min_unit_cell_A is discarded, and the 10 A default is shorter than any protein
    // axis but longer than a small molecule's: a given cell with an 8 A axis was thrown away before
    // it could be compared against the reference it was supposed to match. Lower the floor to reach
    // the cell that was asked for, with the reciprocal of the margin used above.
    constexpr float KNOWN_CELL_FLOOR_MARGIN = 1.0f / KNOWN_CELL_MARGIN;
    const float shortest = std::min({cell->a, cell->b, cell->c});
    const float floor_A = std::max(KNOWN_CELL_FLOOR_MARGIN * shortest,
                                   IndexingSettings::fft_min_unit_cell_limit_A);
    if (floor_A < settings.GetFFT_MinUnitCell_A()) {
        settings.FFT_MinUnitCell_A(floor_A);
        Logger("Rugnux").Info("FFT shortest-axis floor lowered to {:.1f} A to cover the given cell "
                              "(shortest axis {:.1f} A)", floor_A, shortest);
    }
}
}


ProcessResult Rugnux::SpotFindingPass(RugnuxObserver *observer, const DiffractionExperiment &header_experiment,
                                      const SpotFindingSettings &spot_finding) {
    Logger("Rugnux").Info("No lattice: one pass over every image with spot finding and no indexing, for the "
                          "per-image analysis");
    experiment_ = header_experiment;
    config_.spot_finding = spot_finding;
    config_.spot_finding.indexing = false;
    return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
}

ProcessResult Rugnux::Run(RugnuxObserver *observer) {
    // Each pass times only itself, so the canonical result's processing_time_s is the last pass alone -
    // on the rotation two-pass that is under a third of what the run actually took, the pre-scan and the
    // first pass being invisible in it. Time the whole thing here, where every pass is inside.
    const auto run_start = std::chrono::steady_clock::now();
    DiffractionExperiment header_experiment = experiment_;
    // A stored sweep is all there from the start, so its first passes decide their near ties on all of
    // its frames at once. Live, only the frames in so far are: the first passes decide on those, and
    // their near ties are re-checked once the collection is over (decide_late_).
    decide_late_ = !config_.live;
    // The run as it starts, to be made again where a near tie decided early ends elsewhere on the rest
    // of the sweep.
    const auto again = std::make_shared<Rugnux>(*this);
    again->near_tie_rechecks_ = std::make_shared<NearTieRechecks>();
    again->decide_late_ = true;
    again->decide_all_late_ = true;
    const SpotFindingSettings spot_finding = config_.spot_finding;
    Rugnux *run = this;
    ProcessResult result;
    try {
        bool lost = false;
        try {
            result = RunAllPasses(observer);
        } catch (const NoLatticeFound &) {
            // A run that found no lattice after deciding a near tie early is made again deciding late.
            if (!near_tie_rechecks_->carried || config_.fast_live)
                throw;
            lost = true;
        }
        if (lost || (!result.cancelled && !config_.fast_live && NearTieOverturned())) {
            JoinBeamCenterCapture();
            Logger logger("Rugnux");
            logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, "
                        "deciding near ties on all the frames");
            result.outputs.reset();   // replaced: not held through the run made again
            result = again->RunAllPasses(observer);
            run = again.get();
        }
    } catch (NoLatticeFound &e) {
        // No lattice is a statement about every frame, so every frame is reported: what it diffracted, and
        // that it did not index. Only where someone reads the images - the live lattice probe passes no
        // observer.
        if (observer && !cancelled_)
            e.per_image = std::make_shared<ProcessResult>(SpotFindingPass(observer, header_experiment, spot_finding));
        throw;
    }
    // Cut short in fast_live: a pass cut before its image loop read anything has no lattice and no merge,
    // so the run's answer is the last pass that ran to its end. (A canonical pass cut inside its image
    // loop merges the images it read - see ImagePass.)
    if (config_.fast_live && result.cancelled && established_) {
        result = *established_;
        result.cancelled = true;
    }
    result.total_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - run_start).count();
    // For the report's beam-centre warning: what the background measured, and what this crystal asks
    // of the centre - at the file's geometry, as the pre-scan's check asks it, but on the cell the run
    // determined rather than the long axis that check has to assume.
    run->JoinBeamCenterCapture();
    result.background_beam_centre = run->measured_beam_center_;
    if (result.consensus_cell)
        header_experiment.SetUnitCell(*result.consensus_cell);
    result.beam_centre_need_pxl = BeamCenterNeed_pxl(header_experiment);
#ifdef JFJOCH_USE_CUDA
    // The engines went with their passes; the pinned buffers they kept for each other go with the run.
    const size_t pinned_bytes = cuda_release_pinned_pool();
    Logger("Rugnux").Debug("Pinned host memory kept between engines, released: {:.1f} MB", pinned_bytes / 1e6);
#endif
    return result;
}

void Rugnux::StartNearTieRecheck(std::shared_ptr<Rugnux> run, bool geometry_prepass) {
    // The pass again, up to its first pass, as the probe it would be: the same inputs, no memo to
    // answer from, nothing started beside it.
    run->decide_late_ = true;
    run->near_tie_recheck_ = true;
    run->indexing_probe_only_ = true;
    run->first_pass_memo_.clear();
    run->first_pass_outcome_.clear();
    run->speculative_probe_.reset();
    run->speculate_geometry_probe_ = false;
    run->rotation_scale_check_.reset();
    run->gpu_work_beside_ = false;   // it runs after the run, on its own
    std::lock_guard lock(near_tie_rechecks_->m);
    near_tie_rechecks_->pending.push_back({std::move(run), geometry_prepass, first_pass_outcome_,
                                           first_pass_input_key_});
}

bool Rugnux::NearTieOverturned() {
    // After the run: by then every frame is in, live too, and nothing else holds the card.
    Logger logger("Rugnux");
    TimingMark mark(logger, "near-tie re-checks");
    std::lock_guard lock(near_tie_rechecks_->m);
    int elsewhere = 0;
    int run_again = 0;
    auto &pending = near_tie_rechecks_->pending;
    for (size_t i = 0; i < pending.size(); i++) {
        auto &[run, geometry_prepass, outcome, key] = pending[i];
        // A first pass the run made more than once on the same inputs - a probe at the geometry the
        // pass stands at - is re-checked once: the first pass is deterministic in its inputs
        // (FirstPassInputKey), deciding late as deciding early.
        const auto same = std::find_if(pending.begin(), pending.begin() + i, [&](const auto &q) {
            return q.key == key && q.geometry_prepass == geometry_prepass;
        });
        if (same != pending.begin() + i) {
            run->first_pass_outcome_ = same->run->first_pass_outcome_;
        } else {
            run_again++;
            try {
                run->RunPipeline(nullptr, /*write_output=*/false, geometry_prepass);
            } catch (const NoLatticeFound &) {
                run->first_pass_outcome_.clear();   // the pass found one; on all the frames it finds none
            }
        }
        if (run->first_pass_outcome_ != outcome) {
            logger.Info("Near-tie re-check: the first pass ends elsewhere on all the frames - {} against {}",
                        run->first_pass_outcome_, outcome);
            elsewhere++;
        }
    }
    if (!near_tie_rechecks_->pending.empty())
        logger.Info("Near ties re-checked on the rest of the sweep: {} first pass(es), {} of them on inputs "
                    "of their own, {} ending elsewhere", pending.size(), run_again, elsewhere);
    near_tie_rechecks_->pending.clear();
    if (near_tie_rechecks_->radius_elsewhere)
        elsewhere++;
    // The metric-symmetry checks decided early, where no first pass has already sent the run again.
    for (auto &[demoted, promoted, lost] : near_tie_rechecks_->arms) {
        if (elsewhere > 0)
            break;
        ProcessResult r;
        try {
            r = demoted->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false);
        } catch (const std::exception &e) {
            if (IsFatalResourceError(e)) throw;
        }
        const bool lost_late = r.post_refine && HeldOutResidualFell(promoted, *r.post_refine);
        logger.Info("Near-tie re-check: on all the frames the demoted class realises a held-out residual of "
                    "{:.3e} against the promoted class's {:.3e} - the metric-symmetry check {}",
                    r.post_refine ? r.post_refine->held_out_before : NAN, promoted.held_out_before,
                    lost_late == lost ? "ends where it did" : "ends elsewhere");
        if (lost_late != lost)
            elsewhere++;
    }
    near_tie_rechecks_->arms.clear();
    return elsewhere > 0;
}

void Rugnux::AddFirstPassMemo(const FirstPassMemo &memo) {
    for (const auto &m : first_pass_memo_)
        if (m.key == memo.key)
            return;
    // A handful covers every probe a walk can ask for again; the oldest go first.
    constexpr size_t MAX_FIRST_PASS_MEMOS = 8;
    if (first_pass_memo_.size() == MAX_FIRST_PASS_MEMOS)
        first_pass_memo_.erase(first_pass_memo_.begin());
    first_pass_memo_.push_back(memo);
}

int Rugnux::FirstPassIndexerThreads() const {
    return gpu_work_beside_ && gpu_takes_turns() ? 1 : 2;
}

int Rugnux::FirstPassSpotWorkers() const {
    const int gpus = get_gpu_count();
    if (gpus == 0)
        return config_.nthreads;
    return std::min({config_.nthreads, std::max(8, 4 * gpus), GPUWorkersPerCard(8) * gpus});
}

size_t Rugnux::IndexingProbeGPUBytes() const {
    // A worker's engines as an image worker's (GPUWorkersPerCard), and an FFT indexer as measured: the two
    // of a pre-pass held 1.24 GB. On a 16.8 Mpx detector, eight workers and two indexers were measured at
    // 4.7-4.8 GB.
    constexpr size_t WORKER_BYTES_PER_PIXEL = 24, FFT_INDEXER_BYTES = 650'000'000;
    const int gpus = std::max(get_gpu_count(), 1);
    const size_t workers_per_card = (FirstPassSpotWorkers() + gpus - 1) / gpus;
    return workers_per_card * WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum()
           + FirstPassIndexerThreads() * FFT_INDEXER_BYTES;
}

void Rugnux::StartSpeculativeGeometryProbe(std::shared_ptr<Rugnux> run, const std::array<float, 5> &geometry) {
    // What the walk's index_at_geometry sets up on its copy of the run before it indexes at this geometry.
    run->prepass_detector_geometry_ = prepass_detector_geometry_;
    run->speculative_probe_.reset();
    run->speculate_geometry_probe_ = false;
    run->indexing_probe_only_ = true;
    run->postrefine_probe_ = false;
    run->postrefine_probe_only_ = false;
    run->rerun_if_starved_ = false;
    run->gpu_work_beside_ = true;
    // The walk's other probe, at the geometry in hand, on a copy of its own: where this pass's own
    // first pass has not left its memo the walk would index there too, after the pass.
    auto hand = std::make_shared<Rugnux>(*run);
    run->experiment_.BeamX_pxl(geometry[0]).BeamY_pxl(geometry[1]).DetectorDistance_mm(geometry[2])
                    .PoniRot1_rad(geometry[3]).PoniRot2_rad(geometry[4]);
    auto sp = std::make_shared<SpeculativeProbe>();
    sp->run = run;
    sp->hand = hand;
    // A probe that fails leaves no memo, and the walk's own probe then runs it again and meets the
    // failure itself. It runs beside the merge of the pass that started it, on the same card: a merge
    // allocation that does not fit beside it waits for it to end (GPUWorkBeside) - and where the probe's
    // bytes and the merge's do not fit the card together, the probe waits for that merge to end instead
    // (GPUMerge). Its two passes run one after the other, so it holds at most what one does.
    const size_t probe_bytes = run->IndexingProbeGPUBytes();
    Logger("Rugnux").Debug("Speculative geometry probe: {:.2f} GB of the card", probe_bytes / 1e9);
    sp->done = std::async(std::launch::async, [run, hand, probe_bytes] {
        GPUWorkBeside beside(/*after_merges=*/true, probe_bytes);
        for (const auto &r : {run, hand}) {
            try {
                r->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false);
            } catch (const std::exception &e) {
                // Said, not swallowed: a GPU fault here is reported next by whichever user of the card
                // touches it first - the merge beside it - under that user's name.
                Logger("Rugnux").Warning("Speculative geometry probe failed: {}", e.what());
            }
        }
#ifdef JFJOCH_USE_CUDA
        // What its indexers and image workers held went back to the allocation pool; hand it to the card,
        // or it stays reserved under the merge and the validation beside which the probe ran.
        cuda_trim_allocation_pools();
#endif
    });
    speculative_probe_ = std::move(sp);
}

void Rugnux::JoinBeamCenterCapture() {
    if (!background_center_capture_.valid())
        return;
    background_center_ = background_center_capture_.get();
    measured_beam_center_ = background_center_;
    background_center_capture_ = {};
}

void Rugnux::CollectSpeculativeProbe() {
    if (!speculative_probe_)
        return;
    speculative_probe_->done.wait();
    for (const auto &m : speculative_probe_->run->first_pass_memo_)
        AddFirstPassMemo(m);
    for (const auto &m : speculative_probe_->hand->first_pass_memo_)
        AddFirstPassMemo(m);
    speculative_probe_.reset();
}

std::vector<double> Rugnux::ExperimentKey(const DiffractionExperiment &e, bool rotation_angles) const {
    std::vector<double> k{e.GetBeamX_pxl(), e.GetBeamY_pxl(),
                          e.GetDetectorDistance_mm(), e.GetPoniRot1_rad(),
                          e.GetPoniRot2_rad(), e.GetWavelength_A(),
                          static_cast<double>(e.GetMaxSpotCount())};
    if (const auto g = e.GetGoniometer()) {
        const Coord a = g->GetAxis();
        const auto helical = g->GetHelicalStep().value_or(Coord(-1, -1, -1));
        k.insert(k.end(), {1.0, a.x, a.y, a.z, helical.x, helical.y, helical.z});
        if (rotation_angles)
            k.insert(k.end(), {g->GetStart_deg(), g->GetIncrement_deg(), g->GetWedge_deg(),
                               g->GetScreeningWedge().value_or(-1.0f)});
    } else
        k.push_back(0.0);
    if (const auto uc = e.GetUnitCell())
        k.insert(k.end(), {1.0, uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma});
    else
        k.push_back(0.0);
    const auto &x = e.GetIndexingSettings();
    k.insert(k.end(), {x.GetTolerance(), x.GetFFT_MinUnitCell_A(), x.GetFFT_MaxUnitCell_A(),
                       static_cast<double>(x.GetFFT_NumVectors()), x.GetFFT_HighResolution_A(),
                       x.GetFFT_MinAngle_deg(), x.GetFFT_MaxAngle_deg(), x.GetUnitCellDistTolerance(),
                       x.GetUnitCellAngleTolerance_deg(), static_cast<double>(x.GetIndexIceRings()),
                       x.GetRotationIndexingMinAngularRange_deg(), x.GetRotationIndexingAngularStride_deg(),
                       static_cast<double>(x.GetMaxExtraLattices()), static_cast<double>(x.GetLatticeClassDemoted()),
                       static_cast<double>(x.GetViableCellMinSpots()), static_cast<double>(x.GetAlgorithm()),
                       static_cast<double>(x.GetGeomRefinementAlgorithm())});
    const auto &f = config_.spot_finding;
    k.insert(k.end(), {f.signal_to_noise_threshold, static_cast<double>(f.photon_count_threshold),
                       static_cast<double>(f.min_pix_per_spot.value_or(-1)), static_cast<double>(f.max_pix_per_spot),
                       f.high_resolution_limit.value_or(-1.0f)});
    k.insert(k.end(), {static_cast<double>(e.GetSpaceGroupOrP1().number),
                       static_cast<double>(e.GetImageNum())});
    return k;
}

std::vector<double> Rugnux::SpotFindingKey(const DiffractionExperiment &x,
                                           const AzimuthalIntegrationMapping &mapping) const {
    // Without the rotation angles: a frame's spots are found the same under any of them, and the angle
    // each spot carries is stamped again where a list is taken from the store (prefetch_spots). The
    // probes of the rotation-scale walk differ only there, so they find the spots of a frame once.
    std::vector<double> k = ExperimentKey(x, /*rotation_angles=*/false);
    const auto &f = config_.spot_finding;
    k.insert(k.end(), {static_cast<double>(f.enable), f.low_resolution_limit.value_or(-1.0f),
                       f.cutoff_spot_count_low_res, f.high_res_gap_Q_recipA.value_or(-1.0f),
                       f.ice_ring_width_Q_recipA, static_cast<double>(f.adaptive_threshold),
                       f.false_pixels_per_frame, static_cast<double>(f.measured_ring_q_recipA.size())});
    k.insert(k.end(), f.measured_ring_q_recipA.begin(), f.measured_ring_q_recipA.end());
    k.insert(k.end(), {static_cast<double>(x.IsDetectIceRings()),
                       static_cast<double>(pixel_mask_.GetBinaryMaskChecksum()),
                       static_cast<double>(mapping.GetPixelToBinChecksum()),
                       static_cast<double>(mapping.GetCorrectionsChecksum())});
    return k;
}

std::vector<double> Rugnux::FirstPassInputKey() const {
    return FirstPassInputKey(experiment_);
}

std::vector<double> Rugnux::FirstPassInputKey(const DiffractionExperiment &e) const {
    std::vector<double> k = ExperimentKey(e);
    // Run state the first pass reads besides the experiment.
    k.insert(k.end(), {static_cast<double>(lattice_angles_demoted_), prepass_rotation_scale_.value_or(-1.0f),
                       static_cast<double>(force_rotation_result_.has_value()),
                       // a capture still running will have run by the time the pass reads it
                       static_cast<double>(background_center_.has_value() || background_center_capture_.valid()),
                       static_cast<double>(beam_center_alternative_state_.has_value())});
    return k;
}

int Rugnux::PrepassEnd(int start_image, int end_image) const {
    int end = start_image + static_cast<int>(std::lround((end_image - start_image) * config_.prepass_fraction));
    // At least PREPASS_MIN_ROTATION_DEG of rotation, so a narrow sweep is pre-passed whole. The angle is
    // the file's, so every pass of a run picks the same frames, before and after a rotation-scale fit.
    if (const auto g = reader_.GetDataset()->experiment.GetGoniometer(); g && g->GetIncrement_deg() != 0.0f)
        end = std::max(end, start_image + static_cast<int>(
                std::lround(PREPASS_MIN_ROTATION_DEG / std::abs(g->GetIncrement_deg()))));
    return std::min(end, end_image);
}

std::vector<bool> Rugnux::PrepassSubset(int prepass_images) const {
    double arc_deg = 0.0;
    if (const auto g = reader_.GetDataset()->experiment.GetGoniometer())
        arc_deg = std::abs(g->GetIncrement_deg()) * config_.stride * prepass_images;
    const int k = std::max(1, static_cast<int>(std::lround(arc_deg / (3.0 * PREPASS_SUBSET_BLOCK_DEG))));
    const int64_t blocks = 3 * k + 1;
    std::vector<bool> keep(prepass_images);
    for (int64_t o = 0; o < prepass_images; ++o)
        keep[o] = (o * blocks / prepass_images) % 3 == 0;
    return keep;
}

ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
    Logger logger("Rugnux");
    ProcessResult result;
    TimingMark pass_mark(logger, geometry_prepass ? "pass (geometry pre-pass)"
                                 : write_output ? "pass (output)" : "pass (probe)");
    // Only a pass that makes the result is shown as steps; a probe pass is part of the step that ran it.
    steps_ = (write_output || geometry_prepass) ? observer : nullptr;

    // Each pass determines its own space group, so start every one of them from the group the run was
    // ASKED for. A pass that determined one leaves it on experiment_, and the pass after it - the
    // re-runs in Run(), or a second Run() on the same object - would otherwise inherit that answer and
    // skip the search that is the whole point of running again.
    // The run as this pass finds it, for the re-check of a near tie its first pass may decide early.
    std::shared_ptr<Rugnux> recheck;
    if (!decide_late_ && !config_.fast_live && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing
        && config_.two_pass_rotation && !force_rotation_result_.has_value()
        && !config_.forced_rotation_lattice.has_value() && experiment_.IsRotationIndexing())
        recheck = std::make_shared<Rugnux>(*this);

    experiment_.SetSpaceGroup(user_fixed_sg_);

    const auto dataset = reader_.GetDataset();
    if (!dataset)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                              "No experiment dataset found in the input file");

    if (config_.stride <= 0)
        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image stride must be positive");

    const auto total_images_in_file = static_cast<int>(reader_.GetNumberOfImages());
    int end_image = config_.end_image;
    if (end_image < 0 || end_image > total_images_in_file)
        end_image = total_images_in_file;
    const int start_image = config_.start_image;
    // Round UP: the worker loop below walks ordinals until start + ordinal * stride reaches end_image,
    // so a stride that does not divide the range evenly still processes a final partial step. Rounding
    // down here would size the per-image arrays one short of the ordinals that index them.
    const int images_to_process = (end_image - start_image + config_.stride - 1) / config_.stride;
    if (images_to_process <= 0) {
        logger.Warning("No images to process (start {}, end {}, stride {}, total {})",
                       start_image, end_image, config_.stride, total_images_in_file);
        return result;
    }

    const bool full = (config_.mode == ProcessMode::FullAnalysis);
    // A full analysis with indexing off is the per-image pass of a run that found no lattice
    // (SpotFindingPass): the image loop and nothing that needs a lattice.
    const bool indexing = full && config_.spot_finding.indexing;
    // Powder calibration runs the same per-image engine as the full analysis (spot finding lives inside
    // MXAnalysisWithoutFPGA) with indexing switched off in its settings. By rings it needs only the
    // azimuthal profile and the azint worker would do - but the spots are worth their half second even
    // then, because the circle through them places the beam centre without reference to the header at
    // all, which is the one hypothesis that survives a header the profile cannot correct.
    const bool calibration = (config_.mode == ProcessMode::Calibration);
    const bool calibration_spots = calibration && config_.calibration_method == CalibrationMethod::Spots;
    const bool per_image_analysis = full || calibration;
    // The output pass makes what the run returns to its caller (ProcessResult::outputs).
    if (write_output && !geometry_prepass)
        result.outputs = std::make_shared<RugnuxOutputs>();
    // Set below the first merge, once the pass can tell that RunAllPasses is going to throw it away and
    // run again - see there for what that turns off and why it is knowable at that point.
    bool superseded = false;

    // Output/runtime invariants. Algorithm settings (indexing, scaling, integration, polarization,
    // space group, unit cell, ...) are configured on experiment_ by the caller.
    experiment_.BitDepthImage(32).PixelSigned(true);
    experiment_.Mode(DetectorMode::Standard);
    experiment_.OverwriteExistingFiles(true);
    experiment_.SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
    experiment_.ImagesPerTrigger(images_to_process);
    experiment_.NumTriggers(1);
    if (full)
        experiment_.Compression(CompressionAlgorithm::BSHUF_LZ4);

    // The pipeline indexes images 0..N-1 within this run; if we process a sub-range/strided
    // selection, shift the goniometer so local index i maps to the angle of original image
    // start+i*stride (keeping the per-image rotation wedge), otherwise rotation angles would be
    // wrong for any start_image != 0.
    if (const auto g = experiment_.GetGoniometer();
        g.has_value() && (start_image != 0 || config_.stride != 1)) {
        const float incr = g->GetIncrement_deg();
        GoniometerAxis shifted(g->GetName(),
                               g->GetStart_deg() + incr * static_cast<float>(start_image),
                               incr * static_cast<float>(config_.stride),
                               g->GetAxis(), g->GetHelicalStep());
        shifted.ScreeningWedge(g->GetScreeningWedge().value_or(incr));
        experiment_.Goniometer(shifted);
    }

    // Stills global geometry-refinement first pass: bundle-adjust the shared beam/distance/cell from a
    // sample of strongly-indexed frames and apply it to experiment_, so everything below (azimuthal
    // mapping, the main index+integrate pass, scaling) uses the refined geometry. Rotation is skipped
    // inside (it has its own two-pass).
    if (full && config_.refine_geometry.has_value())
        RefineStillsGeometry(start_image, end_image, images_to_process, observer);

    // The geometry pre-pass integrates only the first part of the sweep (pass 2 reads it all), and on a
    // rotation sweep the decisions before pass 2 read only that part too (PrepassEnd).
    const int prepass_end = PrepassEnd(start_image, end_image);
    const int prepass_images = (prepass_end - start_image + config_.stride - 1) / config_.stride;
    const int early_images = experiment_.IsRotationIndexing() ? prepass_images : images_to_process;

    // After any geometry refinement, so the shadow is found about the beam centre actually used,
    // and before the azimuthal mapping and the output mask below, which both read pixel_mask_.
    if (config_.detect_beam_stop.has_value() || config_.adaptive_integration_radius)
        PreScan(start_image, early_images,
                std::min(config_.detect_beam_stop.value_or(BEAM_CENTER_PROJECTION_IMAGES),
                         HotPixelFinder::MAX_SAMPLED_FRAMES), observer);

    // The geometry pre-pass integrates at the radius the run started with; the width the pre-scan just
    // measured is applied to the canonical pass instead (RunAllPasses, between the two). See
    // bragg_adaptive_ for why - in short, the pre-pass keeps only the geometry it fits to the observed
    // reflection positions, and those positions move with the signal disk.
    if (geometry_prepass && bragg_before_adaptive_) {
        bragg_adaptive_ = experiment_.GetBraggIntegrationSettings();
        experiment_.ImportBraggIntegrationSettings(*bragg_before_adaptive_);
    }

    // An indexing probe over inputs a first pass has already been run on (first_pass_memo_) returns what
    // that pass found before anything is built for it - the mapping below alone is 0.15 s on a 16M
    // detector. Nothing between here and the first pass changes what the key reads.
    if (full && indexing_probe_only_ && !force_rotation_result_.has_value()
        && !config_.forced_rotation_lattice.has_value() && config_.rotation_indexing && config_.two_pass_rotation
        && !config_.verify_first_pass_memo) {
        const std::vector<double> first_pass_key = FirstPassInputKey();
        const auto memo = std::find_if(first_pass_memo_.begin(), first_pass_memo_.end(),
                                       [&](const FirstPassMemo &m) { return m.key == first_pass_key; });
        if (memo != first_pass_memo_.end()) {
            logger.Info("Indexing probe: the first pass has already run on these inputs - {}/{} "
                        "validation spots on the lattice, {} at a wrong spindle angle",
                        memo->evidence.on_lattice, memo->evidence.spots, memo->evidence.by_chance);
            result.validation_evidence = memo->evidence;
            return result;
        }
    }

    AzimuthalIntegrationMapping mapping(experiment_, pixel_mask_, *azint_geometry_);

    JFJochReceiverPlots plots;
    plots.Setup(experiment_, mapping);

    // The per-image analysis' start message (an NXmxIntegrated master that links back to the original
    // images); the image sink names the file.
    StartMessage start_message;
    experiment_.FillMessage(start_message);
    start_message.arm_date = dataset->arm_date;
    start_message.az_int_bin_to_q = mapping.GetBinToQ();
    start_message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta();
    start_message.az_int_q_bin_count = mapping.GetQBinCount();
    start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
    if (mapping.GetAzimuthalBinCount() > 1)
        start_message.az_int_bin_to_phi = mapping.GetBinToPhi();
    if (full) {
        start_message.rois = experiment_.ROI().ExportMetadata();
        start_message.max_spot_count = experiment_.GetMaxSpotCount();
    }
    start_message.master_suffix = "process";
    start_message.file_format = FileWriterFormat::NXmxIntegrated;
    start_message.write_master_file = true;
    start_message.write_images = false;
    // The process file LINKS to the original images rather than writing its own, which only an HDF5
    // input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs. So
    // ask for the links only when the source is HDF5, and otherwise write the images into the file -
    // the per-image analysis is the point of it either way, and without pictures nothing downstream
    // could show the frame a reflection came from.
    if (auto *hdf5 = dynamic_cast<JFJochHDF5Reader *>(&reader_)) {
        // With the stride: image i of the process file is source image start_image + i * stride, so
        // the linked pictures line up with the per-image analysis written beside them.
        start_message.hdf5_source_data =
            hdf5->GetHDF5DataSource(start_image, images_to_process, config_.stride);

        // Everything describing the pixels must describe those linked files. experiment_ cannot: it
        // is pinned to signed 32-bit, the container the reader hands images out in, which would type
        // the virtual dataset int32 over unsigned 16- or 32-bit sources and silently convert every
        // value on read. Only consumers going through the virtual view ever saw it - our own reader
        // resolves the mapping and opens the source itself.
        const auto stored = hdf5->GetStoredPixelFormat();
        start_message.bit_depth_image = stored.bit_depth;
        // Unsigned only, for the reason FillMessage gives: DIALS remaps the top two codes whenever
        // the field is present, and on a signed image those land inside the trusted range.
        start_message.bit_depth_readout.reset();
        if (!stored.is_signed)
            start_message.bit_depth_readout = stored.bit_depth;
        start_message.pixel_signed = stored.is_signed;
        start_message.error_value = stored.is_signed
                                    ? -(int64_t(1) << (stored.bit_depth - 1))      // INTx_MIN
                                    : (int64_t(1) << stored.bit_depth) - 1;        // UINTx_MAX
        start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
    } else {
        start_message.write_images = true;   // no file to link to; the process file carries them
    }

    // The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
    // but the process file reads them. Copy them where the start message is handed on rather than here:
    // the default merging path writes no process file at all.
    auto fill_writer_maps = [&] {
        start_message.pixel_mask["default"] = pixel_mask_.GetMask(experiment_);
        if (full && !experiment_.ROI().empty())
            start_message.roi_map = experiment_.ExportROIMap();
    };

    // The output pass streams its per-image analysis to the caller's image sink, where there is one.
    RugnuxImageSink *const sink = write_output && !geometry_prepass ? config_.image_sink : nullptr;
    std::unique_ptr<ProcessFileWriter> writer_queue;
    // An output pass with no sink keeps the per-image scalars of its images, for the end message a run
    // that indexed nothing is written from (the _process.h5 below).
    std::optional<ScanResultGenerator> scan_result;
    if (result.outputs && !sink)
        scan_result.emplace(experiment_);
    // Whether anything is in place to break the indexing ambiguity, for the warning further down: a
    // reference MTZ, or (serial stills) the reference computed from a model just below.
    bool ambiguity_reference = !config_.reference_data.empty();
    if (sink) {
        fill_writer_maps();
        sink->Start(start_message, output_label_);
        // Deep enough that a worker never waits for the writer in the normal case, shallow enough that
        // the backlog is bounded by the worker count rather than by the length of the run.
        writer_queue = std::make_unique<ProcessFileWriter>(*sink, 4 * std::max(1, config_.nthreads));
    }

    const char *mode_name = full ? "full analysis"
                                 : calibration ? "powder calibration" : "azimuthal integration";
    logger.Info("Processing {} images (range {}-{}, stride {}) using {} threads [{}]",
                images_to_process, start_image, end_image, config_.stride, config_.nthreads, mode_name);
    if (observer)
        observer->OnPhase(full ? "Full analysis" : calibration ? "Powder calibration" : "Azimuthal integration");

    // Full-analysis shared engines.
    std::unique_ptr<IndexerThreadPool> indexer_pool;
    // The leftover-lattice census, run in the background (see there); after indexer_pool, which it
    // uses, so that it is waited for before the pool goes.
    std::future<LeftoverLatticeCensus> census_future;
    std::unique_ptr<IndexAndRefine> indexer;
    // The FFT search bound indexer_pool was built with. The long-axis rescue needs it to tell whether a
    // cell it recovered still fits, since a pool's transform is sized once, at construction.
    float indexing_bound_A = 0.0f;
    if (per_image_analysis) {
        auto indexing_settings = experiment_.GetIndexingSettings();
        // Two-pass rotation fires the FFT indexer only twice - the two first-pass schemes, run in
        // parallel (the main per-image loop then uses the forced lattice, no FFT). Size the pool to
        // those two so a rotation run doesn't pay to initialise (cuFFT plan + GPU alloc) indexers it
        // never uses. Stills fire the FFT once per image across all workers, so keep the default there.
        if (config_.rotation_indexing && config_.two_pass_rotation && experiment_.IsRotationIndexing()) {
            indexing_settings.IndexingThreads(FirstPassIndexerThreads());
            // Two indexer threads leave the rest of the machine idle for the whole first pass, which is
            // a third of the run. Split each scheme's candidate-cell refinement over -N/2 threads so the
            // two together use -N. The split is by candidate cell, so the result does not depend on it.
            // Clamped to what IndexingSettings accepts - a big host would otherwise ask for more than
            // the setting's limit and throw before a single image is read.
            indexing_settings.RefineThreads(std::clamp(config_.nthreads / 2, 1, MAX_REFINE_THREADS));
            RaiseFFTBoundForKnownCell(indexing_settings, experiment_);
        } else if (!experiment_.IsRotationIndexing()) {
            // Stills index every frame, and each image worker waits on the pool until its frame is
            // indexed, so the default four threads were what the whole loop ran at: the post-processing
            // of the FFT candidates is CPU work, and four threads of it held sixteen workers back.
            // Measured on a 16 Mpx serial set, de-novo FFT: 83 frames/s with 4, 125 with 8, 133 with 16 -
            // by eight the card is what limits it. Each GPU FFT indexer holds ~0.5 GB of the card
            // (sixteen of them took the run to 12.7 GB), so eight rather than one per worker.
            indexing_settings.IndexingThreads(std::clamp(config_.nthreads, 1, 8));
        }
        indexing_bound_A = indexing_settings.GetFFT_MaxUnitCell_A();
        indexer_pool = std::make_unique<IndexerThreadPool>(indexing_settings, IndexerConstruction::OnFirstUse,
                                                            gpu_work_beside_);
        indexer = std::make_unique<IndexAndRefine>(experiment_, indexer_pool.get());
        // With no per-image file to write, nothing reads the message's reflection list - except the
        // geometry pre-pass, which measures the spots against it (SpotFootprint.h).
        if (!writer_queue && !(geometry_prepass && prescan_footprint_))
            indexer->KeepReflectionsInMessage(false);
        // The reference that breaks the per-image indexing ambiguity: a reference MTZ where there is
        // one, otherwise intensities computed from the model, which carry the same information. Serial
        // stills need it here or not at all - each crystal is indexed in its own hand, so by the time
        // there is a merge for --model to fit, the hands have already been averaged together. Rotation
        // data have one lattice and one hand, so their ambiguity is settled at the merge instead.
        std::vector<MergedReflection> model_reference;
        if (!config_.reference_data.empty()) {
            indexer->ReferenceIntensities(config_.reference_data);
        } else if (config_.model && !experiment_.IsRotationIndexing()
                   && experiment_.GetUnitCell().has_value() && experiment_.GetGemmiSpaceGroup().has_value()) {
            const auto d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A()
                .value_or(experiment_.GetDetectorMaxResolution_A());
            model_reference = ModelReferenceIntensities(
                *config_.model, experiment_.GetUnitCell(),
                &experiment_.GetSpaceGroupOrP1(), d_min, logger);
            if (!model_reference.empty()) {
                indexer->ReferenceIntensities(model_reference);
                ambiguity_reference = true;
            }
        }
        // Second pass of the rotation two-pass: predict with the smoothed mosaicity fitted in the pre-pass.
        if (!geometry_prepass && !prepass_mosaicity_.empty())
            indexer->SetPredictionMosaicityOverride(prepass_mosaicity_);
        // The geometry pre-pass fits the detector geometry to spot POSITIONS and throws its intensities
        // away, so a fixed group's centring absences cost it half its events for nothing. A no-op when
        // no group is fixed - prediction is already in P there - so the de-novo path is untouched.
        //
        // The full pass predicts them too, so a fixed CENTRED group integrates the same reflections a
        // de-novo run would. Both merges then do the right thing on their own: the run's own merge
        // drops the centring absences again (Merge.cpp `IsSystematicallyAbsent`, against the group it
        // is merging in), and a P1 merge keeps them because P1 has none. What it buys is not
        // only a complete P1 merge - an unpredicted node is an unclaimed patch of detector, so a
        // neighbouring reflection sweeps those pixels into its background ring and over-subtracts
        // (see the reflection mask in BraggIntegrationEngineCPU). A fixed group used to cost real
        // intensity accuracy for that reason.
        indexer->PredictAllCentringNodes(true);
    }

    const auto start_time = std::chrono::steady_clock::now();

    const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image;

    // Declared here rather than where they are first filled, so that the phases below can share them
    // through PipelineLocals: the bytes the image loop read, the end message (also handed to the sink),
    // and the unmerged MTZ when it is built beside the reflection files (see there).
    std::atomic<uint64_t> total_uncompressed_bytes = 0;
    EndMessage end_msg;
    std::future<MtzTable> unmerged_mtz;
    PipelineLocals p{
            .logger = logger, .result = result, .observer = observer,
            .geometry_prepass = geometry_prepass, .full = full, .calibration = calibration,
            .calibration_spots = calibration_spots, .per_image_analysis = per_image_analysis,
            .write_output = write_output, .start_image = start_image, .end_image = end_image,
            .images_to_process = images_to_process, .prepass_images = prepass_images, .loop_end = loop_end,
            .dataset = dataset, .mapping = mapping, .plots = plots,
            .sink = sink, .writer_queue = writer_queue, .indexer_pool = indexer_pool, .indexer = indexer,
            .indexing_bound_A = indexing_bound_A, .census_future = census_future, .superseded = superseded,
            .ambiguity_reference = ambiguity_reference, .start_time = start_time,
            .total_uncompressed_bytes = total_uncompressed_bytes, .end_msg = end_msg,
            .scan_result = scan_result, .unmerged_mtz = unmerged_mtz};

    // First pass of two-pass rotation indexing (full analysis only).
    if (indexing && force_rotation_result_.has_value()) {
        // Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
        // search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
        // integrates markedly worse.
        indexer->ForceRotationIndexerResult(*force_rotation_result_);
        logger.Info("Rotation indexer result forced from pass 1 (lattice + geometry) - skipping first pass");
    } else if (indexing && config_.forced_rotation_lattice.has_value()) {
        indexer->ForceRotationIndexerLattice(*config_.forced_rotation_lattice);
        logger.Info("Rotation indexer lattice forced externally - skipping first pass");
    } else if (indexing && config_.rotation_indexing && config_.two_pass_rotation) {
        TimingMark mark(logger, "first-pass indexing");
        const bool probe_done = FirstPassRotationIndexing(p);
        if (recheck && near_tie_carried_)
            StartNearTieRecheck(std::move(recheck), geometry_prepass);
        if (probe_done)
            return result;
    }
    // A rotation pass indexes nothing after its first pass but the leftover-lattice census, which gives
    // them back again when it ends; the image pass and the merge get the card without them.
    if (indexer_pool && experiment_.IsRotationIndexing())
        indexer_pool->ReleaseIndexers();
    // The rotation-scale walk beside this pass starts once its frames are indexed (see RunAllPasses).
    if (rotation_scale_check_)
        rotation_scale_check_->Set(rotation_scale_check_->output_indexed);

    // Whether the lattice this pass integrates on has an index-2 superstructure. Nothing before
    // integration can ask it: every count the lattice decisions use is dominated by the sub-lattice's
    // strong reflections, so a crystal whose cell is doubled by a weak superstructure class is adopted
    // at the half cell by every arbiter. The probe reads the question in intensities instead, on the
    // same frames the first pass validates on (see IndexAndRefine::ProbeSupercell).
    //
    // It decides nothing in this pass. Read on its own it does not tell a weak real class from what sits
    // at the half-integer nodes of crystals whose cell is right, so the run carries the doubled cell as a
    // second hypothesis where the class stands clear out to the limit (DoubledCellHypothesis), and merges
    // both.
    //
    // Only an output pass probes: the answer the run returns is always one (RunAllPasses), and no other
    // pass's probe is read - the pre-pass, the beam-centre arm and the probe passes leave theirs unread.
    if (indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
        std::vector<int64_t> frames;
        for (const int o : select_equally_spaced_image_ordinals(images_to_process, 60))
            frames.push_back(o);
        indexer->ProbeSupercell(frames);
    }

    TimingMark image_pass_mark(logger, "image pass");
    std::optional<GPUImagePass> gpu_image_pass;   // what a probe that does not fit beside it waits for
    if (!gpu_work_beside_)
        gpu_image_pass.emplace();
    ImagePass(p);
    gpu_image_pass.reset();
    image_pass_mark.Stop();

    // Scaling and merging (full analysis only).
    // As ImagePass's stopped(): in fast_live a pass that read all its images merges whatever comes after.
    const bool images_cut = config_.fast_live ? result.cancelled : static_cast<bool>(cancelled_);
    if (full && !images_cut && (result.indexing_rate.value_or(0.0f) > 0 || result.rotation_lattice_found)
        && (config_.run_scaling || !config_.reference_data.empty())) {
        TimingMark mark(logger, "scale, merge and symmetry");
        std::optional<GPUMerge> gpu_merge;   // what the speculative probe waits for where both do not fit
        if (!gpu_work_beside_)
            p.gpu_merge = &gpu_merge.emplace();
        const bool finished = ScaleMergeAndSymmetry(p);
        p.gpu_merge = nullptr;
        gpu_merge.reset();
        if (p.postrefine_ahead.valid()) {
            p.postrefine_ahead.get();
            p.postrefine_log.ReplayInto(logger);
            for (const auto &note : p.postrefine_notes)
                Note(note);
        }
        if (finished)
            return result;
    }

    TimingMark output_mark(logger, "outputs");

    // Unmerged observations. After the merge, because that is where the space group is settled and
    // the file has to be written in it, but from the integrated observations rather than the merged
    // ones: the partiality and the per-image scale are left for the reading program, which fits a
    // scale model of its own.
    if (full && !cancelled_ && result.outputs && !superseded && result.consensus_cell) {
        if (config_.export_unmerged) {
            if (observer) observer->OnPhase("Unmerged reflections");
            MtzTable mtz = unmerged_mtz.valid()
                           ? unmerged_mtz.get()
                           : UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell,
                                              experiment_, true, config_.provenance, config_.nthreads);
            SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
            result.outputs->unmerged = std::move(mtz);
        }
        if (config_.export_unmerged_partials) {
            if (observer) observer->OnPhase("Unmerged reflections");
            MtzTable mtz = UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell,
                                            experiment_, false, config_.provenance, config_.nthreads);
            SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
            result.outputs->unmerged_partials = std::move(mtz);
        }
    }

    // When merging is on, the _process.h5 is skipped because the merged reflections are the wanted
    // output and the per-image file is large. If nothing indexed there are no merged reflections, so
    // the run would finish successfully having written no file at all. Hand the caller one: the
    // per-image messages went past with no sink, but the end message still carries the per-image
    // scalars (spot counts, background, resolution estimate) and the dataset metadata - which is what
    // anyone asking "why did nothing index" needs, and it is small precisely because nothing indexed.
    if (!sink && full && result.outputs && !cancelled_ && result.indexing_rate.value_or(0.0f) <= 0.0f) {
        logger.Warning("No image indexed, so there are no merged reflections - the per-image analysis is "
                       "kept for a _process.h5 instead");
        fill_writer_maps();
        result.outputs->process_start = start_message;
        scan_result->FillEndMessage(end_msg);
    }

    // The per-frame sigma_M the merge wrote back onto the outcomes (the _plots.txt column), for the report.
    if (indexer && experiment_.IsRotationIndexing()) {
        std::vector<float> mos;
        for (const float m : ScalingResult(indexer->GetIntegrationOutcome()).mosaicity_deg)
            if (std::isfinite(m))
                mos.push_back(m);
        if (!mos.empty()) {
            std::ranges::sort(mos);
            result.mosaicity_deg = mos[mos.size() / 2];
            result.mosaicity_deg_p10 = mos[mos.size() / 10];
            result.mosaicity_deg_p90 = mos[mos.size() * 9 / 10];
        }
    }
    // The two files a person checks the run by eye with. Not on a pass that is thrown away. The table
    // is made on a stills run too - it is the only per-image scaling table there is, and the columns
    // a stills run cannot fill come out as nan; the picture is rotation-only at its source, since
    // only the rotation pre-scan builds the projection it draws.
    if (full && result.outputs && !cancelled_ && !superseded) {
        if (detector_diagnostic_jpeg_.valid())
            result.outputs->detector_jpeg = detector_diagnostic_jpeg_.get();

        PerImagePlot plot;
        plots.GetPlotRaw(plot.bkg_estimate, PlotType::BkgEstimate, "");
        plots.GetPlotRaw(plot.resolution_A, PlotType::ResolutionEstimate, "");
        plots.GetPlotRaw(plot.spot_count, PlotType::SpotCount, "");
        if (indexer) {
            // The offline self-scaling result is otherwise not exposed - process.h5's per-image
            // arrays are only filled on the online per-image path - so these columns are the only
            // place a person or a comparison script can read it. Sourced from the partials, which
            // carry the first-pass per-image scale.
            const ScalingResult scaling(indexer->GetIntegrationOutcome());
            plot.scale_g = scaling.image_scale_g;
            plot.mosaicity_deg = scaling.mosaicity_deg;
            plot.cc_to_merge = scaling.image_cc;
            plot.cc_n = scaling.image_cc_n;
        }
        if (result.has_merge_statistics)
            plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition;
        plot.goniometer = experiment_.GetGoniometer();
        plot.image_count = static_cast<size_t>(images_to_process);
        result.outputs->plot = std::move(plot);
    }

    if (sink)
        sink->End(end_msg);
    result.end_message = end_msg;
    // A pass that is to be run again makes nothing.
    if (superseded)
        result.outputs.reset();

    output_mark.Stop();

    if (observer)
        observer->OnPhase(cancelled_ ? "Cancelled" : "Done");

    // Total wall time for the whole run, including scaling/merging/writing above (not just the
    // per-image pass), so the reported rate reflects everything rugnux did.
    result.processing_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - start_time).count();
    if (result.processing_time_s > 0.0) {
        result.frame_rate_hz = static_cast<double>(result.images_processed) / result.processing_time_s;
        result.throughput_MBs = static_cast<double>(total_uncompressed_bytes) / (result.processing_time_s * 1e6);
    }

    logger.Info("{} {} images in {:.2f} s ({:.2f} Hz)", cancelled_ ? "Cancelled after" : "Processed",
                result.images_processed, result.processing_time_s, result.frame_rate_hz);

    // The geometry this pass integrated at, for the report. Taken from experiment_ rather than from the
    // caller's copy: the second pass of the rotation two-pass runs at the post-refined geometry.
    result.used_beam_x_pxl = experiment_.GetBeamX_pxl();
    result.used_beam_y_pxl = experiment_.GetBeamY_pxl();
    result.used_distance_mm = experiment_.GetDetectorDistance_mm();
    {
        const auto &g = experiment_.GetDiffractionGeometry();
        constexpr double DEG = 180.0 / PI;
        result.used_detector_tilt_deg = {g.GetPoniRot1_rad() * DEG, g.GetPoniRot2_rad() * DEG,
                                         g.GetPoniRot3_rad() * DEG};
        const auto direct = g.GetDirectBeam_pxl();
        result.used_direct_beam_x_pxl = direct.first;
        result.used_direct_beam_y_pxl = direct.second;
    }
    if (const auto g = experiment_.GetGoniometer())
        result.used_rotation_axis = g->GetAxis();
    if (census_future.valid()) {
        TimingMark mark(logger, "leftover-lattice census wait");
        result.leftover_lattices = census_future.get();
    }
    return result;
}