Merge order-memory into rc181: model validation after the geometry walk, CELL_MISMATCH stop, one host-memory ledger

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37

common/CUDAWrapper.cpp

@@ -83,6 +83,11 @@ GPUWorkBeside::~GPUWorkBeside() {
     remove(gpu_work_beside_n);
 }
 
+bool gpu_work_fits_beside_merges(size_t bytes) {
+    std::lock_guard lock(gpu_work_m);
+    return gpu_merges_n == 0 || fits_beside_merges(bytes);
+}
+
 bool wait_for_gpu_work_beside(std::chrono::seconds timeout) {
     std::unique_lock lock(gpu_work_m);
     return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; });

common/CUDAWrapper.h

@@ -165,6 +165,12 @@ public:
     void Holds(size_t bytes);
 };
 
+// Whether work holding `bytes` of the card would start beside the main line's merges now (GPUWorkBeside with
+// after_merges): no merge is held, or what they said they hold (GPUMerge::Holds) and `bytes` fit within 80% of
+// the first card's total. For a main-line caller that is about to wait for such work, and must not where that
+// work waits for its merge. Build-independent - it only counts.
+bool gpu_work_fits_beside_merges(size_t bytes);
+
 // Whether the card is too small for a probe pass beside the main line's image pass (12 GB, decided on the
 // first card's total memory), so that they take turns (GPUWorkBeside). Build-independent.
 bool gpu_takes_turns();

common/HostMemory.cpp

@@ -3,6 +3,8 @@
 
 #include "HostMemory.h"
 
+#include <algorithm>
+
 #if defined(_WIN32)
 #ifndef NOMINMAX
 #define NOMINMAX
@@ -47,6 +49,7 @@ HostWorkerPool &HostWorkerPool::Instance() {
 }
 
 HostWorkerPool::HostWorkerPool() {
+    SetBudget(0);
     SetSize(0);
 }
 
@@ -57,14 +60,24 @@ void HostWorkerPool::SetSize(size_t bytes) {
     cv.notify_all();
 }
 
+void HostWorkerPool::SetBudget(size_t bytes) {
+    std::lock_guard lock(m);
+    budget = bytes > 0 ? bytes : DefaultHostMemoryBudget();
+    cv.notify_all();
+}
+
+size_t HostWorkerPool::available() const {
+    return std::min(size, budget > held ? budget - held : 0);
+}
+
 size_t HostWorkerPool::Size() {
     std::lock_guard lock(m);
-    return size;
+    return available();
 }
 
 HostWorkerPool::Reservation::Reservation(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(bytes) {
     std::unique_lock lock(pool.m);
-    pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.size; });
+    pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.available(); });
     pool.reserved += bytes;
 }
 
@@ -75,3 +88,20 @@ HostWorkerPool::Reservation::~Reservation() {
     }
     pool.cv.notify_all();
 }
+
+HostWorkerPool::Held::Held(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(0) {
+    Set(bytes);
+}
+
+HostWorkerPool::Held::~Held() {
+    Set(0);
+}
+
+void HostWorkerPool::Held::Set(size_t new_bytes) {
+    {
+        std::lock_guard lock(pool.m);
+        pool.held = pool.held - bytes + new_bytes;
+        bytes = new_bytes;
+    }
+    pool.cv.notify_all();
+}

common/HostMemory.h

@@ -24,20 +24,26 @@ size_t DefaultHostMemoryBudget();
 // its reflections, its merge, the detector-sized arrays.
 constexpr double HOST_WORKER_POOL_FRACTION = 0.5;
 
-// One pool for those buffers, process-wide. A worker reserves its bytes before it makes them and hands
-// them back when it is done; one that would take the pool past its size waits until others have handed
-// theirs back - except that a reservation made while nothing is reserved is always let through, however
-// large. So how many such workers run at once is decided by their bytes, not by the thread count, and
-// however the steps holding them nest or run side by side; what each worker computes, and where its
-// result goes, does not change. A worker must not reserve again while it holds a reservation, and none
-// may reserve on ParallelFor's pool: a holder's own parallel passes need those threads. Work spread over
-// that pool is bounded by its count instead (Size() / bytes each).
+// One pool for those buffers, process-wide, held in one ledger with what the run holds beside them. The run
+// says what it holds - its reflections, its merge engine, a validation's own grids - as Held bytes, and the
+// workers may hold at once the pool's size or what the budget leaves beside the Held bytes, whichever is
+// less: so the two together never claim more than the budget, however large the run's own part is. A worker
+// reserves its bytes before it makes them and hands them back when it is done; one that would take the pool
+// past that waits until others have handed theirs back - except that a reservation made while no worker holds
+// one is always let through, however large. So how many such workers run at once is decided by their bytes,
+// not by the thread count, and however the steps holding them nest or run side by side; what each worker
+// computes, and where its result goes, does not change. A worker must not reserve again while it holds a
+// reservation, and none may reserve on ParallelFor's pool: a holder's own parallel passes need those threads.
+// Work spread over that pool is bounded by its count instead (Size() / bytes each).
 class HostWorkerPool {
 public:
     static HostWorkerPool &Instance();
 
     // The pool's size in bytes; 0 sets it back to HOST_WORKER_POOL_FRACTION of DefaultHostMemoryBudget().
     void SetSize(size_t bytes);
+    // The budget the pool and the Held bytes share; 0 sets it back to DefaultHostMemoryBudget().
+    void SetBudget(size_t bytes);
+    // What the workers may hold at once now: the pool's size, or the budget less the Held bytes if less.
     [[nodiscard]] size_t Size();
 
     class Reservation {
@@ -51,10 +57,26 @@ public:
         size_t bytes;
     };
 
+    // Bytes the run holds beside the workers for as long as this lives; never waits.
+    class Held {
+    public:
+        Held(HostWorkerPool &pool, size_t bytes);
+        ~Held();
+        Held(const Held &) = delete;
+        Held &operator=(const Held &) = delete;
+        void Set(size_t bytes);   // what it holds from now on
+    private:
+        HostWorkerPool &pool;
+        size_t bytes;
+    };
+
 private:
     HostWorkerPool();
+    size_t available() const;   // Size(), with the lock held
     std::mutex m;
     std::condition_variable cv;
     size_t size = 0;
+    size_t budget = 0;
+    size_t held = 0;
     size_t reserved = 0;
 };

docs/RUGNUX_REPORT.md

@@ -250,7 +250,7 @@ and a run that produced a textbook data set read identically for their first thr
   consumer switches on a code rather than parsing a sentence: `NO_LATTICE`, `INDEXING_AMBIGUITY`,
   `SYMMETRY_AMBIGUITY`, `CENTERING_UNTESTED`, `UNUSABLE_MERGE`, `LOW_COMPLETENESS`, `SWEEP_GAPS`,
   `GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`,
-  `LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
+  `LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_CELL_MISMATCH`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
   `FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`, `SCALING_NOT_CONVERGED`, `HARMONIC_CONTAMINATION`,
   `SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`. `NONE` when nothing fired. A code appears if and only if its
   warning fired, so the flags and the `WARNING:` lines are two renderings of one list. Thresholds are
@@ -973,7 +973,7 @@ tried (a prediction, `--mr`) or its null could not be built.
 
 | key | meaning |
 |---|---|
-| `MODEL_VALIDATION` | `PERFORMED` \| `NOT_PERFORMED` (with `MODEL_VALIDATION_REASON`, and no R-factors) |
+| `MODEL_VALIDATION` | `PERFORMED` \| `NOT_PERFORMED` \| `CELL_MISMATCH` (both with `MODEL_VALIDATION_REASON`, and no R-factors). `CELL_MISMATCH`: no change of basis (an axis permutation, a sign flip, I- against C-centred) carries the model's cell onto the data's within 5% in each length and 3 deg in each angle, and the two primitive cells differ in volume by more than that allows, so the model is of another lattice - a cell of another volume, an axis of another length - and is not put into the data's cell to be scored (a centred description of the same primitive cell is not caught here); the reason names both cells, and the `MODEL_CELL_MISMATCH` warning says so. Molecular replacement is still tried, as for any model that is not placed as given |
 | `MODEL_PLACEMENT` | `RIGID_BODY` (the model fits against its null in the answer's merge or the twin hypothesis' one) \| `MR` (placed by molecular replacement) \| `NOT_PLACED` |
 | `MODEL_FIT` | `ACCEPTED` \| `REJECTED` — the rigid body's verdict: whether the model may decide anything. Absent where no rigid body was tried |
 | `MODEL_FIT_STATISTIC` | What the verdict was taken on; `R_WORK` |

image_analysis/structure_refinement/ModelValidation.cpp

@@ -653,11 +653,34 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
     // each dataset in whichever indexing its own run happened to pick.
     // A model handed over already placed in the data's cell (molecular replacement) is on the data's axes.
     if (data_cell.is_crystal() && placed == nullptr) {
+        // Another lattice, where no change of basis of CellMappingOperators carries the model's cell onto the
+        // data's AND the two primitive cells differ in volume by more than CellsCorrespond lets three lengths
+        // differ. The second condition keeps a description of the same lattice that the unimodular set does not
+        // reach - a centred cell against its primitive one (a C-centred orthorhombic cell and the primitive
+        // monoclinic one of its twin-subgroup), the hexagonal and rhombohedral settings - on the path it took.
+        const std::vector<gemmi::Op> mapping = CellMappingOperators(st.cell, data_cell);
+        const auto primitive_volume = [](const gemmi::UnitCell &c, const gemmi::SpaceGroup *g) {
+            return c.volume / static_cast<double>(g != nullptr ? g->operations().cen_ops.size() : 1);
+        };
+        const double volume_ratio = primitive_volume(st.cell, sg) / primitive_volume(data_cell, data_space_group);
+        constexpr double SAME_VOLUME = 1.05 * 1.05 * 1.05;   // CellsCorrespond's 5% in each length
+        if (mapping.empty() && (volume_ratio > SAME_VOLUME || volume_ratio < 1.0 / SAME_VOLUME)) {
+            const auto cell_text = [](const gemmi::UnitCell &c) {
+                return fmt::format("{:.2f} {:.2f} {:.2f} {:.1f} {:.1f} {:.1f}", c.a, c.b, c.c, c.alpha, c.beta, c.gamma);
+            };
+            result.cell_mismatch = true;
+            result.failure_reason = fmt::format(
+                    "the model's cell {} ({}) is not this lattice's {} (primitive volume ratio {:.2f}): no change "
+                    "of basis carries one onto the other", cell_text(st.cell), sg->xhm(), cell_text(data_cell),
+                    volume_ratio);
+            logger.Warning("Model validation: {}", result.failure_reason);
+            return result;
+        }
         const gemmi::GroupOps model_gops = sg->operations();
         const bool same_point_group =
             data_space_group != nullptr && same_rotations(data_space_group->operations(), model_gops);
         std::vector<gemmi::Op> frames{gemmi::Op::identity()};
-        for (const gemmi::Op &op : CellMappingOperators(st.cell, data_cell)) {
+        for (const gemmi::Op &op : mapping) {
             bool seen = false;
             for (const gemmi::Op &kept : frames)
                 seen = seen || same_frame(kept, op, model_gops);
@@ -687,11 +710,17 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
             std::vector<const std::vector<MergedReflection> *> labellings{&obs};
             for (const auto &v : relabelled)
                 labellings.push_back(&v);
+            // Each holds its grids against the host's worker pool (HostWorkerPool) while it scores: as many
+            // run at once as fit beside what the run holds, and each scores the same either way.
             auto score_frame = [&](size_t i) {
                 gemmi::Structure trial = st;
                 const gemmi::SpaceGroup *trial_sg = sg;
                 if (i > 0 && !change_model_basis(trial, trial_sg, frames[i]))
                     return;   // no origin shift names the transformed group; not a basis we can take
+                const HostWorkerPool::Reservation reserved(
+                        HostWorkerPool::Instance(),
+                        MODEL_FACTORS_BYTES_PER_GRID_POINT
+                                * ModelGridPoints(data_cell, trial_sg, std::max(d_min, FRAME_PROBE_D_MIN)));
                 refractionalize_into(trial, data_cell);
                 trial.setup_cell_images();
                 // Each frame at the data's best indexing: the alternative indexings are only probed
@@ -878,6 +907,12 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
 #else
     (void) rigid_body_gpu;
 #endif
+    // The real model's grids, where its structure factors are made on the CPU, are held for as long as the
+    // validation runs, beside the run's own bytes (HostWorkerPool::Held): the null's replicates below, and any
+    // other worker, take only what the two leave of the host budget.
+    const HostWorkerPool::Held model_grids(
+            HostWorkerPool::Instance(),
+            sf_gpu == nullptr ? MODEL_FACTORS_BYTES_PER_GRID_POINT * ModelGridPoints(ucell, sg, d_min) : 0);
 
     // --- Fcalc (atomic) via electron density on a grid + FFT, plus a flat bulk-solvent mask -> Fmask.
     // A lambda because the rigid-body step below moves the model and then needs both again, and it
@@ -1255,6 +1290,12 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
     if ((sf_gpu != nullptr && null_d_min == sf_gpu->DMin()) || (sf_null != nullptr && null_d_min == sf_null->DMin()))
         replicate_bytes = 0;
 #endif
+    if (replicate_bytes > 0 && decision_pending) {
+        const size_t room = HostWorkerPool::Instance().Size();
+        logger.Info("Model validation: the null's replicates hold {} each on the host, {} of the worker pool beside "
+                    "what the run holds - {} at a time", GBytes(replicate_bytes), GBytes(room),
+                    std::clamp<size_t>(room / replicate_bytes, 1, NULL_REPLICATES));
+    }
     auto run_replicate = [&](int i) {
         const HostWorkerPool::Reservation reserved(HostWorkerPool::Instance(), replicate_bytes);
         std::mt19937 rng(NULL_SEED + 1 + i);

image_analysis/structure_refinement/ModelValidation.h

@@ -35,6 +35,13 @@ struct ModelValidationResult {
     // usable space group. A caller that asked for a model must be able to say so rather than finish
     // quietly with no R-free and no maps, which reads exactly like a run that was never asked.
     std::string failure_reason;
+    // Why it did not run, where the reason is that the model's cell is not the data's lattice: no change of
+    // basis of CellMappingOperators carries one onto the other within CellsCorrespond's tolerance, and the
+    // primitive cells differ in volume beyond it - a model put into a cell four times its volume, an axis a
+    // third longer than the data's. An R-factor of a model refractionalised into another lattice's cell
+    // measures nothing, so the validation stops there, and failure_reason names both cells. A placed model
+    // (molecular replacement) is already in the data's cell.
+    bool cell_mismatch = false;
     double r_work = 0.0;
     double r_free = 0.0;
     double k_sol = 0.0;

rugnux/ReportDocument.h

@@ -35,6 +35,7 @@ namespace PathologyCode {
     constexpr const char *TWINNING = "TWINNING";                         // twinning indicated
     constexpr const char *MODEL_HAND = "MODEL_HAND";                     // data and model in opposite hands
     constexpr const char *MODEL_NOT_VALIDATED = "MODEL_NOT_VALIDATED";   // a model was given, not usable
+    constexpr const char *MODEL_CELL_MISMATCH = "MODEL_CELL_MISMATCH";   // the model's cell is not the data's lattice
     constexpr const char *MODEL_FREE_SET = "MODEL_FREE_SET";             // the model as given was scored on a free set drawn here
     constexpr const char *MODEL_SPACE_GROUP = "MODEL_SPACE_GROUP";       // the model's group replaced the one fixed with -S
     constexpr const char *REFERENCE_MISMATCH = "REFERENCE_MISMATCH";     // the reference MTZ does not match; free flags not inherited

rugnux/ResultReport.cpp

@@ -2661,13 +2661,14 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
         if (!result.model_placement.empty())
             Add(s, KeyEnum("MODEL_PLACEMENT", result.model_placement, {"RIGID_BODY", "MR", "NOT_PLACED"}));
         if (!mv.ok) {
-            Add(s, KeyEnum("MODEL_VALIDATION", "NOT_PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
+            Add(s, KeyEnum("MODEL_VALIDATION", mv.cell_mismatch ? "CELL_MISMATCH" : "NOT_PERFORMED",
+                           {"PERFORMED", "NOT_PERFORMED", "CELL_MISMATCH"}));
             Add(s, KeyText("MODEL_VALIDATION_REASON", mv.failure_reason));
             Add(s, Blank());
             Add(s, Prose(" A model was given but could not be used or placed, so there are no R-factors and no\n"
                          " maps. Everything else in this report is unaffected: the merge does not depend on the model."));
         } else {
-            Add(s, KeyEnum("MODEL_VALIDATION", "PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
+            Add(s, KeyEnum("MODEL_VALIDATION", "PERFORMED", {"PERFORMED", "NOT_PERFORMED", "CELL_MISMATCH"}));
             Add(s, KeyInt("MODEL_SPACE_GROUP_NUMBER", mv.model_space_group_number));
             Add(s, KeyReal("R_WORK", mv.r_work, "{:.4f}"));
             Add(s, KeyReal("R_FREE", mv.r_free, "{:.4f}"));

rugnux/Rugnux.cpp

@@ -331,9 +331,11 @@ Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
 
     // What the run plans its host memory against, decided once from the machine's total (HostMemory.h):
     // work that would not fit it side by side runs one part after the other, and the large per-worker
-    // buffers of the model's steps are held a share of it at a time. Only when things run changes.
+    // buffers of the model's steps are held a share of it at a time - of what the run's own reflections,
+    // merge and grids leave of it (HostWorkerPool::Held). Only when things run changes.
     const size_t machine = get_host_total_memory();
     host_budget_ = config_.host_memory_budget_bytes > 0 ? config_.host_memory_budget_bytes : DefaultHostMemoryBudget();
+    HostWorkerPool::Instance().SetBudget(host_budget_);
     HostWorkerPool::Instance().SetSize(
             std::max<size_t>(1, static_cast<size_t>(HOST_WORKER_POOL_FRACTION * static_cast<double>(host_budget_))));
     Logger("Rugnux").Info("Host memory budget {:.1f} GB ({}): what does not fit it side by side runs in series",
@@ -647,12 +649,13 @@ void Rugnux::StartSpeculativeGeometryProbe(std::shared_ptr<Rugnux> run, const st
     auto sp = std::make_shared<SpeculativeProbe>();
     sp->run = run;
     sp->hand = hand;
+    sp->gpu_bytes = run->IndexingProbeGPUBytes();
     // A probe that fails leaves no memo, and the walk's own probe then runs it again and meets the
     // failure itself. It runs beside the merge of the pass that started it, on the same card: a merge
     // allocation that does not fit beside it waits for it to end (GPUWorkBeside) - and where the probe's
     // bytes and the merge's do not fit the card together, the probe waits for that merge to end instead
     // (GPUMerge). Its two passes run one after the other, so it holds at most what one does.
-    const size_t probe_bytes = run->IndexingProbeGPUBytes();
+    const size_t probe_bytes = sp->gpu_bytes;
     Logger("Rugnux").Debug("Speculative geometry probe: {:.2f} GB of the card", probe_bytes / 1e9);
     sp->done = std::async(std::launch::async, [run, hand, probe_bytes] {
         GPUWorkBeside beside(/*after_merges=*/true, probe_bytes);

rugnux/Rugnux.h

@@ -57,6 +57,7 @@ class JFJochReader;
 class JFJochHDF5Reader;
 struct JFJochReaderRawImage;
 struct PipelineLocals;   // RugnuxPipeline.h, private to the implementation
+class GPUMerge;         // CUDAWrapper.h
 class IndexerThreadPool;
 
 // Offline reprocessing of a stored Jungfraujoch HDF5 dataset, shared by rugnux (all of its --mode
@@ -1350,6 +1351,7 @@ class Rugnux {
         std::shared_ptr<Rugnux> run;
         std::shared_ptr<Rugnux> hand;   // the walk's probe at the geometry in hand, after run's
         std::future<void> done;
+        size_t gpu_bytes = 0;   // what it holds of the card at its most (IndexingProbeGPUBytes)
     };
     std::shared_ptr<SpeculativeProbe> speculative_probe_;
     bool speculate_geometry_probe_ = false;
@@ -1373,6 +1375,12 @@ class Rugnux {
     // Whether the geometry walk after the canonical pass will take a round, asked by that pass from
     // what it already has in hand; `fit` is its post-refinement running beside the merge (RefinedPasses).
     [[nodiscard]] bool GeometryWalkTakesRound(const ProcessResult &result, const std::future<void> &fit) const;
+    // For a pass about to spend its model stage on a merge the walk may replace: waits for `fit`, and says whether
+    // GeometryWalkTakesRound's answer still waits on the speculative probe.
+    [[nodiscard]] bool GeometryWalkProbePending(const ProcessResult &result, const std::future<void> &fit) const;
+    // Wait for that probe, where it can start beside what the merge still holds of the card (`merge_on_card`,
+    // said to GPUMerge); return at once where it cannot - it waits for that merge.
+    void AwaitGeometryWalkProbe(GPUMerge *gpu_merge, size_t merge_on_card) const;
     // The part of FirstPassInputKey that is the experiment and the spot-finding settings; without the
     // goniometer's angles where rotation_angles is false (SpotFindingKey).
     [[nodiscard]] std::vector<double> ExperimentKey(bool rotation_angles = true) const {

rugnux/RugnuxPasses.cpp

@@ -875,6 +875,37 @@ bool Rugnux::GeometryWalkTakesRound(const ProcessResult &result, const std::futu
     return at_hand && at_fit && ValidationEvidencePrefers(*at_hand, *at_fit);
 }
 
+// For a canonical pass with a model to validate: the model stage - validation, molecular replacement and
+// refinement, the largest host and card footprint of a run - is spent on this pass's merge only where the
+// walk is not going to replace it, so the pass asks GeometryWalkTakesRound once its answer is in. That needs
+// the pass's own post-refinement, waited for here, and, for a move of less than a step, the speculative
+// probe: true where the answer still waits on that probe (AwaitGeometryWalkProbe). A pass whose fit does not
+// move the detector started no probe and waits for nothing past its fit.
+bool Rugnux::GeometryWalkProbePending(const ProcessResult &result, const std::future<void> &fit) const {
+    if (!speculate_geometry_probe_ || cancelled_ || !prepass_detector_geometry_)
+        return false;
+    if (fit.valid())
+        fit.wait();
+    return result.post_refine && result.post_refine->detector_refined && !result.post_refine->large_move
+           && !config_.verify_first_pass_memo && speculative_probe_
+           && speculative_probe_->done.wait_for(std::chrono::seconds(0)) != std::future_status::ready;
+}
+
+// The probe waits for the merge where the two do not fit the card together (GPUMerge), so the merge first
+// says what it still holds, and the probe is waited for only where it can then start.
+void Rugnux::AwaitGeometryWalkProbe(GPUMerge *gpu_merge, size_t merge_on_card) const {
+    if (gpu_merge)
+        gpu_merge->Holds(merge_on_card);
+    Logger logger("Rugnux");
+    if (!gpu_work_fits_beside_merges(speculative_probe_->gpu_bytes)) {
+        logger.Debug("Geometry walk: the probe does not fit the card beside the merge's {:.2f} GB - the model "
+                     "stage does not wait for it", static_cast<double>(merge_on_card) / 1e9);
+        return;
+    }
+    TimingMark mark(logger, "geometry walk probe wait (model)");
+    speculative_probe_->done.wait();
+}
+
 // The refined passes of the rotation two-pass, from the second pass on: everything after the
 // rotation-scale walk (see RunAllPasses). Empty where they ran beside the walk and it overturned them.
 std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, ProcessResult pass1,

rugnux/RugnuxScaleMerge.cpp

@@ -34,6 +34,7 @@
 #include "../reader/JFJochHDF5Reader.h"
 #include "../common/JFJochMath.h"
 #include "../common/ParallelFor.h"
+#include "../common/HostMemory.h"
 #include "../common/Logger.h"
 #include "../common/AzimuthalIntegrationMapping.h"
 #include "../common/AzimuthalIntegrationProfile.h"
@@ -1225,6 +1226,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
     }
     const size_t reflection_bytes = n_reflections * sizeof(Reflection);
     const auto host_gb = [](size_t bytes) { return static_cast<double>(bytes) / 1e9; };
+    // ... and what the large per-worker buffers of the model's steps, and any beside the merge, are held
+    // against: the same budget, less this (HostWorkerPool::Held), so the two together never claim more.
+    HostWorkerPool::Held host_held(HostWorkerPool::Instance(), reflection_bytes);
     std::future<ScaleMergeResult> guard_ahead;
     if (merge_first_part && !(postrefine_probe_ && !geometry_prepass && postrefine_probe_only_)) {
         // The guard's merge runs beside the main engine's ingest. Where an engine of the guard's frames and
@@ -1527,6 +1531,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
         constexpr size_t POSTREFINE_RANKED_BYTES = 16;   // PostRefineRotationGeometry's ranking, per partial
         const size_t host_postrefine = n_reflections * (sizeof(PostRefinePartial) + POSTREFINE_RANKED_BYTES);
         bool postrefine_beside = p.postrefine_ahead.valid();
+        host_held.Set(host_engine + host_merge + (postrefine_beside ? host_postrefine : 0));
         if (postrefine_beside && host_engine + host_merge + host_postrefine > host_budget_) {
             logger.Info("Host memory: the post-refinement ({:.1f} GB) does not fit beside the merge ({:.1f} GB, "
                         "{:.1f} GB held) within the {:.1f} GB budget - it is done before the merge",
@@ -4041,6 +4046,25 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
         Note("cut: the model is left out");
         return true;
     };
+    // With a model to validate, the walk's answer is waited for: the model stage is the largest part of such a
+    // run, and spent on a merge the walk replaces it is spent twice (GeometryWalkProbePending). Where the answer
+    // waits on the probe, what the model stage would let go of the card first goes now - the indexers, and the
+    // merge engine where the ledger's merge does not keep it - so that the probe can run.
+    if (result.consensus_cell && write_output && config_.model && !geometry_prepass && !superseded
+        && GeometryWalkProbePending(result, p.postrefine_ahead)) {
+        if (census_future.valid())
+            census_future.wait();
+        p.indexer_pool.reset();
+        if (!ledger_ahead.valid())
+            rsm.reset();
+        AwaitGeometryWalkProbe(p.gpu_merge, rsm && rsm->OnGPU() ? rsm->DeviceBytes() : 0);
+        if (GeometryWalkTakesRound(result, p.postrefine_ahead)) {
+            superseded = true;
+            logger.Info("This pass is going to be re-run, so it validates no model, makes no report and writes "
+                        "no files.");
+            Note("this pass is to be re-run (see the log); it writes nothing");
+        }
+    }
     const bool model_run = result.consensus_cell && write_output && config_.model && !geometry_prepass
                            && !superseded && !model_cut();
     // The merge's own shell bounds, so CC(model, data) is reported on the shells CC1/2 was.
@@ -4097,6 +4121,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
     // that one started on the forecast decides as the one it stands in for would.
     if (model_run && !ledger_ahead.valid())
         rsm.reset();
+    // What the run holds beside the model's steps from here on: its reflections, and the merge engine where the
+    // ledger's merge still runs on it.
+    if (model_run)
+        host_held.Set(reflection_bytes + (rsm ? rsm->HostIngestBytes() + rsm->HostMergeBytes() : 0));
     const size_t merge_on_card = rsm && rsm->OnGPU() ? rsm->DeviceBytes() : 0;
     ModelValidationSchedule first_schedule;
     first_schedule.gpu_resident_bytes = merge_on_card;
@@ -5346,8 +5374,13 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
         std::string why = result.model_validation ? result.model_validation->failure_reason : std::string();
         if (why.empty() && result.model_prior && !result.model_prior->ok)
             why = "the model could not be searched for: " + result.model_prior->failure_reason;
-        if (!(result.model_validation && result.model_validation->ok) && !why.empty())
-            result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED, "Model validation did not run: " + why});
+        if (!(result.model_validation && result.model_validation->ok) && !why.empty()) {
+            if (result.model_validation && result.model_validation->cell_mismatch)
+                result.warnings.push_back({PathologyCode::MODEL_CELL_MISMATCH, "The model does not describe the "
+                                           "data's lattice, so it was not validated: " + why});
+            else
+                result.warnings.push_back({PathologyCode::MODEL_NOT_VALIDATED, "Model validation did not run: " + why});
+        }
         // The outcome, as notes of the step that decided it
         if (result.model_prior) {
             const MolecularReplacementResult &mp = *result.model_prior;

tests/HostMemoryTest.cpp

@@ -46,3 +46,45 @@ TEST_CASE("HostWorkerPool_BoundsWorkersByBytes", "[HostMemory]") {
     pool.SetSize(0);
     CHECK(pool.Size() == static_cast<size_t>(HOST_WORKER_POOL_FRACTION * static_cast<double>(DefaultHostMemoryBudget())));
 }
+
+// One ledger: what the run holds (Held) and what the workers reserve together stay within the budget. Workers
+// take what the Held bytes leave of it, up to the pool's size, and a worker larger than that runs alone rather
+// than never.
+TEST_CASE("HostWorkerPool_HeldBytesShareTheBudget", "[HostMemory]") {
+    HostWorkerPool &pool = HostWorkerPool::Instance();
+    pool.SetBudget(100);
+    pool.SetSize(50);
+    CHECK(pool.Size() == 50);
+    const auto most_at_once = [&](size_t bytes) {
+        std::atomic<int> running{0}, most{0}, done{0};
+        std::vector<std::thread> workers;
+        for (int i = 0; i < 8; i++)
+            workers.emplace_back([&] {
+                const HostWorkerPool::Reservation reserved(pool, bytes);
+                const int now = ++running;
+                int seen = most.load();
+                while (now > seen && !most.compare_exchange_weak(seen, now)) {}
+                std::this_thread::sleep_for(std::chrono::milliseconds(20));
+                --running;
+                ++done;
+            });
+        for (auto &w : workers)
+            w.join();
+        CHECK(done == 8);
+        return most.load();
+    };
+    {
+        HostWorkerPool::Held run(pool, 70);   // leaves 30 of the budget: the pool's 50 does not fit beside it
+        CHECK(pool.Size() == 30);
+        CHECK(most_at_once(10) <= 3);
+        run.Set(20);                          // leaves 80: the pool's own size bounds again
+        CHECK(pool.Size() == 50);
+        CHECK(most_at_once(10) <= 5);
+        run.Set(100);                         // nothing left: one at a time, each still runs
+        CHECK(pool.Size() == 0);
+        CHECK(most_at_once(10) == 1);
+    }
+    CHECK(pool.Size() == 50);
+    pool.SetBudget(0);
+    pool.SetSize(0);
+}

tests/ModelValidationTest.cpp

@@ -13,6 +13,7 @@
 #include <random>
 #include <set>
 #include <sstream>
+#include <utility>
 
 #include <gemmi/mmread_gz.hpp>
 #include <gemmi/fourier.hpp>
@@ -962,9 +963,12 @@ TEST_CASE("ModelValidation_WarmUpChangesNothing", "[ModelValidation]") {
 // reserved for the zone as it is counted, so the validation runs instead of failing on the device.
 TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidation]") {
     Logger logger("ModelValidation_WrongModelInAGroupOffTheDataMetric");
-    const auto data_path = WriteTemp("off_metric_data.pdb", ClusterPdb().c_str());
+    // A cubic model on a cell within the tolerance of the data's orthorhombic one, so that it is scored
+    // (a model of another lattice is not: ModelValidation_StopsOnAModelOfAnotherLattice).
+    const auto data_path = WriteTemp("off_metric_data.pdb",
+        ClusterPdb("CRYST1   36.000   37.000   38.000  90.00  90.00  90.00 P 21 21 21    4\n").c_str());
     const auto wrong_path = WriteTemp("off_metric_wrong.pdb",
-        ClusterPdb("CRYST1   40.000   40.000   40.000  90.00  90.00  90.00 I 21 3       24\n", 7).c_str());
+        ClusterPdb("CRYST1   37.000   37.000   37.000  90.00  90.00  90.00 I 21 3       24\n", 7).c_str());
     auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger);
     REQUIRE(obs.size() > 1000);
     for (size_t i = 0; i < obs.size(); i++) {
@@ -973,7 +977,7 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio
         obs[i].rfree_flag = (i % 20) == 0;
     }
     const auto result =
-        ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
+        ValidateAgainstModel(obs, UnitCell{.a = 36, .b = 37, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
                              ReadModel(wrong_path), logger, gemmi::find_spacegroup_by_name("P 21 21 21"));
     CHECK(result.ok);
     CHECK(!result.adopted_model_enantiomorph);
@@ -982,6 +986,55 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio
     std::filesystem::remove(wrong_path);
 }
 
+// A model whose cell no change of basis carries onto the data's is of another lattice: it is not put into the
+// data's cell to be scored, and the reason names both cells. A synthetic cluster; the model's cells are the
+// data's with the volume four times as large, and with one axis a third longer.
+TEST_CASE("ModelValidation_StopsOnAModelOfAnotherLattice", "[ModelValidation]") {
+    Logger logger("ModelValidation_StopsOnAModelOfAnotherLattice");
+    const auto data_path = WriteTemp("other_lattice_data.pdb", ClusterPdb().c_str());
+    auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.5, logger);
+    REQUIRE(obs.size() > 500);
+    for (size_t i = 0; i < obs.size(); i++) {
+        obs[i].F = std::sqrt(std::max(0.0f, obs[i].I));
+        obs[i].rfree_flag = (i % 20) == 0;
+    }
+    const UnitCell data_cell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90};
+    const gemmi::SpaceGroup *data_sg = gemmi::find_spacegroup_by_name("P 21 21 21");
+
+    const std::pair<const char *, const char *> models[] = {
+            {"CRYST1   60.000   68.000   38.000  90.00  90.00  90.00 P 21 21 21    4\n", "60.00 68.00 38.00"},
+            {"CRYST1   30.000   34.000   52.000  90.00  90.00  90.00 P 21 21 21    4\n", "30.00 34.00 52.00"}};
+    for (const auto &[cryst, model_cell] : models) {
+        CAPTURE(cryst);
+        const auto model_path = WriteTemp("other_lattice_model.pdb", ClusterPdb(cryst).c_str());
+        const auto result = ValidateAgainstModel(obs, data_cell, ReadModel(model_path), logger, data_sg,
+                                                 /*probe_indexing_ambiguity=*/true, 1, 1.0);
+        CHECK_FALSE(result.ok);
+        CHECK(result.cell_mismatch);
+        CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring("30.00 34.00 38.00"));
+        CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring(model_cell));
+        std::filesystem::remove(model_path);
+    }
+
+    // A centred description of a cell of the same primitive volume is not taken for another lattice: it is
+    // left to the scoring, as before (where it reads as a model that does not fit).
+    const auto centred_path = WriteTemp("other_lattice_centred.pdb",
+        ClusterPdb("CRYST1   60.000   34.000   38.000  90.00  90.00  90.00 C 2 2 21     8\n").c_str());
+    const auto centred = ValidateAgainstModel(obs, data_cell, ReadModel(centred_path), logger, data_sg, false, 1, 1.0);
+    CHECK_FALSE(centred.cell_mismatch);
+    std::filesystem::remove(centred_path);
+
+    // The same lattice on other axes is a description of it, and is scored.
+    const auto same_path = WriteTemp("other_lattice_same.pdb",
+        ClusterPdb("CRYST1   31.000   33.500   38.500  90.00  90.00  90.00 P 21 21 21    4\n").c_str());
+    const auto same = ValidateAgainstModel(obs, data_cell, ReadModel(same_path), logger, data_sg, false, 1, 1.0);
+    CHECK(same.ok);
+    CHECK_FALSE(same.cell_mismatch);
+
+    std::filesystem::remove(data_path);
+    std::filesystem::remove(same_path);
+}
+
 // The groups a model may relabel the data into: those the merged intensities cannot separate from the
 // data's - an enantiomorph, or a group whose screws the centring already hides - and no other.
 TEST_CASE("ModelValidation_SameIntensities", "[ModelValidation]") {

tests/RugnuxLargeTest.cpp

@@ -186,4 +186,5 @@ TEST_CASE("Rugnux_HostBudgetChangesOnlyTheSchedule", "[large]") {
     CHECK(slurp((dir / "tight.hkl").string()) == hkl);
     std::filesystem::remove_all(dir);
     HostWorkerPool::Instance().SetSize(0);   // process-wide: back to the machine's for the tests after this one
+    HostWorkerPool::Instance().SetBudget(0);
 }