diff --git a/image_analysis/structure_refinement/ModelStructureFactorsGPU.cpp b/image_analysis/structure_refinement/ModelStructureFactorsGPU.cpp index 02b8bfe3e..a86b8eb9a 100644 --- a/image_analysis/structure_refinement/ModelStructureFactorsGPU.cpp +++ b/image_analysis/structure_refinement/ModelStructureFactorsGPU.cpp @@ -107,7 +107,7 @@ void ModelDensityAtoms(const gemmi::Model &model, const gemmi::DensityCalculator ModelStructureFactorsGPU::ModelStructureFactorsGPU(int device, const gemmi::Model &model, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min) - : device_(device), cell_(cell), sg_(&sg), d_min_(d_min), dc_(Calculator(model, cell, sg, d_min)) { + : device_(device), device_total_(ModelStructureFactorsGPUEngine::TotalMemory(device)), cell_(cell), sg_(&sg), d_min_(d_min), dc_(Calculator(model, cell, sg, d_min)) { const gemmi::Grid grid = RigidBodyZoneGrid(cell, sg, d_min); // DensityCalculator's for d_min setup_.grid.nu = grid.nu; setup_.grid.nv = grid.nv; diff --git a/image_analysis/structure_refinement/ModelStructureFactorsGPU.h b/image_analysis/structure_refinement/ModelStructureFactorsGPU.h index b206997b5..499d4e8aa 100644 --- a/image_analysis/structure_refinement/ModelStructureFactorsGPU.h +++ b/image_analysis/structure_refinement/ModelStructureFactorsGPU.h @@ -44,7 +44,8 @@ void ModelDensityAtoms(const gemmi::Model &model, const gemmi::DensityCalculator class ModelStructureFactorsGPU { public: // For `device`. Host work only: the grid, the reflections and what the device will need for them. - // Nothing is reserved until Reserve(), so DeviceBytes() can decide first whether to. Every call works + // Nothing is reserved until Reserve(), so DeviceBytes() can decide first whether to. Throws + // JFJochException where the device cannot be asked for its memory or its transform sizes. Every call works // on `device` and leaves the calling thread's current device as it found it; which card it is does not // change a number. ModelStructureFactorsGPU(int device, const gemmi::Model &model, const gemmi::UnitCell &cell, @@ -59,7 +60,7 @@ public: double DMin() const { return d_min_; } int Device() const { return device_; } // The device's total memory, which DeviceBytes() is to be judged against. - size_t DeviceTotalMemory() const { return ModelStructureFactorsGPUEngine::TotalMemory(device_); } + size_t DeviceTotalMemory() const { return device_total_; } // The grid the structure factors are computed on. std::array GridSize() const { return {setup_.grid.nu, setup_.grid.nv, setup_.grid.nw}; } std::array MapSizeBound() const { return map_size_; } @@ -81,6 +82,7 @@ public: private: int device_; + size_t device_total_; gemmi::UnitCell cell_; const gemmi::SpaceGroup *sg_; double d_min_; diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index f557b05e3..1f0cb5ce2 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "../../common/CorrelationCoefficient.h" #include "../../common/JFJochMath.h" // PI (M_PI is not standard, and MSVC does not define it) #include "../../common/Logger.h" +#include "../../common/ParallelFor.h" #include "../scale_merge/ReindexAmbiguity.h" // ReindexReflections #include "../scale_merge/CrystalSetting.h" // CellMappingOperators #include "ModelFFT.h" @@ -75,11 +77,20 @@ gemmi::Grid map_from_coefficients(gemmi::AsuData> &co return MapFromFPhi(gemmi::get_f_phi_on_grid(coef, size, true)); } +// A file this run may already have written - the validation in the model's setting writes over the +// first one's maps - is removed before it is written again, not truncated: XFS (and ext4) flush a file +// truncated and rewritten when it is closed, and three 150 MB maps forced out to a disk that way cost +// seconds where a fresh file costs nothing. +void remove_before_rewriting(const std::string &path) { + std::remove(path.c_str()); +} + // Write a map as CCP4; return its RMS (the sigma the map is read in). double write_ccp4(const gemmi::Grid &map, const std::string &path) { gemmi::Ccp4 ccp4; ccp4.grid = map; ccp4.update_ccp4_header(2); + remove_before_rewriting(path); ccp4.write_ccp4_map(path); return ccp4.hstats.rms; } @@ -357,28 +368,29 @@ double frame_probe_r(const gemmi::Structure &st, const gemmi::SpaceGroup *sg, // once, before any of them is computed, and never revisited: nothing is moved to the CPU part way through. // Decided on what the GPU path needs against the card's TOTAL memory and not against what happens to be // free at the moment, so the same input on the same machine always takes the same path, whatever else is -// running beside it. The structure factors may take half the card - `reserved` is what an engine of the -// same validation already holds there; the rigid body's engines take at most a quarter beside them -// (RigidBodyGPUPool::Create), and the last quarter is left to everything else. +// running beside it. A validation plans at most half of its card: its structure-factor engines a quarter +// together - `reserved` is what one of them already holds there - and its rigid-body engines a quarter +// (RigidBodyGPUPool::Create). A second validation runs beside it only where twice that plan fits half of +// the cards together (see on_forecast below), so validations never plan more than half of the memory. std::unique_ptr StructureFactorsGPU(int device, const gemmi::Model &model, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min, size_t reserved, Logger &logger) { - auto sf = std::make_unique(device, model, cell, sg, d_min); + std::unique_ptr sf; + try { + sf = std::make_unique(device, model, cell, sg, d_min); + } catch (const JFJochException &e) { + throw RigidBodyGPUFailure(e.what()); + } const std::array n = sf->GridSize(); if (!sf->Supported()) { logger.Info("Model validation: structure factors to {:.2f} A on the CPU - the cell is too small for the " "GPU's gridding", d_min); return nullptr; } - size_t total = 0; - try { - total = sf->DeviceTotalMemory(); - } catch (const JFJochException &e) { - throw RigidBodyGPUFailure(e.what()); - } - if (reserved + sf->DeviceBytes() > total / 2) { + const size_t total = sf->DeviceTotalMemory(); + if (reserved + sf->DeviceBytes() > total / 4) { logger.Info("Model validation: structure factors to {:.2f} A on the CPU - on a {}x{}x{} grid they need " - "{:.2f} GB of GPU memory beside {:.2f} GB already reserved, over half of the card's {:.2f} GB", + "{:.2f} GB of GPU memory beside {:.2f} GB already reserved, over a quarter of the card's {:.2f} GB", d_min, n[0], n[1], n[2], sf->DeviceBytes() / 1e9, reserved / 1e9, total / 1e9); return nullptr; } @@ -410,6 +422,7 @@ ModelValidationResult Validate(const std::vector &merged, size_t nthreads, double wavelength_A, const std::vector &report_shell_d_min, + const ModelValidationSchedule &schedule, bool rigid_body_gpu) { ModelValidationResult result; result.model_path = model_path; @@ -539,13 +552,15 @@ ModelValidationResult Validate(const std::vector &merged, "is not how the data describe this lattice; scoring {} change(s) of basis", st.cell.a, st.cell.b, st.cell.c, st.cell.alpha, st.cell.beta, st.cell.gamma, sg->xhm(), frames.size() - 1); - size_t best = 0; - double best_r = 0; - for (size_t i = 0; i < frames.size(); i++) { + // The candidates are scored at the same time, each on its own copy of the model, and read + // in their own order below, so the ranking and its tie-break are those of one after another. + std::vector frame_r(frames.size(), 0.0); + std::vector frame_sg(frames.size(), nullptr); + ParallelFor(static_cast(frames.size()), nthreads, [&](int i) { gemmi::Structure trial = st; const gemmi::SpaceGroup *trial_sg = sg; if (i > 0 && !change_model_basis(trial, trial_sg, frames[i])) - continue; // no origin shift names the transformed group; not a basis we can take + return; // no origin shift names the transformed group; not a basis we can take refractionalize_into(trial, data_cell); trial.setup_cell_images(); // Each frame at the data's best indexing: the alternative indexings are only probed @@ -557,8 +572,17 @@ ModelValidationResult Validate(const std::vector &merged, if (same_point_group) for (const gemmi::Op &law : reindex_ops) r = std::min(r, frame_probe_r(trial, trial_sg, ReindexReflections(obs, law), d_min)); + frame_r[i] = r; + frame_sg[i] = trial_sg; + }); + size_t best = 0; + double best_r = 0; + for (size_t i = 0; i < frames.size(); i++) { + if (frame_sg[i] == nullptr) + continue; + const double r = frame_r[i]; logger.Info("Model validation: {:<12} -> {:<12} R {:.4f} on the coarse shell", - frames[i].triplet(), trial_sg->xhm(), r); + frames[i].triplet(), frame_sg[i]->xhm(), r); if (i == 0 || r < best_r) { best_r = r; best = i; @@ -880,6 +904,27 @@ ModelValidationResult Validate(const std::vector &merged, const bool decision_pending = result.model_enantiomorph_candidate || !(indexing.op == gemmi::Op::identity()) || !(result.change_of_basis_op == gemmi::Op::identity()); + if (schedule.on_forecast) { + // Two validations at once take twice the engines, and are allowed where twice what this one plans - + // its rigid-body engines, its structure-factor engine and, where a null is coming, the null's, + // which is no larger - fits half of the cards' memory taken together (see StructureFactorsGPU). + bool may_run_beside = true; + size_t planned = 0, card = 0; + if (rigid_body_pool != nullptr) { + planned += rigid_body_pool->PlannedBytes(); + card = rigid_body_pool->CardBytes(); + } +#ifdef JFJOCH_USE_CUDA + if (sf_gpu != nullptr) { + planned += (decision_pending ? 2 : 1) * sf_gpu->DeviceBytes(); + card = sf_gpu->DeviceTotalMemory(); + } +#endif + if (planned > 0) + may_run_beside = 2 * planned <= static_cast(get_gpu_count()) * card / 2; + schedule.on_forecast({result.change_of_basis_op, indexing.op, result.model_enantiomorph_candidate, + may_run_beside}); + } std::vector equivalent; // the orientations the crystal cannot tell from the model's double reach_deg = 0; std::vector null_rotation; @@ -1024,7 +1069,11 @@ ModelValidationResult Validate(const std::vector &merged, std::vector> replicate_threads; if (decision_pending) for (size_t t = 0; t < std::min(std::max(nthreads, 1), NULL_REPLICATES); t++) - replicate_threads.push_back(std::async(std::launch::async, [&] { + replicate_threads.push_back(std::async(std::launch::async, [&, t] { + // Each on a card in turn from the one after the real fit's, which its engines are + // spread the same way over (RigidBodyGPUPool::Create). + if (rigid_body_pool != nullptr) + pin_gpu((rigid_body_pool->Device() + 1 + static_cast(t)) % get_gpu_count()); for (int i = next_replicate++; i < NULL_REPLICATES; i = next_replicate++) run_replicate(i); })); @@ -1354,6 +1403,13 @@ ModelValidationResult Validate(const std::vector &merged, mapfofc.v.push_back({terms[i].hkl, delfwt[i] * ph}); } + // A validation started before the one it follows had decided writes nothing until that one has, + // and nothing at all where it decided otherwise. + if (schedule.write_gate.valid() && !schedule.write_gate.get()) { + result.failure_reason = "superseded before its files were written"; + return result; + } + // --- write the maps and score the 2mFo-DFc map at atom centres (a real map peaks there) --- // The three maps - this one, the difference map and the anomalous map - share nothing but what // they read, so the other two are made and written on threads of their own beside this one. @@ -1525,6 +1581,7 @@ ModelValidationResult Validate(const std::vector &merged, } mtz.nreflections = static_cast(terms.size()); mtz.data = std::move(data); + remove_before_rewriting(output_prefix + "_maps.mtz"); mtz.write_to_file(output_prefix + "_maps.mtz"); } catch (const std::exception &e) { logger.Warning("Model validation: could not write map MTZ: {}", e.what()); @@ -1581,14 +1638,15 @@ ModelValidationResult ValidateAgainstModel(const std::vector & bool probe_indexing_ambiguity, size_t nthreads, double wavelength_A, - const std::vector &report_shell_d_min) { + const std::vector &report_shell_d_min, + const ModelValidationSchedule &schedule) { #ifdef JFJOCH_USE_CUDA // A CUDA failure ends the validation, not the run, and nothing is moved to the CPU part way through: // the validation reports why it did not finish, and a validation that did not finish decides nothing, // so the reflection files are those of a run without a model. try { return Validate(merged, cell, model_path, output_prefix, logger, data_space_group, - probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, true); + probe_indexing_ambiguity, nthreads, wavelength_A, report_shell_d_min, schedule, true); } catch (const RigidBodyGPUFailure &e) { cuda_clear_error(); ModelValidationResult failed; @@ -1599,7 +1657,7 @@ ModelValidationResult ValidateAgainstModel(const std::vector & } #else return Validate(merged, cell, model_path, output_prefix, logger, data_space_group, probe_indexing_ambiguity, - nthreads, wavelength_A, report_shell_d_min, false); + nthreads, wavelength_A, report_shell_d_min, schedule, false); #endif } diff --git a/image_analysis/structure_refinement/ModelValidation.h b/image_analysis/structure_refinement/ModelValidation.h index 47bffdd26..a2f24f3cc 100644 --- a/image_analysis/structure_refinement/ModelValidation.h +++ b/image_analysis/structure_refinement/ModelValidation.h @@ -4,6 +4,8 @@ #pragma once #include +#include +#include #include #include #include @@ -219,13 +221,33 @@ struct ModelValidationResult { // the runner-up beats the lead the same model in a random orientation takes, since a random model // also picks a winner. Holohedral crystals (no twin laws) are unaffected either way. // -// nthreads is what the null's replicates run on - they are independent of each other and of the real -// model, so they run at once. Nothing else here is threaded, and the answer does not depend on it. +// nthreads is what the null's replicates and the candidate changes of basis run on - each is +// independent of the others, so they run at once. The answer does not depend on it. // // report_shell_d_min is the merge statistics' own shell bounds, coarse to fine, and it is what // cc_model_shells is binned on. Sharing the grid is the point: a reader has to be able to put a // CC(model, data) row beside that shell's CC1/2 and know the two describe the same reflections. // Empty (the default) means no shells were given and none are reported. +// What a validation knows of its outcome before it has one: the setting the model was put into and +// the indexing the probe prefers - which is what it decides where the model turns out to fit and the +// probe's lead turns out to be real. Enough for a caller to start what follows on that forecast and +// keep it only where the outcome agrees. +struct ModelFrameForecast { + gemmi::Op change_of_basis_op = gemmi::Op::identity(); + gemmi::Op indexing_op = gemmi::Op::identity(); + bool enantiomorph_candidate = false; + // Whether a second validation of this size may run beside this one: decided on the sizes of its + // GPU engines against the memory of the cards, never on what happens to be free. + bool may_run_beside = true; +}; + +struct ModelValidationSchedule { + // Called once, before the null, with the forecast above. + std::function on_forecast; + // Where valid, waited on before the first file is written: false returns without writing any. + std::shared_future write_gate; +}; + ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, const std::string &model_path, @@ -235,7 +257,8 @@ ModelValidationResult ValidateAgainstModel(const std::vector & bool probe_indexing_ambiguity = true, size_t nthreads = 1, double wavelength_A = 0.0, - const std::vector &report_shell_d_min = {}); + const std::vector &report_shell_d_min = {}, + const ModelValidationSchedule &schedule = {}); // Reindex `merged` into the frame ValidateAgainstModel reported, so the reflection files that are // written describe the same indexing as the R-factors and the maps. Returns the space group they are diff --git a/image_analysis/structure_refinement/RigidBodyGPU.cpp b/image_analysis/structure_refinement/RigidBodyGPU.cpp index 2c0b5eb16..9ba4c2d7e 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.cpp +++ b/image_analysis/structure_refinement/RigidBodyGPU.cpp @@ -163,9 +163,16 @@ std::unique_ptr RigidBodyGPUPool::Create(const gemmi::Model &m const size_t budget = std::min(total / 4, free > HEADROOM ? free - HEADROOM : 0); const size_t fit = bytes > 0 ? budget / bytes : 0; const size_t want = std::min(fit, std::max(max_engines, 1)); + // Engine i on the i-th card from the calling thread's, so a validation uses every card there is; + // the engines give the same bits on any card of one model, so this changes only where they run. const int device = RigidBodyGPUEngine::CurrentDevice(); + const int cards = get_gpu_count(); + pool->device_ = device; + pool->planned_bytes_ = bytes * std::max(max_engines, 1); + pool->card_bytes_ = total; for (size_t i = 0; i < want; i++) - pool->engines_.push_back(std::make_unique(cap, device)); + pool->engines_.push_back(std::make_unique(cap, (device + static_cast(i)) % cards)); + set_gpu(device); // each engine was made on its own card if (pool->engines_.empty()) { logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB and the " "budget is {:.0f} MB", bytes / 1e6, budget / 1e6); @@ -216,10 +223,19 @@ const RigidBodyGPUZone &RigidBodyGPUPool::Zone(const gemmi::Model &model, const } RigidBodyGPUEngine &RigidBodyGPUPool::Acquire() { + // One on the calling thread's own card where one is idle - the threads that drive a validation are + // pinned to the cards in turn - and any other otherwise. + const int device = RigidBodyGPUEngine::CurrentDevice(); std::unique_lock lock(m_); cv_.wait(lock, [this] { return !idle_.empty(); }); - RigidBodyGPUEngine *e = idle_.back(); - idle_.pop_back(); + size_t pick = idle_.size() - 1; + for (size_t i = idle_.size(); i-- > 0;) + if (idle_[i]->Device() == device) { + pick = i; + break; + } + RigidBodyGPUEngine *e = idle_[pick]; + idle_.erase(idle_.begin() + static_cast(pick)); return *e; } diff --git a/image_analysis/structure_refinement/RigidBodyGPU.cu b/image_analysis/structure_refinement/RigidBodyGPU.cu index 689ab5513..6740a4710 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.cu +++ b/image_analysis/structure_refinement/RigidBodyGPU.cu @@ -511,7 +511,18 @@ RigidBodyGPUEngine::RigidBodyGPUEngine(const RigidBodyGPUCapacity &capacity, int impl_ = std::make_unique(capacity, device); } -RigidBodyGPUEngine::~RigidBodyGPUEngine() = default; +// Released on its own card, which need not be the one the destroying thread is on. +RigidBodyGPUEngine::~RigidBodyGPUEngine() { + int current = 0; + cudaGetDevice(¤t); + cudaSetDevice(impl_->device); + impl_.reset(); + cudaSetDevice(current); +} + +int RigidBodyGPUEngine::Device() const { + return impl_->device; +} void RigidBodyGPUEngine::SetBody(const std::vector> &relative) { RigidBodyGPUEngineImpl &e = *impl_; diff --git a/image_analysis/structure_refinement/RigidBodyGPU.h b/image_analysis/structure_refinement/RigidBodyGPU.h index c9c38d855..c2b8cb37f 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.h +++ b/image_analysis/structure_refinement/RigidBodyGPU.h @@ -50,6 +50,12 @@ public: ~RigidBodyGPUPool(); size_t Engines() const { return engines_.size(); } + // The card the pool was made from, which its first engine is on and the others count on from. + int Device() const { return device_; } + // What max_engines engines take, however many fitted, and the memory of the card: the sizes a + // caller decides by whether a second validation may run beside this one. + size_t PlannedBytes() const { return planned_bytes_; } + size_t CardBytes() const { return card_bytes_; } RigidBodyGPUEngine &Acquire(); void Release(RigidBodyGPUEngine &engine); @@ -65,6 +71,9 @@ public: private: RigidBodyGPUPool() = default; + int device_ = 0; + size_t planned_bytes_ = 0; + size_t card_bytes_ = 0; std::vector> engines_; std::vector idle_; std::mutex m_; diff --git a/image_analysis/structure_refinement/RigidBodyGPUEngine.h b/image_analysis/structure_refinement/RigidBodyGPUEngine.h index 9a1919858..8c86b20a6 100644 --- a/image_analysis/structure_refinement/RigidBodyGPUEngine.h +++ b/image_analysis/structure_refinement/RigidBodyGPUEngine.h @@ -98,6 +98,7 @@ public: RigidBodyGPUEngine(const RigidBodyGPUCapacity &capacity, int device); ~RigidBodyGPUEngine(); + int Device() const; // Per fit: each atom's position relative to the model centroid, which the placements rotate. void SetBody(const std::vector> &relative); diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 00dd32f2d..3845d2821 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -3247,14 +3247,55 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { std::vector report_shell_d_min; for (const auto &sh : sm.statistics.shells) report_shell_d_min.push_back(sh.d_min); - const auto validate = [&, cell = *result.consensus_cell, - wavelength = experiment_.GetWavelength_A()](Logger &log) { - return ValidateAgainstModel(sm.merged, cell, config_.model_path, - config_.output_prefix, log, - data_sg ? &*data_sg : nullptr, - /*probe_indexing_ambiguity=*/config_.reference_data.empty(), - static_cast(config_.nthreads), - wavelength, report_shell_d_min); + // The validation in the model's setting further down depends on this one only through the + // setting and the indexing it settles, and both are forecast well before this one has finished + // (ModelFrameForecast). So it is started on that forecast, beside this one - on the next card + // where there is one - and kept only where the decision is the one forecast; its log is held, + // and its files wait, until then. On any other decision it is dropped, unwritten. The relabelling + // it runs on is the one AdoptModelFrame and relabel_output below make, made here on a copy. + // A model asserting the other enantiomorph is not forecast: whether the label is taken is + // only known at the end, from the anomalous map. + struct Ahead { + gemmi::Op change_of_basis_op, indexing_op; + Logger log = Logger::Buffered(); + std::future run; + std::promise write; // destroyed before `run`, so a run left waiting on it is released + } ahead; + ModelValidationSchedule first_schedule; + first_schedule.on_forecast = [&](const ModelFrameForecast &f) { + if (ahead.run.valid() || !config_.reference_data.empty() || !data_sg.has_value() + || !f.may_run_beside || f.enantiomorph_candidate + || f.change_of_basis_op == gemmi::Op::identity()) + return; + ahead.change_of_basis_op = f.change_of_basis_op; + ahead.indexing_op = f.indexing_op; + ahead.run = std::async(std::launch::async, + [&, merged = sm.merged, sg = experiment_.GetSpaceGroupOrP1(), cell = *result.consensus_cell, + friedel = experiment_.GetScalingSettings().GetMergeFriedel(), + rfree_fraction = experiment_.GetScalingSettings().GetRfreeFraction(), + wavelength = experiment_.GetWavelength_A(), + gate = ahead.write.get_future().share()]() mutable { + if (const int32_t cards = get_gpu_count(); cards > 0) + pin_gpu(1 % cards); + if (!(ahead.indexing_op == gemmi::Op::identity())) + merged = ReindexMergedIntoAsu(merged, ahead.indexing_op, sg, friedel); + gemmi::Op to_model = ahead.change_of_basis_op; + to_model.tran = {0, 0, 0}; + const gemmi::Op cob = to_model.inverse(); + const gemmi::SpaceGroup *in = &sg; + if (const gemmi::SpaceGroup *moved = SpaceGroupInBasis(sg, cob)) { + merged = ReindexMergedIntoAsu(merged, HklOperator(cob), *moved, friedel); + cell = CellInBasis(cell, cob); + AssignRfreeFlags(merged, *moved, rfree_fraction, 500, cell, config_.nthreads); + in = moved; + } + ModelValidationSchedule schedule; + schedule.write_gate = gate; + return ValidateAgainstModel(merged, cell, config_.model_path, config_.output_prefix, ahead.log, + in, /*probe_indexing_ambiguity=*/false, + static_cast(config_.nthreads), wavelength, + report_shell_d_min, schedule); + }); }; // The ledger's P1 merge (started above) is taken before anything below acts on what the // validation decides: relabelling the outcomes and the group is what it must not see happen. @@ -3264,7 +3305,12 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // validation decides, from the card's total, whether its GPU path fits. if (!ledger_ahead.valid()) rsm.reset(); - ModelValidationResult validation = validate(logger); + ModelValidationResult validation = + ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, + config_.output_prefix, logger, data_sg ? &*data_sg : nullptr, + /*probe_indexing_ambiguity=*/config_.reference_data.empty(), + static_cast(config_.nthreads), experiment_.GetWavelength_A(), + report_shell_d_min, first_schedule); take_ledger_p1(); // A model that was asked for and could not be used has to say so where anyone will see // it. Without this the run ends successfully with no R-free, no maps and nothing in the @@ -3320,18 +3366,33 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // validation is then made again on the relabelled data, so the maps and the placed model // come out on the same axes as the reflections; what it had decided is kept, since the // second one, facing a model already in the data's setting, has nothing left to decide. - if (config_.reference_data.empty() && validation.model_fits - && !(validation.change_of_basis_op == gemmi::Op::identity())) { + const bool remake = config_.reference_data.empty() && validation.model_fits + && !(validation.change_of_basis_op == gemmi::Op::identity()); + const bool forecast_held = remake && ahead.run.valid() && !validation.adopted_model_enantiomorph + && validation.change_of_basis_op == ahead.change_of_basis_op + && validation.indexing_op == ahead.indexing_op; + if (ahead.run.valid() && !forecast_held) { + ahead.write.set_value(false); + ahead.run.wait(); + } + if (remake) { gemmi::Op to_model = validation.change_of_basis_op; to_model.tran = {0, 0, 0}; relabel_output(to_model.inverse(), "MODEL"); const auto sg_now = experiment_.GetGemmiSpaceGroup(); - auto remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, - config_.output_prefix, logger, - sg_now ? &*sg_now : nullptr, - /*probe_indexing_ambiguity=*/false, - static_cast(config_.nthreads), - experiment_.GetWavelength_A(), report_shell_d_min); + ModelValidationResult remade; + if (forecast_held) { + ahead.write.set_value(true); + remade = ahead.run.get(); + ahead.log.ReplayInto(logger); + } else { + remade = ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, + config_.output_prefix, logger, + sg_now ? &*sg_now : nullptr, + /*probe_indexing_ambiguity=*/false, + static_cast(config_.nthreads), + experiment_.GetWavelength_A(), report_shell_d_min); + } if (remade.ok) { KeepModelVerdict(remade, validation); validation = std::move(remade);