diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index 1f8913118..9354f8053 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -2192,9 +2192,14 @@ void WarmUpModelValidation(std::shared_ptr model, size_t nthr // The refinement's kernels that a validation does not launch, on every card: lazily their modules would be // loaded at their first launch, in the refinement, from the executable - 1-3 s in a live run where its // pages had left the page cache and the disk was busy. - for (int d = 0; d < get_gpu_count(); d++) { - const ScopedGpuPin pin(d); - RefinementGPUEngine::LoadModules(); + try { + for (int d = 0; d < get_gpu_count(); d++) { + const ScopedGpuPin pin(d); + RefinementGPUEngine::LoadModules(); + } + } catch (const std::exception &e) { + cuda_clear_error(); + logger.Warning("Refinement modules not loaded ahead: {}", e.what()); } Logger quiet = Logger::Buffered(); if (const SearchModel search = PrepareSearchModel(*model); search.failure_reason.empty() && search.predicted) { diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 7e8034a1e..7cba1a50e 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3584,7 +3584,9 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector budget) + // Nothing held on a card: the untwinned arm is on the CPU, where a second arm only shares its threads. + const size_t held_on_card = held_bytes.get(); + if (held_on_card == 0 || 2 * held_on_card > budget) untwinned.wait(); DecideTwin(placed, data, cell, sg, wavelength_A, twin_logger, settings, result, untwinned); }); diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index 7cc1ca12a..383da3146 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -30,12 +30,14 @@ namespace { // machine, and a margin - at most the machine's budget. Both come off the machine's total, not off the // budget: the budget's own fraction is the reserve for the rest of the machine, which the margin already // keeps for the broker, and taken off twice it left nothing (1 byte) with a buffer sized to most of the - // machine. Never 0, which would take the whole machine's budget: at worst what could run side by side + // machine. A tenth of the machine stays out of it for the system and the other services on the host (a + // writer, for one). Never 0, which would take the whole machine's budget: at worst what could run side by side // runs one part after the other. size_t RugnuxHostBudget(const ImageBuffer &buffer) { const size_t total = get_host_total_memory(); const size_t taken = buffer.GetBufferSize() + RUGNUX_HOST_MEMORY_MARGIN; - return total > taken ? std::min(DefaultHostMemoryBudget(), total - taken) : 1; + const size_t usable = total / 10 * 9; + return usable > taken ? std::min(DefaultHostMemoryBudget(), usable - taken) : 1; } double SecondsSince(std::chrono::steady_clock::time_point t) { diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index f63d8eb7a..85c0fae1c 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -491,8 +491,14 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { if (!long_axis_asked) best = run.LongAxisHypothesis(std::move(best)); // What later passes of the run carry (Rugnux::long_axis_answer_). - if (run.long_axis_asked_here || run.long_axis_adopted) - long_axis_answer_ = LongAxisAnswer{run.long_axis_adopted}; + // Where the beam-centre check asked at both centres, the adoption carried is the one the pass ended on. + if (run.long_axis_asked_here || !run.long_axis_adopted.empty()) { + LongAxisAnswer answer; + for (const auto &a : run.long_axis_adopted) + if (best.result && SameLattice(a, *best.result)) + answer.adopted = a.lattice.GetUnitCell(); + long_axis_answer_ = answer; + } // The end wedge, at the geometry the rescues settled on. best = run.AddEndWedge(std::move(best), *p.indexer_pool, *p.indexer); run.ReleaseSpotEngines(); // no more frames to read; give the cards their memory back @@ -1521,8 +1527,8 @@ void Rugnux::FirstPassRun::StartCentreSpeculation() { auto Rugnux::FirstPassRun::IndexSchemes() -> FirstPass { return PickBest(*indexer_pool, *indexer, [&] { + StartCentreSpeculation(); // on the critical path with two or more cards; the coarse search is not StartLongAxisCoarse(); - StartCentreSpeculation(); }); } diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index 5dcf13326..42183ca8b 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -356,7 +356,7 @@ struct Rugnux::FirstPassRun { std::unique_ptr experiment; std::unique_ptr own_pool; // none where the run's pool is wide enough SchemeIndexers ris; - std::future done; // last, so it is waited for before the rest goes + std::shared_future done; // last, so it is waited for before the rest goes }; std::vector long_axis_spec; // The experiment the coarse search was started on. @@ -367,5 +367,5 @@ struct Rugnux::FirstPassRun { // Whether this pass asked the long-axis question itself, and the cell it adopted (see // Rugnux::long_axis_answer_). bool long_axis_asked_here = false; - std::optional long_axis_adopted; + std::vector long_axis_adopted; // what each centre asked here adopted }; diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index 6e89db80d..1c3cca12b 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -197,7 +197,18 @@ void Rugnux::FirstPassRun::StartLongAxisSearch(std::unique_ptr ris; for (const auto &ri : s.ris) ris.push_back(ri.get()); - s.done = std::async(std::launch::async, [ris] { RunIndexing(ris); }); + // One after the other, as the hypothesis asks them, each handing its card memory back when done: no more + // wide indexers on a card at once than the hypothesis itself would build. + const std::shared_future before = + long_axis_spec.size() > 1 ? long_axis_spec[long_axis_spec.size() - 2].done : std::shared_future(); + IndexerThreadPool *pool = s.own_pool.get(); + s.done = std::async(std::launch::async, [ris, before, pool] { + if (before.valid()) + before.wait(); + RunIndexing(ris); + if (pool != nullptr) + pool->ReleaseIndexers(); + }).share(); } void Rugnux::FirstPassRun::StartLongAxisCoarse() { @@ -378,7 +389,7 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { choice ? "adopting " + cell_text(*constrained[*choice].result) : std::string("keeping the standing cell"), seconds); if (choice) { - long_axis_adopted = constrained[*choice].result->lattice.GetUnitCell(); + long_axis_adopted.push_back(*constrained[*choice].result); return std::move(constrained[*choice]); } return standing;