From 2cd3da67be95e23be751baf507e7e725a582d436 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 23:55:35 +0100 Subject: [PATCH] Speed-up follow-ups: card memory, carried long-axis answer, CPU twin arms, live host reserve - Long-axis speculation: the re-indexes of the offered cells ran at once, one wide FFT pool each, beside the measured-centre spot finding; with two offers on an 8 GB card the latter's allocations could fail where they used to fit. They now run one after the other, each releasing its indexers when done, so no more wide pools are alive at once than the hypothesis itself builds. - Long-axis answer carried to later passes: where the beam-centre check asked at both centres, the run recorded the adoption of whichever arm adopted, even if the other centre won. Each arm's adoption is kept and only the one on the lattice the pass ends on is carried. - Twinned arm beside the untwinned one: with the untwinned arm on the CPU (CPU-only build, or a model too big for the card) nothing is held on a card and the memory gate let two full CPU refinements run at once. It now waits in that case. - Broker model warm-up: a CUDA error while loading the refinement modules skipped the whole validation warm-up; it is now caught, cleared and logged. - Live host budget: nine tenths of the machine, not all of it, is shared between the image buffer, the broker's margin and rugnux, leaving a tenth to the system and other services on the host. - First pass: the measured-centre speculation starts before the coarse long-axis search, since it is on the critical path with two or more cards. Results unchanged: thau, lyso and cytc reports, .hkl and refined models identical to rc184-port, also with every pinned host buffer filled with 0xA5 on allocation (reused pool pieces carry no stale data into results); the ~1000 A test sweep identical to the run without these fixes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- .../structure_refinement/ModelValidation.cpp | 11 ++++++++--- .../structure_refinement/Refinement.cpp | 4 +++- receiver/JFJochReceiverRugnux.cpp | 6 ++++-- rugnux/RugnuxFirstPass.cpp | 12 +++++++++--- rugnux/RugnuxFirstPass.h | 4 ++-- rugnux/RugnuxFirstPassRescues.cpp | 15 +++++++++++++-- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index 1f8913118..9354f8053 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -2192,9 +2192,14 @@ void WarmUpModelValidation(std::shared_ptr model, size_t nthr // The refinement's kernels that a validation does not launch, on every card: lazily their modules would be // loaded at their first launch, in the refinement, from the executable - 1-3 s in a live run where its // pages had left the page cache and the disk was busy. - for (int d = 0; d < get_gpu_count(); d++) { - const ScopedGpuPin pin(d); - RefinementGPUEngine::LoadModules(); + try { + for (int d = 0; d < get_gpu_count(); d++) { + const ScopedGpuPin pin(d); + RefinementGPUEngine::LoadModules(); + } + } catch (const std::exception &e) { + cuda_clear_error(); + logger.Warning("Refinement modules not loaded ahead: {}", e.what()); } Logger quiet = Logger::Buffered(); if (const SearchModel search = PrepareSearchModel(*model); search.failure_reason.empty() && search.predicted) { diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 7e8034a1e..7cba1a50e 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3584,7 +3584,9 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector budget) + // Nothing held on a card: the untwinned arm is on the CPU, where a second arm only shares its threads. + const size_t held_on_card = held_bytes.get(); + if (held_on_card == 0 || 2 * held_on_card > budget) untwinned.wait(); DecideTwin(placed, data, cell, sg, wavelength_A, twin_logger, settings, result, untwinned); }); diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index 7cc1ca12a..383da3146 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -30,12 +30,14 @@ namespace { // machine, and a margin - at most the machine's budget. Both come off the machine's total, not off the // budget: the budget's own fraction is the reserve for the rest of the machine, which the margin already // keeps for the broker, and taken off twice it left nothing (1 byte) with a buffer sized to most of the - // machine. Never 0, which would take the whole machine's budget: at worst what could run side by side + // machine. A tenth of the machine stays out of it for the system and the other services on the host (a + // writer, for one). Never 0, which would take the whole machine's budget: at worst what could run side by side // runs one part after the other. size_t RugnuxHostBudget(const ImageBuffer &buffer) { const size_t total = get_host_total_memory(); const size_t taken = buffer.GetBufferSize() + RUGNUX_HOST_MEMORY_MARGIN; - return total > taken ? std::min(DefaultHostMemoryBudget(), total - taken) : 1; + const size_t usable = total / 10 * 9; + return usable > taken ? std::min(DefaultHostMemoryBudget(), usable - taken) : 1; } double SecondsSince(std::chrono::steady_clock::time_point t) { diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index f63d8eb7a..85c0fae1c 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -491,8 +491,14 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { if (!long_axis_asked) best = run.LongAxisHypothesis(std::move(best)); // What later passes of the run carry (Rugnux::long_axis_answer_). - if (run.long_axis_asked_here || run.long_axis_adopted) - long_axis_answer_ = LongAxisAnswer{run.long_axis_adopted}; + // Where the beam-centre check asked at both centres, the adoption carried is the one the pass ended on. + if (run.long_axis_asked_here || !run.long_axis_adopted.empty()) { + LongAxisAnswer answer; + for (const auto &a : run.long_axis_adopted) + if (best.result && SameLattice(a, *best.result)) + answer.adopted = a.lattice.GetUnitCell(); + long_axis_answer_ = answer; + } // The end wedge, at the geometry the rescues settled on. best = run.AddEndWedge(std::move(best), *p.indexer_pool, *p.indexer); run.ReleaseSpotEngines(); // no more frames to read; give the cards their memory back @@ -1521,8 +1527,8 @@ void Rugnux::FirstPassRun::StartCentreSpeculation() { auto Rugnux::FirstPassRun::IndexSchemes() -> FirstPass { return PickBest(*indexer_pool, *indexer, [&] { + StartCentreSpeculation(); // on the critical path with two or more cards; the coarse search is not StartLongAxisCoarse(); - StartCentreSpeculation(); }); } diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index 5dcf13326..42183ca8b 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -356,7 +356,7 @@ struct Rugnux::FirstPassRun { std::unique_ptr experiment; std::unique_ptr own_pool; // none where the run's pool is wide enough SchemeIndexers ris; - std::future done; // last, so it is waited for before the rest goes + std::shared_future done; // last, so it is waited for before the rest goes }; std::vector long_axis_spec; // The experiment the coarse search was started on. @@ -367,5 +367,5 @@ struct Rugnux::FirstPassRun { // Whether this pass asked the long-axis question itself, and the cell it adopted (see // Rugnux::long_axis_answer_). bool long_axis_asked_here = false; - std::optional long_axis_adopted; + std::vector long_axis_adopted; // what each centre asked here adopted }; diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index 6e89db80d..1c3cca12b 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -197,7 +197,18 @@ void Rugnux::FirstPassRun::StartLongAxisSearch(std::unique_ptr ris; for (const auto &ri : s.ris) ris.push_back(ri.get()); - s.done = std::async(std::launch::async, [ris] { RunIndexing(ris); }); + // One after the other, as the hypothesis asks them, each handing its card memory back when done: no more + // wide indexers on a card at once than the hypothesis itself would build. + const std::shared_future before = + long_axis_spec.size() > 1 ? long_axis_spec[long_axis_spec.size() - 2].done : std::shared_future(); + IndexerThreadPool *pool = s.own_pool.get(); + s.done = std::async(std::launch::async, [ris, before, pool] { + if (before.valid()) + before.wait(); + RunIndexing(ris); + if (pool != nullptr) + pool->ReleaseIndexers(); + }).share(); } void Rugnux::FirstPassRun::StartLongAxisCoarse() { @@ -378,7 +389,7 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { choice ? "adopting " + cell_text(*constrained[*choice].result) : std::string("keeping the standing cell"), seconds); if (choice) { - long_axis_adopted = constrained[*choice].result->lattice.GetUnitCell(); + long_axis_adopted.push_back(*constrained[*choice].result); return std::move(constrained[*choice]); } return standing;