From 9d6a3f1e04cecc4fd2e679ce1609515c5a2c6b89 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 11 Oct 2026 08:09:50 +0200 Subject: [PATCH] rugnux merge engines: count-based device-memory estimate for the all-observation arm beside The gate for merging the space-group search's all-observation arm on a copy of the engine, beside the first arm, judged an engine at a fixed 256 B per observation. It now counts what the ingest knows - observations, rocking events (at most one full each) and distinct hkl (at most one ASU group each) - at 80 / 260 / 200 B, the envelope of the measured per-engine peaks (helpers included: error-model fit, Wilson outlier test, pair sort), plus a fifth, and keeps 3 GB of the card for what else stays resident (context, the run's indexers: measured up to 2.3 GB at the gate with no speculative pass running). Measured per-engine peaks (scratch build tracking each engine's device allocations): 4nwv 7.98 GB, 8a1a 8.15, 8tyy 9.36, 8qaw 13.3 - two engines do not fit a 16 GB card on any of these, so they stay serial. The old constant was not an over-estimate there (fulls cost ~260 B each at the surface fit). What changes: 9 battery sets of 33-53 M observations at high multiplicity (8s38, 8sqt, 7ph1, 9min, 9i80, 9sl0, 9h0q, 8v4o, 6ze4) now merge the arm beside; none goes the other way. Their merge-step VRAM peak is 11.0-14.1 GiB of 15.9, no out-of-memory, no failed speculative probe. Every engine peak measured on 11 sets is at most 81% of the estimate. Output identical (p.hkl md5 and MTZ data, 40 sets incl. ci tier). Gain is small: the two merges share one GPU, so the arm beside slows the main P1 merge by ~0.5 s while saving ~0.7-1 s of wait at the search; alternating A/B on 5 of the sets: -1.1 s output pass in total. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- .../scale_merge/RotationScaleMerge.cpp | 12 ++++++++++++ image_analysis/scale_merge/RotationScaleMerge.h | 17 ++++++++--------- rugnux/RugnuxScaleMerge.cpp | 12 +++++++----- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index dcec6a2ba..e89f95803 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -706,6 +706,18 @@ void RotationScaleMerge::StageObservations() { } #endif +size_t RotationScaleMerge::DeviceBytes() const { + // What each costs at the peak, measured: an observation's arrays and its group CSR; a full's arrays + // with the surface terms, or with the outlier test's and the error model's samples; an hkl's raw run + // and its ASU group's sums. The events are counted on the device path's ingest; elsewhere every + // observation may be an event of its own. + constexpr size_t BYTES_PER_OBS = 80, BYTES_PER_EVENT = 260, BYTES_PER_HKL = 200; + const auto events = static_cast(resident_ingest ? rocking_events_at_ingest : n_partials_obs); + const size_t bytes = BYTES_PER_OBS * static_cast(n_partials_obs) + BYTES_PER_EVENT * events + + BYTES_PER_HKL * rawrun_start.size(); + return bytes + bytes / 5; +} + std::unique_ptr RotationScaleMerge::CloneIngest(int device, Logger &log) const { TimingMark mark(*logger, "rsm ingest copy"); std::unique_ptr c(new RotationScaleMerge(*this)); diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index ae68d4560..9d1ac125e 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -162,15 +162,14 @@ public: // The copy does not write its per-frame scales back (WriteBackPerFrameScale does). [[nodiscard]] std::unique_ptr CloneIngest(int device, Logger &log) const; - // The partial observations the ingest kept. - [[nodiscard]] int64_t ObservationCount() const { return n_partials_obs; } - - // The card memory an engine may hold during a search merge, per observation the ingest kept: what - // a caller goes by when it decides whether a second engine fits beside one already on the card. - // Between what two engines merging side by side were measured at, ~150 bytes per observation each - // (20.6M observations, 4.2M fulls), and the layout's worst case of ~280 (71 bytes per observation, - // 83 per full, ~130 per ASU group, with a full and a group for every observation). - static constexpr size_t DEVICE_BYTES_PER_OBSERVATION = 256; + // The card memory an engine may hold during a merge, the device helpers its merge calls included: + // what a caller goes by when it decides whether a second engine fits beside one already on the + // card. Call after IngestHost(). It counts what the ingest knows - the observations, the rocking + // events (the combine makes at most one full of each) and the distinct hkl (at most one ASU group + // each) - at what each costs at the engine's peak (the correction-surface terms, or the merge with + // its outlier test), plus a fifth. Measured on 11 data sets of 13-123 M observations, every + // engine's peak was at most 81% of this. + [[nodiscard]] size_t DeviceBytes() const; // Merge only the first n frames of the outcomes (by default all of them). Set before Ingest(). void SetFrameLimit(int n) { frame_limit = n; } diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 3e3a63028..fb4d8f8a9 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -1033,15 +1033,17 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // and the speculative passes). Otherwise on the last card, beside the engine already there - // this one, or the guard's, which merges at most as many observations - and only where two // engines of this size fit the card's memory, judged on its total, not on what happens to be - // free: the answer must not depend on the moment. On one card that is the small and middling - // data sets; the large ones merge their arms one after the other. + // free: the answer must not depend on the moment. What else stays on the card meanwhile - the + // context, the run's indexers - was measured at up to 2.3 GB; a speculative pass running beside + // holds more, but a merge that does not fit beside it waits for it to end. On a 16 GB card that + // is the data sets up to about 50 M observations at high multiplicity; the large ones merge + // their arms one after the other. const int32_t cards = get_gpu_count(); if (!geometry_prepass && !experiment_.GetGemmiSpaceGroup().has_value() && rsm->GetSearchMinZeta() > 0.0 && cards > 0) { const int32_t device = std::min(cards - 1, 2); - const size_t engine_bytes = static_cast(rsm->ObservationCount()) - * RotationScaleMerge::DEVICE_BYTES_PER_OBSERVATION; - if (cards >= 3 || 2 * engine_bytes <= get_gpu_total_memory(device)) { + constexpr size_t CARD_BYTES_RESERVED = size_t(3) << 30; + if (cards >= 3 || 2 * rsm->DeviceBytes() + CARD_BYTES_RESERVED <= get_gpu_total_memory(device)) { rsm_all = rsm->CloneIngest(device, all_arm_log); rsm_all->SetSearchMinZeta(0.0); rsm_all->SetExportScaledFulls(false); // as scale_and_merge sets them for a search merge