From d449a9dfd6047a732beee2282b5bc8fd33087e07 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 10 Oct 2026 23:52:56 +0200 Subject: [PATCH] rugnux pre-scan: spot-width workers spread over every card The pre-scan's spot measurement (spot width, bandwidth, powder rings) built all of its GPU engines on card 0 from unpinned threads. Worker t now runs on card t % gpus with pin_gpu(), as the image-loop workers do, which also binds it to that card's NUMA node where binding is enabled. Every tier runs on fresh threads, so each tier re-pins worker t to the card its engine was built on, and the engines are freed on threads pinned the same way (as the first pass's ReleaseSpotEngines does). Worker count, width tiers (8/15/30/60 on a 60-frame sample) and the frames read are unchanged. A frame's spots do not depend on the card that found them (same model), and the per-frame results are stitched in sample order as before, so the output does not depend on the number of cards. The projection and defective-pixel passes are left on card 0: both fold every frame into a single device accumulator (ShadowAccumulatorGPU, HotPixelFinderGPU) that lives there. Verified on one card: CI-tier battery p.hkl md5 and MTZ reflection data identical to the base binary on every set. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- rugnux/RugnuxPreScan.cpp | 76 +++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/rugnux/RugnuxPreScan.cpp b/rugnux/RugnuxPreScan.cpp index c5020f201..e965f3ca7 100644 --- a/rugnux/RugnuxPreScan.cpp +++ b/rugnux/RugnuxPreScan.cpp @@ -459,6 +459,21 @@ void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, Ru std::vector workers(nworkers); std::vector worker_built(nworkers, 0); + // fn(t) on threads of their own, thread t on card t % gpus as an image-loop worker is: worker + // t's engine is built on that card and must be used, and freed, with it current - and each + // tier runs on fresh threads. Which card finds a frame's spots does not change them. + const auto on_workers = [&](auto &&fn) { + std::vector> futures; + futures.reserve(nworkers); + for (size_t t = 0; t < nworkers; t++) + futures.emplace_back(std::async(std::launch::async, [&, t] { + if (const int32_t gpus = get_gpu_count(); gpus > 0) + pin_gpu(static_cast(t) % gpus); + fn(t); + })); + for (auto &f : futures) f.get(); + }; + bool width_settled = !want_width; // The bandwidth reads the same spots, but its slope needs more of them than the width does: // the tiers carry on past the width's for it alone, until the pool is large enough. @@ -467,40 +482,36 @@ void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, Ru size_t phase_begin = 0; for (const size_t phase_end : tier_end) { std::atomic next{phase_begin}; - std::vector> futures; - futures.reserve(nworkers); - for (size_t t = 0; t < nworkers; t++) - futures.emplace_back(std::async(std::launch::async, [&, t] { - if (!worker_built[t]) { - workers[t] = make_worker(); - worker_built[t] = 1; + on_workers([&](size_t t) { + if (!worker_built[t]) { + workers[t] = make_worker(); + worker_built[t] = 1; + } + PreScanWorker &w = workers[t]; + for (size_t v = next.fetch_add(1); v < phase_end; v = next.fetch_add(1)) { + const size_t i = visit[v]; + const int ordinal = ordinals[i]; + const int image_idx = start_image + ordinal * config_.stride; + const bool for_width = !width_settled; + const bool for_shape = width_settled && !shapes_settled; + if (!(for_width || for_shape)) + continue; // nothing to measure on this frame + bool read = false; + try { + read = reader_.ReadRawImage(image_idx, w.raw_image); + } catch (const std::exception &e) { + if (IsFatalResourceError(e)) throw; + logger.Warning("Pre-scan: failed to load image {}: {}", image_idx, + e.what()); + continue; } - PreScanWorker &w = workers[t]; - for (size_t v = next.fetch_add(1); v < phase_end; v = next.fetch_add(1)) { - const size_t i = visit[v]; - const int ordinal = ordinals[i]; - const int image_idx = start_image + ordinal * config_.stride; - const bool for_width = !width_settled; - const bool for_shape = width_settled && !shapes_settled; - if (!(for_width || for_shape)) - continue; // nothing to measure on this frame - bool read = false; - try { - read = reader_.ReadRawImage(image_idx, w.raw_image); - } catch (const std::exception &e) { - if (IsFatalResourceError(e)) throw; - logger.Warning("Pre-scan: failed to load image {}: {}", image_idx, - e.what()); - continue; - } - if (!read) continue; + if (!read) continue; - if (for_shape) shape_only[i] = 1; - find_spots(w, w.raw_image.image, image_idx, for_width || for_shape, curves_of[i], - spot_q_of[i], footprint_of[i]); - } - })); - for (auto &f : futures) f.get(); + if (for_shape) shape_only[i] = 1; + find_spots(w, w.raw_image.image, image_idx, for_width || for_shape, curves_of[i], + spot_q_of[i], footprint_of[i]); + } + }); phase_begin = phase_end; // What this tier makes of the width, pooled in sample order so the answer does not @@ -522,6 +533,7 @@ void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, Ru shape_curves.insert(shape_curves.end(), c.begin(), c.end()); shapes_settled = width_settled && shape_curves.size() >= spot_width::BANDWIDTH_POOL_SPOTS; } + on_workers([&](size_t t) { PreScanWorker released = std::move(workers[t]); }); for (size_t i = 0; i < ordinals.size(); i++) if (!shape_only[i])