From 21845e91f5bac4bac7ff6831ba0d97c1a797f3bf Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 23:01:29 +0100 Subject: [PATCH] Pre-scan tail: the beam stop's sector medians in parallel, the pass's mapping built beside the masks The polarization plane (measured_plane in ShadowFinder) put the 18 M pixels of a 16M detector into their sectors on one thread - an atan2, a ring position and a push_back per pixel - and pooled them over a box with two serial pixel loops. The pixels now go into their sectors by blocks in parallel, through the same counting scatter the rings use (bin_by_ring, with the sector as the key), and the pooling loops are ParallelChunks. A sector's median is an order statistic of its values, so neither the order they come in nor the full sort bin_by_ring does (nth_element before) can change it. The pass after the pre-scan builds its azimuthal mapping at a geometry that is final once the beam stop has decided the polarization plane: the shadow and defective-pixel bits added after it do not enter what the geometry cache keeps (AzimuthalIntegrationMapping applies the mask to its copy). So that mapping is now built into the run's geometry cache there, beside the defective-pixel pass (Rugnux::MappingsAhead), and the pass finds it in the cache - waiting under the cache's lock where it is still being built. Offline the pre-scan's own spot measurement has built it already and this is a cache hit; live (no spot measurement) it was built in the gap before the first pass. Thau, offline (-N 28 -e 1000 --model 5a47), "pre-scan beam stop" 1.33/1.16 s -> 0.77/0.63 s; cytc 1.21/1.12 -> 0.79/0.59 s; lyso_x06da_ref 0.20/0.21 -> 0.13/0.14 s. The offline pre-scan step does not move: the spot measurement beside it is the longer branch. Live replay (thau, 1000 EIGER 16M frames at 100 Hz, idx-simplon harness, the quiet runs): from the main line's last pre-scan frame read to its first pass 2.58-2.63 s -> 1.85-1.94 s (beam stop 1.25-1.31 -> 0.70; the gap after the defective pixels 0.67-0.68 -> 0.44-0.51); the probe's beam stop 1.29-1.31 -> 0.80-0.82 s. Outputs identical to the base binary (thau, cytc, lyso offline; MTZ data, maps, models, blobs and the live status live). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- image_analysis/beam_stop/ShadowFinder.cpp | 67 ++++++++++++++--------- rugnux/Rugnux.h | 19 +++++++ rugnux/RugnuxPreScan.cpp | 3 + 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/image_analysis/beam_stop/ShadowFinder.cpp b/image_analysis/beam_stop/ShadowFinder.cpp index aa8fbc4c9..21774e2a4 100644 --- a/image_analysis/beam_stop/ShadowFinder.cpp +++ b/image_analysis/beam_stop/ShadowFinder.cpp @@ -442,15 +442,19 @@ PlaneEvidence measured_plane(const std::vector &mean, int W, int H, float { Plane num(n_pixels); Plane den(n_pixels); - for (int i = 0; i < n_pixels; i++) { - num[i] = valid[i] ? value[i] : 0.0; - den[i] = valid[i] ? 1 : 0; - } + ParallelChunks(n_pixels, nthreads, [&](int lo, int hi) { + for (int i = lo; i < hi; i++) { + num[i] = valid[i] ? value[i] : 0.0; + den[i] = valid[i] ? 1 : 0; + } + }); const auto pooled_sum = box_sum(num, W, H, PLANE_POOL_PX, nthreads); const auto pooled_count = box_sum(den, W, H, PLANE_POOL_PX, nthreads); - for (int i = 0; i < n_pixels; i++) - if (valid[i]) - value[i] = static_cast(pooled_sum[i] / pooled_count[i]); + ParallelChunks(n_pixels, nthreads, [&](int lo, int hi) { + for (int i = lo; i < hi; i++) + if (valid[i]) + value[i] = static_cast(pooled_sum[i] / pooled_count[i]); + }); } // Each ring is taken relative to its upper decile rather than its median: where a shadow covers @@ -463,27 +467,38 @@ PlaneEvidence measured_plane(const std::vector &mean, int W, int H, float level[r] = rings.values[rings.offset[r] + n * 9 / 10]; } - // The median of each sector of each band, relative to its rings. + // The median of each sector of each band, relative to its rings. The pixels are put in their sectors in + // parallel, the same way as in their rings (bin_by_ring, with the sector as the key); a median does not + // depend on the order the values come in. const size_t n_sectors = static_cast(n_bands) * HARMONIC_SECTORS; - std::vector> sector_values(n_sectors); - for (int i = 0; i < n_pixels; i++) { - if (!valid[i] || level[radius[i]] <= 0.0f) - continue; - const RingPosition ring = ring_position(det_matrix, distance_px, static_cast(i % W) - centre_x, - static_cast(i / W) - centre_y); - const double phi = std::atan2(ring.ly, ring.lx) + std::numbers::pi; - const int sector = std::min(HARMONIC_SECTORS - 1, static_cast(phi / (2.0 * std::numbers::pi) * HARMONIC_SECTORS)); - sector_values[static_cast(radius[i] / HARMONIC_BAND_PX) * HARMONIC_SECTORS + sector] - .push_back(value[i] / level[radius[i]]); - } - std::vector sector_median(n_sectors, -1.0); - ParallelFor(static_cast(n_sectors), nthreads, [&](int k) { - auto &v = sector_values[k]; - if (v.size() >= MIN_SECTOR_PIXELS) { - std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end()); - sector_median[k] = v[v.size() / 2]; - } + Plane ratio(n_pixels); + Plane in_sector(n_pixels); + Plane sector_of(n_pixels); + ParallelChunks(H, nthreads, [&](int ylo, int yhi) { + for (int y = ylo; y < yhi; y++) + for (int x = 0; x < W; x++) { + const int i = y * W + x; + in_sector[i] = valid[i] && level[radius[i]] > 0.0f; + ratio[i] = 0.0f; + sector_of[i] = 0; + if (!in_sector[i]) + continue; + const RingPosition ring = ring_position(det_matrix, distance_px, static_cast(x) - centre_x, + static_cast(y) - centre_y); + const double phi = std::atan2(ring.ly, ring.lx) + std::numbers::pi; + const int sector = std::min(HARMONIC_SECTORS - 1, + static_cast(phi / (2.0 * std::numbers::pi) * HARMONIC_SECTORS)); + sector_of[i] = (radius[i] / HARMONIC_BAND_PX) * HARMONIC_SECTORS + sector; + ratio[i] = value[i] / level[radius[i]]; + } }); + const RingValues sectors = bin_by_ring(ratio, in_sector, sector_of, static_cast(n_sectors) - 1, nthreads); + std::vector sector_median(n_sectors, -1.0); + for (size_t k = 0; k < n_sectors; k++) { + const int n = sectors.offset[k + 1] - sectors.offset[k]; + if (n >= MIN_SECTOR_PIXELS) + sector_median[k] = sectors.values[sectors.offset[k] + n / 2]; + } // The modulation a fully polarized source gives each band, at the 2 theta of its centre. std::vector amplitude(n_bands); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index b062da167..a1e4fe70d 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -1567,6 +1567,25 @@ class Rugnux { // not evaluate it again. Shared with the copies the run makes of itself. std::shared_ptr azint_geometry_ = std::make_shared(); + // Mappings built into azint_geometry_ ahead of the step that reads them, as soon as their geometry is known: + // the step then finds the geometry there - or waits for it, under the cache's lock - instead of building it + // on its own path. The mask does not enter what is kept (the cache keeps the geometry alone). Waited for + // when the last copy of the run goes. + struct MappingsAhead { + std::mutex m; + std::vector> built; + void Build(DiffractionExperiment x, PixelMask mask, std::shared_ptr cache, + int nthreads) { + if (!x.IsGeometryTransformed()) // only a converted geometry is kept (AzimuthalIntegrationMapping) + return; + std::lock_guard lock(m); + built.push_back(std::async(std::launch::async, [x = std::move(x), mask = std::move(mask), + cache = std::move(cache), nthreads] { + AzimuthalIntegrationMapping(x, mask, *cache, static_cast(std::max(nthreads, 1))); + })); + } + }; + std::shared_ptr mappings_ahead_ = std::make_shared(); [[nodiscard]] std::vector SpotFindingKey(const AzimuthalIntegrationMapping &mapping) const { return SpotFindingKey(experiment_, mapping); } diff --git a/rugnux/RugnuxPreScan.cpp b/rugnux/RugnuxPreScan.cpp index 4b74efcda..f11216aaf 100644 --- a/rugnux/RugnuxPreScan.cpp +++ b/rugnux/RugnuxPreScan.cpp @@ -1111,6 +1111,9 @@ void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, Ru experiment_.PolarizationFactor(polarization); polarization_from_background_ = polarization; } + // The geometry of the pass after the pre-scan is final here (the masks added below do not enter its + // mapping's geometry), so its mapping is built now, beside them. + mappings_ahead_->Build(experiment_, pixel_mask_, azint_geometry_, config_.nthreads); // A pixel whose mean is negative holds the detector's own mark for no data - a module gap a PILATUS // CBF leaves at -1 - and is the darkest pixel there is, so the shadow finder reports it. It is dead // rather than shadowed, and goes to the defective pixels (after MaskDefectivePixels below), so that