diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 6d48a4b82..b21475948 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -586,18 +586,24 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, // caller's callback binds it to the right image (GPU-resident buffer, host buffer, or the assembled // FPGA image read straight on the CPU). auto integration_start_time = std::chrono::steady_clock::now(); - i_outcome.reflections = integrate(prediction.GetReflections(), nrefl, msg.number); - msg.integrated_reflections = i_outcome.reflections.size(); + const std::vector integrated = integrate(prediction.GetReflections(), nrefl, msg.number); + msg.integrated_reflections = integrated.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); - CalcISigmaAndWilsonBFactor(msg, i_outcome.reflections); + CalcISigmaAndWilsonBFactor(msg, integrated); + + // A retained outcome is kept in the pass's arena (see ReflectionArena), not beside the worker's + // other allocations. + i_outcome.reflections = ReflectionVector(integrated.begin(), integrated.end(), + ArenaAllocator(retain_outcomes_ ? &reflection_arena + : nullptr)); ScaleImage(msg, i_outcome); // Copy reflections to outgoing message if (keep_reflections_in_message_) - msg.reflections = i_outcome.reflections; + msg.reflections.assign(i_outcome.reflections.begin(), i_outcome.reflections.end()); // Persist the per-image result for the whole-run scaling/merge pass, unless the caller opted out // (viewer interactive use only needs the current image, returned above via msg). diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index 0fb0d1fdd..56904f78e 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -62,6 +62,8 @@ class IndexAndRefine { }; mutable std::mutex reflections_mutex; + // Holds the retained outcomes' reflections; declared before them, so it outlives them. + ReflectionArena reflection_arena; std::vector integration_outcome; std::vector mosaicity; // Optional per-frame mosaicity used for Bragg prediction, indexed by image number. When set (the diff --git a/image_analysis/IntegrationOutcome.h b/image_analysis/IntegrationOutcome.h index c0d8855fa..a16f41957 100644 --- a/image_analysis/IntegrationOutcome.h +++ b/image_analysis/IntegrationOutcome.h @@ -8,11 +8,16 @@ #include "../common/Reflection.h" #include "../common/CrystalLattice.h" #include "../common/DiffractionGeometry.h" +#include "ReflectionArena.h" + +// A frame's integrated reflections. The whole-run passes keep them in a ReflectionArena (see there); +// anywhere else the allocator is plain new/delete. +using ReflectionVector = std::vector>; struct IntegrationOutcome { DiffractionGeometry geom; CrystalLattice latt; - std::vector reflections; + ReflectionVector reflections; std::optional mosaicity_deg; std::optional image_scale_cc; std::optional image_scale_cc_n; diff --git a/image_analysis/ReflectionArena.h b/image_analysis/ReflectionArena.h new file mode 100644 index 000000000..193dc71b7 --- /dev/null +++ b/image_analysis/ReflectionArena.h @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include +#include +#include + +// Large blocks that a whole pass's per-image reflection vectors are carved from. +// +// A rotation pass keeps every frame's integrated reflections until its scaling is done - thousands of +// vectors of a few megabytes each, gigabytes together, allocated by the image workers in the +// allocator's per-thread arenas. When the pass hands them back, that memory mostly stays in those +// arenas, as holes between whatever else the workers allocated, and the next pass's workers, being +// new threads, do not reuse it: on a fine-sliced long axis gigabytes of freed reflections were carried +// to the end of the run. Here a block is allocated in one piece (large enough that it is its own +// mapping) and returned in one piece once the last vector in it is gone, so a pass's reflections +// leave nothing behind. +// +// A vector is carved from the current block by bumping a pointer; the space of a vector freed early +// is only reclaimed with its whole block. That suits the use here: the vectors are written once, +// when the frame is integrated, and handed back together. +class ReflectionArena { +public: + ReflectionArena() = default; + ReflectionArena(const ReflectionArena &) = delete; + ReflectionArena &operator=(const ReflectionArena &) = delete; + ~ReflectionArena() { + for (auto &b : blocks) + ::operator delete(b.data); + } + + void *Allocate(size_t bytes) { + bytes = (bytes + kAlign - 1) / kAlign * kAlign; + std::unique_lock ul(m); + if (blocks.empty() || blocks.back().size - blocks.back().used < bytes) { + const size_t size = std::max(kBlockBytes, bytes); + blocks.push_back(Block{static_cast(::operator new(size)), size, 0, 0}); + } + Block &b = blocks.back(); + void *p = b.data + b.used; + b.used += bytes; + b.live++; + return p; + } + + void Deallocate(void *p) { + std::unique_lock ul(m); + for (size_t i = 0; i < blocks.size(); ++i) { + Block &b = blocks[i]; + if (static_cast(p) < b.data || static_cast(p) >= b.data + b.size) + continue; + if (--b.live == 0) { + if (i + 1 == blocks.size()) { + b.used = 0; // the block being filled: keep it for the next vector + } else { + ::operator delete(b.data); + blocks.erase(blocks.begin() + static_cast(i)); + } + } + return; + } + } + +private: + // 64 MiB: above the largest size glibc ever serves from its heaps (32 MiB), so every block is a + // mapping of its own and goes back to the system when freed. A frame's reflections are a few + // megabytes, so the tail a block cannot fit is a few percent of it. + static constexpr size_t kBlockBytes = size_t(64) << 20; + static constexpr size_t kAlign = alignof(std::max_align_t); + struct Block { + char *data; + size_t size, used, live; + }; + std::mutex m; + std::vector blocks; +}; + +// A std::allocator that carves from a ReflectionArena, or is plain new/delete without one. A copy of +// a container is made on the heap (select_on_container_copy_construction), so only the containers the +// arena's owner fills itself live in the arena; a move carries the arena along with the storage. +template +struct ArenaAllocator { + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + + ReflectionArena *arena = nullptr; + + ArenaAllocator() = default; + explicit ArenaAllocator(ReflectionArena *a) : arena(a) {} + template ArenaAllocator(const ArenaAllocator &o) : arena(o.arena) {} + + T *allocate(size_t n) { + if (arena) + return static_cast(arena->Allocate(n * sizeof(T))); + return static_cast(::operator new(n * sizeof(T))); + } + void deallocate(T *p, size_t) { + if (arena) + arena->Deallocate(p); + else + ::operator delete(p); + } + ArenaAllocator select_on_container_copy_construction() const { return ArenaAllocator(); } + + template bool operator==(const ArenaAllocator &o) const { return arena == o.arena; } + template bool operator!=(const ArenaAllocator &o) const { return arena != o.arena; } +}; diff --git a/image_analysis/geom_refinement/PostRefine.cpp b/image_analysis/geom_refinement/PostRefine.cpp index fee879231..d2c86d4b8 100644 --- a/image_analysis/geom_refinement/PostRefine.cpp +++ b/image_analysis/geom_refinement/PostRefine.cpp @@ -219,13 +219,15 @@ PostRefineObservations GatherPostRefineObservations(std::vector(r.h), static_cast(r.k), + static_cast(r.l), r.image_number, + r.I, r.sigma, ox, oy}; } // The caller said it will never read this image's reflections again, so hand the // vector back the moment it is consumed: the payload shrinks image by image as it // moves into the (much narrower) partials instead of doubling up in full. if (release_reflections) - std::vector().swap(outcomes[o].reflections); + ReflectionVector().swap(outcomes[o].reflections); } }); logger.Info("Post-refine: {} partials gathered", n_pts); diff --git a/image_analysis/geom_refinement/PostRefine.h b/image_analysis/geom_refinement/PostRefine.h index 5abfe9ce7..682107817 100644 --- a/image_analysis/geom_refinement/PostRefine.h +++ b/image_analysis/geom_refinement/PostRefine.h @@ -103,7 +103,10 @@ struct PostRefineSettings { // per-bucket sort move all of it. The goniometer angle is not stored - it is a function of the // image number alone, and is rebuilt from it where it is needed. struct PostRefinePartial { - int h, k, l; + // Sixteen bits are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the + // longest axis at atomic resolution - and here four bytes of every partial are four bytes of an + // array that is gigabytes on a large cell. + int16_t h, k, l; float img; float I, sigma; float obs_x, obs_y; // observed spot centroid (pixels); NAN if the box sum found no centroid diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 2945ce2f7..dea9f923f 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -414,7 +414,7 @@ struct ShellAccum { CorrelationCoefficient cc_ref; }; -std::pair ImageReferenceCC(const std::vector &reflections, +std::pair ImageReferenceCC(std::span reflections, const std::map &reference, const HKLKeyGenerator &generator, std::optional d_min_limit, diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index 025dfe7cb..cf7b6df80 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -308,7 +309,7 @@ std::vector MergeAll(const DiffractionExperiment &x, // This is the per-image image_scale_cc: ScaleOnTheFly sets it, and StillsPartialityRefine recomputes it // after refining the partiality model, so the reported CC always describes the corrections that will be // merged - which matters because --min-image-cc drops images by it. -std::pair ImageReferenceCC(const std::vector &reflections, +std::pair ImageReferenceCC(std::span reflections, const std::map &reference, const HKLKeyGenerator &generator, std::optional d_min_limit, diff --git a/image_analysis/scale_merge/ReindexAmbiguity.cpp b/image_analysis/scale_merge/ReindexAmbiguity.cpp index 672bf4bfb..d776097ea 100644 --- a/image_analysis/scale_merge/ReindexAmbiguity.cpp +++ b/image_analysis/scale_merge/ReindexAmbiguity.cpp @@ -164,7 +164,7 @@ bool ReindexAmbiguityResolver::Accept(const Reflection &r) const { return AcceptReflection(r, s.GetHighResolutionLimit_A(), s.GetLowResolutionLimit_A()); } -double ReindexAmbiguityResolver::ReferenceCC(const std::vector &reflections, +double ReindexAmbiguityResolver::ReferenceCC(std::span reflections, const gemmi::Op &op) const { double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; size_t n = 0; @@ -195,7 +195,7 @@ double ReindexAmbiguityResolver::ReferenceCC(const std::vector &refl // Serial stills index each crystal in one of the merohedrally-equivalent hands at random; pick, for this // image alone, the reindexing whose intensities correlate best with the external reference and apply it. -void ReindexAmbiguityResolver::Resolve(std::vector &reflections) const { +void ReindexAmbiguityResolver::Resolve(std::span reflections) const { if (ops.empty()) return; diff --git a/image_analysis/scale_merge/ReindexAmbiguity.h b/image_analysis/scale_merge/ReindexAmbiguity.h index 4b8f6f32f..76c8b6bc0 100644 --- a/image_analysis/scale_merge/ReindexAmbiguity.h +++ b/image_analysis/scale_merge/ReindexAmbiguity.h @@ -5,6 +5,7 @@ #include #include +#include #include #include "gemmi/symmetry.hpp" @@ -82,7 +83,7 @@ class ReindexAmbiguityResolver { // Pearson correlation of this image's partiality/Lorentz-corrected intensities against the reference, // matched by ASU index under `op`. Scale-invariant (the unknown per-image scale cancels). NaN when // too few reflections match. - double ReferenceCC(const std::vector &reflections, const gemmi::Op &op) const; + double ReferenceCC(std::span reflections, const gemmi::Op &op) const; public: // The reference must be an external, correctly-handed dataset (not the data's own running merge). @@ -90,5 +91,5 @@ public: // Reindex this image's reflections in place into the hand best-correlated with the reference (done // once per image, for good). No-op when there is no ambiguity. - void Resolve(std::vector &reflections) const; + void Resolve(std::span reflections) const; }; diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 2a6409224..8bf050f1c 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -291,8 +291,8 @@ void RotationScaleMerge::Ingest() { partials_released = false; resident_ingest = false; #ifdef JFJOCH_USE_CUDA - // Probed before the observation array is built: which record the build lays down (the narrow - // resident one or the full Obs) depends on whether the device pipeline will be active. + // Probed before the observations are built: whether the build lays down the full Obs array or + // only the ingest arrays depends on whether the device pipeline will be active. gpu_ = std::make_unique(); gpu_active_ = gpu_->Available(); resident_ingest = gpu_active_ && observation_dump_path.empty(); @@ -420,7 +420,8 @@ void RotationScaleMerge::Ingest() { for (int o = lo; o < hi; ++o) { int32_t at = src_start[o]; for (const auto &r : partials_out[o].reflections) { - keys[fill[r.h - h_min]++] = SortKey{r.h, r.k, r.l, r.image_number, at, r.d}; + keys[fill[r.h - h_min]++] = SortKey{static_cast(r.h), static_cast(r.k), + static_cast(r.l), r.image_number, at, r.d}; ++at; } } @@ -488,6 +489,15 @@ void RotationScaleMerge::Ingest() { n_partials_obs, n_frames, rawrun_start.size()); SmoothMosaicityAndPartiality(); + if (!resident_ingest) { + ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), + [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) { + partials[i].delta_phi = ingest_delta_phi[i]; + partials[i].partiality = ingest_partiality[i]; + } + }); + } // The corr every pass starts from (see the header). The resident build filled it already, // straight from the source reflections (and the smoothing between there and here never @@ -513,7 +523,7 @@ void RotationScaleMerge::Ingest() { for (int at = 0; at < n; at += slice) { const int cnt = std::min(n - at, slice); ParallelChunks(cnt, nthreads, [&](int lo, int hi) { - for (int i = lo; i < hi; ++i) buf[i] = partials_ingest[at + i].partiality; + for (int i = lo; i < hi; ++i) buf[i] = ingest_partiality[at + i]; }); gpu_->SetObsField(F::Partiality, at, cnt, buf.get()); } @@ -565,16 +575,21 @@ void RotationScaleMerge::Ingest() { gpu_->SetFrameCellOk(frame_cell_ok.data()); logger.Info("RotationScaleMerge: GPU scaling + combine + scale-fulls + merge active"); } - // The device now holds every per-obs field the resident pipeline reads, so the host record's + // The device now holds every per-obs field the resident pipeline reads, so the ingest arrays' // one remaining reader is RockingEventFrames - which on this path sees only ingest-time values - // (see the header). Take its answer now and hand the record back. The dump path keeps the full - // Obs array: the CPU combine is what writes the dump. + // (see the header). Take its answer now and hand them back. The dump path keeps the full Obs + // array: the CPU combine is what writes the dump. if (resident_ingest) { rocking_event_frames_at_ingest = RockingEventFrames(); - std::vector().swap(partials_ingest); partials_released = true; } #endif + std::vector().swap(ingest_delta_phi); + std::vector().swap(ingest_partiality); + std::vector().swap(ingest_image_number); + std::vector().swap(ingest_rock_ok); + std::vector().swap(ingest_keep); + std::vector().swap(ingest_src_start); } // Build `partials` (and the per-frame CSR, the finiteness mask and `perm`) from the source reflections, @@ -611,14 +626,15 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector &keys) { // Where each frame's reflections sit in the key numbering, and how many of them survive. Building in // index order keeps each frame's observations contiguous and in the order they had, so the per-frame // ranges still describe them and every per-frame sum keeps its sequence of roundings. - std::vector src_start(n_frames, 0); + ingest_src_start.assign(n_frames, 0); { int32_t at = 0; for (int o = 0; o < n_frames; ++o) { - src_start[o] = at; + ingest_src_start[o] = at; at += static_cast(partials_out[o].reflections.size()); } } + const std::vector &src_start = ingest_src_start; frame_count.assign(n_frames, 0); ParallelFor(n_frames, nthreads, [&](int o) { int32_t c = 0; @@ -684,14 +700,16 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector &keys) { // finiteness (immutable) - it lets ComputeAsuGroups stamp the ASU-group id per obs from a flat // 1-byte array instead of re-reading the fat Obs struct for every space group. finite_ok.assign(n_keep, 0); + ingest_delta_phi.resize(n_keep); + ingest_partiality.resize(n_keep); + ingest_image_number.resize(n_keep); #ifdef JFJOCH_USE_CUDA if (resident_ingest) { - // The resident path: build the narrow host record, then stage every immutable per-obs field - // to the device straight from the source reflections, in frame-aligned bounded slices. The - // 80-byte Obs is never materialised here - beside the source reflections it was another - // copy of the whole payload - and the slice buffers are bounded whatever the run. - partials_ingest.clear(); - partials_ingest.resize(n_keep); + // The resident path: fill the ingest arrays, then stage every immutable per-obs field to the + // device straight from the source reflections, in frame-aligned bounded slices. The 80-byte + // Obs is never materialised here - beside the source reflections it was another copy of the + // whole payload - and the slice buffers are bounded whatever the run. + ingest_rock_ok.resize(n_keep); corr_ingested.resize(n_keep); ParallelFor(n_frames, nthreads, [&](int o) { int32_t src = src_start[o]; @@ -699,25 +717,18 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector &keys) { for (const auto &r : partials_out[o].reflections) { if (!keep[src]) { ++src; continue; } ++src; - ObsIngest &obs = partials_ingest[at]; - obs.h = static_cast(r.h); - obs.k = static_cast(r.k); - obs.l = static_cast(r.l); - obs.frame = o; - obs.image_number = r.image_number; - obs.d = r.d; - obs.zeta = r.zeta; - obs.delta_phi = r.delta_phi_deg; - obs.partiality = r.partiality; + ingest_delta_phi[at] = r.delta_phi_deg; + ingest_partiality[at] = r.partiality; + ingest_image_number[at] = r.image_number; const float prescaling_corr = static_cast( r.prescaling_corr * r.qe_corr * r.flight_corr / frame_flux[o]); corr_ingested[at] = r.image_scale_corr; finite_ok[at] = (std::isfinite(r.I) && std::isfinite(prescaling_corr) && prescaling_corr != 0.0f && std::isfinite(r.sigma) && r.sigma > 0.0f) ? 1 : 0; - obs.rock_ok = (std::isfinite(r.image_scale_corr) && r.image_scale_corr > 0.0f - && std::isfinite(r.I) && std::isfinite(r.sigma) && r.sigma > 0.0f) - ? 1 : 0; + ingest_rock_ok[at] = (std::isfinite(r.image_scale_corr) && r.image_scale_corr > 0.0f + && std::isfinite(r.I) && std::isfinite(r.sigma) && r.sigma > 0.0f) + ? 1 : 0; ++at; } }); @@ -804,6 +815,9 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector &keys) { obs.group = -1; finite_ok[at] = (std::isfinite(obs.I) && std::isfinite(obs.prescaling_corr) && obs.prescaling_corr != 0.0f && std::isfinite(obs.sigma) && obs.sigma > 0.0f) ? 1 : 0; + ingest_delta_phi[at] = r.delta_phi_deg; + ingest_partiality[at] = r.partiality; + ingest_image_number[at] = r.image_number; ++at; } }); @@ -812,6 +826,7 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector &keys) { #endif n_partials_obs = n_keep; + ingest_keep = std::move(keep); if (n_keep < n_obs) logger.Info("RotationScaleMerge: dropped {} of {} observations ({} of {} distinct hkl) outside the " @@ -879,7 +894,7 @@ namespace { // window would over-smooth a fast-slipping crystal and under-smooth a sparse one. The frame's own // value is excluded when scoring, so a window is only preferred if it PREDICTS that frame better, // which is exactly the noise-versus-drift trade. -template void RotationScaleMerge::SmoothGeometry(std::vector &obs) { +void RotationScaleMerge::SmoothGeometry() { const auto gon = x.GetGoniometer(); if (!gon || n_frames < 5) return; @@ -986,37 +1001,31 @@ template void RotationScaleMerge::SmoothGeometry(std::vector &obs) } std::atomic changed{0}; - ParallelChunks(static_cast(obs.size()), nthreads, [&](int lo, int hi) { + ParallelFor(n_frames, nthreads, [&](int f) { + if (!ok[f]) + return; + const std::array &rc = frame_recip[f]; int64_t local = 0; - for (int i = lo; i < hi; ++i) { - auto &o = obs[i]; - if (o.frame < 0 || o.frame >= n_frames || !ok[o.frame]) - continue; - const std::array &r = frame_recip[o.frame]; - const Coord p0 = r[0] * static_cast(o.h) + r[1] * static_cast(o.k) - + r[2] * static_cast(o.l); - const float phi_near = o.delta_phi * static_cast(PI) / 180.0f; + ForEachKeptIn(f, [&](int i, const Reflection &r) { + const Coord p0 = rc[0] * static_cast(r.h) + rc[1] * static_cast(r.k) + + rc[2] * static_cast(r.l); + const float phi_near = ingest_delta_phi[i] * static_cast(PI) / 180.0f; const auto phi = SolvePhi(p0, S0, m1, m2, m3, m2_S0, m3_S0, phi_near); if (!phi || !std::isfinite(*phi)) - continue; - o.delta_phi = *phi * 180.0f / static_cast(PI); + return; + ingest_delta_phi[i] = *phi * 180.0f / static_cast(PI); ++local; - } + }); changed += local; }); logger.Info("Smoothed per-frame geometry over +-{} frames (chosen by cross-validation); " "recomputed delta_phi for {} of {} partials", best_half, changed.load(), - obs.size()); + n_partials_obs); } void RotationScaleMerge::SmoothMosaicityAndPartiality() { - if (resident_ingest) SmoothMosaicityAndPartialityOn(partials_ingest); - else SmoothMosaicityAndPartialityOn(partials); -} - -template void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std::vector &obs) { - SmoothGeometry(obs); + SmoothGeometry(); // One rocking event, one exact-Bragg angle. Every partial's delta_phi is solved from its OWN // frame's lattice - by the predictor, and again by SmoothGeometry above - so the per-frame @@ -1042,25 +1051,25 @@ template void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std:: int i = lo; while (i < hi) { int j = i + 1; - while (j < hi && obs[perm[j]].image_number - - obs[perm[j - 1]].image_number <= max_frame_gap) + while (j < hi && ingest_image_number[perm[j]] + - ingest_image_number[perm[j - 1]] <= max_frame_gap) ++j; if (j - i >= 2) { - const float f0 = obs[perm[i]].image_number; + const float f0 = ingest_image_number[perm[i]]; double sum = 0.0; int n = 0; for (int m = i; m < j; ++m) { - const auto &o = obs[perm[m]]; - if (!std::isfinite(o.delta_phi)) continue; - sum += o.delta_phi + increment_deg * (o.image_number - f0); + const int p = perm[m]; + if (!std::isfinite(ingest_delta_phi[p])) continue; + sum += ingest_delta_phi[p] + increment_deg * (ingest_image_number[p] - f0); ++n; } if (n >= 2) { const double c = sum / n; for (int m = i; m < j; ++m) { - auto &o = obs[perm[m]]; - o.delta_phi = static_cast(c - increment_deg - * (o.image_number - f0)); + const int p = perm[m]; + ingest_delta_phi[p] = static_cast(c - increment_deg + * (ingest_image_number[p] - f0)); } ++l_ev; l_pa += j - i; @@ -1133,20 +1142,19 @@ template void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std:: // partiality recomputed here matches the one integration was based on. sin(theta_B) = lambda/(2d). const double bandwidth_sigma = x.GetBandwidthFWHM().value_or(0.0f) / 2.3548; const double half_wavelength_A = x.GetWavelength_A() / 2.0; - ParallelChunks(static_cast(obs.size()), nthreads, [&](int lo, int hi) { - for (int i = lo; i < hi; ++i) { - auto &o = obs[i]; - float mos = mos_smooth[o.frame]; - if (bandwidth_sigma > 0.0 && o.d > 0.0f) { - const double sin_theta = half_wavelength_A / o.d; + ParallelFor(n_frames, nthreads, [&](int f) { + ForEachKeptIn(f, [&](int i, const Reflection &r) { + float mos = mos_smooth[f]; + if (bandwidth_sigma > 0.0 && r.d > 0.0f) { + const double sin_theta = half_wavelength_A / r.d; const double sigma_bw = (180.0 / PI) * bandwidth_sigma * sin_theta / std::sqrt(1.0 - sin_theta * sin_theta); mos = static_cast(std::sqrt(mos * mos + sigma_bw * sigma_bw)); } - if (std::isfinite(mos) && mos > 1e-6f && std::isfinite(o.zeta) && o.zeta > 0.0f - && std::isfinite(o.delta_phi)) - o.partiality = RotationPartiality(o.delta_phi, o.zeta, mos, wedge); - } + if (std::isfinite(mos) && mos > 1e-6f && std::isfinite(r.zeta) && r.zeta > 0.0f + && std::isfinite(ingest_delta_phi[i])) + ingest_partiality[i] = RotationPartiality(ingest_delta_phi[i], r.zeta, mos, wedge); + }); }); logger.Info("Recomputed partiality from frame-order-smoothed mosaicity"); } @@ -2233,10 +2241,10 @@ namespace { int RotationScaleMerge::RockingEventFrames() const { if (partials_released) return rocking_event_frames_at_ingest; - if (resident_ingest) // Ingest's own call, before the narrow record is handed back + if (resident_ingest) // Ingest's own call, before the ingest arrays are handed back return RockingEventFramesOver( - [&](int i) { return partials_ingest[i].rock_ok != 0; }, - [&](int i) { return partials_ingest[i].image_number; }); + [&](int i) { return ingest_rock_ok[i] != 0; }, + [&](int i) { return ingest_image_number[i]; }); return RockingEventFramesOver( [&](int i) { const Obs &o = partials[i]; @@ -4055,36 +4063,40 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool bool did_gpu_acc = false; #ifdef JFJOCH_USE_CUDA if (use_gpu_merge) { - // Resized, not sized vectors: the kernel writes every one of these for every group, so - // value-initialising them first is a hundred megabytes of zeroes faulted in on one thread - // and overwritten on the next line - the same trap Ingest calls out. Members for the same - // reason as FullsStaging. - MergeAccumStaging &ms = merge_accum; - ms.Resize(n_groups); gpu_->MergeAccum(error_model_a, error_model_b, error_model_active, reject_outliers, reject_nsigma, reject_median.data(), merge_half.data(), - frame_cc_factor.data(), - ms.swI.data(), ms.sw.data(), ms.swIh0.data(), ms.swIh1.data(), - ms.swh0.data(), ms.swh1.data(), ms.swh_typ0.data(), ms.swh_typ1.data(), - ms.nh0.data(), ms.nh1.data(), - ms.d.data(), ms.rej.data(), ms.on_ice.data(), rejected_obs.data()); + frame_cc_factor.data(), rejected_obs.data()); + // Downloaded a slice of groups at a time and unpacked straight into acc, so the host never + // holds a second full-length copy of the accumulators - on a large cell that is most of a + // gigabyte beside acc itself. Resized, not sized vectors: the download writes every element + // (the kernel fills the empty groups too), and a member for the same reason as FullsStaging. // Per-group and independent, so the unpack splits; the rejected count is an integer sum and // does not care in which order the chunks add theirs in. + constexpr int ACC_SLICE = 1 << 20; + MergeAccumStaging &ms = merge_accum; + ms.Resize(std::min(n_groups, ACC_SLICE)); std::atomic rejected_total{0}; - ParallelChunks(n_groups, ThreadsForWork(n_groups, nthreads), [&](int glo, int ghi) { - size_t rej = 0; - for (int g = glo; g < ghi; ++g) { - Accum &a = acc[g]; - a.swI = ms.swI[g]; a.sw = ms.sw[g]; a.swIh[0] = ms.swIh0[g]; a.swIh[1] = ms.swIh1[g]; - a.swh[0] = ms.swh0[g]; a.swh[1] = ms.swh1[g]; - a.swh_typ[0] = ms.swh_typ0[g]; a.swh_typ[1] = ms.swh_typ1[g]; - a.nh[0] = static_cast(ms.nh0[g]); a.nh[1] = static_cast(ms.nh1[g]); - a.d = static_cast(ms.d[g]); - a.on_ice = ms.on_ice[g] != 0; - rej += static_cast(ms.rej[g]); - } - rejected_total += rej; - }); + for (int g0 = 0; g0 < n_groups; g0 += ACC_SLICE) { + const int cnt = std::min(ACC_SLICE, n_groups - g0); + gpu_->MergeAccumRange(g0, cnt, ms.swI.data(), ms.sw.data(), ms.swIh0.data(), ms.swIh1.data(), + ms.swh0.data(), ms.swh1.data(), ms.swh_typ0.data(), ms.swh_typ1.data(), + ms.nh0.data(), ms.nh1.data(), ms.d.data(), ms.rej.data(), + ms.on_ice.data()); + ParallelChunks(cnt, ThreadsForWork(cnt, nthreads), [&](int lo, int hi) { + size_t rej = 0; + for (int i = lo; i < hi; ++i) { + Accum &a = acc[g0 + i]; + a.swI = ms.swI[i]; a.sw = ms.sw[i]; a.swIh[0] = ms.swIh0[i]; a.swIh[1] = ms.swIh1[i]; + a.swh[0] = ms.swh0[i]; a.swh[1] = ms.swh1[i]; + a.swh_typ[0] = ms.swh_typ0[i]; a.swh_typ[1] = ms.swh_typ1[i]; + a.nh[0] = static_cast(ms.nh0[i]); a.nh[1] = static_cast(ms.nh1[i]); + a.d = static_cast(ms.d[i]); + a.on_ice = ms.on_ice[i] != 0; + rej += static_cast(ms.rej[i]); + } + rejected_total += rej; + }); + } reject_count = rejected_total; did_gpu_acc = true; } diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index 5bc743ff0..6cc953c67 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -148,32 +148,14 @@ private: // re-run over the samples that survive the automatic resolution cutoff. See MergeAndStats. struct Sample { double s2, I2, dev2; float d; }; - // The record the ingest keeps on the host when the device pipeline is resident (a GPU and no - // observation dump): every immutable per-obs field is uploaded straight from the source - // reflections inside BuildInRangeObservations, and the host keeps only what its own remaining - // stages touch - the geometry smoothing (delta_phi, then the recomputed partiality), plus the - // rocking-event walk's inputs. Thirty-two bytes against the Obs's eighty, and on that path the fat - // record is never materialised at all: on a fine-sliced long axis it alone is many gigabytes, - // held beside the source reflections it was copied from. - struct ObsIngest { - // Sixteen bits are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the - // longest axis at atomic resolution - and on a long axis this array sets the ingest's memory - // high-water mark, so each index costs two bytes here, not four. - int16_t h, k, l; - // The rocking-event usability test (RockingEventFrames), evaluated on the ingest-time - // values - which is what that walk reads on the resident path, where the host corr is - // never refreshed (see partials_released). Beside the indices, where it fills their padding. - uint8_t rock_ok; - int32_t frame; - float image_number, d, zeta, delta_phi, partiality; - }; - // The narrow per-observation record the ingest sort orders: the raw hkl the runs are cut on, the // frame position that breaks a tie inside one, the observation's own index (which makes the order - // total - see the .cpp), and the resolution the range test reads. Twenty-four bytes against the - // Obs's eighty, and it is all the ingest needs before it knows which observations survive. + // total - see the .cpp), and the resolution the range test reads. Twenty bytes against the Obs's + // eighty, and it is all the ingest needs before it knows which observations survive. Sixteen bits + // are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the longest axis at + // atomic resolution - and on a long axis this array sets the sort's memory high-water mark. struct SortKey { - int32_t h, k, l; + int16_t h, k, l; float image_number; int32_t idx; float d; @@ -233,12 +215,36 @@ private: int report_shell_count = 9; std::optional report_range; // the second, reference-range table - // Flat buffers, allocated once by Ingest() and reused across Run() calls. Exactly one of the - // two observation arrays is built: the narrow resident record when the device pipeline is - // active and no observation dump was asked for (resident_ingest), the full Obs otherwise. + // Flat buffers, allocated once by Ingest() and reused across Run() calls. The full Obs array is + // built only when the device pipeline is NOT resident (no GPU, or an observation dump was asked + // for); on the resident path (resident_ingest) the device holds the observations and the host + // keeps only the ingest arrays below, until the end of Ingest. std::vector partials; // all per-frame partials, grouped by frame - std::vector partials_ingest; // the resident-path record (same order) bool resident_ingest = false; + // What the ingest-time geometry smoothing (SmoothMosaicityAndPartiality) needs beyond the source + // reflections, one array per field in `partials` order: the two fields it rewrites, and the two the + // rocking-event walks read in raw-hkl order, where a source reflection is out of reach. Every other + // field it reads - hkl, frame, d, zeta - is read from the source reflection itself (ForEachKeptIn), + // so on the resident path no per-observation record is built at all: on a fine-sliced long axis + // that record was gigabytes, held beside the source reflections it was copied from. Filled by + // BuildInRangeObservations and handed back at the end of Ingest. + std::vector ingest_delta_phi, ingest_partiality, ingest_image_number; + // The rocking-event usability test (RockingEventFrames) on the ingest-time values - which is what + // that walk reads on the resident path, where the host corr is never refreshed (see + // partials_released). Resident path only. + std::vector ingest_rock_ok; + // Which source reflections the resolution range kept (indexed like the sort keys: frame by frame, + // from ingest_src_start), so that the i-th kept one of a frame can be found again. + std::vector ingest_keep; + std::vector ingest_src_start; + // Call fn(observation index, source reflection) for each reflection of frame o that the ingest + // kept, in order. + template void ForEachKeptIn(int o, Fn fn) const { + int32_t src = ingest_src_start[o]; + int32_t at = frame_start[o]; + for (const auto &r : partials_out[o].reflections) + if (ingest_keep[src++]) fn(at++, r); + } // How many observations Ingest built (== partials.size() until any release below), so the // sizes survive when the array itself does not. int n_partials_obs = 0; @@ -470,9 +476,8 @@ private: // per-frame partials of one rocking event tile the curve consistently (they sum toward 1) before the // 3D combine. Deterministic (frame order); replaces the old arrival-order mosaicity moving average // that prediction applied. SG-independent, so done once in Ingest. - template void SmoothGeometry(std::vector &obs); - void SmoothMosaicityAndPartiality(); // dispatch: the resident narrow record or the full Obs - template void SmoothMosaicityAndPartialityOn(std::vector &obs); + void SmoothGeometry(); + void SmoothMosaicityAndPartiality(); void Combine(); // partials -> fulls (CPU) diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.cu b/image_analysis/scale_merge/RotationScaleMergeGPU.cu index 6e161058d..09b1bbd88 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.cu +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.cu @@ -879,10 +879,7 @@ void RotationScaleMergeGPU::MergeEmSamples(bool for_search, double min_partialit void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_b, bool error_model_active, bool reject_outliers, double reject_nsigma, const float *reject_median, const uint8_t *half, const double *frame_cc_factor, - double *swI, double *sw, double *swIh0, double *swIh1, - double *swh0, double *swh1, double *swh_typ0, double *swh_typ1, - int32_t *nh0, int32_t *nh1, double *d_out, - int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs) { + uint8_t *rejected_obs) { DeviceGuard guard(impl_->device, impl_->available); auto &d = *impl_; const int ng = d.n_groups; @@ -924,17 +921,23 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_ MergeAccumKernel<<>>(p); CudaCheck(cudaGetLastError(), "merge accum launch"); CudaCheck(cudaDeviceSynchronize(), "merge accum sync"); - auto dl = [&](void *h, const CudaDevicePtr &s) { - CudaCheck(cudaMemcpy(h, s.get(), size_t(ng) * sizeof(double), cudaMemcpyDeviceToHost), "dl accum"); }; + CudaCheck(cudaMemcpy(rejected_obs, d.m_rejected.get(), size_t(d.n_fulls) * sizeof(uint8_t), cudaMemcpyDeviceToHost), + "dl rejected_obs"); +} + +void RotationScaleMergeGPU::MergeAccumRange(int g0, int n, double *swI, double *sw, double *swIh0, + double *swIh1, double *swh0, double *swh1, double *swh_typ0, + double *swh_typ1, int32_t *nh0, int32_t *nh1, double *d_out, + int32_t *rejected, uint8_t *on_ice_out) { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + auto dl = [&](void *h, const auto &s) { + CudaCheck(cudaMemcpy(h, s.get() + g0, size_t(n) * sizeof(*s.get()), cudaMemcpyDeviceToHost), + "dl accum"); }; dl(swI, d.a_swI); dl(sw, d.a_sw); dl(swIh0, d.a_swIh0); dl(swIh1, d.a_swIh1); dl(swh0, d.a_swh0); dl(swh1, d.a_swh1); dl(swh_typ0, d.a_swht0); dl(swh_typ1, d.a_swht1); dl(d_out, d.a_d); - CudaCheck(cudaMemcpy(nh0, d.a_nh0.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh0"); - CudaCheck(cudaMemcpy(nh1, d.a_nh1.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh1"); - CudaCheck(cudaMemcpy(rejected, d.a_rejected.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl rej"); - CudaCheck(cudaMemcpy(on_ice_out, d.a_on_ice.get(), size_t(ng) * sizeof(uint8_t), cudaMemcpyDeviceToHost), "dl on_ice"); - CudaCheck(cudaMemcpy(rejected_obs, d.m_rejected.get(), size_t(d.n_fulls) * sizeof(uint8_t), cudaMemcpyDeviceToHost), - "dl rejected_obs"); + dl(nh0, d.a_nh0); dl(nh1, d.a_nh1); dl(rejected, d.a_rejected); dl(on_ice_out, d.a_on_ice); } void RotationScaleMergeGPU::MergeRmeas(const double *merged_I, double *absdev, double *sumI, diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.h b/image_analysis/scale_merge/RotationScaleMergeGPU.h index 49c4c889d..d1be12791 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.h +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.h @@ -91,18 +91,20 @@ public: // Per-group merge accumulators (inv-var sums + deterministic half-sets, error-model-corrected sigma // from a/b). `half` is the per-full CC1/2 half-set (length n_fulls) assigned on the host, so this // kernel and the host merge loop cannot disagree about it. - // Outputs length n_groups; rejected[g] counts outliers dropped (reject_median uploaded, NAN - // where none). rejected_obs is the per-full flag (length n_fulls): the host needs it for the reductions - // it still does itself, above all the anomalous I(+)/I(-) split. frame_cc_factor (length n_frames) is - // each frame's (G_ref/G)^2 floored at 1; swh_typ0/1 are the half-set weights multiplied by it. - // Requires MergeEmSamples first (em_mean resident). + // The per-group results stay on the device for MergeAccumRange to download; reject_median is + // uploaded (NAN where none). rejected_obs is the per-full flag (length n_fulls): the host needs it + // for the reductions it still does itself, above all the anomalous I(+)/I(-) split. + // frame_cc_factor (length n_frames) is each frame's (G_ref/G)^2 floored at 1; swh_typ0/1 are the + // half-set weights multiplied by it. Requires MergeEmSamples first (em_mean resident). void MergeAccum(double error_model_a, double error_model_b, bool error_model_active, bool reject_outliers, double reject_nsigma, const float *reject_median, - const uint8_t *half, const double *frame_cc_factor, - double *swI, double *sw, double *swIh0, double *swIh1, - double *swh0, double *swh1, double *swh_typ0, double *swh_typ1, - int32_t *nh0, int32_t *nh1, double *d_out, - int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs); + const uint8_t *half, const double *frame_cc_factor, uint8_t *rejected_obs); + // Download groups [g0, g0 + n) of what MergeAccum left on the device; every output has length n. + // rejected[g] counts the outliers dropped from the group. + void MergeAccumRange(int g0, int n, double *swI, double *sw, double *swIh0, double *swIh1, + double *swh0, double *swh1, double *swh_typ0, double *swh_typ1, + int32_t *nh0, int32_t *nh1, double *d_out, + int32_t *rejected, uint8_t *on_ice_out); // Per-group R_meas accumulators (sum|I_corr-merged_I|, sum_I, n, and the count this looser walk // accepted - which the host uses only to skip empty groups, NOT as the per-shell diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index e65edee4e..7c7de1169 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -303,9 +303,10 @@ CrystalLattice ApplyReindex(const CrystalLattice &latt, const std::optional bool ReadReflectionsFromGroup(HDF5Object &file, const std::string &image_group_name, - std::vector &reflections, + ReflectionContainer &reflections, const std::optional> &reindex) { if (!file.Exists("/entry/reflections") || !file.Exists(image_group_name)) return false; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index a36c21b72..1e353fcb7 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -6210,7 +6210,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b } else { for (auto &io : indexer->GetIntegrationOutcome()) { io.latt = io.latt.Multiply(reduce); - std::vector kept; + ReflectionVector kept; kept.reserve(io.reflections.size()); for (auto r : io.reflections) if (reduce_hkl(r))