Merge branch 'c-mem' into pool-C

This commit is contained in:
leonarski_f committed 2026-09-20 18:45:17 +02:00
commit 3ca22c07c1
16 files changed
+320 -164

No files matched your search

+10 -4
View File
@@ -586,18 +586,24 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg,
// caller's callback binds it to the right image (GPU-resident buffer, host buffer, or the assembled
// FPGA image read straight on the CPU).
auto integration_start_time = std::chrono::steady_clock::now();
i_outcome.reflections = integrate(prediction.GetReflections(), nrefl, msg.number);
msg.integrated_reflections = i_outcome.reflections.size();
const std::vector<Reflection> integrated = integrate(prediction.GetReflections(), nrefl, msg.number);
msg.integrated_reflections = integrated.size();
auto integration_end_time = std::chrono::steady_clock::now();
msg.integration_time_s = std::chrono::duration<float>(integration_end_time - integration_start_time).count();
CalcISigmaAndWilsonBFactor(msg, i_outcome.reflections);
CalcISigmaAndWilsonBFactor(msg, integrated);
// A retained outcome is kept in the pass's arena (see ReflectionArena), not beside the worker's
// other allocations.
i_outcome.reflections = ReflectionVector(integrated.begin(), integrated.end(),
ArenaAllocator<Reflection>(retain_outcomes_ ? &reflection_arena
: nullptr));
ScaleImage(msg, i_outcome);
// Copy reflections to outgoing message
if (keep_reflections_in_message_)
msg.reflections = i_outcome.reflections;
msg.reflections.assign(i_outcome.reflections.begin(), i_outcome.reflections.end());
// Persist the per-image result for the whole-run scaling/merge pass, unless the caller opted out
// (viewer interactive use only needs the current image, returned above via msg).
+2
View File
@@ -62,6 +62,8 @@ class IndexAndRefine {
};
mutable std::mutex reflections_mutex;
// Holds the retained outcomes' reflections; declared before them, so it outlives them.
ReflectionArena reflection_arena;
std::vector<IntegrationOutcome> integration_outcome;
std::vector<float> mosaicity;
// Optional per-frame mosaicity used for Bragg prediction, indexed by image number. When set (the
+6 -1
View File
@@ -8,11 +8,16 @@
#include "../common/Reflection.h"
#include "../common/CrystalLattice.h"
#include "../common/DiffractionGeometry.h"
#include "ReflectionArena.h"
// A frame's integrated reflections. The whole-run passes keep them in a ReflectionArena (see there);
// anywhere else the allocator is plain new/delete.
using ReflectionVector = std::vector<Reflection, ArenaAllocator<Reflection>>;
struct IntegrationOutcome {
DiffractionGeometry geom;
CrystalLattice latt;
std::vector<Reflection> reflections;
ReflectionVector reflections;
std::optional<float> mosaicity_deg;
std::optional<float> image_scale_cc;
std::optional<int64_t> image_scale_cc_n;
+113
View File
@@ -0,0 +1,113 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <algorithm>
#include <cstddef>
#include <mutex>
#include <new>
#include <type_traits>
#include <vector>
// Large blocks that a whole pass's per-image reflection vectors are carved from.
//
// A rotation pass keeps every frame's integrated reflections until its scaling is done - thousands of
// vectors of a few megabytes each, gigabytes together, allocated by the image workers in the
// allocator's per-thread arenas. When the pass hands them back, that memory mostly stays in those
// arenas, as holes between whatever else the workers allocated, and the next pass's workers, being
// new threads, do not reuse it: on a fine-sliced long axis gigabytes of freed reflections were carried
// to the end of the run. Here a block is allocated in one piece (large enough that it is its own
// mapping) and returned in one piece once the last vector in it is gone, so a pass's reflections
// leave nothing behind.
//
// A vector is carved from the current block by bumping a pointer; the space of a vector freed early
// is only reclaimed with its whole block. That suits the use here: the vectors are written once,
// when the frame is integrated, and handed back together.
class ReflectionArena {
public:
ReflectionArena() = default;
ReflectionArena(const ReflectionArena &) = delete;
ReflectionArena &operator=(const ReflectionArena &) = delete;
~ReflectionArena() {
for (auto &b : blocks)
::operator delete(b.data);
}
void *Allocate(size_t bytes) {
bytes = (bytes + kAlign - 1) / kAlign * kAlign;
std::unique_lock ul(m);
if (blocks.empty() || blocks.back().size - blocks.back().used < bytes) {
const size_t size = std::max(kBlockBytes, bytes);
blocks.push_back(Block{static_cast<char *>(::operator new(size)), size, 0, 0});
}
Block &b = blocks.back();
void *p = b.data + b.used;
b.used += bytes;
b.live++;
return p;
}
void Deallocate(void *p) {
std::unique_lock ul(m);
for (size_t i = 0; i < blocks.size(); ++i) {
Block &b = blocks[i];
if (static_cast<char *>(p) < b.data || static_cast<char *>(p) >= b.data + b.size)
continue;
if (--b.live == 0) {
if (i + 1 == blocks.size()) {
b.used = 0; // the block being filled: keep it for the next vector
} else {
::operator delete(b.data);
blocks.erase(blocks.begin() + static_cast<std::ptrdiff_t>(i));
}
}
return;
}
}
private:
// 64 MiB: above the largest size glibc ever serves from its heaps (32 MiB), so every block is a
// mapping of its own and goes back to the system when freed. A frame's reflections are a few
// megabytes, so the tail a block cannot fit is a few percent of it.
static constexpr size_t kBlockBytes = size_t(64) << 20;
static constexpr size_t kAlign = alignof(std::max_align_t);
struct Block {
char *data;
size_t size, used, live;
};
std::mutex m;
std::vector<Block> blocks;
};
// A std::allocator that carves from a ReflectionArena, or is plain new/delete without one. A copy of
// a container is made on the heap (select_on_container_copy_construction), so only the containers the
// arena's owner fills itself live in the arena; a move carries the arena along with the storage.
template <class T>
struct ArenaAllocator {
using value_type = T;
using propagate_on_container_move_assignment = std::true_type;
using propagate_on_container_swap = std::true_type;
ReflectionArena *arena = nullptr;
ArenaAllocator() = default;
explicit ArenaAllocator(ReflectionArena *a) : arena(a) {}
template <class U> ArenaAllocator(const ArenaAllocator<U> &o) : arena(o.arena) {}
T *allocate(size_t n) {
if (arena)
return static_cast<T *>(arena->Allocate(n * sizeof(T)));
return static_cast<T *>(::operator new(n * sizeof(T)));
}
void deallocate(T *p, size_t) {
if (arena)
arena->Deallocate(p);
else
::operator delete(p);
}
ArenaAllocator select_on_container_copy_construction() const { return ArenaAllocator(); }
template <class U> bool operator==(const ArenaAllocator<U> &o) const { return arena == o.arena; }
template <class U> bool operator!=(const ArenaAllocator<U> &o) const { return arena != o.arena; }
};
@@ -219,13 +219,15 @@ PostRefineObservations GatherPostRefineObservations(std::vector<IntegrationOutco
if (!std::isfinite(r.I) || !std::isfinite(r.sigma) || r.sigma <= 0.0f) continue;
const float ox = std::isfinite(r.observed_x) ? r.observed_x : NAN;
const float oy = std::isfinite(r.observed_y) ? r.observed_y : NAN;
pts[fill[r.h - h_lo]++] = Partial{r.h, r.k, r.l, r.image_number, r.I, r.sigma, ox, oy};
pts[fill[r.h - h_lo]++] = Partial{static_cast<int16_t>(r.h), static_cast<int16_t>(r.k),
static_cast<int16_t>(r.l), r.image_number,
r.I, r.sigma, ox, oy};
}
// The caller said it will never read this image's reflections again, so hand the
// vector back the moment it is consumed: the payload shrinks image by image as it
// moves into the (much narrower) partials instead of doubling up in full.
if (release_reflections)
std::vector<Reflection>().swap(outcomes[o].reflections);
ReflectionVector().swap(outcomes[o].reflections);
}
});
logger.Info("Post-refine: {} partials gathered", n_pts);
+4 -1
View File
@@ -103,7 +103,10 @@ struct PostRefineSettings {
// per-bucket sort move all of it. The goniometer angle is not stored - it is a function of the
// image number alone, and is rebuilt from it where it is needed.
struct PostRefinePartial {
int h, k, l;
// Sixteen bits are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the
// longest axis at atomic resolution - and here four bytes of every partial are four bytes of an
// array that is gigabytes on a large cell.
int16_t h, k, l;
float img;
float I, sigma;
float obs_x, obs_y; // observed spot centroid (pixels); NAN if the box sum found no centroid
+1 -1
View File
@@ -414,7 +414,7 @@ struct ShellAccum {
CorrelationCoefficient cc_ref;
};
std::pair<double, size_t> ImageReferenceCC(const std::vector<Reflection> &reflections,
std::pair<double, size_t> ImageReferenceCC(std::span<const Reflection> reflections,
const std::map<HKLKey, double> &reference,
const HKLKeyGenerator &generator,
std::optional<double> d_min_limit,
+2 -1
View File
@@ -6,6 +6,7 @@
#include <algorithm>
#include <cmath>
#include <map>
#include <span>
#include <unordered_map>
#include <vector>
@@ -308,7 +309,7 @@ std::vector<MergedReflection> MergeAll(const DiffractionExperiment &x,
// This is the per-image image_scale_cc: ScaleOnTheFly sets it, and StillsPartialityRefine recomputes it
// after refining the partiality model, so the reported CC always describes the corrections that will be
// merged - which matters because --min-image-cc drops images by it.
std::pair<double, size_t> ImageReferenceCC(const std::vector<Reflection> &reflections,
std::pair<double, size_t> ImageReferenceCC(std::span<const Reflection> reflections,
const std::map<HKLKey, double> &reference,
const HKLKeyGenerator &generator,
std::optional<double> d_min_limit,
@@ -164,7 +164,7 @@ bool ReindexAmbiguityResolver::Accept(const Reflection &r) const {
return AcceptReflection(r, s.GetHighResolutionLimit_A(), s.GetLowResolutionLimit_A());
}
double ReindexAmbiguityResolver::ReferenceCC(const std::vector<Reflection> &reflections,
double ReindexAmbiguityResolver::ReferenceCC(std::span<const Reflection> reflections,
const gemmi::Op &op) const {
double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
size_t n = 0;
@@ -195,7 +195,7 @@ double ReindexAmbiguityResolver::ReferenceCC(const std::vector<Reflection> &refl
// Serial stills index each crystal in one of the merohedrally-equivalent hands at random; pick, for this
// image alone, the reindexing whose intensities correlate best with the external reference and apply it.
void ReindexAmbiguityResolver::Resolve(std::vector<Reflection> &reflections) const {
void ReindexAmbiguityResolver::Resolve(std::span<Reflection> reflections) const {
if (ops.empty())
return;
@@ -5,6 +5,7 @@
#include <functional>
#include <map>
#include <span>
#include <vector>
#include "gemmi/symmetry.hpp"
@@ -82,7 +83,7 @@ class ReindexAmbiguityResolver {
// Pearson correlation of this image's partiality/Lorentz-corrected intensities against the reference,
// matched by ASU index under `op`. Scale-invariant (the unknown per-image scale cancels). NaN when
// too few reflections match.
double ReferenceCC(const std::vector<Reflection> &reflections, const gemmi::Op &op) const;
double ReferenceCC(std::span<const Reflection> reflections, const gemmi::Op &op) const;
public:
// The reference must be an external, correctly-handed dataset (not the data's own running merge).
@@ -90,5 +91,5 @@ public:
// Reindex this image's reflections in place into the hand best-correlated with the reference (done
// once per image, for good). No-op when there is no ambiguity.
void Resolve(std::vector<Reflection> &reflections) const;
void Resolve(std::span<Reflection> reflections) const;
};
+108 -96
View File
@@ -291,8 +291,8 @@ void RotationScaleMerge::Ingest() {
partials_released = false;
resident_ingest = false;
#ifdef JFJOCH_USE_CUDA
// Probed before the observation array is built: which record the build lays down (the narrow
// resident one or the full Obs) depends on whether the device pipeline will be active.
// Probed before the observations are built: whether the build lays down the full Obs array or
// only the ingest arrays depends on whether the device pipeline will be active.
gpu_ = std::make_unique<RotationScaleMergeGPU>();
gpu_active_ = gpu_->Available();
resident_ingest = gpu_active_ && observation_dump_path.empty();
@@ -420,7 +420,8 @@ void RotationScaleMerge::Ingest() {
for (int o = lo; o < hi; ++o) {
int32_t at = src_start[o];
for (const auto &r : partials_out[o].reflections) {
keys[fill[r.h - h_min]++] = SortKey{r.h, r.k, r.l, r.image_number, at, r.d};
keys[fill[r.h - h_min]++] = SortKey{static_cast<int16_t>(r.h), static_cast<int16_t>(r.k),
static_cast<int16_t>(r.l), r.image_number, at, r.d};
++at;
}
}
@@ -488,6 +489,15 @@ void RotationScaleMerge::Ingest() {
n_partials_obs, n_frames, rawrun_start.size());
SmoothMosaicityAndPartiality();
if (!resident_ingest) {
ParallelChunks(static_cast<int>(partials.size()), ThreadsForWork(partials.size(), nthreads),
[&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
partials[i].delta_phi = ingest_delta_phi[i];
partials[i].partiality = ingest_partiality[i];
}
});
}
// The corr every pass starts from (see the header). The resident build filled it already,
// straight from the source reflections (and the smoothing between there and here never
@@ -513,7 +523,7 @@ void RotationScaleMerge::Ingest() {
for (int at = 0; at < n; at += slice) {
const int cnt = std::min(n - at, slice);
ParallelChunks(cnt, nthreads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) buf[i] = partials_ingest[at + i].partiality;
for (int i = lo; i < hi; ++i) buf[i] = ingest_partiality[at + i];
});
gpu_->SetObsField(F::Partiality, at, cnt, buf.get());
}
@@ -565,16 +575,21 @@ void RotationScaleMerge::Ingest() {
gpu_->SetFrameCellOk(frame_cell_ok.data());
logger.Info("RotationScaleMerge: GPU scaling + combine + scale-fulls + merge active");
}
// The device now holds every per-obs field the resident pipeline reads, so the host record's
// The device now holds every per-obs field the resident pipeline reads, so the ingest arrays'
// one remaining reader is RockingEventFrames - which on this path sees only ingest-time values
// (see the header). Take its answer now and hand the record back. The dump path keeps the full
// Obs array: the CPU combine is what writes the dump.
// (see the header). Take its answer now and hand them back. The dump path keeps the full Obs
// array: the CPU combine is what writes the dump.
if (resident_ingest) {
rocking_event_frames_at_ingest = RockingEventFrames();
std::vector<ObsIngest>().swap(partials_ingest);
partials_released = true;
}
#endif
std::vector<float>().swap(ingest_delta_phi);
std::vector<float>().swap(ingest_partiality);
std::vector<float>().swap(ingest_image_number);
std::vector<uint8_t>().swap(ingest_rock_ok);
std::vector<uint8_t>().swap(ingest_keep);
std::vector<int32_t>().swap(ingest_src_start);
}
// Build `partials` (and the per-frame CSR, the finiteness mask and `perm`) from the source reflections,
@@ -611,14 +626,15 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
// Where each frame's reflections sit in the key numbering, and how many of them survive. Building in
// index order keeps each frame's observations contiguous and in the order they had, so the per-frame
// ranges still describe them and every per-frame sum keeps its sequence of roundings.
std::vector<int32_t> src_start(n_frames, 0);
ingest_src_start.assign(n_frames, 0);
{
int32_t at = 0;
for (int o = 0; o < n_frames; ++o) {
src_start[o] = at;
ingest_src_start[o] = at;
at += static_cast<int32_t>(partials_out[o].reflections.size());
}
}
const std::vector<int32_t> &src_start = ingest_src_start;
frame_count.assign(n_frames, 0);
ParallelFor(n_frames, nthreads, [&](int o) {
int32_t c = 0;
@@ -684,14 +700,16 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
// finiteness (immutable) - it lets ComputeAsuGroups stamp the ASU-group id per obs from a flat
// 1-byte array instead of re-reading the fat Obs struct for every space group.
finite_ok.assign(n_keep, 0);
ingest_delta_phi.resize(n_keep);
ingest_partiality.resize(n_keep);
ingest_image_number.resize(n_keep);
#ifdef JFJOCH_USE_CUDA
if (resident_ingest) {
// The resident path: build the narrow host record, then stage every immutable per-obs field
// to the device straight from the source reflections, in frame-aligned bounded slices. The
// 80-byte Obs is never materialised here - beside the source reflections it was another
// copy of the whole payload - and the slice buffers are bounded whatever the run.
partials_ingest.clear();
partials_ingest.resize(n_keep);
// The resident path: fill the ingest arrays, then stage every immutable per-obs field to the
// device straight from the source reflections, in frame-aligned bounded slices. The 80-byte
// Obs is never materialised here - beside the source reflections it was another copy of the
// whole payload - and the slice buffers are bounded whatever the run.
ingest_rock_ok.resize(n_keep);
corr_ingested.resize(n_keep);
ParallelFor(n_frames, nthreads, [&](int o) {
int32_t src = src_start[o];
@@ -699,25 +717,18 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
for (const auto &r : partials_out[o].reflections) {
if (!keep[src]) { ++src; continue; }
++src;
ObsIngest &obs = partials_ingest[at];
obs.h = static_cast<int16_t>(r.h);
obs.k = static_cast<int16_t>(r.k);
obs.l = static_cast<int16_t>(r.l);
obs.frame = o;
obs.image_number = r.image_number;
obs.d = r.d;
obs.zeta = r.zeta;
obs.delta_phi = r.delta_phi_deg;
obs.partiality = r.partiality;
ingest_delta_phi[at] = r.delta_phi_deg;
ingest_partiality[at] = r.partiality;
ingest_image_number[at] = r.image_number;
const float prescaling_corr = static_cast<float>(
r.prescaling_corr * r.qe_corr * r.flight_corr / frame_flux[o]);
corr_ingested[at] = r.image_scale_corr;
finite_ok[at] = (std::isfinite(r.I) && std::isfinite(prescaling_corr)
&& prescaling_corr != 0.0f
&& std::isfinite(r.sigma) && r.sigma > 0.0f) ? 1 : 0;
obs.rock_ok = (std::isfinite(r.image_scale_corr) && r.image_scale_corr > 0.0f
&& std::isfinite(r.I) && std::isfinite(r.sigma) && r.sigma > 0.0f)
? 1 : 0;
ingest_rock_ok[at] = (std::isfinite(r.image_scale_corr) && r.image_scale_corr > 0.0f
&& std::isfinite(r.I) && std::isfinite(r.sigma) && r.sigma > 0.0f)
? 1 : 0;
++at;
}
});
@@ -804,6 +815,9 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
obs.group = -1;
finite_ok[at] = (std::isfinite(obs.I) && std::isfinite(obs.prescaling_corr) && obs.prescaling_corr != 0.0f
&& std::isfinite(obs.sigma) && obs.sigma > 0.0f) ? 1 : 0;
ingest_delta_phi[at] = r.delta_phi_deg;
ingest_partiality[at] = r.partiality;
ingest_image_number[at] = r.image_number;
++at;
}
});
@@ -812,6 +826,7 @@ void RotationScaleMerge::BuildInRangeObservations(std::vector<SortKey> &keys) {
#endif
n_partials_obs = n_keep;
ingest_keep = std::move(keep);
if (n_keep < n_obs)
logger.Info("RotationScaleMerge: dropped {} of {} observations ({} of {} distinct hkl) outside the "
@@ -879,7 +894,7 @@ namespace {
// window would over-smooth a fast-slipping crystal and under-smooth a sparse one. The frame's own
// value is excluded when scoring, so a window is only preferred if it PREDICTS that frame better,
// which is exactly the noise-versus-drift trade.
template <class T> void RotationScaleMerge::SmoothGeometry(std::vector<T> &obs) {
void RotationScaleMerge::SmoothGeometry() {
const auto gon = x.GetGoniometer();
if (!gon || n_frames < 5)
return;
@@ -986,37 +1001,31 @@ template <class T> void RotationScaleMerge::SmoothGeometry(std::vector<T> &obs)
}
std::atomic<int64_t> changed{0};
ParallelChunks(static_cast<int>(obs.size()), nthreads, [&](int lo, int hi) {
ParallelFor(n_frames, nthreads, [&](int f) {
if (!ok[f])
return;
const std::array<Coord, 3> &rc = frame_recip[f];
int64_t local = 0;
for (int i = lo; i < hi; ++i) {
auto &o = obs[i];
if (o.frame < 0 || o.frame >= n_frames || !ok[o.frame])
continue;
const std::array<Coord, 3> &r = frame_recip[o.frame];
const Coord p0 = r[0] * static_cast<float>(o.h) + r[1] * static_cast<float>(o.k)
+ r[2] * static_cast<float>(o.l);
const float phi_near = o.delta_phi * static_cast<float>(PI) / 180.0f;
ForEachKeptIn(f, [&](int i, const Reflection &r) {
const Coord p0 = rc[0] * static_cast<float>(r.h) + rc[1] * static_cast<float>(r.k)
+ rc[2] * static_cast<float>(r.l);
const float phi_near = ingest_delta_phi[i] * static_cast<float>(PI) / 180.0f;
const auto phi = SolvePhi(p0, S0, m1, m2, m3, m2_S0, m3_S0, phi_near);
if (!phi || !std::isfinite(*phi))
continue;
o.delta_phi = *phi * 180.0f / static_cast<float>(PI);
return;
ingest_delta_phi[i] = *phi * 180.0f / static_cast<float>(PI);
++local;
}
});
changed += local;
});
logger.Info("Smoothed per-frame geometry over +-{} frames (chosen by cross-validation); "
"recomputed delta_phi for {} of {} partials", best_half, changed.load(),
obs.size());
n_partials_obs);
}
void RotationScaleMerge::SmoothMosaicityAndPartiality() {
if (resident_ingest) SmoothMosaicityAndPartialityOn(partials_ingest);
else SmoothMosaicityAndPartialityOn(partials);
}
template <class T> void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std::vector<T> &obs) {
SmoothGeometry(obs);
SmoothGeometry();
// One rocking event, one exact-Bragg angle. Every partial's delta_phi is solved from its OWN
// frame's lattice - by the predictor, and again by SmoothGeometry above - so the per-frame
@@ -1042,25 +1051,25 @@ template <class T> void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std::
int i = lo;
while (i < hi) {
int j = i + 1;
while (j < hi && obs[perm[j]].image_number
- obs[perm[j - 1]].image_number <= max_frame_gap)
while (j < hi && ingest_image_number[perm[j]]
- ingest_image_number[perm[j - 1]] <= max_frame_gap)
++j;
if (j - i >= 2) {
const float f0 = obs[perm[i]].image_number;
const float f0 = ingest_image_number[perm[i]];
double sum = 0.0;
int n = 0;
for (int m = i; m < j; ++m) {
const auto &o = obs[perm[m]];
if (!std::isfinite(o.delta_phi)) continue;
sum += o.delta_phi + increment_deg * (o.image_number - f0);
const int p = perm[m];
if (!std::isfinite(ingest_delta_phi[p])) continue;
sum += ingest_delta_phi[p] + increment_deg * (ingest_image_number[p] - f0);
++n;
}
if (n >= 2) {
const double c = sum / n;
for (int m = i; m < j; ++m) {
auto &o = obs[perm[m]];
o.delta_phi = static_cast<float>(c - increment_deg
* (o.image_number - f0));
const int p = perm[m];
ingest_delta_phi[p] = static_cast<float>(c - increment_deg
* (ingest_image_number[p] - f0));
}
++l_ev;
l_pa += j - i;
@@ -1133,20 +1142,19 @@ template <class T> void RotationScaleMerge::SmoothMosaicityAndPartialityOn(std::
// partiality recomputed here matches the one integration was based on. sin(theta_B) = lambda/(2d).
const double bandwidth_sigma = x.GetBandwidthFWHM().value_or(0.0f) / 2.3548;
const double half_wavelength_A = x.GetWavelength_A() / 2.0;
ParallelChunks(static_cast<int>(obs.size()), nthreads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
auto &o = obs[i];
float mos = mos_smooth[o.frame];
if (bandwidth_sigma > 0.0 && o.d > 0.0f) {
const double sin_theta = half_wavelength_A / o.d;
ParallelFor(n_frames, nthreads, [&](int f) {
ForEachKeptIn(f, [&](int i, const Reflection &r) {
float mos = mos_smooth[f];
if (bandwidth_sigma > 0.0 && r.d > 0.0f) {
const double sin_theta = half_wavelength_A / r.d;
const double sigma_bw = (180.0 / PI) * bandwidth_sigma * sin_theta
/ std::sqrt(1.0 - sin_theta * sin_theta);
mos = static_cast<float>(std::sqrt(mos * mos + sigma_bw * sigma_bw));
}
if (std::isfinite(mos) && mos > 1e-6f && std::isfinite(o.zeta) && o.zeta > 0.0f
&& std::isfinite(o.delta_phi))
o.partiality = RotationPartiality(o.delta_phi, o.zeta, mos, wedge);
}
if (std::isfinite(mos) && mos > 1e-6f && std::isfinite(r.zeta) && r.zeta > 0.0f
&& std::isfinite(ingest_delta_phi[i]))
ingest_partiality[i] = RotationPartiality(ingest_delta_phi[i], r.zeta, mos, wedge);
});
});
logger.Info("Recomputed partiality from frame-order-smoothed mosaicity");
}
@@ -2233,10 +2241,10 @@ namespace {
int RotationScaleMerge::RockingEventFrames() const {
if (partials_released)
return rocking_event_frames_at_ingest;
if (resident_ingest) // Ingest's own call, before the narrow record is handed back
if (resident_ingest) // Ingest's own call, before the ingest arrays are handed back
return RockingEventFramesOver(
[&](int i) { return partials_ingest[i].rock_ok != 0; },
[&](int i) { return partials_ingest[i].image_number; });
[&](int i) { return ingest_rock_ok[i] != 0; },
[&](int i) { return ingest_image_number[i]; });
return RockingEventFramesOver(
[&](int i) {
const Obs &o = partials[i];
@@ -4055,36 +4063,40 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
bool did_gpu_acc = false;
#ifdef JFJOCH_USE_CUDA
if (use_gpu_merge) {
// Resized, not sized vectors: the kernel writes every one of these for every group, so
// value-initialising them first is a hundred megabytes of zeroes faulted in on one thread
// and overwritten on the next line - the same trap Ingest calls out. Members for the same
// reason as FullsStaging.
MergeAccumStaging &ms = merge_accum;
ms.Resize(n_groups);
gpu_->MergeAccum(error_model_a, error_model_b, error_model_active,
reject_outliers, reject_nsigma, reject_median.data(), merge_half.data(),
frame_cc_factor.data(),
ms.swI.data(), ms.sw.data(), ms.swIh0.data(), ms.swIh1.data(),
ms.swh0.data(), ms.swh1.data(), ms.swh_typ0.data(), ms.swh_typ1.data(),
ms.nh0.data(), ms.nh1.data(),
ms.d.data(), ms.rej.data(), ms.on_ice.data(), rejected_obs.data());
frame_cc_factor.data(), rejected_obs.data());
// Downloaded a slice of groups at a time and unpacked straight into acc, so the host never
// holds a second full-length copy of the accumulators - on a large cell that is most of a
// gigabyte beside acc itself. Resized, not sized vectors: the download writes every element
// (the kernel fills the empty groups too), and a member for the same reason as FullsStaging.
// Per-group and independent, so the unpack splits; the rejected count is an integer sum and
// does not care in which order the chunks add theirs in.
constexpr int ACC_SLICE = 1 << 20;
MergeAccumStaging &ms = merge_accum;
ms.Resize(std::min(n_groups, ACC_SLICE));
std::atomic<size_t> rejected_total{0};
ParallelChunks(n_groups, ThreadsForWork(n_groups, nthreads), [&](int glo, int ghi) {
size_t rej = 0;
for (int g = glo; g < ghi; ++g) {
Accum &a = acc[g];
a.swI = ms.swI[g]; a.sw = ms.sw[g]; a.swIh[0] = ms.swIh0[g]; a.swIh[1] = ms.swIh1[g];
a.swh[0] = ms.swh0[g]; a.swh[1] = ms.swh1[g];
a.swh_typ[0] = ms.swh_typ0[g]; a.swh_typ[1] = ms.swh_typ1[g];
a.nh[0] = static_cast<size_t>(ms.nh0[g]); a.nh[1] = static_cast<size_t>(ms.nh1[g]);
a.d = static_cast<float>(ms.d[g]);
a.on_ice = ms.on_ice[g] != 0;
rej += static_cast<size_t>(ms.rej[g]);
}
rejected_total += rej;
});
for (int g0 = 0; g0 < n_groups; g0 += ACC_SLICE) {
const int cnt = std::min(ACC_SLICE, n_groups - g0);
gpu_->MergeAccumRange(g0, cnt, ms.swI.data(), ms.sw.data(), ms.swIh0.data(), ms.swIh1.data(),
ms.swh0.data(), ms.swh1.data(), ms.swh_typ0.data(), ms.swh_typ1.data(),
ms.nh0.data(), ms.nh1.data(), ms.d.data(), ms.rej.data(),
ms.on_ice.data());
ParallelChunks(cnt, ThreadsForWork(cnt, nthreads), [&](int lo, int hi) {
size_t rej = 0;
for (int i = lo; i < hi; ++i) {
Accum &a = acc[g0 + i];
a.swI = ms.swI[i]; a.sw = ms.sw[i]; a.swIh[0] = ms.swIh0[i]; a.swIh[1] = ms.swIh1[i];
a.swh[0] = ms.swh0[i]; a.swh[1] = ms.swh1[i];
a.swh_typ[0] = ms.swh_typ0[i]; a.swh_typ[1] = ms.swh_typ1[i];
a.nh[0] = static_cast<size_t>(ms.nh0[i]); a.nh[1] = static_cast<size_t>(ms.nh1[i]);
a.d = static_cast<float>(ms.d[i]);
a.on_ice = ms.on_ice[i] != 0;
rej += static_cast<size_t>(ms.rej[i]);
}
rejected_total += rej;
});
}
reject_count = rejected_total;
did_gpu_acc = true;
}
+35 -30
View File
@@ -148,32 +148,14 @@ private:
// re-run over the samples that survive the automatic resolution cutoff. See MergeAndStats.
struct Sample { double s2, I2, dev2; float d; };
// The record the ingest keeps on the host when the device pipeline is resident (a GPU and no
// observation dump): every immutable per-obs field is uploaded straight from the source
// reflections inside BuildInRangeObservations, and the host keeps only what its own remaining
// stages touch - the geometry smoothing (delta_phi, then the recomputed partiality), plus the
// rocking-event walk's inputs. Thirty-two bytes against the Obs's eighty, and on that path the fat
// record is never materialised at all: on a fine-sliced long axis it alone is many gigabytes,
// held beside the source reflections it was copied from.
struct ObsIngest {
// Sixteen bits are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the
// longest axis at atomic resolution - and on a long axis this array sets the ingest's memory
// high-water mark, so each index costs two bytes here, not four.
int16_t h, k, l;
// The rocking-event usability test (RockingEventFrames), evaluated on the ingest-time
// values - which is what that walk reads on the resident path, where the host corr is
// never refreshed (see partials_released). Beside the indices, where it fills their padding.
uint8_t rock_ok;
int32_t frame;
float image_number, d, zeta, delta_phi, partiality;
};
// The narrow per-observation record the ingest sort orders: the raw hkl the runs are cut on, the
// frame position that breaks a tie inside one, the observation's own index (which makes the order
// total - see the .cpp), and the resolution the range test reads. Twenty-four bytes against the
// Obs's eighty, and it is all the ingest needs before it knows which observations survive.
// total - see the .cpp), and the resolution the range test reads. Twenty bytes against the Obs's
// eighty, and it is all the ingest needs before it knows which observations survive. Sixteen bits
// are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the longest axis at
// atomic resolution - and on a long axis this array sets the sort's memory high-water mark.
struct SortKey {
int32_t h, k, l;
int16_t h, k, l;
float image_number;
int32_t idx;
float d;
@@ -233,12 +215,36 @@ private:
int report_shell_count = 9;
std::optional<ReportResolutionRange> report_range; // the second, reference-range table
// Flat buffers, allocated once by Ingest() and reused across Run() calls. Exactly one of the
// two observation arrays is built: the narrow resident record when the device pipeline is
// active and no observation dump was asked for (resident_ingest), the full Obs otherwise.
// Flat buffers, allocated once by Ingest() and reused across Run() calls. The full Obs array is
// built only when the device pipeline is NOT resident (no GPU, or an observation dump was asked
// for); on the resident path (resident_ingest) the device holds the observations and the host
// keeps only the ingest arrays below, until the end of Ingest.
std::vector<Obs> partials; // all per-frame partials, grouped by frame
std::vector<ObsIngest> partials_ingest; // the resident-path record (same order)
bool resident_ingest = false;
// What the ingest-time geometry smoothing (SmoothMosaicityAndPartiality) needs beyond the source
// reflections, one array per field in `partials` order: the two fields it rewrites, and the two the
// rocking-event walks read in raw-hkl order, where a source reflection is out of reach. Every other
// field it reads - hkl, frame, d, zeta - is read from the source reflection itself (ForEachKeptIn),
// so on the resident path no per-observation record is built at all: on a fine-sliced long axis
// that record was gigabytes, held beside the source reflections it was copied from. Filled by
// BuildInRangeObservations and handed back at the end of Ingest.
std::vector<float> ingest_delta_phi, ingest_partiality, ingest_image_number;
// The rocking-event usability test (RockingEventFrames) on the ingest-time values - which is what
// that walk reads on the resident path, where the host corr is never refreshed (see
// partials_released). Resident path only.
std::vector<uint8_t> ingest_rock_ok;
// Which source reflections the resolution range kept (indexed like the sort keys: frame by frame,
// from ingest_src_start), so that the i-th kept one of a frame can be found again.
std::vector<uint8_t> ingest_keep;
std::vector<int32_t> ingest_src_start;
// Call fn(observation index, source reflection) for each reflection of frame o that the ingest
// kept, in order.
template <class Fn> void ForEachKeptIn(int o, Fn fn) const {
int32_t src = ingest_src_start[o];
int32_t at = frame_start[o];
for (const auto &r : partials_out[o].reflections)
if (ingest_keep[src++]) fn(at++, r);
}
// How many observations Ingest built (== partials.size() until any release below), so the
// sizes survive when the array itself does not.
int n_partials_obs = 0;
@@ -470,9 +476,8 @@ private:
// per-frame partials of one rocking event tile the curve consistently (they sum toward 1) before the
// 3D combine. Deterministic (frame order); replaces the old arrival-order mosaicity moving average
// that prediction applied. SG-independent, so done once in Ingest.
template <class T> void SmoothGeometry(std::vector<T> &obs);
void SmoothMosaicityAndPartiality(); // dispatch: the resident narrow record or the full Obs
template <class T> void SmoothMosaicityAndPartialityOn(std::vector<T> &obs);
void SmoothGeometry();
void SmoothMosaicityAndPartiality();
void Combine(); // partials -> fulls (CPU)
@@ -879,10 +879,7 @@ void RotationScaleMergeGPU::MergeEmSamples(bool for_search, double min_partialit
void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_b, bool error_model_active,
bool reject_outliers, double reject_nsigma, const float *reject_median,
const uint8_t *half, const double *frame_cc_factor,
double *swI, double *sw, double *swIh0, double *swIh1,
double *swh0, double *swh1, double *swh_typ0, double *swh_typ1,
int32_t *nh0, int32_t *nh1, double *d_out,
int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs) {
uint8_t *rejected_obs) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
const int ng = d.n_groups;
@@ -924,17 +921,23 @@ void RotationScaleMergeGPU::MergeAccum(double error_model_a, double error_model_
MergeAccumKernel<<<grp_blocks, BLK>>>(p);
CudaCheck(cudaGetLastError(), "merge accum launch");
CudaCheck(cudaDeviceSynchronize(), "merge accum sync");
auto dl = [&](void *h, const CudaDevicePtr<double> &s) {
CudaCheck(cudaMemcpy(h, s.get(), size_t(ng) * sizeof(double), cudaMemcpyDeviceToHost), "dl accum"); };
CudaCheck(cudaMemcpy(rejected_obs, d.m_rejected.get(), size_t(d.n_fulls) * sizeof(uint8_t), cudaMemcpyDeviceToHost),
"dl rejected_obs");
}
void RotationScaleMergeGPU::MergeAccumRange(int g0, int n, double *swI, double *sw, double *swIh0,
double *swIh1, double *swh0, double *swh1, double *swh_typ0,
double *swh_typ1, int32_t *nh0, int32_t *nh1, double *d_out,
int32_t *rejected, uint8_t *on_ice_out) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
auto dl = [&](void *h, const auto &s) {
CudaCheck(cudaMemcpy(h, s.get() + g0, size_t(n) * sizeof(*s.get()), cudaMemcpyDeviceToHost),
"dl accum"); };
dl(swI, d.a_swI); dl(sw, d.a_sw); dl(swIh0, d.a_swIh0); dl(swIh1, d.a_swIh1);
dl(swh0, d.a_swh0); dl(swh1, d.a_swh1); dl(swh_typ0, d.a_swht0); dl(swh_typ1, d.a_swht1);
dl(d_out, d.a_d);
CudaCheck(cudaMemcpy(nh0, d.a_nh0.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh0");
CudaCheck(cudaMemcpy(nh1, d.a_nh1.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl nh1");
CudaCheck(cudaMemcpy(rejected, d.a_rejected.get(), size_t(ng) * sizeof(int32_t), cudaMemcpyDeviceToHost), "dl rej");
CudaCheck(cudaMemcpy(on_ice_out, d.a_on_ice.get(), size_t(ng) * sizeof(uint8_t), cudaMemcpyDeviceToHost), "dl on_ice");
CudaCheck(cudaMemcpy(rejected_obs, d.m_rejected.get(), size_t(d.n_fulls) * sizeof(uint8_t), cudaMemcpyDeviceToHost),
"dl rejected_obs");
dl(nh0, d.a_nh0); dl(nh1, d.a_nh1); dl(rejected, d.a_rejected); dl(on_ice_out, d.a_on_ice);
}
void RotationScaleMergeGPU::MergeRmeas(const double *merged_I, double *absdev, double *sumI,
@@ -91,18 +91,20 @@ public:
// Per-group merge accumulators (inv-var sums + deterministic half-sets, error-model-corrected sigma
// from a/b). `half` is the per-full CC1/2 half-set (length n_fulls) assigned on the host, so this
// kernel and the host merge loop cannot disagree about it.
// Outputs length n_groups; rejected[g] counts outliers dropped (reject_median uploaded, NAN
// where none). rejected_obs is the per-full flag (length n_fulls): the host needs it for the reductions
// it still does itself, above all the anomalous I(+)/I(-) split. frame_cc_factor (length n_frames) is
// each frame's (G_ref/G)^2 floored at 1; swh_typ0/1 are the half-set weights multiplied by it.
// Requires MergeEmSamples first (em_mean resident).
// The per-group results stay on the device for MergeAccumRange to download; reject_median is
// uploaded (NAN where none). rejected_obs is the per-full flag (length n_fulls): the host needs it
// for the reductions it still does itself, above all the anomalous I(+)/I(-) split.
// frame_cc_factor (length n_frames) is each frame's (G_ref/G)^2 floored at 1; swh_typ0/1 are the
// half-set weights multiplied by it. Requires MergeEmSamples first (em_mean resident).
void MergeAccum(double error_model_a, double error_model_b, bool error_model_active,
bool reject_outliers, double reject_nsigma, const float *reject_median,
const uint8_t *half, const double *frame_cc_factor,
double *swI, double *sw, double *swIh0, double *swIh1,
double *swh0, double *swh1, double *swh_typ0, double *swh_typ1,
int32_t *nh0, int32_t *nh1, double *d_out,
int32_t *rejected, uint8_t *on_ice_out, uint8_t *rejected_obs);
const uint8_t *half, const double *frame_cc_factor, uint8_t *rejected_obs);
// Download groups [g0, g0 + n) of what MergeAccum left on the device; every output has length n.
// rejected[g] counts the outliers dropped from the group.
void MergeAccumRange(int g0, int n, double *swI, double *sw, double *swIh0, double *swIh1,
double *swh0, double *swh1, double *swh_typ0, double *swh_typ1,
int32_t *nh0, int32_t *nh1, double *d_out,
int32_t *rejected, uint8_t *on_ice_out);
// Per-group R_meas accumulators (sum|I_corr-merged_I|, sum_I, n, and the count this looser walk
// accepted - which the host uses only to skip empty groups, NOT as the per-shell
+2 -1
View File
@@ -303,9 +303,10 @@ CrystalLattice ApplyReindex(const CrystalLattice &latt, const std::optional<std:
return latt.Multiply(gemmi::Mat33(v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8]));
}
template <class ReflectionContainer>
bool ReadReflectionsFromGroup(HDF5Object &file,
const std::string &image_group_name,
std::vector<Reflection> &reflections,
ReflectionContainer &reflections,
const std::optional<std::array<int32_t, 9>> &reindex) {
if (!file.Exists("/entry/reflections") || !file.Exists(image_group_name))
return false;
+1 -1
View File
@@ -6210,7 +6210,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
} else {
for (auto &io : indexer->GetIntegrationOutcome()) {
io.latt = io.latt.Multiply(reduce);
std::vector<Reflection> kept;
ReflectionVector kept;
kept.reserve(io.reflections.size());
for (auto r : io.reflections)
if (reduce_hkl(r))