Merge branch 'perf-f-gpu-surfaces' into perf-merge

This commit is contained in:
leonarski_f committed 2026-10-03 13:55:04 +02:00
commit 581e1c8ca2
5 files changed
+493 -41

No files matched your search

+106 -41
View File
@@ -3,6 +3,7 @@
#include "../../common/ParallelFor.h"
#include "RotationScaleMerge.h"
#include "RotationScaleMergeGPU.h" // SurfaceTerm, the surface fit's term on both paths
#include <algorithm>
#include <atomic>
@@ -3334,7 +3335,7 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
// traffic of the data it touched, over hundreds of megabytes. Copying once turns them into
// sequential walks of a compact array. The copy keeps fulls order, so each sum below is formed
// from exactly the same terms in exactly the same order.
struct Term { float I, sigma, corr, d; int32_t cell, group; };
using Term = RotationScaleMergeGPU::SurfaceTerm; // float I, sigma, corr, d; int32_t cell, group
std::vector<Term> term;
std::vector<uint8_t> term_parity; // frame parity, read only by the group-ordered copy below
// Which terms each pass walks, as positions in `term`. The cross-validated halves then cost half a
@@ -3437,22 +3438,27 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
std::vector<double> shw_shell(nshell, 0.0), shw_cell(nshell > 0 ? ncell : 0, 0.0);
std::vector<int32_t> group_shell(n_groups, 0); // the shell each ASU group sits in, for the gate
if (nshell > 0) {
std::vector<float> s2;
s2.reserve(term.size());
for (const Term &t : term)
s2.push_back(t.d > 0.0f ? 1.0f / (t.d * t.d) : 0.0f);
const int n_term = static_cast<int>(term.size());
std::vector<float> s2(n_term);
ParallelChunks(n_term, nt, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k)
s2[k] = term[k].d > 0.0f ? 1.0f / (term[k].d * term[k].d) : 0.0f;
});
// The edges are order statistics of s2, so they are the same values however the sort orders
// equal elements among themselves.
std::vector<float> sorted = s2;
ParallelSort(sorted.begin(), sorted.end(), nt, std::less<float>());
std::vector<float> edge(nshell - 1);
size_t prev = 0;
for (int i = 1; i < nshell; ++i) {
const size_t pos = s2.size() * static_cast<size_t>(i) / static_cast<size_t>(nshell);
std::nth_element(s2.begin() + prev, s2.begin() + pos, s2.end());
edge[i - 1] = s2[pos];
prev = pos;
}
for (int i = 1; i < nshell; ++i)
edge[i - 1] = sorted[sorted.size() * static_cast<size_t>(i) / static_cast<size_t>(nshell)];
std::vector<uint8_t> term_shell(n_term);
ParallelChunks(n_term, nt, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k)
term_shell[k] = static_cast<uint8_t>(std::upper_bound(edge.begin(), edge.end(), s2[k]) - edge.begin());
});
for (size_t k = 0; k < term.size(); ++k) {
const Term &t = term[k];
const float v = t.d > 0.0f ? 1.0f / (t.d * t.d) : 0.0f;
const int s = static_cast<int>(std::upper_bound(edge.begin(), edge.end(), v) - edge.begin());
const int s = term_shell[k];
group_shell[t.group] = s;
const double sc = static_cast<double>(t.sigma) * t.corr;
if (!(sc > 0.0)) continue;
@@ -3475,8 +3481,41 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
std::vector<int32_t> g_start(n_groups + 1, 0);
for (const Term &t : term) ++g_start[t.group + 1];
for (int g = 0; g < n_groups; ++g) g_start[g + 1] += g_start[g];
std::vector<RefTerm> gterm(term.size());
{
std::vector<RefTerm> gterm;
// With a GPU the two passes of every round run there (RotationScaleMergeGPU::Surface*), on the same
// terms in the same order: the device gets the terms with the group order as a permutation, and each
// subset cut into this fit's reduction blocks with every block's terms ordered by cell, so that one
// device thread forms what one block of the host loop sums into one cell.
bool on_gpu = false;
#ifdef JFJOCH_USE_CUDA
on_gpu = gpu_active_;
if (on_gpu) {
std::vector<int32_t> gperm(term.size());
std::vector<int32_t> fill(g_start.begin(), g_start.end() - 1);
for (size_t k = 0; k < term.size(); ++k)
gperm[fill[term[k].group]++] = static_cast<int32_t>(k);
gpu_->SurfaceSetTerms(static_cast<int>(term.size()), term.data(), term_parity.data(),
n_groups, gperm.data(), g_start.data(), ncell);
for (int parity : {0, 1, -1}) {
const std::vector<int32_t> &sel = subset(parity);
const int n = static_cast<int>(sel.size());
const int nb = ReductionBlocks(n, SURFACE_BLOCK);
std::vector<int32_t> perm(n), seg_start(static_cast<size_t>(nb) * ncell + 1, n);
ParallelFor(nb, nt, [&](int b) {
const int lo = static_cast<int>(static_cast<int64_t>(n) * b / nb);
const int hi = static_cast<int>(static_cast<int64_t>(n) * (b + 1) / nb);
std::vector<int32_t> pos(ncell + 1, 0);
for (int k = lo; k < hi; ++k) ++pos[term[sel[k]].cell + 1];
for (int c = 0; c < ncell; ++c) pos[c + 1] += pos[c];
for (int c = 0; c < ncell; ++c) seg_start[static_cast<size_t>(b) * ncell + c] = lo + pos[c];
for (int k = lo; k < hi; ++k) perm[lo + pos[term[sel[k]].cell]++] = sel[k];
});
gpu_->SurfaceSetSubset(parity < 0 ? 2 : parity, nb, perm.data(), seg_start.data());
}
}
#endif
if (!on_gpu) {
gterm.resize(term.size());
std::vector<int32_t> fill(g_start.begin(), g_start.end() - 1);
for (size_t k = 0; k < term.size(); ++k) {
const Term &t = term[k];
@@ -3488,6 +3527,12 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
// and the score.
std::vector<double> sw(n_groups), swI(n_groups);
auto reference = [&](int parity, const std::vector<double> &A) {
#ifdef JFJOCH_USE_CUDA
if (on_gpu) {
gpu_->SurfaceReference(parity, A.data());
return;
}
#endif
ParallelChunks(n_groups, nt, [&](int glo, int ghi) {
for (int g = glo; g < ghi; ++g) {
double s_w = 0.0, s_wI = 0.0;
@@ -3520,7 +3565,7 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
const std::vector<int32_t> &sel = subset(parity);
std::vector<double> A(ncell, 1.0);
// Per-block cell accumulators, allocated once for the whole fit rather than per round.
const int nb = ReductionBlocks(static_cast<int>(sel.size()), SURFACE_BLOCK);
const int nb = on_gpu ? 0 : ReductionBlocks(static_cast<int>(sel.size()), SURFACE_BLOCK);
std::vector<std::vector<double>> tcross(nb, std::vector<double>(ncell)), tref2(nb, std::vector<double>(ncell));
settled = false;
n_clamped = 0;
@@ -3555,22 +3600,28 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
// there is no ordering that keeps threads off each other's bins, and there are only ncell
// of them, so per-block copies are cheap and the fixed blocks keep it the same at any -N.
std::vector<double> cross(ncell, 0.0), ref2(ncell, 0.0);
ParallelBlocks(static_cast<int>(sel.size()), nt, [&](int b, int lo, int hi) {
std::vector<double> &xcross = tcross[b], &xref2 = tref2[b];
std::fill(xcross.begin(), xcross.end(), 0.0);
std::fill(xref2.begin(), xref2.end(), 0.0);
for (int k = lo; k < hi; ++k) {
const Term &o = term[sel[k]];
if (sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group], a = A[o.cell];
const double Is = static_cast<double>(o.I) * o.corr * a, sc = static_cast<double>(o.sigma) * o.corr * a;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue;
const double w = 1.0 / (sc * sc);
xcross[o.cell] += w * Is * Iref; xref2[o.cell] += w * Iref * Iref;
}
}, SURFACE_BLOCK);
for (int b = 0; b < nb; ++b)
for (int c = 0; c < ncell; ++c) { cross[c] += tcross[b][c]; ref2[c] += tref2[b][c]; }
#ifdef JFJOCH_USE_CUDA
if (on_gpu)
gpu_->SurfaceFitSums(parity < 0 ? 2 : parity, cross.data(), ref2.data());
#endif
if (!on_gpu) {
ParallelBlocks(static_cast<int>(sel.size()), nt, [&](int b, int lo, int hi) {
std::vector<double> &xcross = tcross[b], &xref2 = tref2[b];
std::fill(xcross.begin(), xcross.end(), 0.0);
std::fill(xref2.begin(), xref2.end(), 0.0);
for (int k = lo; k < hi; ++k) {
const Term &o = term[sel[k]];
if (sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group], a = A[o.cell];
const double Is = static_cast<double>(o.I) * o.corr * a, sc = static_cast<double>(o.sigma) * o.corr * a;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue;
const double w = 1.0 / (sc * sc);
xcross[o.cell] += w * Is * Iref; xref2[o.cell] += w * Iref * Iref;
}
}, SURFACE_BLOCK);
for (int b = 0; b < nb; ++b)
for (int c = 0; c < ncell; ++c) { cross[c] += tcross[b][c]; ref2[c] += tref2[b][c]; }
}
std::vector<double> dsorted = cross;
std::nth_element(dsorted.begin(), dsorted.begin() + dsorted.size() / 2, dsorted.end());
const double lambda = 0.1 * std::max(1e-30, dsorted[dsorted.size() / 2]);
@@ -3672,20 +3723,33 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
const int nsh_cc = nshell > 0 ? nshell : 1;
auto half_means = [&](int parity, const std::vector<double> &A) {
reference(parity, A);
#ifdef JFJOCH_USE_CUDA
if (on_gpu)
gpu_->SurfaceGetReference(sw.data(), swI.data());
#endif
std::vector<double> m(n_groups, std::numeric_limits<double>::quiet_NaN());
for (int g = 0; g < n_groups; ++g)
if (sw[g] > 0.0) m[g] = swI[g] / sw[g];
return m;
};
auto shell_cc = [&](const std::vector<double> &x, const std::vector<double> &y, int s) {
double n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
// The correlation within every shell, in one walk over the groups: each shell still sums its own
// groups in group order.
auto shell_cc = [&](const std::vector<double> &x, const std::vector<double> &y) {
struct Sums { double n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; };
std::vector<Sums> sum(nsh_cc);
for (int g = 0; g < n_groups; ++g) {
if (group_shell[g] != s || !std::isfinite(x[g]) || !std::isfinite(y[g])) continue;
n += 1; sx += x[g]; sy += y[g]; sxx += x[g] * x[g]; syy += y[g] * y[g]; sxy += x[g] * y[g];
if (!std::isfinite(x[g]) || !std::isfinite(y[g])) continue;
Sums &u = sum[group_shell[g]];
u.n += 1; u.sx += x[g]; u.sy += y[g]; u.sxx += x[g] * x[g]; u.syy += y[g] * y[g]; u.sxy += x[g] * y[g];
}
const double vx = sxx - sx * sx / n, vy = syy - sy * sy / n;
return (n >= 50 && vx > 0.0 && vy > 0.0) ? (sxy - sx * sy / n) / std::sqrt(vx * vy)
: std::numeric_limits<double>::quiet_NaN();
std::vector<double> cc(nsh_cc, std::numeric_limits<double>::quiet_NaN());
for (int s = 0; s < nsh_cc; ++s) {
const Sums &u = sum[s];
const double vx = u.sxx - u.sx * u.sx / u.n, vy = u.syy - u.sy * u.sy / u.n;
if (u.n >= 50 && vx > 0.0 && vy > 0.0)
cc[s] = (u.sxy - u.sx * u.sy / u.n) / std::sqrt(vx * vy);
}
return cc;
};
const std::vector<double> ident(ncell, 1.0);
const std::vector<double> A_even = fit_surface(0), A_odd = fit_surface(1);
@@ -3702,8 +3766,9 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector<int32_t> &cell, int
// Following Fisher (1915) Biometrika 10, 507-521
double gain = 0.0;
int n_cc = 0;
const std::vector<double> cc0 = shell_cc(odd0, even0), cc1 = shell_cc(odd1, even1);
for (int sh = 0; sh < nsh_cc; ++sh) {
const double c0 = shell_cc(odd0, even0, sh), c1 = shell_cc(odd1, even1, sh);
const double c0 = cc0[sh], c1 = cc1[sh];
if (std::isfinite(c0) && std::isfinite(c1)) { gain += std::atanh(c1) - std::atanh(c0); ++n_cc; }
}
if (n_cc > 0) gain /= n_cc;
@@ -639,6 +639,100 @@ namespace {
}
}
// --- correction-surface fit (ApplyCellSurface) ---
// The host fit is the reference here: these kernels form the same sums from the same terms in the
// same order, and every rounding is spelled out (__dmul_rn / __dadd_rn round each step on its own,
// fma rounds once) to be the one the host build makes - nvcc would otherwise contract a multiply
// and an add wherever it sees them, and GCC at -march=x86-64-v3 does so only in some of them. The
// products are the host's (I * corr) * a and (sigma * corr) * a.
using SurfaceTerm = RotationScaleMergeGPU::SurfaceTerm;
__device__ __forceinline__ double SurfaceIs(const SurfaceTerm &t, double a) {
return __dmul_rn(__dmul_rn(double(t.I), double(t.corr)), a);
}
__device__ __forceinline__ double SurfaceSigma(const SurfaceTerm &t, double a) {
return __dmul_rn(__dmul_rn(double(t.sigma), double(t.corr)), a);
}
// One thread per ASU group: the group's inverse-variance sums over its terms of frame parity `parity`
// (< 0 = all), in fulls order - the host's `reference`. The host builds that loop twice, and only
// the parity-filtered copy fuses the multiply-add of swI; the unfiltered one rounds the product.
__global__ void SurfaceReferenceKernel(int n_groups, int parity, const int32_t *__restrict__ gperm,
const int32_t *__restrict__ gstart,
const SurfaceTerm *__restrict__ term,
const uint8_t *__restrict__ term_parity,
const double *__restrict__ A,
double *__restrict__ sw, double *__restrict__ swI) {
for (int g = blockIdx.x * blockDim.x + threadIdx.x; g < n_groups; g += gridDim.x * blockDim.x) {
double s_w = 0.0, s_wI = 0.0;
for (int k = gstart[g]; k < gstart[g + 1]; ++k) {
const int i = gperm[k];
if (parity >= 0 && term_parity[i] != parity) continue;
const SurfaceTerm t = term[i];
const double a = A[t.cell];
const double Is = SurfaceIs(t, a), sc = SurfaceSigma(t, a);
const double w = 1.0 / __dmul_rn(sc, sc);
s_w = __dadd_rn(s_w, w);
s_wI = parity >= 0 ? fma(Is, w, s_wI) : __dadd_rn(s_wI, __dmul_rn(Is, w));
}
sw[g] = s_w; swI[g] = s_wI;
}
}
// The fit's sums of one round. Each term's contribution is formed on its own thread (one per term,
// in segment order), and only the two multiply-adds that sum them are left to the thread of each
// (reduction block, cell) - the per-block accumulators of the host fit, one slot at a time, walking
// its segment in term order. A term the host skips is marked by a NaN Iref (a kept one is finite).
__global__ void SurfaceFitTermKernel(int n, const int32_t *__restrict__ perm,
const SurfaceTerm *__restrict__ term,
const double *__restrict__ A,
const double *__restrict__ sw, const double *__restrict__ swI,
double *__restrict__ w_Is, double *__restrict__ w_Iref,
double *__restrict__ Iref_out) {
for (int k = blockIdx.x * blockDim.x + threadIdx.x; k < n; k += gridDim.x * blockDim.x) {
const SurfaceTerm t = term[perm[k]];
Iref_out[k] = NAN;
if (sw[t.group] <= 0.0) continue;
const double Iref = swI[t.group] / sw[t.group], a = A[t.cell];
const double Is = SurfaceIs(t, a), sc = SurfaceSigma(t, a);
if (!isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue;
const double w = 1.0 / __dmul_rn(sc, sc);
w_Is[k] = __dmul_rn(w, Is);
w_Iref[k] = __dmul_rn(w, Iref);
Iref_out[k] = Iref;
}
}
__global__ void SurfaceFitSegmentKernel(int n_seg, const int32_t *__restrict__ seg_start,
const double *__restrict__ w_Is, const double *__restrict__ w_Iref,
const double *__restrict__ Iref,
double *__restrict__ tcross, double *__restrict__ tref2) {
for (int s = blockIdx.x * blockDim.x + threadIdx.x; s < n_seg; s += gridDim.x * blockDim.x) {
double xcross = 0.0, xref2 = 0.0;
for (int k = seg_start[s]; k < seg_start[s + 1]; ++k) {
if (isnan(Iref[k])) continue;
xcross = fma(w_Is[k], Iref[k], xcross);
xref2 = fma(w_Iref[k], Iref[k], xref2);
}
tcross[s] = xcross; tref2[s] = xref2;
}
}
// One thread per cell: the blocks' sums added up in block order, as the host adds its slots.
__global__ void SurfaceFitCellKernel(int n_blocks, int ncell, const double *__restrict__ tcross,
const double *__restrict__ tref2,
double *__restrict__ cross, double *__restrict__ ref2) {
for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < ncell; c += gridDim.x * blockDim.x) {
double sc = 0.0, sr = 0.0;
for (int b = 0; b < n_blocks; ++b) {
sc = __dadd_rn(sc, tcross[b * ncell + c]);
sr = __dadd_rn(sr, tref2[b * ncell + c]);
}
cross[c] = sc; ref2[c] = sr;
}
}
void CudaCheck(cudaError_t e, const char *what);
// A copy on the instance's stream, waited for - what cudaMemcpy on the NULL stream was, without also
@@ -817,6 +911,18 @@ struct RotationScaleMergeGPU::Impl {
CudaDevicePtr<uint8_t> f_sco_ok;
CudaDevicePtr<int32_t> f_frame_perm, f_frame_start, f_frame_count;
CudaDevicePtr<int32_t> f_gperm, f_gstart, f_gcount;
// correction-surface fit (one ApplyCellSurface call at a time): its terms and their group CSR, the
// three subsets' per-(block, cell) segments, the surface and the sums of the round
int s_ncell = 0, s_n_groups = 0;
int s_n_blocks[3] = {0, 0, 0};
int s_n_sel[3] = {0, 0, 0};
size_t s_slots = 0; // capacity of s_tcross / s_tref2
CudaDevicePtr<RotationScaleMergeGPU::SurfaceTerm> s_term;
CudaDevicePtr<uint8_t> s_parity;
CudaDevicePtr<int32_t> s_gperm, s_gstart;
CudaDevicePtr<int32_t> s_perm[3], s_seg_start[3];
CudaDevicePtr<double> s_A, s_sw, s_swI, s_tcross, s_tref2, s_cross, s_ref2;
CudaDevicePtr<double> s_w_Is, s_w_Iref, s_Iref; // per term of a subset, in segment order
};
// Set the device this instance's memory lives on for the duration of a call, and put the caller's
@@ -1460,3 +1566,88 @@ void RotationScaleMergeGPU::SetFullsCorr(const float *corr) {
CopyAndWait(d.f_corr.get(), corr, size_t(d.n_fulls) * sizeof(float),
cudaMemcpyHostToDevice, impl_->s(), "upload f_corr");
}
void RotationScaleMergeGPU::SurfaceSetTerms(int n_terms, const SurfaceTerm *term, const uint8_t *parity,
int n_groups, const int32_t *gperm, const int32_t *gstart,
int ncell) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
d.s_ncell = ncell;
d.s_n_groups = n_groups;
d.Upload(d.s_term, term, n_terms);
d.Upload(d.s_parity, parity, n_terms);
d.Upload(d.s_gperm, gperm, n_terms);
d.Upload(d.s_gstart, gstart, n_groups + 1);
d.s_A = d.Alloc<double>(std::max(1, ncell));
d.s_cross = d.Alloc<double>(std::max(1, ncell));
d.s_ref2 = d.Alloc<double>(std::max(1, ncell));
d.s_sw = d.Alloc<double>(std::max(1, n_groups));
d.s_swI = d.Alloc<double>(std::max(1, n_groups));
d.s_w_Is = d.Alloc<double>(std::max(1, n_terms));
d.s_w_Iref = d.Alloc<double>(std::max(1, n_terms));
d.s_Iref = d.Alloc<double>(std::max(1, n_terms));
for (int &nb : d.s_n_blocks) nb = 0;
}
void RotationScaleMergeGPU::SurfaceSetSubset(int subset, int n_blocks, const int32_t *perm,
const int32_t *seg_start) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
const int n_seg = n_blocks * d.s_ncell;
d.s_n_sel[subset] = n_blocks > 0 ? seg_start[n_seg] : 0;
d.Upload(d.s_perm[subset], perm, d.s_n_sel[subset]);
d.Upload(d.s_seg_start[subset], seg_start, n_seg + 1);
d.s_n_blocks[subset] = n_blocks;
if (size_t(n_seg) > d.s_slots) {
d.s_tcross = d.Alloc<double>(n_seg);
d.s_tref2 = d.Alloc<double>(n_seg);
d.s_slots = n_seg;
}
}
void RotationScaleMergeGPU::SurfaceReference(int parity, const double *A) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
CopyAndWait(d.s_A.get(), A, size_t(d.s_ncell) * sizeof(double), cudaMemcpyHostToDevice, impl_->s(),
"upload surface");
const int grp_blocks = std::min(65535, (d.s_n_groups + BLK - 1) / BLK);
if (grp_blocks > 0)
SurfaceReferenceKernel<<<grp_blocks, BLK, 0, impl_->s()>>>(d.s_n_groups, parity, d.s_gperm.get(),
d.s_gstart.get(), d.s_term.get(), d.s_parity.get(), d.s_A.get(), d.s_sw.get(), d.s_swI.get());
CudaCheck(cudaGetLastError(), "surface reference launch");
CudaCheck(cudaStreamSynchronize(impl_->s()), "surface reference sync");
}
void RotationScaleMergeGPU::SurfaceGetReference(double *sw, double *swI) const {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
const size_t bytes = size_t(d.s_n_groups) * sizeof(double);
CopyAndWait(sw, d.s_sw.get(), bytes, cudaMemcpyDeviceToHost, impl_->s(), "dl surface sw");
CopyAndWait(swI, d.s_swI.get(), bytes, cudaMemcpyDeviceToHost, impl_->s(), "dl surface swI");
}
void RotationScaleMergeGPU::SurfaceFitSums(int subset, double *cross, double *ref2) {
DeviceGuard guard(impl_->device, impl_->available);
auto &d = *impl_;
const int ncell = d.s_ncell, nb = d.s_n_blocks[subset], n_seg = nb * ncell;
if (nb == 0) {
std::fill(cross, cross + ncell, 0.0);
std::fill(ref2, ref2 + ncell, 0.0);
return;
}
SurfaceFitTermKernel<<<std::min(65535, (d.s_n_sel[subset] + BLK - 1) / BLK), BLK, 0, impl_->s()>>>(
d.s_n_sel[subset], d.s_perm[subset].get(), d.s_term.get(), d.s_A.get(), d.s_sw.get(), d.s_swI.get(),
d.s_w_Is.get(), d.s_w_Iref.get(), d.s_Iref.get());
CudaCheck(cudaGetLastError(), "surface fit term launch");
SurfaceFitSegmentKernel<<<std::min(65535, (n_seg + BLK - 1) / BLK), BLK, 0, impl_->s()>>>(n_seg,
d.s_seg_start[subset].get(), d.s_w_Is.get(), d.s_w_Iref.get(), d.s_Iref.get(),
d.s_tcross.get(), d.s_tref2.get());
CudaCheck(cudaGetLastError(), "surface fit segment launch");
SurfaceFitCellKernel<<<std::min(65535, (ncell + BLK - 1) / BLK), BLK, 0, impl_->s()>>>(nb, ncell,
d.s_tcross.get(), d.s_tref2.get(), d.s_cross.get(), d.s_ref2.get());
CudaCheck(cudaGetLastError(), "surface cell sum launch");
CopyAndWait(cross, d.s_cross.get(), size_t(ncell) * sizeof(double), cudaMemcpyDeviceToHost, impl_->s(),
"dl surface cross");
CopyAndWait(ref2, d.s_ref2.get(), size_t(ncell) * sizeof(double), cudaMemcpyDeviceToHost, impl_->s(),
"dl surface ref2");
}
@@ -192,6 +192,34 @@ public:
// Download the fulls' working corr (length = n_fulls), valid after ScaleFulls.
void GetFullsCorr(float *corr) const;
// --- correction-surface fit (RotationScaleMerge::ApplyCellSurface) ---
// The two passes every round of the fit makes over its terms, on the device; the host keeps the
// per-cell step, the gauge and the cross-validation. Both sums are formed in exactly the order the
// host forms them, so the fitted surface is the host's to the last bit (see the kernels).
// One observation as the surface fit sees it: the host's term, uploaded as it stands.
struct SurfaceTerm { float I, sigma, corr, d; int32_t cell, group; };
// The terms (in fulls order) with each one's frame parity, the ASU-group CSR over them (gperm lists
// the terms of group g at [gstart[g], gstart[g+1]), in fulls order) and the cell count.
void SurfaceSetTerms(int n_terms, const SurfaceTerm *term, const uint8_t *parity,
int n_groups, const int32_t *gperm, const int32_t *gstart, int ncell);
// One subset of the terms (0 = even frames, 1 = odd, 2 = all), cut into the host's n_blocks
// reduction blocks and ordered within each block by cell, keeping term order inside a cell:
// block b, cell c is perm[seg_start[b * ncell + c], seg_start[b * ncell + c + 1]).
void SurfaceSetSubset(int subset, int n_blocks, const int32_t *perm, const int32_t *seg_start);
// The per-group reference sums sw / swI over the terms of frame parity `parity` (< 0 = all) with
// the surface A (length ncell) applied. They stay on the device for SurfaceFitSums;
// SurfaceGetReference downloads them (length n_groups each).
void SurfaceReference(int parity, const double *A);
void SurfaceGetReference(double *sw, double *swI) const;
// The fit's per-cell sums over one subset against the last SurfaceReference and its A:
// cross = sum w Is Iref and ref2 = sum w Iref^2 (length ncell each).
void SurfaceFitSums(int subset, double *cross, double *ref2);
private:
struct Impl;
std::unique_ptr<Impl> impl_;
+1
View File
@@ -148,6 +148,7 @@ ADD_EXECUTABLE(jfjoch_test
AnisotropyAnalysisTest.cpp
ModelScalingTest.cpp
ModelScaleGPUTest.cpp
CorrectionSurfaceGPUTest.cpp
TwinningAnalysisTest.cpp
TranslationalNCSTest.cpp
RfreeFlagsTest.cpp
+167
View File
@@ -0,0 +1,167 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../common/CUDAWrapper.h"
#ifdef JFJOCH_USE_CUDA
#include <cmath>
#include <cstring>
#include <random>
#include <vector>
#include "../common/ParallelFor.h"
#include "../image_analysis/scale_merge/RotationScaleMergeGPU.h"
namespace {
using Term = RotationScaleMergeGPU::SurfaceTerm;
// The roundings RotationScaleMerge::ApplyCellSurface makes on the host (x86-64-v3 build), spelled out:
// a volatile result is rounded on its own and never fused into the next operation, std::fma is fused.
double Mul(double a, double b) { volatile double r = a * b; return r; }
double Add(double a, double b) { volatile double r = a + b; return r; }
constexpr int SURFACE_BLOCK = 32768; // ApplyCellSurface's reduction block
struct Surface {
int n_groups = 0, ncell = 0;
std::vector<Term> term;
std::vector<uint8_t> parity;
std::vector<int32_t> gperm, gstart;
std::vector<int32_t> sel[3]; // even, odd, all - in term order
};
Surface MakeSurface(int n_terms, int n_groups, int ncell, uint32_t seed) {
std::mt19937 rng(seed);
std::uniform_int_distribution<int> group(0, n_groups - 1), cell(0, ncell - 1), bit(0, 1);
std::uniform_real_distribution<float> u(0.0f, 1.0f);
Surface s;
s.n_groups = n_groups;
s.ncell = ncell;
for (int i = 0; i < n_terms; ++i) {
// Negative intensities, and now and then a sigma of zero: both reach the host sums as they are.
const float I = 1000.0f * u(rng) - 100.0f;
const float sigma = (i % 997 == 0) ? 0.0f : 1.0f + 30.0f * u(rng);
// Every 50th group gets no terms at all, so its reference is empty.
int g = group(rng);
if (g % 50 == 0) g = (g + 1) % n_groups;
s.term.push_back({I, sigma, 0.5f + u(rng), 1.0f + 3.0f * u(rng), cell(rng), g});
s.parity.push_back(static_cast<uint8_t>(bit(rng)));
s.sel[s.parity.back()].push_back(i);
s.sel[2].push_back(i);
}
s.gstart.assign(n_groups + 1, 0);
for (const Term &t : s.term) ++s.gstart[t.group + 1];
for (int g = 0; g < n_groups; ++g) s.gstart[g + 1] += s.gstart[g];
s.gperm.resize(n_terms);
std::vector<int32_t> fill(s.gstart.begin(), s.gstart.end() - 1);
for (int i = 0; i < n_terms; ++i) s.gperm[fill[s.term[i].group]++] = i;
return s;
}
void HostReference(const Surface &s, int parity, const std::vector<double> &A,
std::vector<double> &sw, std::vector<double> &swI) {
sw.assign(s.n_groups, 0.0);
swI.assign(s.n_groups, 0.0);
for (int g = 0; g < s.n_groups; ++g) {
double s_w = 0.0, s_wI = 0.0;
for (int k = s.gstart[g]; k < s.gstart[g + 1]; ++k) {
const int i = s.gperm[k];
if (parity >= 0 && s.parity[i] != parity) continue;
const Term &t = s.term[i];
const double a = A[t.cell];
const double Is = Mul(Mul(t.I, t.corr), a), sc = Mul(Mul(t.sigma, t.corr), a);
const double w = 1.0 / Mul(sc, sc);
s_w = Add(s_w, w);
s_wI = parity >= 0 ? std::fma(Is, w, s_wI) : Add(s_wI, Mul(Is, w));
}
sw[g] = s_w;
swI[g] = s_wI;
}
}
void HostFitSums(const Surface &s, const std::vector<int32_t> &sel, const std::vector<double> &A,
const std::vector<double> &sw, const std::vector<double> &swI,
std::vector<double> &cross, std::vector<double> &ref2) {
cross.assign(s.ncell, 0.0);
ref2.assign(s.ncell, 0.0);
const int n = static_cast<int>(sel.size()), nb = ReductionBlocks(n, SURFACE_BLOCK);
for (int b = 0; b < nb; ++b) {
std::vector<double> xcross(s.ncell, 0.0), xref2(s.ncell, 0.0);
const int lo = static_cast<int>(int64_t(n) * b / nb), hi = static_cast<int>(int64_t(n) * (b + 1) / nb);
for (int k = lo; k < hi; ++k) {
const Term &t = s.term[sel[k]];
if (sw[t.group] <= 0.0) continue;
const double Iref = swI[t.group] / sw[t.group], a = A[t.cell];
const double Is = Mul(Mul(t.I, t.corr), a), sc = Mul(Mul(t.sigma, t.corr), a);
if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue;
const double w = 1.0 / Mul(sc, sc);
xcross[t.cell] = std::fma(Mul(w, Is), Iref, xcross[t.cell]);
xref2[t.cell] = std::fma(Mul(w, Iref), Iref, xref2[t.cell]);
}
for (int c = 0; c < s.ncell; ++c) {
cross[c] = Add(cross[c], xcross[c]);
ref2[c] = Add(ref2[c], xref2[c]);
}
}
}
// The device side of one subset: ApplyCellSurface's per-block counting sort by cell.
void UploadSubset(RotationScaleMergeGPU &gpu, const Surface &s, int id, const std::vector<int32_t> &sel) {
const int n = static_cast<int>(sel.size()), nb = ReductionBlocks(n, SURFACE_BLOCK);
std::vector<int32_t> perm(n), seg_start(static_cast<size_t>(nb) * s.ncell + 1, n);
for (int b = 0; b < nb; ++b) {
const int lo = static_cast<int>(int64_t(n) * b / nb), hi = static_cast<int>(int64_t(n) * (b + 1) / nb);
std::vector<int32_t> pos(s.ncell + 1, 0);
for (int k = lo; k < hi; ++k) ++pos[s.term[sel[k]].cell + 1];
for (int c = 0; c < s.ncell; ++c) pos[c + 1] += pos[c];
for (int c = 0; c < s.ncell; ++c) seg_start[size_t(b) * s.ncell + c] = lo + pos[c];
for (int k = lo; k < hi; ++k) perm[lo + pos[s.term[sel[k]].cell]++] = sel[k];
}
gpu.SurfaceSetSubset(id, nb, perm.data(), seg_start.data());
}
bool SameBits(const std::vector<double> &a, const std::vector<double> &b) {
return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0;
}
} // namespace
TEST_CASE("CorrectionSurfaceGPU_SumsMatchHostBitForBit", "[RotationScale][gpu]") {
if (get_gpu_count() == 0)
SKIP("No GPU");
// Enough terms for several reduction blocks in every subset.
const Surface s = MakeSurface(250000, 4000, 144, 7);
RotationScaleMergeGPU gpu;
REQUIRE(gpu.Available());
gpu.SurfaceSetTerms(static_cast<int>(s.term.size()), s.term.data(), s.parity.data(), s.n_groups,
s.gperm.data(), s.gstart.data(), s.ncell);
for (int id = 0; id < 3; ++id)
UploadSubset(gpu, s, id, s.sel[id]);
std::mt19937 rng(11);
std::uniform_real_distribution<double> u(0.7, 1.4);
std::vector<double> A(s.ncell);
for (double &a : A) a = u(rng);
for (int parity : {0, 1, -1}) {
const int id = parity < 0 ? 2 : parity;
std::vector<double> sw, swI, cross, ref2;
HostReference(s, parity, A, sw, swI);
HostFitSums(s, s.sel[id], A, sw, swI, cross, ref2);
REQUIRE(ReductionBlocks(static_cast<int>(s.sel[id].size()), SURFACE_BLOCK) > 1);
gpu.SurfaceReference(parity, A.data());
std::vector<double> dsw(s.n_groups), dswI(s.n_groups), dcross(s.ncell), dref2(s.ncell);
gpu.SurfaceGetReference(dsw.data(), dswI.data());
gpu.SurfaceFitSums(id, dcross.data(), dref2.data());
CHECK(SameBits(sw, dsw));
CHECK(SameBits(swI, dswI));
CHECK(SameBits(cross, dcross));
CHECK(SameBits(ref2, dref2));
}
}
#endif