diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 6eff70b3e..81dabcd22 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -3,6 +3,7 @@ #include "../../common/ParallelFor.h" #include "RotationScaleMerge.h" +#include "RotationScaleMergeGPU.h" // SurfaceTerm, the surface fit's term on both paths #include #include @@ -3334,7 +3335,7 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int // traffic of the data it touched, over hundreds of megabytes. Copying once turns them into // sequential walks of a compact array. The copy keeps fulls order, so each sum below is formed // from exactly the same terms in exactly the same order. - struct Term { float I, sigma, corr, d; int32_t cell, group; }; + using Term = RotationScaleMergeGPU::SurfaceTerm; // float I, sigma, corr, d; int32_t cell, group std::vector term; std::vector term_parity; // frame parity, read only by the group-ordered copy below // Which terms each pass walks, as positions in `term`. The cross-validated halves then cost half a @@ -3437,22 +3438,27 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int std::vector shw_shell(nshell, 0.0), shw_cell(nshell > 0 ? ncell : 0, 0.0); std::vector group_shell(n_groups, 0); // the shell each ASU group sits in, for the gate if (nshell > 0) { - std::vector s2; - s2.reserve(term.size()); - for (const Term &t : term) - s2.push_back(t.d > 0.0f ? 1.0f / (t.d * t.d) : 0.0f); + const int n_term = static_cast(term.size()); + std::vector s2(n_term); + ParallelChunks(n_term, nt, [&](int lo, int hi) { + for (int k = lo; k < hi; ++k) + s2[k] = term[k].d > 0.0f ? 1.0f / (term[k].d * term[k].d) : 0.0f; + }); + // The edges are order statistics of s2, so they are the same values however the sort orders + // equal elements among themselves. + std::vector sorted = s2; + ParallelSort(sorted.begin(), sorted.end(), nt, std::less()); std::vector edge(nshell - 1); - size_t prev = 0; - for (int i = 1; i < nshell; ++i) { - const size_t pos = s2.size() * static_cast(i) / static_cast(nshell); - std::nth_element(s2.begin() + prev, s2.begin() + pos, s2.end()); - edge[i - 1] = s2[pos]; - prev = pos; - } + for (int i = 1; i < nshell; ++i) + edge[i - 1] = sorted[sorted.size() * static_cast(i) / static_cast(nshell)]; + std::vector term_shell(n_term); + ParallelChunks(n_term, nt, [&](int lo, int hi) { + for (int k = lo; k < hi; ++k) + term_shell[k] = static_cast(std::upper_bound(edge.begin(), edge.end(), s2[k]) - edge.begin()); + }); for (size_t k = 0; k < term.size(); ++k) { const Term &t = term[k]; - const float v = t.d > 0.0f ? 1.0f / (t.d * t.d) : 0.0f; - const int s = static_cast(std::upper_bound(edge.begin(), edge.end(), v) - edge.begin()); + const int s = term_shell[k]; group_shell[t.group] = s; const double sc = static_cast(t.sigma) * t.corr; if (!(sc > 0.0)) continue; @@ -3475,8 +3481,41 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int std::vector g_start(n_groups + 1, 0); for (const Term &t : term) ++g_start[t.group + 1]; for (int g = 0; g < n_groups; ++g) g_start[g + 1] += g_start[g]; - std::vector gterm(term.size()); - { + std::vector gterm; + // With a GPU the two passes of every round run there (RotationScaleMergeGPU::Surface*), on the same + // terms in the same order: the device gets the terms with the group order as a permutation, and each + // subset cut into this fit's reduction blocks with every block's terms ordered by cell, so that one + // device thread forms what one block of the host loop sums into one cell. + bool on_gpu = false; +#ifdef JFJOCH_USE_CUDA + on_gpu = gpu_active_; + if (on_gpu) { + std::vector gperm(term.size()); + std::vector fill(g_start.begin(), g_start.end() - 1); + for (size_t k = 0; k < term.size(); ++k) + gperm[fill[term[k].group]++] = static_cast(k); + gpu_->SurfaceSetTerms(static_cast(term.size()), term.data(), term_parity.data(), + n_groups, gperm.data(), g_start.data(), ncell); + for (int parity : {0, 1, -1}) { + const std::vector &sel = subset(parity); + const int n = static_cast(sel.size()); + const int nb = ReductionBlocks(n, SURFACE_BLOCK); + std::vector perm(n), seg_start(static_cast(nb) * ncell + 1, n); + ParallelFor(nb, nt, [&](int b) { + const int lo = static_cast(static_cast(n) * b / nb); + const int hi = static_cast(static_cast(n) * (b + 1) / nb); + std::vector pos(ncell + 1, 0); + for (int k = lo; k < hi; ++k) ++pos[term[sel[k]].cell + 1]; + for (int c = 0; c < ncell; ++c) pos[c + 1] += pos[c]; + for (int c = 0; c < ncell; ++c) seg_start[static_cast(b) * ncell + c] = lo + pos[c]; + for (int k = lo; k < hi; ++k) perm[lo + pos[term[sel[k]].cell]++] = sel[k]; + }); + gpu_->SurfaceSetSubset(parity < 0 ? 2 : parity, nb, perm.data(), seg_start.data()); + } + } +#endif + if (!on_gpu) { + gterm.resize(term.size()); std::vector fill(g_start.begin(), g_start.end() - 1); for (size_t k = 0; k < term.size(); ++k) { const Term &t = term[k]; @@ -3488,6 +3527,12 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int // and the score. std::vector sw(n_groups), swI(n_groups); auto reference = [&](int parity, const std::vector &A) { +#ifdef JFJOCH_USE_CUDA + if (on_gpu) { + gpu_->SurfaceReference(parity, A.data()); + return; + } +#endif ParallelChunks(n_groups, nt, [&](int glo, int ghi) { for (int g = glo; g < ghi; ++g) { double s_w = 0.0, s_wI = 0.0; @@ -3520,7 +3565,7 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int const std::vector &sel = subset(parity); std::vector A(ncell, 1.0); // Per-block cell accumulators, allocated once for the whole fit rather than per round. - const int nb = ReductionBlocks(static_cast(sel.size()), SURFACE_BLOCK); + const int nb = on_gpu ? 0 : ReductionBlocks(static_cast(sel.size()), SURFACE_BLOCK); std::vector> tcross(nb, std::vector(ncell)), tref2(nb, std::vector(ncell)); settled = false; n_clamped = 0; @@ -3555,22 +3600,28 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int // there is no ordering that keeps threads off each other's bins, and there are only ncell // of them, so per-block copies are cheap and the fixed blocks keep it the same at any -N. std::vector cross(ncell, 0.0), ref2(ncell, 0.0); - ParallelBlocks(static_cast(sel.size()), nt, [&](int b, int lo, int hi) { - std::vector &xcross = tcross[b], &xref2 = tref2[b]; - std::fill(xcross.begin(), xcross.end(), 0.0); - std::fill(xref2.begin(), xref2.end(), 0.0); - for (int k = lo; k < hi; ++k) { - const Term &o = term[sel[k]]; - if (sw[o.group] <= 0.0) continue; - const double Iref = swI[o.group] / sw[o.group], a = A[o.cell]; - const double Is = static_cast(o.I) * o.corr * a, sc = static_cast(o.sigma) * o.corr * a; - if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue; - const double w = 1.0 / (sc * sc); - xcross[o.cell] += w * Is * Iref; xref2[o.cell] += w * Iref * Iref; - } - }, SURFACE_BLOCK); - for (int b = 0; b < nb; ++b) - for (int c = 0; c < ncell; ++c) { cross[c] += tcross[b][c]; ref2[c] += tref2[b][c]; } +#ifdef JFJOCH_USE_CUDA + if (on_gpu) + gpu_->SurfaceFitSums(parity < 0 ? 2 : parity, cross.data(), ref2.data()); +#endif + if (!on_gpu) { + ParallelBlocks(static_cast(sel.size()), nt, [&](int b, int lo, int hi) { + std::vector &xcross = tcross[b], &xref2 = tref2[b]; + std::fill(xcross.begin(), xcross.end(), 0.0); + std::fill(xref2.begin(), xref2.end(), 0.0); + for (int k = lo; k < hi; ++k) { + const Term &o = term[sel[k]]; + if (sw[o.group] <= 0.0) continue; + const double Iref = swI[o.group] / sw[o.group], a = A[o.cell]; + const double Is = static_cast(o.I) * o.corr * a, sc = static_cast(o.sigma) * o.corr * a; + if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue; + const double w = 1.0 / (sc * sc); + xcross[o.cell] += w * Is * Iref; xref2[o.cell] += w * Iref * Iref; + } + }, SURFACE_BLOCK); + for (int b = 0; b < nb; ++b) + for (int c = 0; c < ncell; ++c) { cross[c] += tcross[b][c]; ref2[c] += tref2[b][c]; } + } std::vector dsorted = cross; std::nth_element(dsorted.begin(), dsorted.begin() + dsorted.size() / 2, dsorted.end()); const double lambda = 0.1 * std::max(1e-30, dsorted[dsorted.size() / 2]); @@ -3672,20 +3723,33 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int const int nsh_cc = nshell > 0 ? nshell : 1; auto half_means = [&](int parity, const std::vector &A) { reference(parity, A); +#ifdef JFJOCH_USE_CUDA + if (on_gpu) + gpu_->SurfaceGetReference(sw.data(), swI.data()); +#endif std::vector m(n_groups, std::numeric_limits::quiet_NaN()); for (int g = 0; g < n_groups; ++g) if (sw[g] > 0.0) m[g] = swI[g] / sw[g]; return m; }; - auto shell_cc = [&](const std::vector &x, const std::vector &y, int s) { - double n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; + // The correlation within every shell, in one walk over the groups: each shell still sums its own + // groups in group order. + auto shell_cc = [&](const std::vector &x, const std::vector &y) { + struct Sums { double n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; }; + std::vector sum(nsh_cc); for (int g = 0; g < n_groups; ++g) { - if (group_shell[g] != s || !std::isfinite(x[g]) || !std::isfinite(y[g])) continue; - n += 1; sx += x[g]; sy += y[g]; sxx += x[g] * x[g]; syy += y[g] * y[g]; sxy += x[g] * y[g]; + if (!std::isfinite(x[g]) || !std::isfinite(y[g])) continue; + Sums &u = sum[group_shell[g]]; + u.n += 1; u.sx += x[g]; u.sy += y[g]; u.sxx += x[g] * x[g]; u.syy += y[g] * y[g]; u.sxy += x[g] * y[g]; } - const double vx = sxx - sx * sx / n, vy = syy - sy * sy / n; - return (n >= 50 && vx > 0.0 && vy > 0.0) ? (sxy - sx * sy / n) / std::sqrt(vx * vy) - : std::numeric_limits::quiet_NaN(); + std::vector cc(nsh_cc, std::numeric_limits::quiet_NaN()); + for (int s = 0; s < nsh_cc; ++s) { + const Sums &u = sum[s]; + const double vx = u.sxx - u.sx * u.sx / u.n, vy = u.syy - u.sy * u.sy / u.n; + if (u.n >= 50 && vx > 0.0 && vy > 0.0) + cc[s] = (u.sxy - u.sx * u.sy / u.n) / std::sqrt(vx * vy); + } + return cc; }; const std::vector ident(ncell, 1.0); const std::vector A_even = fit_surface(0), A_odd = fit_surface(1); @@ -3702,8 +3766,9 @@ void RotationScaleMerge::ApplyCellSurface(const std::vector &cell, int // Following Fisher (1915) Biometrika 10, 507-521 double gain = 0.0; int n_cc = 0; + const std::vector cc0 = shell_cc(odd0, even0), cc1 = shell_cc(odd1, even1); for (int sh = 0; sh < nsh_cc; ++sh) { - const double c0 = shell_cc(odd0, even0, sh), c1 = shell_cc(odd1, even1, sh); + const double c0 = cc0[sh], c1 = cc1[sh]; if (std::isfinite(c0) && std::isfinite(c1)) { gain += std::atanh(c1) - std::atanh(c0); ++n_cc; } } if (n_cc > 0) gain /= n_cc; diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.cu b/image_analysis/scale_merge/RotationScaleMergeGPU.cu index 6e54fb380..66aeabacb 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.cu +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.cu @@ -639,6 +639,100 @@ namespace { } } + // --- correction-surface fit (ApplyCellSurface) --- + // The host fit is the reference here: these kernels form the same sums from the same terms in the + // same order, and every rounding is spelled out (__dmul_rn / __dadd_rn round each step on its own, + // fma rounds once) to be the one the host build makes - nvcc would otherwise contract a multiply + // and an add wherever it sees them, and GCC at -march=x86-64-v3 does so only in some of them. The + // products are the host's (I * corr) * a and (sigma * corr) * a. + using SurfaceTerm = RotationScaleMergeGPU::SurfaceTerm; + + __device__ __forceinline__ double SurfaceIs(const SurfaceTerm &t, double a) { + return __dmul_rn(__dmul_rn(double(t.I), double(t.corr)), a); + } + + __device__ __forceinline__ double SurfaceSigma(const SurfaceTerm &t, double a) { + return __dmul_rn(__dmul_rn(double(t.sigma), double(t.corr)), a); + } + + // One thread per ASU group: the group's inverse-variance sums over its terms of frame parity `parity` + // (< 0 = all), in fulls order - the host's `reference`. The host builds that loop twice, and only + // the parity-filtered copy fuses the multiply-add of swI; the unfiltered one rounds the product. + __global__ void SurfaceReferenceKernel(int n_groups, int parity, const int32_t *__restrict__ gperm, + const int32_t *__restrict__ gstart, + const SurfaceTerm *__restrict__ term, + const uint8_t *__restrict__ term_parity, + const double *__restrict__ A, + double *__restrict__ sw, double *__restrict__ swI) { + for (int g = blockIdx.x * blockDim.x + threadIdx.x; g < n_groups; g += gridDim.x * blockDim.x) { + double s_w = 0.0, s_wI = 0.0; + for (int k = gstart[g]; k < gstart[g + 1]; ++k) { + const int i = gperm[k]; + if (parity >= 0 && term_parity[i] != parity) continue; + const SurfaceTerm t = term[i]; + const double a = A[t.cell]; + const double Is = SurfaceIs(t, a), sc = SurfaceSigma(t, a); + const double w = 1.0 / __dmul_rn(sc, sc); + s_w = __dadd_rn(s_w, w); + s_wI = parity >= 0 ? fma(Is, w, s_wI) : __dadd_rn(s_wI, __dmul_rn(Is, w)); + } + sw[g] = s_w; swI[g] = s_wI; + } + } + + // The fit's sums of one round. Each term's contribution is formed on its own thread (one per term, + // in segment order), and only the two multiply-adds that sum them are left to the thread of each + // (reduction block, cell) - the per-block accumulators of the host fit, one slot at a time, walking + // its segment in term order. A term the host skips is marked by a NaN Iref (a kept one is finite). + __global__ void SurfaceFitTermKernel(int n, const int32_t *__restrict__ perm, + const SurfaceTerm *__restrict__ term, + const double *__restrict__ A, + const double *__restrict__ sw, const double *__restrict__ swI, + double *__restrict__ w_Is, double *__restrict__ w_Iref, + double *__restrict__ Iref_out) { + for (int k = blockIdx.x * blockDim.x + threadIdx.x; k < n; k += gridDim.x * blockDim.x) { + const SurfaceTerm t = term[perm[k]]; + Iref_out[k] = NAN; + if (sw[t.group] <= 0.0) continue; + const double Iref = swI[t.group] / sw[t.group], a = A[t.cell]; + const double Is = SurfaceIs(t, a), sc = SurfaceSigma(t, a); + if (!isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue; + const double w = 1.0 / __dmul_rn(sc, sc); + w_Is[k] = __dmul_rn(w, Is); + w_Iref[k] = __dmul_rn(w, Iref); + Iref_out[k] = Iref; + } + } + + __global__ void SurfaceFitSegmentKernel(int n_seg, const int32_t *__restrict__ seg_start, + const double *__restrict__ w_Is, const double *__restrict__ w_Iref, + const double *__restrict__ Iref, + double *__restrict__ tcross, double *__restrict__ tref2) { + for (int s = blockIdx.x * blockDim.x + threadIdx.x; s < n_seg; s += gridDim.x * blockDim.x) { + double xcross = 0.0, xref2 = 0.0; + for (int k = seg_start[s]; k < seg_start[s + 1]; ++k) { + if (isnan(Iref[k])) continue; + xcross = fma(w_Is[k], Iref[k], xcross); + xref2 = fma(w_Iref[k], Iref[k], xref2); + } + tcross[s] = xcross; tref2[s] = xref2; + } + } + + // One thread per cell: the blocks' sums added up in block order, as the host adds its slots. + __global__ void SurfaceFitCellKernel(int n_blocks, int ncell, const double *__restrict__ tcross, + const double *__restrict__ tref2, + double *__restrict__ cross, double *__restrict__ ref2) { + for (int c = blockIdx.x * blockDim.x + threadIdx.x; c < ncell; c += gridDim.x * blockDim.x) { + double sc = 0.0, sr = 0.0; + for (int b = 0; b < n_blocks; ++b) { + sc = __dadd_rn(sc, tcross[b * ncell + c]); + sr = __dadd_rn(sr, tref2[b * ncell + c]); + } + cross[c] = sc; ref2[c] = sr; + } + } + void CudaCheck(cudaError_t e, const char *what); // A copy on the instance's stream, waited for - what cudaMemcpy on the NULL stream was, without also @@ -817,6 +911,18 @@ struct RotationScaleMergeGPU::Impl { CudaDevicePtr f_sco_ok; CudaDevicePtr f_frame_perm, f_frame_start, f_frame_count; CudaDevicePtr f_gperm, f_gstart, f_gcount; + // correction-surface fit (one ApplyCellSurface call at a time): its terms and their group CSR, the + // three subsets' per-(block, cell) segments, the surface and the sums of the round + int s_ncell = 0, s_n_groups = 0; + int s_n_blocks[3] = {0, 0, 0}; + int s_n_sel[3] = {0, 0, 0}; + size_t s_slots = 0; // capacity of s_tcross / s_tref2 + CudaDevicePtr s_term; + CudaDevicePtr s_parity; + CudaDevicePtr s_gperm, s_gstart; + CudaDevicePtr s_perm[3], s_seg_start[3]; + CudaDevicePtr s_A, s_sw, s_swI, s_tcross, s_tref2, s_cross, s_ref2; + CudaDevicePtr s_w_Is, s_w_Iref, s_Iref; // per term of a subset, in segment order }; // Set the device this instance's memory lives on for the duration of a call, and put the caller's @@ -1460,3 +1566,88 @@ void RotationScaleMergeGPU::SetFullsCorr(const float *corr) { CopyAndWait(d.f_corr.get(), corr, size_t(d.n_fulls) * sizeof(float), cudaMemcpyHostToDevice, impl_->s(), "upload f_corr"); } + +void RotationScaleMergeGPU::SurfaceSetTerms(int n_terms, const SurfaceTerm *term, const uint8_t *parity, + int n_groups, const int32_t *gperm, const int32_t *gstart, + int ncell) { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + d.s_ncell = ncell; + d.s_n_groups = n_groups; + d.Upload(d.s_term, term, n_terms); + d.Upload(d.s_parity, parity, n_terms); + d.Upload(d.s_gperm, gperm, n_terms); + d.Upload(d.s_gstart, gstart, n_groups + 1); + d.s_A = d.Alloc(std::max(1, ncell)); + d.s_cross = d.Alloc(std::max(1, ncell)); + d.s_ref2 = d.Alloc(std::max(1, ncell)); + d.s_sw = d.Alloc(std::max(1, n_groups)); + d.s_swI = d.Alloc(std::max(1, n_groups)); + d.s_w_Is = d.Alloc(std::max(1, n_terms)); + d.s_w_Iref = d.Alloc(std::max(1, n_terms)); + d.s_Iref = d.Alloc(std::max(1, n_terms)); + for (int &nb : d.s_n_blocks) nb = 0; +} + +void RotationScaleMergeGPU::SurfaceSetSubset(int subset, int n_blocks, const int32_t *perm, + const int32_t *seg_start) { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + const int n_seg = n_blocks * d.s_ncell; + d.s_n_sel[subset] = n_blocks > 0 ? seg_start[n_seg] : 0; + d.Upload(d.s_perm[subset], perm, d.s_n_sel[subset]); + d.Upload(d.s_seg_start[subset], seg_start, n_seg + 1); + d.s_n_blocks[subset] = n_blocks; + if (size_t(n_seg) > d.s_slots) { + d.s_tcross = d.Alloc(n_seg); + d.s_tref2 = d.Alloc(n_seg); + d.s_slots = n_seg; + } +} + +void RotationScaleMergeGPU::SurfaceReference(int parity, const double *A) { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + CopyAndWait(d.s_A.get(), A, size_t(d.s_ncell) * sizeof(double), cudaMemcpyHostToDevice, impl_->s(), + "upload surface"); + const int grp_blocks = std::min(65535, (d.s_n_groups + BLK - 1) / BLK); + if (grp_blocks > 0) + SurfaceReferenceKernel<<s()>>>(d.s_n_groups, parity, d.s_gperm.get(), + d.s_gstart.get(), d.s_term.get(), d.s_parity.get(), d.s_A.get(), d.s_sw.get(), d.s_swI.get()); + CudaCheck(cudaGetLastError(), "surface reference launch"); + CudaCheck(cudaStreamSynchronize(impl_->s()), "surface reference sync"); +} + +void RotationScaleMergeGPU::SurfaceGetReference(double *sw, double *swI) const { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + const size_t bytes = size_t(d.s_n_groups) * sizeof(double); + CopyAndWait(sw, d.s_sw.get(), bytes, cudaMemcpyDeviceToHost, impl_->s(), "dl surface sw"); + CopyAndWait(swI, d.s_swI.get(), bytes, cudaMemcpyDeviceToHost, impl_->s(), "dl surface swI"); +} + +void RotationScaleMergeGPU::SurfaceFitSums(int subset, double *cross, double *ref2) { + DeviceGuard guard(impl_->device, impl_->available); + auto &d = *impl_; + const int ncell = d.s_ncell, nb = d.s_n_blocks[subset], n_seg = nb * ncell; + if (nb == 0) { + std::fill(cross, cross + ncell, 0.0); + std::fill(ref2, ref2 + ncell, 0.0); + return; + } + SurfaceFitTermKernel<<s()>>>( + d.s_n_sel[subset], d.s_perm[subset].get(), d.s_term.get(), d.s_A.get(), d.s_sw.get(), d.s_swI.get(), + d.s_w_Is.get(), d.s_w_Iref.get(), d.s_Iref.get()); + CudaCheck(cudaGetLastError(), "surface fit term launch"); + SurfaceFitSegmentKernel<<s()>>>(n_seg, + d.s_seg_start[subset].get(), d.s_w_Is.get(), d.s_w_Iref.get(), d.s_Iref.get(), + d.s_tcross.get(), d.s_tref2.get()); + CudaCheck(cudaGetLastError(), "surface fit segment launch"); + SurfaceFitCellKernel<<s()>>>(nb, ncell, + d.s_tcross.get(), d.s_tref2.get(), d.s_cross.get(), d.s_ref2.get()); + CudaCheck(cudaGetLastError(), "surface cell sum launch"); + CopyAndWait(cross, d.s_cross.get(), size_t(ncell) * sizeof(double), cudaMemcpyDeviceToHost, impl_->s(), + "dl surface cross"); + CopyAndWait(ref2, d.s_ref2.get(), size_t(ncell) * sizeof(double), cudaMemcpyDeviceToHost, impl_->s(), + "dl surface ref2"); +} diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.h b/image_analysis/scale_merge/RotationScaleMergeGPU.h index 730a79add..766e2f640 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.h +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.h @@ -192,6 +192,34 @@ public: // Download the fulls' working corr (length = n_fulls), valid after ScaleFulls. void GetFullsCorr(float *corr) const; + // --- correction-surface fit (RotationScaleMerge::ApplyCellSurface) --- + // The two passes every round of the fit makes over its terms, on the device; the host keeps the + // per-cell step, the gauge and the cross-validation. Both sums are formed in exactly the order the + // host forms them, so the fitted surface is the host's to the last bit (see the kernels). + + // One observation as the surface fit sees it: the host's term, uploaded as it stands. + struct SurfaceTerm { float I, sigma, corr, d; int32_t cell, group; }; + + // The terms (in fulls order) with each one's frame parity, the ASU-group CSR over them (gperm lists + // the terms of group g at [gstart[g], gstart[g+1]), in fulls order) and the cell count. + void SurfaceSetTerms(int n_terms, const SurfaceTerm *term, const uint8_t *parity, + int n_groups, const int32_t *gperm, const int32_t *gstart, int ncell); + + // One subset of the terms (0 = even frames, 1 = odd, 2 = all), cut into the host's n_blocks + // reduction blocks and ordered within each block by cell, keeping term order inside a cell: + // block b, cell c is perm[seg_start[b * ncell + c], seg_start[b * ncell + c + 1]). + void SurfaceSetSubset(int subset, int n_blocks, const int32_t *perm, const int32_t *seg_start); + + // The per-group reference sums sw / swI over the terms of frame parity `parity` (< 0 = all) with + // the surface A (length ncell) applied. They stay on the device for SurfaceFitSums; + // SurfaceGetReference downloads them (length n_groups each). + void SurfaceReference(int parity, const double *A); + void SurfaceGetReference(double *sw, double *swI) const; + + // The fit's per-cell sums over one subset against the last SurfaceReference and its A: + // cross = sum w Is Iref and ref2 = sum w Iref^2 (length ncell each). + void SurfaceFitSums(int subset, double *cross, double *ref2); + private: struct Impl; std::unique_ptr impl_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 799c06c0d..c6d6df0bb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -148,6 +148,7 @@ ADD_EXECUTABLE(jfjoch_test AnisotropyAnalysisTest.cpp ModelScalingTest.cpp ModelScaleGPUTest.cpp + CorrectionSurfaceGPUTest.cpp TwinningAnalysisTest.cpp TranslationalNCSTest.cpp RfreeFlagsTest.cpp diff --git a/tests/CorrectionSurfaceGPUTest.cpp b/tests/CorrectionSurfaceGPUTest.cpp new file mode 100644 index 000000000..bfe767db0 --- /dev/null +++ b/tests/CorrectionSurfaceGPUTest.cpp @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include "../common/CUDAWrapper.h" + +#ifdef JFJOCH_USE_CUDA + +#include +#include +#include +#include + +#include "../common/ParallelFor.h" +#include "../image_analysis/scale_merge/RotationScaleMergeGPU.h" + +namespace { + +using Term = RotationScaleMergeGPU::SurfaceTerm; + +// The roundings RotationScaleMerge::ApplyCellSurface makes on the host (x86-64-v3 build), spelled out: +// a volatile result is rounded on its own and never fused into the next operation, std::fma is fused. +double Mul(double a, double b) { volatile double r = a * b; return r; } +double Add(double a, double b) { volatile double r = a + b; return r; } + +constexpr int SURFACE_BLOCK = 32768; // ApplyCellSurface's reduction block + +struct Surface { + int n_groups = 0, ncell = 0; + std::vector term; + std::vector parity; + std::vector gperm, gstart; + std::vector sel[3]; // even, odd, all - in term order +}; + +Surface MakeSurface(int n_terms, int n_groups, int ncell, uint32_t seed) { + std::mt19937 rng(seed); + std::uniform_int_distribution group(0, n_groups - 1), cell(0, ncell - 1), bit(0, 1); + std::uniform_real_distribution u(0.0f, 1.0f); + Surface s; + s.n_groups = n_groups; + s.ncell = ncell; + for (int i = 0; i < n_terms; ++i) { + // Negative intensities, and now and then a sigma of zero: both reach the host sums as they are. + const float I = 1000.0f * u(rng) - 100.0f; + const float sigma = (i % 997 == 0) ? 0.0f : 1.0f + 30.0f * u(rng); + // Every 50th group gets no terms at all, so its reference is empty. + int g = group(rng); + if (g % 50 == 0) g = (g + 1) % n_groups; + s.term.push_back({I, sigma, 0.5f + u(rng), 1.0f + 3.0f * u(rng), cell(rng), g}); + s.parity.push_back(static_cast(bit(rng))); + s.sel[s.parity.back()].push_back(i); + s.sel[2].push_back(i); + } + s.gstart.assign(n_groups + 1, 0); + for (const Term &t : s.term) ++s.gstart[t.group + 1]; + for (int g = 0; g < n_groups; ++g) s.gstart[g + 1] += s.gstart[g]; + s.gperm.resize(n_terms); + std::vector fill(s.gstart.begin(), s.gstart.end() - 1); + for (int i = 0; i < n_terms; ++i) s.gperm[fill[s.term[i].group]++] = i; + return s; +} + +void HostReference(const Surface &s, int parity, const std::vector &A, + std::vector &sw, std::vector &swI) { + sw.assign(s.n_groups, 0.0); + swI.assign(s.n_groups, 0.0); + for (int g = 0; g < s.n_groups; ++g) { + double s_w = 0.0, s_wI = 0.0; + for (int k = s.gstart[g]; k < s.gstart[g + 1]; ++k) { + const int i = s.gperm[k]; + if (parity >= 0 && s.parity[i] != parity) continue; + const Term &t = s.term[i]; + const double a = A[t.cell]; + const double Is = Mul(Mul(t.I, t.corr), a), sc = Mul(Mul(t.sigma, t.corr), a); + const double w = 1.0 / Mul(sc, sc); + s_w = Add(s_w, w); + s_wI = parity >= 0 ? std::fma(Is, w, s_wI) : Add(s_wI, Mul(Is, w)); + } + sw[g] = s_w; + swI[g] = s_wI; + } +} + +void HostFitSums(const Surface &s, const std::vector &sel, const std::vector &A, + const std::vector &sw, const std::vector &swI, + std::vector &cross, std::vector &ref2) { + cross.assign(s.ncell, 0.0); + ref2.assign(s.ncell, 0.0); + const int n = static_cast(sel.size()), nb = ReductionBlocks(n, SURFACE_BLOCK); + for (int b = 0; b < nb; ++b) { + std::vector xcross(s.ncell, 0.0), xref2(s.ncell, 0.0); + const int lo = static_cast(int64_t(n) * b / nb), hi = static_cast(int64_t(n) * (b + 1) / nb); + for (int k = lo; k < hi; ++k) { + const Term &t = s.term[sel[k]]; + if (sw[t.group] <= 0.0) continue; + const double Iref = swI[t.group] / sw[t.group], a = A[t.cell]; + const double Is = Mul(Mul(t.I, t.corr), a), sc = Mul(Mul(t.sigma, t.corr), a); + if (!std::isfinite(Iref) || Iref <= 0.0 || !(sc > 0.0)) continue; + const double w = 1.0 / Mul(sc, sc); + xcross[t.cell] = std::fma(Mul(w, Is), Iref, xcross[t.cell]); + xref2[t.cell] = std::fma(Mul(w, Iref), Iref, xref2[t.cell]); + } + for (int c = 0; c < s.ncell; ++c) { + cross[c] = Add(cross[c], xcross[c]); + ref2[c] = Add(ref2[c], xref2[c]); + } + } +} + +// The device side of one subset: ApplyCellSurface's per-block counting sort by cell. +void UploadSubset(RotationScaleMergeGPU &gpu, const Surface &s, int id, const std::vector &sel) { + const int n = static_cast(sel.size()), nb = ReductionBlocks(n, SURFACE_BLOCK); + std::vector perm(n), seg_start(static_cast(nb) * s.ncell + 1, n); + for (int b = 0; b < nb; ++b) { + const int lo = static_cast(int64_t(n) * b / nb), hi = static_cast(int64_t(n) * (b + 1) / nb); + std::vector pos(s.ncell + 1, 0); + for (int k = lo; k < hi; ++k) ++pos[s.term[sel[k]].cell + 1]; + for (int c = 0; c < s.ncell; ++c) pos[c + 1] += pos[c]; + for (int c = 0; c < s.ncell; ++c) seg_start[size_t(b) * s.ncell + c] = lo + pos[c]; + for (int k = lo; k < hi; ++k) perm[lo + pos[s.term[sel[k]].cell]++] = sel[k]; + } + gpu.SurfaceSetSubset(id, nb, perm.data(), seg_start.data()); +} + +bool SameBits(const std::vector &a, const std::vector &b) { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0; +} + +} // namespace + +TEST_CASE("CorrectionSurfaceGPU_SumsMatchHostBitForBit", "[RotationScale][gpu]") { + if (get_gpu_count() == 0) + SKIP("No GPU"); + // Enough terms for several reduction blocks in every subset. + const Surface s = MakeSurface(250000, 4000, 144, 7); + RotationScaleMergeGPU gpu; + REQUIRE(gpu.Available()); + gpu.SurfaceSetTerms(static_cast(s.term.size()), s.term.data(), s.parity.data(), s.n_groups, + s.gperm.data(), s.gstart.data(), s.ncell); + for (int id = 0; id < 3; ++id) + UploadSubset(gpu, s, id, s.sel[id]); + + std::mt19937 rng(11); + std::uniform_real_distribution u(0.7, 1.4); + std::vector A(s.ncell); + for (double &a : A) a = u(rng); + + for (int parity : {0, 1, -1}) { + const int id = parity < 0 ? 2 : parity; + std::vector sw, swI, cross, ref2; + HostReference(s, parity, A, sw, swI); + HostFitSums(s, s.sel[id], A, sw, swI, cross, ref2); + REQUIRE(ReductionBlocks(static_cast(s.sel[id].size()), SURFACE_BLOCK) > 1); + + gpu.SurfaceReference(parity, A.data()); + std::vector dsw(s.n_groups), dswI(s.n_groups), dcross(s.ncell), dref2(s.ncell); + gpu.SurfaceGetReference(dsw.data(), dswI.data()); + gpu.SurfaceFitSums(id, dcross.data(), dref2.data()); + CHECK(SameBits(sw, dsw)); + CHECK(SameBits(swI, dswI)); + CHECK(SameBits(cross, dcross)); + CHECK(SameBits(ref2, dref2)); + } +} + +#endif