v1.0.0-rc.173 (#83)

* jfjoch_broker: Optional per-dataset authentication - statistics, images and plots can require a bearer token, which jfjoch_viewer supports.
* jfjoch_viewer: Dark mode and a theme-matched colour scheme, a magnifier panel, and simpler contrast and background controls.
* Rugnux: Multiple performance improvements on GPU and CPU (CPU-only processing up to 40% faster, faster image decoding on ARM), with unchanged results.
* Rugnux: `--model` rigid-body refinement runs on the GPU, and the model-validation check is faster and more reliable.
* Rugnux: Improved scaling and merging - error model, outlier rejection, absorption correction and French-Wilson amplitudes now agree more closely with XDS and ctruncate.
* Rugnux: Improved integration - radial background on powder and ice rings, crowded rotation data keep their reflections, and CPU-only builds integrate large unit cells as GPU builds do.
* Rugnux: More robust detector geometry - measured beam centre, X-ray bandwidth and goniometer rate, and geometry refinement accepted only on significant evidence.
* Rugnux: Merged files are written in the standard setting, or in the setting of a reference MTZ, structure-factor mmCIF or model, with its free-R flags.
* Rugnux: Richer report - ice and powder rings, further lattices, superstructure candidates and mosaicity, with warnings worded as prompts to check.
* Rugnux: Clear error messages when a data set needs more GPU or host memory than is available.

Reviewed-on: #83
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

image_analysis/bragg_integration/BraggIntegrationEngineCPU.cpp

@@ -9,6 +9,10 @@
 #include <limits>
 #include <vector>
 
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+#include <xmmintrin.h>
+#endif
+
 #include "../../common/CompressedImage.h"
 #include "../../common/JFJochException.h"
 
@@ -19,10 +23,24 @@ namespace {
 // The engine reads pixels in the INT32_MIN(masked)/INT32_MAX(saturated) convention.
 inline bool valid(int32_t v) { return v != INT32_MIN && v != INT32_MAX; }
 
+// Ask for the cache lines from first to last ahead of their use. Only a hint: it changes no result.
+inline void prefetch(const void *p) {
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
+    _mm_prefetch(static_cast<const char *>(p), _MM_HINT_T0);
+#elif defined(__GNUC__) || defined(__clang__)
+    __builtin_prefetch(p);
+#endif
+}
+inline void prefetch_row(const void *first, const void *last) {
+    for (const char *a = static_cast<const char *>(first); a < last; a += 64) prefetch(a);
+    prefetch(last);
+}
+
 // Identity sampler over the preprocessed int32 buffer (already in that convention).
 struct BufferSampler {
     const int32_t *p;
     int32_t operator[](size_t i) const { return p[i]; }
+    const void *address(size_t i) const { return p + i; }
 };
 
 // Sampler over a raw detector image of pixel type T: masked pixels carry the type minimum, saturated
@@ -39,12 +57,15 @@ struct ImageSampler {
         if (v == saturation) return INT32_MAX;
         return static_cast<int32_t>(v);
     }
+    const void *address(size_t i) const { return p + i; }
 };
 
 }   // namespace
 
 BraggIntegrationEngineCPU::BraggIntegrationEngineCPU(const DiffractionExperiment &experiment)
-    : BraggIntegrationEngine(experiment) {}
+    : BraggIntegrationEngine(experiment),
+      refl_mask(static_cast<int>(xpixel), static_cast<int>(ypixel), 0),
+      owner(static_cast<int>(xpixel), static_cast<int>(ypixel), BRAGG_OWNER_NONE) {}
 
 template <class Sampler>
 std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
@@ -67,13 +88,18 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
 
     auto grid_idx = [this](int dx, int dy) { return (dy + R) * G + (dx + R); };
 
-    // --- Reflection mask: mark the r2 signal region of every predicted reflection so a neighbour's
-    //     signal is excluded from this reflection's r2..r3 background ring. The region is the INNER
-    //     stencil ellipse, taken in each neighbour's OWN frame - an elongated neighbour whose streak
-    //     is still marked as a disk would leak its tails into this reflection's ring. ---
-    std::vector<uint8_t> refl_mask(npixel, 0);
+    // --- Reflection mask: mark the r2 signal region of every predicted reflection, so a neighbour's
+    //     signal is excluded from this reflection's r2..r3 background ring. MASK_FLUX where the
+    //     neighbour puts flux on this frame (NEIGHBOUR_MASK_MIN_PARTIALITY) - only those are excluded
+    //     - and MASK_TAIL where only the tail of a rocking curve does, which is counted for the density
+    //     guard but read as background. The region is the INNER stencil ellipse, taken in each
+    //     neighbour's OWN frame - an elongated neighbour whose streak is still marked as a disk would
+    //     leak its tails into this reflection's ring. ---
+    refl_mask.Clear();
+    owner.Clear();
     for (size_t i = 0; i < npredicted; ++i) {
         const auto &r = predicted[i];
+        const uint8_t level = r.partiality >= NEIGHBOUR_MASK_MIN_PARTIALITY ? MASK_FLUX : MASK_TAIL;
         const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
         const int x0 = std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_in - 1.0f)));
         const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_in + 1.0f)));
@@ -82,7 +108,10 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
         for (int y = y0; y <= y1; ++y)
             for (int x = x0; x <= x1; ++x) {
                 const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
-                if (d.inner < r2_sq) refl_mask[y * W + x] = 1;
+                if (d.inner < r2_sq) {
+                    uint8_t &m = refl_mask.At(x, y);
+                    m = std::max(m, level);
+                }
             }
     }
 
@@ -92,9 +121,8 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
     //     pixel (BraggStencil.h). Built only when an overlap treatment is asked for. ---
     // A box sum has no profile to renormalise a disk it has taken pixels out of, so it never excludes.
     const bool exclude = overlap == OverlapMode::Exclude && mode != IntegratorMode::BoxSum;
-    std::vector<uint32_t> owner;
-    if (overlap != OverlapMode::Off) {
-        owner.assign(npixel, BRAGG_OWNER_NONE);
+    const bool use_owner = overlap != OverlapMode::Off;
+    if (use_owner) {
         const float claim_sq = claim * claim;
         for (size_t i = 0; i < npredicted; ++i) {
             const auto &r = predicted[i];
@@ -107,14 +135,14 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
                     const float dx = x - r.predicted_x, dy = y - r.predicted_y;
                     const float d2 = dx * dx + dy * dy;
                     if (d2 >= claim_sq) continue;
-                    uint32_t &o = owner[y * W + x];
+                    uint32_t &o = owner.At(x, y);
                     o = std::min(o, BraggOwnerKey(std::sqrt(d2), inv_claim, static_cast<int>(i)));
                 }
         }
     }
     auto clean = [&](int x, int y, size_t i) {
-        return owner.empty()
-               || BraggOwnedBy(owner[static_cast<size_t>(y) * W + x], static_cast<int>(i));
+        return !use_owner
+               || BraggOwnedBy(owner.Get(x, y), static_cast<int>(i));
     };
 
     // --- Pass A: box-sum every reflection (rough I, background, centroid, strong flag). ---
@@ -147,17 +175,34 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
         rad_cnt.assign(n_rad, 0);
     }
 
+    // The box pass A reads: the OUTER ellipse, tightly - taking the largest semi-axis in both
+    // directions instead would read up to 60% more pixels for nothing.
+    struct Box { int x0, x1, y0, y1; };
+    auto box_of = [&](const Reflection &r, const BraggStencil &st) {
+        return Box{std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f))),
+                    std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_out + 1.0f))),
+                    std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_out - 1.0f))),
+                    std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_out + 1.0f)))};
+    };
     for (size_t i = 0; i < npredicted; ++i) {
+        // The next reflection's image rows, asked for while this one is worked on: the reflections
+        // are spread over a frame far larger than the caches, so each box would otherwise wait on
+        // memory row by row. (Written out here: GCC drops a call to a function that does nothing but
+        // prefetch.)
+        if (i + 1 < npredicted) {
+            const Reflection &n = predicted[i + 1];
+            const Box b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil));
+            for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y)
+                prefetch_row(img.address(static_cast<size_t>(y) * W + b.x0),
+                             img.address(static_cast<size_t>(y) * W + b.x1));
+        }
         const auto &r = predicted[i];
         Rough out;
         // This reflection's stencil: the r1 signal disk, and the r2..r3 ring elongated radially by
-        // the analytic smear. The bounding box spans the OUTER ellipse, tightly - taking the largest
-        // semi-axis in both directions instead would read up to 60% more pixels for nothing.
+        // the analytic smear.
         const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
-        const int x0 = std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f)));
-        const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_out + 1.0f)));
-        const int y0 = std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_out - 1.0f)));
-        const int y1 = std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_out + 1.0f)));
+        const Box box = box_of(r, st);
+        const int x0 = box.x0, x1 = box.x1, y0 = box.y0, y1 = box.y1;
 
         // Both from the stencil's own radius, so nothing downstream is derived from a second one.
         out.r_bin = std::clamp(static_cast<int>(std::lround(st.r0)), 0, n_rad - 1);
@@ -168,11 +213,12 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
         int n_disk = 0, n_own = 0;   // pixels in the signal disk, and how many are this reflection's
         double bkg_sum = 0.0;
         int n_bkg = 0;
-        // Ring pixels a NEIGHBOUR's signal region occupies. refl_mask marks d.inner < r2_sq and the
-        // ring is d.inner >= r2_sq, so the two are complementary and a masked ring pixel is always
-        // someone else's - never this reflection's own core. That makes this an exact count of what
-        // the pattern's density took, separable from what the detector took.
-        int n_bkg_neighbour = 0;
+        // Readable ring pixels, and those of them no predicted neighbour's signal region covers.
+        // refl_mask marks d.inner < r2_sq and the ring is d.inner >= r2_sq, so the two are
+        // complementary and a masked ring pixel is always someone else's - never this reflection's
+        // own core. That makes the difference an exact count of what the pattern's density took,
+        // separable from what the detector took.
+        int n_ring = 0, n_clean = 0;
         bkg_vals.clear();
         for (int y = y0; y <= y1; ++y)
             for (int x = x0; x <= x1; ++x) {
@@ -199,8 +245,11 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
                     y_sum += y;
                     ++n_inner_valid;
                 } else if (d.inner >= r2_sq && d.outer < r3_sq) {
-                    if (refl_mask[y * W + x]) { ++n_bkg_neighbour; continue; }
                     if (!valid(px)) continue;
+                    ++n_ring;
+                    const uint8_t m = refl_mask.Get(x, y);
+                    if (m == 0) ++n_clean;
+                    if (m == MASK_FLUX) continue;
                     bkg_sum += static_cast<double>(px);
                     if (bkg_trim_frac > 0.0) bkg_vals.push_back(px);
                     ++n_bkg;
@@ -216,17 +265,16 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
         // survived to constrain the amplitude (XDS's MINPK, dials' valid_foreground_threshold). A box
         // sum has no profile to renormalise with, so there it stays all or nothing.
         const bool full = n_inner_valid == n_inner;
-        // A ring left with five or fewer clean pixels cannot estimate a background, so the reflection
-        // is dropped whole - the one thing the stencil geometry does to the DATA rather than to a
-        // measurement. Counted here because it is the only direct evidence of a radius that has
+        // A ring left with five or fewer readable pixels cannot estimate a background, so the
+        // reflection is dropped whole - the one thing the stencil geometry does to the DATA rather
+        // than to a measurement. Counted here because it is the only direct evidence of a radius that has
         // outgrown the pattern it is integrating (BraggIntegrationCounts).
         const bool keep_partial = full || mode != IntegratorMode::BoxSum;
-        if (keep_partial && n_bkg <= 5) {
-            ++counts.bkg_starved;
-            // Would the ring have been enough without the neighbours? Then it is the pattern, not the
-            // detector, that took it.
-            if (n_bkg + n_bkg_neighbour > 5) ++counts.bkg_starved_by_neighbour;
-        }
+        if (keep_partial && n_bkg <= 5) ++counts.bkg_starved;
+        // Would every predicted neighbour, the tails included, leave the ring starved that the
+        // detector alone would not? That is the pattern's density, what the guard on a widened
+        // radius reads (BraggIntegrationCounts).
+        if (keep_partial && n_clean <= 5 && n_ring > 5) ++counts.bkg_starved_by_neighbour;
         if (keep_partial && n_bkg > 5) {
             out.bkg = bkg_sum / n_bkg;
             if (bkg_trim_frac > 0.0 && bkg_vals.size() > 5
@@ -253,7 +301,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
                     for (int x = x0; x <= x1; ++x) {
                         const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
                         if (!(d.inner >= r2_sq && d.outer < r3_sq)) continue;
-                        if (refl_mask[y * W + x]) continue;
+                        if (refl_mask.Get(x, y) == MASK_FLUX) continue;
                         const int32_t px = img[y * W + x];
                         if (!valid(px)) continue;
                         if (static_cast<double>(px) <= thr) {
@@ -460,6 +508,10 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
     //     reweighting is the Kabsch/Otwinowski iteration: Kabsch, Acta Cryst D66, 133-144 (2010);
     //     Otwinowski & Minor, Methods Enzymol 276, 307-326 (1997). ---
     std::vector<double> Pbuf;
+    // The pixels the fit reads, gathered once while p_valid walks the grid: the profile value and the
+    // background-subtracted count, in the grid's order.
+    struct FitPixel { double Pp, signal; };
+    std::vector<FitPixel> fit_pixels;
     for (size_t i = 0; i < npredicted; ++i) {
         const auto &rh = rough[i];
         if (!rh.ok) continue;
@@ -491,6 +543,12 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
             // needed) radially elongated, on a grid grown to hold the streak.
             const double fx = predicted[i].predicted_x - rh.cx, fy = predicted[i].predicted_y - rh.cy;
             Rf = elong ? std::min(3 * R, static_cast<int>(std::ceil(r2 + 2.0 * std::sqrt(s2r)))) : R;
+            // The grid's image rows, asked for now (as in pass A): building the profile below takes
+            // long enough for them to arrive before the fit reads them.
+            const int gx0 = std::max(0, rh.cx - Rf), gx1 = std::min(W - 1, rh.cx + Rf);
+            for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y)
+                prefetch_row(img.address(static_cast<size_t>(y) * W + gx0),
+                             img.address(static_cast<size_t>(y) * W + gx1));
             const int Gf = 2 * Rf + 1;
             Pbuf.assign(static_cast<size_t>(Gf) * Gf, 0.0);
             double gs = 0.0;
@@ -517,6 +575,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
         //     and nothing excluded it is 1 and the guard is untouched. ---
         double p_grid = 0.0, p_valid = 0.0, p_own = 0.0, m_all = 0.0, m_read = 0.0;
         double p_peak = 0.0, p_lost_peak = 0.0;
+        fit_pixels.clear();
         for (int dy = -Rf; dy <= Rf; ++dy)
             for (int dx = -Rf; dx <= Rf; ++dx) {
                 const double Pp = (*Pvec)[(dy + Rf) * Gf + (dx + Rf)];
@@ -535,6 +594,8 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
                 const bool own = clean(x, y, i);
                 if (own) p_own += Pp;
                 if (in_disk && (own || !exclude)) m_read += Pp;
+                if (own || !exclude)
+                    fit_pixels.push_back({Pp, static_cast<double>(img[y * W + x]) - rh.bkg});
             }
         if (p_valid < overlap_min_peak * p_grid) continue;
         // A hole in the profile's PEAK is a different defect from a hole in its wings, and the mass
@@ -550,20 +611,12 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
             double num = 0.0;
             den = 0.0;
             wsum = 0.0;
-            for (int dy = -Rf; dy <= Rf; ++dy)
-                for (int dx = -Rf; dx <= Rf; ++dx) {
-                    const double Pp = (*Pvec)[(dy + Rf) * Gf + (dx + Rf)];
-                    if (Pp <= 0.0) continue;
-                    const int x = rh.cx + dx, y = rh.cy + dy;
-                    if (x < 0 || y < 0 || x >= W || y >= H) continue;
-                    const int32_t px = img[y * W + x];
-                    if (!valid(px)) continue;
-                    if (exclude && !clean(x, y, i)) continue;
-                    const double v = std::max(B + I * Pp, WEIGHT_VARIANCE_MIN_FRACTION * B);
-                    num += Pp * (static_cast<double>(px) - rh.bkg) / v;
-                    den += Pp * Pp / v;
-                    wsum += Pp / v;
-                }
+            for (const auto &[Pp, signal] : fit_pixels) {
+                const double v = std::max(B + I * Pp, WEIGHT_VARIANCE_MIN_FRACTION * B);
+                num += Pp * signal / v;
+                den += Pp * Pp / v;
+                wsum += Pp / v;
+            }
             if (den > 0.0) I = num / den; else break;
         }
         if (!(den > 0.0)) continue;