v1.0.0-rc.179 (#90)

* Rugnux: Stills processing needs only the unit cell and space group beyond what the file records - detector distance, beam centre, tilt, panel placement, saturation level, gain and polarisation plane are found from the images - and is up to twice as fast.
* Rugnux: Stills can be indexed without a GPU, with a CPU implementation of the fast-feedback indexer (macOS, computers without a GPU).
* Rugnux: Rotation and stills data are integrated in the detector's own units with one gain model, improving data from CCD detectors.
* Web frontend: The Rugnux live page shows the placed model with its electron density and anomalous difference map in a Mol* viewer.

Reviewed-on: #90
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/BraggIntegrationEngineGPUTest.cpp

@@ -15,6 +15,7 @@
 #include "../common/BraggIntegrationSettings.h"
 #include "../common/DetectorSetup.h"
 #include "../common/DiffractionExperiment.h"
+#include "../common/JFJochMath.h"
 #include "../common/Reflection.h"
 #include "../image_analysis/bragg_integration/BraggIntegrationEngineCPU.h"
 #include "../image_analysis/bragg_integration/BraggIntegrationEngineGPU.h"
@@ -149,13 +150,15 @@ double CompareCpuVsGpu(IntegratorMode mode, std::optional<float> bandwidth_fwhm,
                      float r1 = 0.0f, float r2 = 0.0f, float r3 = 0.0f,
                      OverlapMode overlap = OverlapMode::Off, float companion_dx = 0.0f,
                      bool clip_spots = false, float odd_partiality = 1.0f,
-                     const SpotFootprint *footprint = nullptr) {
+                     const SpotFootprint *footprint = nullptr, float pixel_gain = 1.0f) {
     DiffractionExperiment experiment =
             MakeExperiment(mode, bandwidth_fwhm, clip_nsigma, radial, DetJF(2), stencil_k, r1, r2, r3,
                            overlap);
-    if (footprint) {
+    {
         BraggIntegrationSettings settings = experiment.GetBraggIntegrationSettings();
-        settings.Footprint(*footprint);
+        if (footprint)
+            settings.Footprint(*footprint);
+        settings.PixelGain(pixel_gain);
         experiment.ImportBraggIntegrationSettings(settings);
     }
     const size_t width = experiment.GetXPixelsNum();
@@ -204,9 +207,10 @@ double CompareCpuVsGpu(IntegratorMode mode, std::optional<float> bandwidth_fwhm,
         const auto out_clean = clean_cpu.Run(clean_image, clean_scene.predicted,
                                              clean_scene.predicted.size(), 5);
         // An unreadable pixel the (empty) mask does not explain reads as an overload, which keeps the
-        // reflection, flagged - so the cost shows in the measured ones.
+        // reflection - flagged, or on a still as a lower bound, clipped either way - so the cost shows
+        // in the complete ones.
         const auto measured = std::count_if(out_cpu.begin(), out_cpu.end(),
-                                            [](const Reflection &r) { return !r.overloaded; });
+                                            [](const Reflection &r) { return !r.clipped; });
         CHECK(static_cast<size_t>(measured) < out_clean.size());
     }
     for (size_t i = 0; i < out_cpu.size(); ++i) {
@@ -322,6 +326,16 @@ TEST_CASE("BraggIntegrationEngineGPU_MatchesCPU") {
         CompareCpuVsGpu(IntegratorMode::ProfileGaussian, std::nullopt, 4.0f, false, 8, 0.0f,
                         0.0f, 0.0f, 0.0f, OverlapMode::Off, 0.0f, false, 0.01f);
     }
+    // A CCD's gain: every variance at it, so the strong flag and the guard decide in photons, and the
+    // reflections leave in photons - computed independently in the two engines.
+    SECTION("ProfileGaussian pixel gain") {
+        CompareCpuVsGpu(IntegratorMode::ProfileGaussian, std::nullopt, 4.0f, false, 60, 0.0f,
+                        0.0f, 0.0f, 0.0f, OverlapMode::Off, 0.0f, false, 1.0f, nullptr, 4.0f);
+    }
+    SECTION("BoxSum pixel gain") {
+        CompareCpuVsGpu(IntegratorMode::BoxSum, std::nullopt, 4.0f, false, 60, 0.0f,
+                        0.0f, 0.0f, 0.0f, OverlapMode::Off, 0.0f, false, 1.0f, nullptr, 4.0f);
+    }
     SECTION("ProfileGaussian mono trim") { CompareCpuVsGpu(IntegratorMode::ProfileGaussian, std::nullopt, 0.0f); }
     // Unreadable pixels inside the signal disks themselves: the MINPK rescue keeps the reflection and
     // fits it over what is left, and the peak-loss rule throws back the ones that lost the profile's
@@ -362,6 +376,58 @@ TEST_CASE("BraggIntegrationEngineGPU_MatchesCPU") {
 // next one, so a reused engine has to give the same answer as a fresh one. A first frame whose spots
 // are somewhere else entirely is what would show a leftover mark: a stale mask pixel is read as a
 // neighbour's signal and dropped from the background ring, a stale owner steals a pixel outright.
+// The still's overload decision on the device: a clipped spot is kept as a lower bound, an overload by
+// an unreadable pixel is not - as on the CPU (BraggIntegrationEngineCPU_StillKeepsOnlySaturatedOverloads).
+TEST_CASE("BraggIntegrationEngineGPU_StillOverloadsMatchCPU", "[Integration]") {
+    if (get_gpu_count() == 0) {
+        WARN("No CUDA GPU present. Skipping BraggIntegrationEngineGPU_StillOverloadsMatchCPU");
+        return;
+    }
+    DiffractionExperiment experiment(DetJF(2));
+    experiment.DetectorDistance_mm(100.0f).IncidentEnergy_keV(WVL_1A_IN_KEV).BeamX_pxl(400.0f).BeamY_pxl(400.0f);
+    experiment.ImportBraggIntegrationSettings(BraggIntegrationSettings());
+    const int32_t limit = 500, bkg = 12;
+    experiment.Detector().SaturationLimit(limit);
+    const size_t width = experiment.GetXPixelsNum(), npixel = experiment.GetPixelsNum();
+
+    std::vector<int32_t> image(npixel, bkg);
+    auto spot = [&](int cx, int cy, double amplitude) {
+        for (int dy = -6; dy <= 6; ++dy)
+            for (int dx = -6; dx <= 6; ++dx) {
+                const int32_t v = bkg + static_cast<int32_t>(std::lround(amplitude * std::exp(-(dx * dx + dy * dy) / (2 * 1.3 * 1.3))));
+                image[(cy + dy) * width + cx + dx] = v >= limit ? INT32_MAX : v;
+            }
+    };
+    spot(600, 300, 2000.0);
+    spot(300, 300, 40.0);
+    image[300 * width + 300] = INT32_MIN;
+    std::vector<Reflection> predicted;
+    for (const auto &[x, y] : {std::pair{600, 300}, std::pair{300, 300}})
+        predicted.push_back(MakeReflection(static_cast<float>(x), static_cast<float>(y), 2.0f, x));
+
+    ImagePreprocessorBuffer cpu_image(npixel);
+    for (size_t i = 0; i < npixel; ++i)
+        cpu_image[i] = image[i];
+    BraggIntegrationEngineCPU cpu(experiment, PixelMask(experiment));
+    const auto out_cpu = cpu.Run(cpu_image, predicted, predicted.size(), 0);
+
+    auto stream = std::make_shared<CudaStream>();
+    ImagePreprocessorBufferGPU gpu_image(npixel);
+    for (size_t i = 0; i < npixel; ++i)
+        gpu_image[i] = image[i];
+    REQUIRE(cudaMemcpyAsync(gpu_image.getGPUBuffer(), gpu_image.getBuffer().data(),
+                            npixel * sizeof(int32_t), cudaMemcpyHostToDevice, *stream) == cudaSuccess);
+    BraggIntegrationEngineGPU gpu(experiment, stream, PixelMask(experiment));
+    const auto out_gpu = gpu.Run(gpu_image, predicted, predicted.size(), 0);
+
+    REQUIRE(out_cpu.size() == 2);
+    REQUIRE(out_gpu.size() == 2);
+    CHECK_FALSE(out_gpu[0].overloaded);
+    CHECK(out_gpu[1].overloaded);
+    CHECK(gpu.Counts().overloads_kept == 1);
+    CHECK(out_gpu[0].I == Catch::Approx(out_cpu[0].I).epsilon(0.001));
+}
+
 TEST_CASE("BraggIntegrationEngineGPU_ReusedEngineMatchesFresh") {
     if (get_gpu_count() == 0) {
         WARN("No CUDA GPU present. Skipping BraggIntegrationEngineGPU_ReusedEngineMatchesFresh");
@@ -413,6 +479,67 @@ TEST_CASE("BraggIntegrationEngineGPU_ReusedEngineMatchesFresh") {
     }
 }
 
+// The radial correction with its curve handed over from a per-ring profile (RadialBackgroundFromRings), as
+// on stills: the GPU reads the same curve and corrects the same empty aperture as the CPU.
+TEST_CASE("BraggIntegrationEngineGPU_RadialCurveFromRingsMatchesCPU", "[Integration]") {
+    if (get_gpu_count() == 0) {
+        WARN("No CUDA GPU present. Skipping");
+        return;
+    }
+    DiffractionExperiment experiment(DetJF(2));
+    experiment.DetectorDistance_mm(100.0f).IncidentEnergy_keV(WVL_1A_IN_KEV).BeamX_pxl(400.0f).BeamY_pxl(400.0f);
+    const float bx = 400.0f, by = 400.0f;
+    // Curved, plus a slope steep enough that the ring's flat clip would cut its beam-side half: the
+    // GPU must follow the curve in the clip as the CPU does (RadialClipOffset).
+    const auto background = [](double r) { return 40000.0 - 100.0 * r + 0.05 * (r - 300.0) * (r - 300.0); };
+    const size_t width = experiment.GetXPixelsNum(), npixel = experiment.GetPixelsNum();
+    ImagePreprocessorBuffer cpu_image(npixel);
+    ImagePreprocessorBufferGPU gpu_image(npixel);
+    for (size_t i = 0; i < npixel; ++i) {
+        const double r = std::hypot(static_cast<double>(i % width) - bx, static_cast<double>(i / width) - by);
+        cpu_image[i] = r < 296.0 ? INT32_MIN : static_cast<int32_t>(std::lround(background(r)));
+        gpu_image[i] = cpu_image[i];
+    }
+    auto stream = std::make_shared<CudaStream>();
+    REQUIRE(cudaMemcpyAsync(gpu_image.getGPUBuffer(), gpu_image.getBuffer().data(),
+                            npixel * sizeof(int32_t), cudaMemcpyHostToDevice, *stream) == cudaSuccess);
+
+    const float low_q = 0.05f, dq = 0.01f;
+    const double F_px = 100.0 / 0.075, lambda = 1.0;
+    std::vector<float> rings(500);
+    for (size_t i = 0; i < rings.size(); ++i) {
+        const double q = low_q + (i + 0.5) * dq;
+        rings[i] = static_cast<float>(background(F_px * std::tan(2.0 * std::asin(q * lambda / (4.0 * PI)))));
+    }
+
+    // Half of them next to a masked disc around the beam - a beam-stop shadow - whose edge their rings
+    // reach, so the correction is taken over a ring that lost part of itself (BraggCurveCorrection).
+    std::vector<Reflection> predicted;
+    for (int i = 0; i < 8; ++i) {
+        const float rr = i % 2 ? 306.0f : 300.0f;
+        const float d = static_cast<float>(lambda / (2.0 * std::sin(0.5 * std::atan(rr / F_px))));
+        predicted.push_back(MakeReflection(bx + rr * std::cos(0.7f * i), by + rr * std::sin(0.7f * i), d, i + 1));
+    }
+
+    for (const auto mode : {IntegratorMode::BoxSum, IntegratorMode::ProfileGaussian}) {
+        experiment.ImportBraggIntegrationSettings(BraggIntegrationSettings().Integrator(mode).BackgroundRadialCorrection(true));
+        BraggIntegrationEngineCPU cpu(experiment, PixelMask(experiment));
+        BraggIntegrationEngineGPU gpu(experiment, stream, PixelMask(experiment));
+        cpu.RadialBackgroundFromRings(rings, static_cast<int32_t>(rings.size()), low_q, dq);
+        gpu.RadialBackgroundFromRings(rings, static_cast<int32_t>(rings.size()), low_q, dq);
+        const auto out_cpu = cpu.Run(cpu_image, predicted, predicted.size(), 0);
+        const auto out_gpu = gpu.Run(gpu_image, predicted, predicted.size(), 0);
+        REQUIRE(out_cpu.size() == predicted.size());
+        REQUIRE(out_gpu.size() == out_cpu.size());
+        for (size_t i = 0; i < out_cpu.size(); ++i) {
+            INFO("mode " << static_cast<int>(mode) << " reflection " << i);
+            CHECK(out_gpu[i].bkg == Catch::Approx(out_cpu[i].bkg).epsilon(0.001).margin(0.05));
+            CHECK(out_gpu[i].I == Catch::Approx(out_cpu[i].I).epsilon(0.01).margin(2.0));
+            CHECK(out_gpu[i].var_bkg == Catch::Approx(out_cpu[i].var_bkg).epsilon(0.001));
+        }
+    }
+}
+
 // Hidden ([.]) benchmark: the raison d'etre of the GPU port is < 2 ms/frame (vs ~142 ms on the CPU
 // for ProfileIntegrate2D). Run explicitly with:  ./jfjoch_test "[bragg_bench]"
 TEST_CASE("BraggIntegrationEngineGPU_Benchmark", "[.][bragg_bench]") {