v1.0.0-rc.173 (#83)

* jfjoch_broker: Optional per-dataset authentication - statistics, images and plots can require a bearer token, which jfjoch_viewer supports.
* jfjoch_viewer: Dark mode and a theme-matched colour scheme, a magnifier panel, and simpler contrast and background controls.
* Rugnux: Multiple performance improvements on GPU and CPU (CPU-only processing up to 40% faster, faster image decoding on ARM), with unchanged results.
* Rugnux: `--model` rigid-body refinement runs on the GPU, and the model-validation check is faster and more reliable.
* Rugnux: Improved scaling and merging - error model, outlier rejection, absorption correction and French-Wilson amplitudes now agree more closely with XDS and ctruncate.
* Rugnux: Improved integration - radial background on powder and ice rings, crowded rotation data keep their reflections, and CPU-only builds integrate large unit cells as GPU builds do.
* Rugnux: More robust detector geometry - measured beam centre, X-ray bandwidth and goniometer rate, and geometry refinement accepted only on significant evidence.
* Rugnux: Merged files are written in the standard setting, or in the setting of a reference MTZ, structure-factor mmCIF or model, with its free-R flags.
* Rugnux: Richer report - ice and powder rings, further lattices, superstructure candidates and mosaicity, with warnings worded as prompts to check.
* Rugnux: Clear error messages when a data set needs more GPU or host memory than is available.

Reviewed-on: #83
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/MergeScaleTest.cpp

@@ -4,6 +4,7 @@
 #include <catch2/catch_all.hpp>
 
 #include <random>
+#include "../image_analysis/scale_merge/ErrorModel.h"
 #include "../image_analysis/scale_merge/HKLKey.h"
 #include "../image_analysis/scale_merge/Merge.h"
 #include "../image_analysis/scale_merge/ResolutionCutoff.h"
@@ -387,6 +388,27 @@ TEST_CASE("CCHalfFrameFactors_DeadStretchDoesNotSetTheTypicalScale") {
     CHECK(factor[100] == Catch::Approx(1.0).epsilon(1e-3));
 }
 
+// Equal weights give the ordinary R_meas terms, whatever their common value.
+TEST_CASE("WeightedRmeas_EqualWeightsAreTheOrdinaryRmeas") {
+    // Three observations 9, 10, 11 of a reflection with <I> = 10: sum|dev| = 2, sum I = 30.
+    double num, den;
+    REQUIRE(WeightedRmeasTerms(0.5 * 2.0, 0.5 * 30.0, 0.5 * 3, 0.25 * 3, num, den));
+    CHECK(num == Catch::Approx(std::sqrt(3.0 / 2.0) * 2.0));
+    CHECK(den == Catch::Approx(30.0));
+}
+
+// An observation that carries almost no information counts for almost nothing: the terms tend to those
+// of the reflection without it, and a reflection left with one effective observation has none.
+TEST_CASE("WeightedRmeas_NegligibleWeightDropsOut") {
+    // 9 and 11 at weight 1, <I> = 10; a third at 40 with weight 1e-6.
+    const double v = 1e-6;
+    double num, den;
+    REQUIRE(WeightedRmeasTerms(2.0 + v * 30.0, 20.0 + v * 40.0, 2.0 + v, 2.0 + v * v, num, den));
+    CHECK(num == Catch::Approx(std::sqrt(2.0) * 2.0).epsilon(1e-4));
+    CHECK(den == Catch::Approx(20.0).epsilon(1e-4));
+    CHECK_FALSE(WeightedRmeasTerms(0.0, 10.0, 1.0, 1.0, num, den));
+}
+
 // A fall-off region a logistic cannot follow: CC1/2 drops through the target and comes straight back
 // up. The fitted crossing is then an extrapolation far past the bins it was made over, and reading
 // the cut off it writes the data deep into the noise; the crossing the bins themselves show is where
@@ -406,3 +428,72 @@ TEST_CASE("ResolutionCutoff_RaggedFallOffIsReadOffTheBins") {
     REQUIRE(rc.d_cut);
     CHECK(*rc.d_cut > 2.30);                        // coarser than the band that correlates again
 }
+
+namespace {
+    // Samples of var = a*s2 + b^2*I2 over four decades of counting I/sigma, with counting variances
+    // spread over a factor of 100 at every intensity, and a fraction of gross outliers.
+    std::vector<ErrorModelSample> SyntheticErrorModelSamples(double a, double b, double max_snr,
+                                                             double outlier_fraction) {
+        std::mt19937 rng(7);
+        std::uniform_real_distribution<double> u(0.0, 1.0);
+        std::normal_distribution<double> z(0.0, 1.0);
+        std::vector<ErrorModelSample> out;
+        for (int i = 0; i < 200000; ++i) {
+            const double s2 = std::pow(10.0, 2.0 * u(rng));
+            const double snr = max_snr * std::pow(10.0, -4.0 * u(rng));
+            const double I2 = snr * snr * s2;
+            const double e = z(rng);
+            double dev2 = (a * s2 + b * b * I2) * e * e;
+            if (u(rng) < outlier_fraction) dev2 *= 400.0;
+            out.push_back({s2, I2, dev2, 2.0f});
+        }
+        return out;
+    }
+}
+
+// The fit recovers a and b from observations whose counting variances differ widely inside every
+// intensity bin (where a ratio of bin medians does not), with 0.3% gross outliers in the pool.
+TEST_CASE("ErrorModel_RecoversAAndBThroughOutliers") {
+    const auto pool = SyntheticErrorModelSamples(1.3, 0.03, 300.0, 0.003);
+    std::vector<ErrorModelBinned> scratch;
+    const auto fit = FitErrorModel(pool, scratch, 4);
+    REQUIRE(fit.active);
+    CHECK(fit.b_measured);
+    CHECK(fit.b_resolved);
+    CHECK(fit.a == Catch::Approx(1.3).epsilon(0.03));
+    CHECK(1.0 / std::sqrt(fit.b2) == Catch::Approx(1.0 / 0.03).epsilon(0.05));
+}
+
+// A few observations whose counting variance is hugely overstated - their scatter is ordinary - do not
+// set the fit: each sample counts by its deviation relative to its own variance.
+TEST_CASE("ErrorModel_OverstatedCountingVarianceDoesNotSetTheFit") {
+    auto pool = SyntheticErrorModelSamples(1.3, 0.03, 300.0, 0.0);
+    for (size_t i = 0; i < pool.size(); i += 200)
+        pool[i].s2 *= 1e4;
+    std::vector<ErrorModelBinned> scratch;
+    const auto fit = FitErrorModel(pool, scratch, 4);
+    REQUIRE(fit.active);
+    CHECK(fit.a == Catch::Approx(1.3).epsilon(0.03));
+    CHECK(1.0 / std::sqrt(fit.b2) == Catch::Approx(1.0 / 0.03).epsilon(0.05));
+}
+
+// The same pool on one thread and on many gives the same numbers.
+TEST_CASE("ErrorModel_SameOnAnyNumberOfThreads") {
+    const auto pool = SyntheticErrorModelSamples(1.1, 0.05, 100.0, 0.001);
+    std::vector<ErrorModelBinned> scratch;
+    const auto one = FitErrorModel(pool, scratch, 1);
+    const auto many = FitErrorModel(pool, scratch, 16);
+    CHECK(one.a == many.a);
+    CHECK(one.b2 == many.b2);
+}
+
+// Data that never reach where b could be seen: a is fitted alone and b held at 0.
+TEST_CASE("ErrorModel_BNotMeasuredOnWeakData") {
+    const auto pool = SyntheticErrorModelSamples(0.9, 0.05, 1.5, 0.0);
+    std::vector<ErrorModelBinned> scratch;
+    const auto fit = FitErrorModel(pool, scratch, 4);
+    REQUIRE(fit.active);
+    CHECK(!fit.b_measured);
+    CHECK(fit.b2 == 0.0);
+    CHECK(fit.a == Catch::Approx(0.9).epsilon(0.03));
+}