v1.0.0-rc.173 (#83)

* jfjoch_broker: Optional per-dataset authentication - statistics, images and plots can require a bearer token, which jfjoch_viewer supports.
* jfjoch_viewer: Dark mode and a theme-matched colour scheme, a magnifier panel, and simpler contrast and background controls.
* Rugnux: Multiple performance improvements on GPU and CPU (CPU-only processing up to 40% faster, faster image decoding on ARM), with unchanged results.
* Rugnux: `--model` rigid-body refinement runs on the GPU, and the model-validation check is faster and more reliable.
* Rugnux: Improved scaling and merging - error model, outlier rejection, absorption correction and French-Wilson amplitudes now agree more closely with XDS and ctruncate.
* Rugnux: Improved integration - radial background on powder and ice rings, crowded rotation data keep their reflections, and CPU-only builds integrate large unit cells as GPU builds do.
* Rugnux: More robust detector geometry - measured beam centre, X-ray bandwidth and goniometer rate, and geometry refinement accepted only on significant evidence.
* Rugnux: Merged files are written in the standard setting, or in the setting of a reference MTZ, structure-factor mmCIF or model, with its free-R flags.
* Rugnux: Richer report - ice and powder rings, further lattices, superstructure candidates and mosaicity, with warnings worded as prompts to check.
* Rugnux: Clear error messages when a data set needs more GPU or host memory than is available.

Reviewed-on: #83
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/ModelValidationTest.cpp

@@ -5,19 +5,28 @@
 
 #include <cmath>
 #include <cstdio>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
 #include <map>
 #include <random>
+#include <set>
 #include <sstream>
 
 #include <gemmi/mmread_gz.hpp>
 #include <gemmi/fourier.hpp>
+#include <gemmi/solmask.hpp>
 
 #include "../common/Logger.h"
 #include "../rugnux/ModelFFT.h"
+#include "../rugnux/ModelGrid.h"
 #include "../rugnux/ModelValidation.h"
 #include "../rugnux/RigidBodyRefine.h"
+#ifdef JFJOCH_USE_CUDA
+#include "../rugnux/RigidBodyGPU.h"
+#include "../rugnux/RigidBodyGPUEngine.h"
+#include "../common/CUDAWrapper.h"
+#endif
 #include "../rugnux/SigmaA.h"
 #include "../rugnux/WriteModel.h"
 #include "../image_analysis/scale_merge/ReindexAmbiguity.h"
@@ -811,3 +820,598 @@ TEST_CASE("ModelValidation_RigidBodySameOnAnyNumberOfThreads", "[ModelValidation
 
     std::filesystem::remove(path);
 }
+
+// The rigid body puts each probe placement on the grid with its own parallel copy of gemmi's gridding
+// (density, solvent mask and their symmetrization). It must give gemmi's grids bit for bit, on any
+// number of threads, for isotropic and anisotropic atoms and for groups with and without centring.
+TEST_CASE("ModelValidation_ParallelGriddingMatchesGemmi", "[ModelValidation]") {
+    const char *crysts[] = {
+        "CRYST1   40.000   50.000   60.000  90.00  90.00  90.00 P 1           1\n",
+        "CRYST1   40.000   50.000   60.000  90.00 100.00  90.00 C 1 2 1       4\n",
+        "CRYST1   40.000   50.000   60.000  90.00  90.00  90.00 P 21 21 21    4\n",
+        "CRYST1   60.000   60.000   60.000  90.00  90.00  90.00 I 2 3        24\n",
+        "CRYST1   80.000   80.000   80.000  90.00  90.00  90.00 F 41 3 2     96\n",
+    };
+    for (const char *cryst : crysts) {
+        const auto path = WriteTemp("parallel_gridding_test.pdb", ClusterPdb(cryst).c_str());
+        gemmi::Structure st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect);
+        std::filesystem::remove(path);
+        const gemmi::SpaceGroup *sg = st.find_spacegroup();
+        REQUIRE(sg != nullptr);
+        int i = 0;
+        for (gemmi::Chain &ch : st.models[0].chains)
+            for (gemmi::Residue &r : ch.residues)
+                for (gemmi::Atom &a : r.atoms)
+                    if (i++ % 3 == 0)
+                        a.aniso = {0.30f, 0.25f, 0.20f, 0.02f, -0.01f, 0.03f};
+
+        const auto same = [](const gemmi::Grid<float> &a, const gemmi::Grid<float> &b) {
+            return a.data.size() == b.data.size() &&
+                   std::memcmp(a.data.data(), b.data.data(), a.data.size() * sizeof(float)) == 0;
+        };
+        for (double d_min : {6.0, 3.5}) {
+            gemmi::DensityCalculator<gemmi::IT92<float>, float> gemmi_dc;
+            gemmi_dc.d_min = d_min;
+            gemmi_dc.rate = 1.5;
+            gemmi_dc.grid.unit_cell = st.cell;
+            gemmi_dc.grid.spacegroup = sg;
+            gemmi_dc.set_refmac_compatible_blur(st.models[0]);
+            gemmi_dc.put_model_density_on_grid(st.models[0]);
+
+            gemmi::Grid<float> gemmi_mask;
+            gemmi_mask.unit_cell = st.cell;
+            gemmi_mask.spacegroup = sg;
+            gemmi_mask.set_size_from_spacing(gemmi_dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
+            gemmi::SolventMasker(gemmi::AtomicRadiiSet::Refmac).put_mask_on_grid(gemmi_mask, st.models[0]);
+
+            for (size_t nthreads : {1, 4}) {
+                const std::vector<size_t> leaders = OrbitLeaders(gemmi_mask, nthreads);
+
+                gemmi::DensityCalculator<gemmi::IT92<float>, float> dc;
+                dc.d_min = d_min;
+                dc.rate = 1.5;
+                dc.grid.unit_cell = st.cell;
+                dc.grid.spacegroup = sg;
+                dc.set_refmac_compatible_blur(st.models[0]);
+                PutModelDensityOnGrid(dc, st.models[0], leaders, nthreads);
+                CHECK(same(dc.grid, gemmi_dc.grid));
+
+                gemmi::Grid<float> mask;
+                mask.unit_cell = st.cell;
+                mask.spacegroup = sg;
+                mask.set_size_from_spacing(dc.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
+                PutMaskOnGrid(mask, st.models[0], leaders, nthreads);
+                CHECK(same(mask, gemmi_mask));
+            }
+        }
+    }
+}
+
+namespace {
+    // The five groups of ModelValidation_ParallelGriddingMatchesGemmi: none, a centring, screws, a
+    // cubic body centring and a cubic face centring with a diamond glide.
+    const char *kRigidBodyCrysts[] = {
+        "CRYST1   40.000   50.000   60.000  90.00  90.00  90.00 P 1           1\n",
+        "CRYST1   40.000   50.000   60.000  90.00 100.00  90.00 C 1 2 1       4\n",
+        "CRYST1   40.000   50.000   60.000  90.00  90.00  90.00 P 21 21 21    4\n",
+        "CRYST1   60.000   60.000   60.000  90.00  90.00  90.00 I 2 3        24\n",
+        "CRYST1   80.000   80.000   80.000  90.00  90.00  90.00 F 41 3 2     96\n",
+    };
+
+    // ClusterPdb() in `cryst`, every third atom anisotropic.
+    gemmi::Structure AnisoCluster(const char *cryst) {
+        const auto path = WriteTemp("rigid_body_composition_test.pdb", ClusterPdb(cryst).c_str());
+        gemmi::Structure st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect);
+        std::filesystem::remove(path);
+        st.setup_cell_images();
+        int i = 0;
+        for (gemmi::Chain &ch : st.models[0].chains)
+            for (gemmi::Residue &r : ch.residues)
+                for (gemmi::Atom &a : r.atoms)
+                    if (i++ % 3 == 0)
+                        a.aniso = {0.30f, 0.25f, 0.20f, 0.02f, -0.01f, 0.03f};
+        return st;
+    }
+
+    // The zone's density calculator, as the rigid body sets it up for `model`.
+    gemmi::DensityCalculator<gemmi::IT92<float>, float> ZoneDensity(const gemmi::Structure &st,
+                                                                    const gemmi::Model &model, double d_min) {
+        gemmi::DensityCalculator<gemmi::IT92<float>, float> dc;
+        dc.d_min = d_min;
+        dc.rate = 1.5;
+        dc.grid.unit_cell = st.cell;
+        dc.grid.spacegroup = st.find_spacegroup();
+        dc.set_refmac_compatible_blur(model);
+        return dc;
+    }
+}
+
+// The rigid body's Fcalc is composed from the transform of one copy of the model instead of being
+// taken from the symmetrized grid. The two are the same sum rearranged, so they must agree to
+// rounding, and must give a value to exactly the reflections prepare_asu_data() does - the
+// systematic absences of the group and a reflection sitting exactly on d_min (the (10 0 0) of the
+// 60 A cubic cell at 6 A) are left out by both.
+TEST_CASE("ModelValidation_RigidBodyP1FcalcMatchesSymmetrized", "[ModelValidation]") {
+    for (const char *cryst : kRigidBodyCrysts) {
+        const gemmi::Structure st = AnisoCluster(cryst);
+        const gemmi::Model &model = st.models[0];
+        for (double d_min : {6.0, 3.5}) {
+            auto sym = ZoneDensity(st, model, d_min);
+            sym.put_model_density_on_grid(model);
+            gemmi::FPhiGrid<float> sym_f = MapToFPhi(sym.grid);
+            const auto ref = sym_f.prepare_asu_data(d_min, sym.blur, false, false, false);
+            // Every index of the ASU in the grid's box, whatever its resolution, absences and 000 too.
+            std::vector<gemmi::Miller> candidates;
+            for (const auto &hv : sym_f.prepare_asu_data(0, 0, true, true, false).v)
+                candidates.push_back(hv.hkl);
+
+            auto copy = ZoneDensity(st, model, d_min);
+            PutModelDensityOnGrid(copy, model, {}, 4);
+            const SymmetryComposition composition(copy.grid, d_min, candidates);
+            std::vector<std::complex<double>> f;
+            composition.Compose(MapToFPhi(copy.grid), copy.blur, f, nullptr, 4);
+
+            REQUIRE(composition.Hkl().size() == ref.v.size());
+            double mean = 0, worst = 0;
+            for (size_t i = 0; i < ref.v.size(); i++) {
+                REQUIRE(composition.Hkl()[i] == ref.v[i].hkl);
+                mean += std::abs(ref.v[i].value) / static_cast<double>(ref.v.size());
+                worst = std::max(worst, std::abs(f[i] - std::complex<double>(ref.v[i].value)));
+            }
+            INFO(cryst << " at " << d_min << " A: worst " << worst / mean << " of the mean |F|");
+            CHECK(worst <= 1e-4 * mean);
+        }
+    }
+}
+
+// The derivative of the composed Fcalc with respect to a translation of the body is a phase factor
+// per term, exact - against a central difference of the regridded, recomposed Fcalc. Along a
+// direction the origin is free in (all three in P1, b in C2) the amplitude does not change at all.
+TEST_CASE("ModelValidation_RigidBodyTranslationDerivativeIsExact", "[ModelValidation]") {
+    for (const char *cryst : kRigidBodyCrysts) {
+        const gemmi::Structure st = AnisoCluster(cryst);
+        const double d_min = 3.5;
+        const gemmi::DensityCalculator<gemmi::IT92<float>, float> zone = ZoneDensity(st, st.models[0], d_min);
+        gemmi::Grid<float> grid;
+        grid.unit_cell = zone.grid.unit_cell;
+        grid.spacegroup = zone.grid.spacegroup;
+        grid.set_size_from_spacing(zone.requested_grid_spacing(), gemmi::GridSizeRounding::Up);
+        std::vector<gemmi::Miller> candidates;
+        const gemmi::ReciprocalAsu asu(grid.spacegroup);
+        for (int h = -25; h <= 25; h++)
+            for (int k = -25; k <= 25; k++)
+                for (int l = -25; l <= 25; l++)
+                    if (asu.is_in({{h, k, l}}))
+                        candidates.push_back({{h, k, l}});
+        const SymmetryComposition composition(grid, d_min, candidates);
+        REQUIRE(composition.Hkl().size() > 200);
+
+        auto fcalc = [&](const gemmi::Vec3 &shift, std::vector<std::array<std::complex<double>, 3>> *df_dt) {
+            gemmi::Model model = st.models[0];
+            for (gemmi::Chain &ch : model.chains)
+                for (gemmi::Residue &r : ch.residues)
+                    for (gemmi::Atom &a : r.atoms)
+                        a.pos += gemmi::Position(shift);
+            auto dc = ZoneDensity(st, model, d_min);
+            PutModelDensityOnGrid(dc, model, {}, 4);
+            std::vector<std::complex<double>> f;
+            composition.Compose(MapToFPhi(dc.grid), dc.blur, f, df_dt, 4);
+            return f;
+        };
+        std::vector<std::array<std::complex<double>, 3>> df_dt;
+        const std::vector<std::complex<double>> f = fcalc({}, &df_dt);
+        // A five-point difference, whose truncation error goes as eps^4: the grid is float, and at the
+        // 1e-3 A of a plain central difference its rounding alone reads as 3e-3 of the derivative. What
+        // is left, about 5e-4, is the sampling - an atom gridded after a move is the moved atom only to
+        // the accuracy of the grid - which the exact derivative does not have and the difference does.
+        const double eps = 0.05;
+        const bool p1 = st.find_spacegroup()->number == 1, c2 = st.find_spacegroup()->number == 5;
+        for (int k = 0; k < 3; k++) {
+            gemmi::Vec3 e;
+            e.at(k) = eps;
+            const auto p_1 = fcalc(e, nullptr), m_1 = fcalc(-e, nullptr);
+            const auto p_2 = fcalc(2 * e, nullptr), m_2 = fcalc(-2 * e, nullptr);
+            double diff = 0, norm = 0, amplitude = 0;
+            for (size_t m = 0; m < f.size(); m++) {
+                const std::complex<double> numeric = (8.0 * (p_1[m] - m_1[m]) - (p_2[m] - m_2[m])) / (12 * eps);
+                diff += std::norm(numeric - df_dt[m][k]);
+                norm += std::norm(df_dt[m][k]);
+                amplitude += gemmi::sq(std::real(std::conj(f[m]) * df_dt[m][k]) / std::abs(f[m]));
+            }
+            INFO(cryst << " axis " << k << ": relative error " << std::sqrt(diff / norm)
+                       << ", amplitude part " << std::sqrt(amplitude / norm));
+            CHECK(std::sqrt(diff / norm) <= 1e-3);
+            if (p1 || (c2 && k == 1))
+                CHECK(std::sqrt(amplitude / norm) <= 1e-5);
+        }
+    }
+}
+
+#ifdef JFJOCH_USE_CUDA
+namespace {
+    // A pool of `engines`, or a SKIP where there is none because the card is busy - over half its memory
+    // taken by something else, where the pool falls back to the CPU by design. Anywhere else no pool fails.
+    std::unique_ptr<RigidBodyGPUPool> TestPool(const gemmi::Model &model, const gemmi::UnitCell &cell,
+                                               const gemmi::SpaceGroup &sg, double d_min, size_t observations,
+                                               size_t engines, Logger &logger) {
+        auto pool = RigidBodyGPUPool::Create(model, cell, sg, d_min, observations, engines, logger);
+        if (!pool) {
+            size_t free = 0, total = 0;
+            RigidBodyGPUEngine::MemoryInfo(free, total);
+            if (free < total / 2)
+                SKIP("No GPU engine: the card is busy (" << free / 1000000 << " of " << total / 1000000 << " MB free)");
+        }
+        REQUIRE(pool);
+        return pool;
+    }
+}
+#endif
+
+namespace {
+    // The rigid body's Jacobian at q against a central difference of its own residuals - which
+    // re-fit the scale at every evaluation - with the bulk-solvent mask held, as the Jacobian holds
+    // it. Per column (those in `columns`): the cosine between the two and the ratio of their norms.
+    // Also logged, not checked: how far the difference moves once the mask is let move with the body,
+    // which is what holding it costs.
+    // The model's own amplitudes to `d_min`: "observed" data whose minimum is where the model is.
+    gemmi::AsuData<gemmi::ValueSigma<float>> OwnAmplitudes(const char *cryst, const gemmi::Structure &st,
+                                                           double d_min, Logger &logger) {
+        const auto path = WriteTemp("rigid_body_own_amplitudes.pdb", ClusterPdb(cryst).c_str());
+        const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
+        std::filesystem::remove(path);
+        gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
+        fobs.unit_cell_ = st.cell;
+        fobs.spacegroup_ = st.find_spacegroup();
+        for (const auto &r : ref)
+            if (st.cell.calculate_d({{r.h, r.k, r.l}}) >= d_min)
+                fobs.v.push_back({{{r.h, r.k, r.l}}, {std::sqrt(r.I), 1.0f}});
+        fobs.ensure_sorted();
+        return fobs;
+    }
+
+    // The rigid body's target on the CPU, or on an engine of `gpu` where it is given.
+    std::unique_ptr<RigidBodyTargetBase> MakeTarget(gemmi::Model &model, const gemmi::Structure &st,
+                                                    RigidBodyGPUPool *gpu) {
+#ifdef JFJOCH_USE_CUDA
+        if (gpu != nullptr)
+            return std::make_unique<RigidBodyTargetGPU>(*gpu, model, st.cell, *st.find_spacegroup(), 4);
+#endif
+        return std::make_unique<RigidBodyTarget>(model, st.cell, *st.find_spacegroup(), 4);
+    }
+
+    void CheckRigidBodyJacobian(const char *cryst, const double q0[6], const std::vector<int> &columns,
+                                double min_cosine, double max_norm_error, bool gpu = false) {
+        Logger logger("CheckRigidBodyJacobian");
+        const gemmi::Structure st = AnisoCluster(cryst);
+        const gemmi::SpaceGroup *sg = st.find_spacegroup();
+        // "Observed" amplitudes: the model's own, as placed at q = 0.
+        const auto path = WriteTemp("rigid_body_jacobian_test.pdb", ClusterPdb(cryst).c_str());
+        const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
+        std::filesystem::remove(path);
+        REQUIRE_FALSE(ref.empty());
+
+        for (double zone : {6.0, 3.5}) {
+            gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
+            fobs.unit_cell_ = st.cell;
+            fobs.spacegroup_ = sg;
+            for (const auto &r : ref)
+                if (st.cell.calculate_d({{r.h, r.k, r.l}}) >= zone)
+                    fobs.v.push_back({{{r.h, r.k, r.l}}, {std::sqrt(r.I), 1.0f}});
+            fobs.ensure_sorted();
+
+            gemmi::Model model = st.models[0];
+            RigidBodyGPUPool *pool = nullptr;
+#ifdef JFJOCH_USE_CUDA
+            std::unique_ptr<RigidBodyGPUPool> engines;
+            if (gpu) {
+                engines = TestPool(model, st.cell, *sg, zone, fobs.v.size(), 1, logger);
+                pool = engines.get();
+            }
+#else
+            (void) gpu;
+#endif
+            const std::unique_ptr<RigidBodyTargetBase> target_backend = MakeTarget(model, st, pool);
+            RigidBodyTargetBase &target = *target_backend;
+            target.SetZone(fobs, zone);
+            const size_t n = target.NumObservations();
+            std::vector<double> r(n), jacobian(n * 6);
+            REQUIRE(target.Residuals(q0, r.data()));
+            REQUIRE(target.Jacobian(q0, jacobian.data()));
+
+            auto difference = [&](int j, double h) {
+                double qp[6], qm[6];
+                std::copy(q0, q0 + 6, qp);
+                std::copy(q0, q0 + 6, qm);
+                qp[j] += h;
+                qm[j] -= h;
+                std::vector<double> rp(n), rm(n), d(n);
+                REQUIRE(target.Residuals(qp, rp.data()));
+                REQUIRE(target.Residuals(qm, rm.data()));
+                for (size_t i = 0; i < n; i++)
+                    d[i] = (rp[i] - rm[i]) / (2 * h);
+                return d;
+            };
+            std::vector<std::vector<double>> held(6), moving(6);
+            target.hold_mask = true;
+            REQUIRE(target.Residuals(q0, r.data()));   // the mask the difference holds is q0's
+            for (int j : columns)
+                held[j] = difference(j, 0.005);
+            // The mask is binary on the grid, so with it moving the difference is taken at the step the
+            // Jacobian had when every column moved it (0.01 of the zone's resolution): at 0.005 A it
+            // mostly measures which grid points a few atoms happened to cross.
+            target.hold_mask = false;
+            for (int j : columns)
+                moving[j] = difference(j, 0.01 * zone);
+
+            double largest = 0;
+            for (int j : columns) {
+                double s = 0;
+                for (size_t i = 0; i < n; i++)
+                    s += gemmi::sq(held[j][i]);
+                largest = std::max(largest, std::sqrt(s));
+            }
+            for (int j : columns) {
+                double dot = 0, ours = 0, theirs = 0, dot_moving = 0, norm_moving = 0;
+                for (size_t i = 0; i < n; i++) {
+                    dot += jacobian[i * 6 + j] * held[j][i];
+                    ours += gemmi::sq(jacobian[i * 6 + j]);
+                    theirs += gemmi::sq(held[j][i]);
+                    dot_moving += moving[j][i] * held[j][i];
+                    norm_moving += gemmi::sq(moving[j][i]);
+                }
+                ours = std::sqrt(ours);
+                theirs = std::sqrt(theirs);
+                norm_moving = std::sqrt(norm_moving);
+                const double cosine = dot / (ours * theirs);
+                logger.Info("{} at {:.1f} A, column {}: cosine {:.5f}, norm {:.4f} of the difference's; "
+                            "with the mask moving, cosine {:.4f} and norm {:.4f}",
+                            std::string(cryst).substr(55, 11), zone, j, cosine, ours / theirs,
+                            dot_moving / (norm_moving * theirs), norm_moving / theirs);
+                INFO(cryst << " at " << zone << " A, column " << j << ": cosine " << cosine
+                           << ", norm ratio " << ours / theirs);
+                if (theirs < 1e-3 * largest) {
+                    // A direction the origin is free in: |F| does not change, so neither may the column.
+                    CHECK(ours < 1e-2 * largest);
+                    continue;
+                }
+                CHECK(cosine >= min_cosine);
+                CHECK(std::fabs(ours / theirs - 1) <= max_norm_error);
+            }
+        }
+    }
+}
+
+// The rotation columns at a placement already rotated (about 1.4 deg, so the step is taken from a
+// rotated body and not from the model as read) with anisotropic atoms, whose U the placement does not
+// turn with the body - the columns must differentiate exactly the function the residuals evaluate.
+TEST_CASE("ModelValidation_RigidBodyRotationDerivativeMatchesDifferences", "[ModelValidation]") {
+    const double q0[6] = {0.12, -0.09, 0.07, 0.15, -0.10, 0.05};
+    for (const char *cryst : kRigidBodyCrysts)
+        CheckRigidBodyJacobian(cryst, q0, {0, 1, 2}, 0.99, 0.03);
+}
+
+// The whole Jacobian - analytic translation, forward-difference rotation, the scale re-fit folded in by
+// projection - against the central difference of the residuals it describes. The projection is
+// Kaufman's, which leaves out a term that grows with the residuals (the scale's derivatives depend on
+// the placement too, weighted by how badly the model fits): with the body 0.07 A off it agrees with the
+// difference to about 2e-3 in cosine, with it 0.35 A off to about 2e-2, and the bounds follow.
+TEST_CASE("ModelValidation_RigidBodyJacobianMatchesNumericScaleRefit", "[ModelValidation]") {
+    const double close[6] = {0.0, 0.0, 0.0, 0.05, -0.04, 0.03};
+    const double far[6] = {0.0, 0.0, 0.0, 0.25, -0.20, 0.15};
+    for (const char *cryst : kRigidBodyCrysts) {
+        CheckRigidBodyJacobian(cryst, close, {0, 1, 2, 3, 4, 5}, 0.99, 0.02);
+        CheckRigidBodyJacobian(cryst, far, {0, 1, 2, 3, 4, 5}, 0.975, 0.03);
+    }
+}
+
+// The null checks where a replicate ENDED against the orientations equivalent to the model's, from the
+// rotation the rigid body reports - which must be the rotation it applied: every atom's offset from the
+// centroid after the refinement is that rotation of its offset before.
+TEST_CASE("ModelValidation_RigidBodyReportsTheRotationItApplied", "[ModelValidation]") {
+    Logger logger("ModelValidation_RigidBodyReportsTheRotationItApplied");
+    const auto path = WriteTemp("rigid_body_rotation_test.pdb", ClusterPdb().c_str());
+    gemmi::Structure st = gemmi::read_structure_gz(path, gemmi::CoorFormat::Detect);
+    const gemmi::SpaceGroup *sg = st.find_spacegroup();
+    REQUIRE(sg != nullptr);
+    st.setup_cell_images();
+    const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
+    std::filesystem::remove(path);
+    REQUIRE_FALSE(ref.empty());
+    gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
+    fobs.unit_cell_ = st.cell;
+    fobs.spacegroup_ = sg;
+    for (const auto &r : ref)
+        fobs.v.push_back({{{r.h, r.k, r.l}}, {std::sqrt(r.I), 1.0f}});
+    fobs.ensure_sorted();
+
+    // Turned 3 deg about z through the centroid, so there is a rotation to take back.
+    std::vector<gemmi::Position> turned = ModelPositions(st.models[0]);
+    gemmi::Vec3 centre;
+    for (const gemmi::Position &p : turned)
+        centre += p;
+    centre *= 1.0 / static_cast<double>(turned.size());
+    const double a = 3.0 * PI / 180.0;
+    const gemmi::Mat33 rz(std::cos(a), -std::sin(a), 0, std::sin(a), std::cos(a), 0, 0, 0, 1);
+    for (gemmi::Position &p : turned)
+        p = gemmi::Position(rz.multiply(gemmi::Vec3(p) - centre) + centre);
+    SetModelPositions(st.models[0], turned);
+
+    const RigidBodyRefineResult result = RefineRigidBody(st.models[0], st.cell, *sg, fobs, 3.0, logger);
+    REQUIRE(result.angle_deg > 1.0);
+    const std::vector<gemmi::Position> refined = ModelPositions(st.models[0]);
+    gemmi::Vec3 moved_centre;
+    for (const gemmi::Position &p : refined)
+        moved_centre += p;
+    moved_centre *= 1.0 / static_cast<double>(refined.size());
+    double worst = 0;
+    for (size_t i = 0; i < refined.size(); i++) {
+        const gemmi::Vec3 expected = result.rotation.multiply(gemmi::Vec3(turned[i]) - centre);
+        worst = std::max(worst, (gemmi::Vec3(refined[i]) - moved_centre - expected).length());
+    }
+    CHECK(worst < 1e-6);
+    const double trace = result.rotation[0][0] + result.rotation[1][1] + result.rotation[2][2];
+    CHECK(std::acos(std::clamp((trace - 1) / 2, -1.0, 1.0)) * 180 / PI ==
+          Catch::Approx(result.angle_deg).margin(1e-6));
+}
+
+#ifdef JFJOCH_USE_CUDA
+
+// The gather's bound on the bricks a box falls in along one axis, against every box on small grids -
+// wrapped boxes on grids that are not a multiple of the brick included (n = 17: 15, 16, 0 fall in bricks
+// 1, 2 and 0). Host arithmetic, no device needed.
+TEST_CASE("RigidBodyGPU_AxisBrickBoundCoversWrappedBoxes", "[ModelValidation][gpu]") {
+    CHECK(RigidBodyGPUEngine::AxisBrickBound(1, 17) == 3);
+    const int brick = 8;
+    for (int n = 1; n <= 64; n++)
+        for (int d = 0; 2 * d + 1 <= n; d++)
+            for (int c = 0; c < n; c++) {
+                std::set<int> bricks;
+                for (int p = c - d; p <= c + d; p++)
+                    bricks.insert(((p % n) + n) % n / brick);
+                INFO("n " << n << ", d " << d << ", c " << c);
+                CHECK(bricks.size() <= RigidBodyGPUEngine::AxisBrickBound(d, n));
+            }
+}
+
+// The GPU target is the CPU target's function, computed on the device: at the same placement the two
+// give the same residuals and the same Jacobian to rounding (float distances on the device, cuFFT for
+// FFTW), on every group of the composition tests, both zones, anisotropic atoms included.
+TEST_CASE("RigidBodyGPU_MatchesCPU", "[ModelValidation][gpu]") {
+    if (get_gpu_count() == 0)
+        SKIP("No GPU");
+    Logger logger("RigidBodyGPU_MatchesCPU");
+    const double q0[6] = {0.12, -0.09, 0.07, 0.15, -0.10, 0.05};
+    for (const char *cryst : kRigidBodyCrysts) {
+        const gemmi::Structure st = AnisoCluster(cryst);
+        const gemmi::SpaceGroup *sg = st.find_spacegroup();
+        for (double zone : {6.0, 3.5}) {
+            const auto fobs = OwnAmplitudes(cryst, st, zone, logger);
+            gemmi::Model cpu_model = st.models[0], gpu_model = st.models[0];
+            auto pool = TestPool(gpu_model, st.cell, *sg, zone, fobs.v.size(), 1, logger);
+            RigidBodyTarget cpu(cpu_model, st.cell, *sg, 4);
+            RigidBodyTargetGPU gpu(*pool, gpu_model, st.cell, *sg, 4);
+            cpu.SetZone(fobs, zone);
+            gpu.SetZone(fobs, zone);
+            const size_t n = cpu.NumObservations();
+            REQUIRE(gpu.NumObservations() == n);
+            std::vector<double> rc(n), rg(n), jc(6 * n), jg(6 * n);
+            REQUIRE(cpu.Residuals(q0, rc.data()));
+            REQUIRE(gpu.Residuals(q0, rg.data()));
+            REQUIRE(cpu.Jacobian(q0, jc.data()));
+            REQUIRE(gpu.Jacobian(q0, jg.data()));
+            CHECK(gpu.unmatched == cpu.unmatched);
+            CHECK(gpu.k_sol == cpu.k_sol);
+            CHECK(gpu.b_sol == cpu.b_sol);
+
+            double worst = 0, rms = 0;
+            for (size_t i = 0; i < n; i++) {
+                worst = std::max(worst, std::fabs(rc[i] - rg[i]));
+                rms += rc[i] * rc[i] / static_cast<double>(n);
+            }
+            INFO(cryst << " at " << zone << " A: residuals differ by at most " << worst << ", rms residual "
+                       << std::sqrt(rms));
+            // Most cases agree to a few 1e-6 of <Fobs>. The bound is set by gemmi's scale fit instead: its
+            // Levenberg-Marquardt stops at a relative change of 1e-5, so a near-tie in its accept or stop
+            // decision - which a 1e-5 change of Fcalc can flip, on the CPU alone - moves the scale by up
+            // to about 1e-4 of |F|.
+            CHECK(worst <= 2e-4);
+            for (int j = 0; j < 6; j++) {
+                double diff = 0, norm = 0;
+                for (size_t i = 0; i < n; i++) {
+                    diff += gemmi::sq(jc[6 * i + j] - jg[6 * i + j]);
+                    norm += gemmi::sq(jc[6 * i + j]);
+                }
+                INFO(cryst << " at " << zone << " A, column " << j << ": relative difference "
+                           << std::sqrt(diff / norm));
+                // Measured at up to 5e-5 (the rotation columns, a forward difference of float-gridded
+                // density; the translation columns agree to about 1e-6). Every column is proportional to
+                // the overall scale, which the near-tie above can move by 1e-4, so that is the floor of
+                // an honest bound; 5e-4 leaves room for the difference's own rounding on another card.
+                CHECK(std::sqrt(diff) <= 5e-4 * std::sqrt(norm) + 1e-9);
+            }
+        }
+    }
+}
+
+// The whole Jacobian on the GPU against the central difference of the GPU's own residuals, with the
+// bounds of ModelValidation_RigidBodyJacobianMatchesNumericScaleRefit.
+TEST_CASE("RigidBodyGPU_JacobianMatchesNumericScaleRefit", "[ModelValidation][gpu]") {
+    if (get_gpu_count() == 0)
+        SKIP("No GPU");
+    const double close[6] = {0.0, 0.0, 0.0, 0.05, -0.04, 0.03};
+    const double rotated[6] = {0.12, -0.09, 0.07, 0.15, -0.10, 0.05};
+    for (const char *cryst : kRigidBodyCrysts) {
+        CheckRigidBodyJacobian(cryst, close, {0, 1, 2, 3, 4, 5}, 0.99, 0.02, true);
+        CheckRigidBodyJacobian(cryst, rotated, {0, 1, 2}, 0.99, 0.03, true);
+    }
+}
+
+namespace {
+    // A whole placement on the GPU from the model displaced by 0.54 A and turned 2 deg, against the
+    // model's own amplitudes; the model is left where the fit put it.
+    RigidBodyRefineResult DisplacedFit(const char *cryst, gemmi::Structure &st, RigidBodyGPUPool *pool,
+                                       Logger &logger) {
+        const gemmi::SpaceGroup *sg = st.find_spacegroup();
+        const auto fobs = OwnAmplitudes(cryst, st, 3.0, logger);
+        std::vector<gemmi::Position> moved = ModelPositions(st.models[0]);
+        gemmi::Vec3 centre;
+        for (const gemmi::Position &p : moved)
+            centre += p;
+        centre *= 1.0 / static_cast<double>(moved.size());
+        const double a = 2.0 * PI / 180.0;
+        const gemmi::Mat33 rz(std::cos(a), -std::sin(a), 0, std::sin(a), std::cos(a), 0, 0, 0, 1);
+        for (gemmi::Position &p : moved)
+            p = gemmi::Position(rz.multiply(gemmi::Vec3(p) - centre) + centre + gemmi::Vec3(0.40, -0.30, 0.20));
+        SetModelPositions(st.models[0], moved);
+        return RefineRigidBody(st.models[0], st.cell, *sg, fobs, 3.0, logger, 4, pool);
+    }
+}
+
+// The GPU fit walks where the CPU fit walks: from the same displaced start, to the same placement.
+TEST_CASE("RigidBodyGPU_FitAgreesWithCPU", "[ModelValidation][gpu]") {
+    if (get_gpu_count() == 0)
+        SKIP("No GPU");
+    Logger logger("RigidBodyGPU_FitAgreesWithCPU");
+    for (const char *cryst : {kCryst, kPolarCryst, kRigidBodyCrysts[3]}) {
+        gemmi::Structure cpu_st = AnisoCluster(cryst), gpu_st = AnisoCluster(cryst);
+        auto pool = TestPool(gpu_st.models[0], gpu_st.cell, *gpu_st.find_spacegroup(), 3.0, 100000, 1, logger);
+        const RigidBodyRefineResult cpu = DisplacedFit(cryst, cpu_st, nullptr, logger);
+        const RigidBodyRefineResult gpu = DisplacedFit(cryst, gpu_st, pool.get(), logger);
+        CHECK(gpu.converged == cpu.converged);
+        const std::vector<gemmi::Position> pc = ModelPositions(cpu_st.models[0]), pg = ModelPositions(gpu_st.models[0]);
+        double rmsd = 0;
+        for (size_t i = 0; i < pc.size(); i++)
+            rmsd += pc[i].dist_sq(pg[i]) / static_cast<double>(pc.size());
+        INFO(cryst << ": CPU " << cpu.angle_deg << " deg " << cpu.shift_A << " A, GPU " << gpu.angle_deg << " deg "
+                   << gpu.shift_A << " A, " << std::sqrt(rmsd) << " A apart");
+        // Measured at up to 5e-7 A: the endpoint moves only by what the residuals differ by (a few
+        // 1e-6 of <Fobs>) over the target's curvature. The bound leaves a factor 20 for another card's
+        // cuFFT and float rounding, and is still four orders under anything a placement is judged at.
+        CHECK(std::sqrt(rmsd) < 1e-5);
+    }
+}
+
+// Deterministic: the same fit twice, and on a pool of one engine and of four, gives the same placement
+// bit for bit.
+TEST_CASE("RigidBodyGPU_Deterministic", "[ModelValidation][gpu]") {
+    if (get_gpu_count() == 0)
+        SKIP("No GPU");
+    Logger logger("RigidBodyGPU_Deterministic");
+    const char *cryst = kRigidBodyCrysts[4];
+    std::vector<std::vector<gemmi::Position>> placed;
+    for (size_t engines : {1, 1, 4}) {
+        gemmi::Structure st = AnisoCluster(cryst);
+        auto pool = TestPool(st.models[0], st.cell, *st.find_spacegroup(), 3.0, 100000, engines, logger);
+        const RigidBodyRefineResult r = DisplacedFit(cryst, st, pool.get(), logger);
+        CHECK(r.evaluations > 0);
+        placed.push_back(ModelPositions(st.models[0]));
+    }
+    for (size_t k = 1; k < placed.size(); k++)
+        for (size_t i = 0; i < placed[0].size(); i++) {
+            CHECK(placed[k][i].x == placed[0][i].x);
+            CHECK(placed[k][i].y == placed[0][i].y);
+            CHECK(placed[k][i].z == placed[0][i].z);
+        }
+}
+
+#endif