v1.0.0-rc.175 (#85)

* Rugnux: Processes a collection live while jfjoch_broker writes it (`rugnux <master.h5> <broker URL>`).
* Rugnux: Processing starts on the first images and confirms its choices once all are in, so analysis can begin during collection.
* Rugnux: Faster processing, especially on servers with several GPUs and CPU sockets and on computers without a GPU (`--no-gpu`, macOS).
* Rugnux: Uses much less GPU memory, so large data sets stay on the GPU of smaller cards, and a data set too large for the card is processed on the CPU instead.
* Rugnux: On macOS, Bragg integration runs on the Apple GPU.
* Rugnux: Better handling of twinning and pseudo-symmetry, with the twin hypothesis written as `<prefix>_twin_<SG>.mtz`, and a wrong `--model` no longer confirms the data or decides the hand.
* Rugnux: `--model` accepts predicted (AlphaFold) models and falls back to molecular replacement when the model is not placed as a rigid body (experimental `--mr` forces it), settling space group, hand and twinning and writing a different answer as `<prefix>_mr.mtz`.
* Rugnux: Molecular replacement is faster and settles the space group in more cases with several copies in the asymmetric unit.
* Rugnux: Deposition-ready mmCIF, a richer report, and numbered steps on the terminal with the full log in `<prefix>_log.txt`; `_P1.mtz`, `--estimate-beam-center` and `--beam-center-search` are removed.
* Rugnux: The beam-stop shadow is found correctly on tilted detectors.
* Rugnux: Stills files that record a stationary goniometer axis get the default stills geometry refinement.
* jfjoch_viewer: "Analyze dataset" uses the same defaults as a bare `rugnux <file>`, and shows the results of processed CBF, marCCD and SMV sweeps.
* jfjoch_viewer: The Files panel refreshes on F5 and every few seconds, and the plot type and other selections stay put during live collection.
* jfjoch_writer: Memory stays bounded and old data files are closed efficiently on long runs, any failure ends the run in an error state the broker is told about, and the master file appears only after its data files; the ZeroMQ pusher logs when a stalled writer makes it drop images.
* Fixes: The TCP image pusher no longer blocks the broker's receiver, GPU out-of-memory failures stop with a clear message, and the XDS plugin masks module and chip edges.

---------

Co-authored-by: Filip Leonarski <jungfrau@filips-mac-mini.home>
Reviewed-on: #85
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/ModelMaskGPUTest.cpp

@@ -10,14 +10,15 @@
 #include <cstring>
 #include <random>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include <gemmi/pdb.hpp>
 #include <gemmi/solmask.hpp>
 #include <gemmi/symmetry.hpp>
 
-#include "../rugnux/ModelGrid.h"
-#include "../rugnux/ModelMaskGPU.h"
+#include "../image_analysis/structure_refinement/ModelGrid.h"
+#include "../image_analysis/structure_refinement/ModelMaskGPU.h"
 
 namespace {
 
@@ -202,18 +203,65 @@ TEST_CASE("ModelMaskGPU_MatchesGemmi", "[ModelValidation][gpu]") {
     CHECK(any_odd_size);
 }
 
-// The shrink is not implemented: a grid fine enough for it to change anything is refused.
-TEST_CASE("ModelMaskGPU_RefusesAGridTheShrinkWouldChange", "[ModelValidation][gpu]") {
+// Grids fine enough for gemmi's shrink to act. The shrink alone, on gemmi's own mask after its island
+// step, is exact; the whole mask may differ from gemmi's only where the masking does (points at an atom's
+// radius) and at the neighbours the shrink then treats differently - measured, it does not differ at
+// all here - and is the same bits twice.
+TEST_CASE("ModelMaskGPU_ShrinkMatchesGemmi", "[ModelValidation][gpu]") {
     if (get_gpu_count() == 0)
         SKIP("No GPU");
     CudaStream stream;
-    const gemmi::Structure st = ClusterModel(kCrysts[0]);
-    const std::vector<ModelMaskOp> ops = MaskOps(*st.find_spacegroup());
-    const gemmi::Grid<float> coarse = MaskGrid(st, 3.5);
-    const gemmi::Grid<float> fine = MaskGrid(st, 2.0);
-    ModelMaskGPU mask(stream, fine.point_count());
-    CHECK_NOTHROW(mask.SetGrid(MaskGridDescription(coarse), ops));
-    CHECK_THROWS(mask.SetGrid(MaskGridDescription(fine), ops));
+    // The five groups at 1.5 A, and a strongly oblique monoclinic cell at 3.5 A, where the planes of the
+    // grid along a and c are closer than r_shrink although its steps along the axes are not.
+    std::vector<std::pair<const char *, double>> cases;
+    for (const char *cryst : kCrysts)
+        cases.emplace_back(cryst, 1.5);
+    cases.emplace_back("CRYST1   40.000   50.000   60.000  90.00 141.00  90.00 P 1 21 1       2\n", 3.5);
+    for (const auto &[cryst, d_min] : cases) {
+        const gemmi::Structure st = ClusterModel(cryst);
+        const gemmi::SpaceGroup *sg = st.find_spacegroup();
+        REQUIRE(sg != nullptr);
+        const std::vector<ModelMaskAtom> atoms = MaskAtoms(st);
+        const std::vector<ModelMaskOp> ops = MaskOps(*sg);
+        CudaDevicePtr<ModelMaskAtom> atoms_d(atoms.size());
+        REQUIRE(cudaMemcpyAsync(atoms_d, atoms.data(), atoms.size() * sizeof(ModelMaskAtom), cudaMemcpyHostToDevice, stream) == cudaSuccess);
+
+        const gemmi::SolventMasker masker(gemmi::AtomicRadiiSet::Refmac);
+        gemmi::Grid<float> islands = MaskGrid(st, d_min);
+        masker.clear(islands);
+        masker.mask_points(islands, st.models[0]);
+        masker.symmetrize(islands);
+        masker.remove_islands(islands);
+        gemmi::Grid<float> ref = islands;
+        masker.shrink(ref);
+        const size_t n = ref.data.size();
+        REQUIRE(std::memcmp(ref.data.data(), islands.data.data(), n * sizeof(float)) != 0);   // the shrink acts
+
+        ModelMaskGPU mask(stream, n);
+        mask.SetGrid(MaskGridDescription(ref), ops);
+        CudaDevicePtr<float> mask_d(n);
+        std::vector<float> out(n), first;
+        REQUIRE(cudaMemcpyAsync(mask_d, islands.data.data(), n * sizeof(float), cudaMemcpyHostToDevice, stream) == cudaSuccess);
+        mask.Shrink(mask_d);
+        REQUIRE(cudaMemcpyAsync(out.data(), mask_d, n * sizeof(float), cudaMemcpyDeviceToHost, stream) == cudaSuccess);
+        REQUIRE(cudaStreamSynchronize(stream) == cudaSuccess);
+        CHECK(std::memcmp(out.data(), ref.data.data(), n * sizeof(float)) == 0);
+
+        for (int repeat = 0; repeat < 2; repeat++) {
+            mask.Compute(atoms_d, static_cast<int>(atoms.size()), mask_d);
+            REQUIRE(cudaMemcpyAsync(out.data(), mask_d, n * sizeof(float), cudaMemcpyDeviceToHost, stream) == cudaSuccess);
+            REQUIRE(cudaStreamSynchronize(stream) == cudaSuccess);
+            if (repeat == 0) {
+                first = out;
+                size_t differ = 0;
+                for (size_t i = 0; i < n; i++)
+                    differ += out[i] != ref.data[i];
+                INFO(sg->hm << " grid " << ref.nu << "x" << ref.nv << "x" << ref.nw << ": " << differ << " points differ");
+                CHECK(differ <= 27 * (15 * n / 10000000 + 1));
+            } else
+                CHECK(std::memcmp(out.data(), first.data(), n * sizeof(float)) == 0);
+        }
+    }
 }
 
 #endif