rugnux live: merging statistics by resolution shell in the live status

The live status carries the shell table of the latest merge (RugnuxLiveShell:
d range, N_obs, N_unique, completeness, multiplicity, <I/sigma>, R_meas on
rotation, CC1/2, CCref, R_split on stills, R-free of the placed model), filled
by LiveShells from the same MergeStatistics the report prints, with the same
rows. Rotation sets it when the run is over; stills at each provisional merge
and at the end, with the R-free added once the model step on that same merge
returns. A quantity the run did not measure is left unset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK

receiver/JFJochReceiverRugnuxRotation.cpp

@@ -300,6 +300,9 @@ void JFJochReceiverRugnuxRotation::RugnuxThread() {
         if (!result.merge_statistics.stills && result.error_model_isa_resolved && result.error_model_isa > 0.0)
             isa = result.error_model_isa;
         live_status.Headline(result.space_group ? result.space_group->xhm() : "", d_min, isa);
+        if (result.has_merge_statistics)
+            live_status.Shells("the collection", LiveShells(result.merge_statistics,
+                                                            result.model_validation ? &*result.model_validation : nullptr));
         // For the broker's viewer routes; a copy of the map coefficients, as the end message takes the originals
         if (result.outputs && result.outputs->placed_model) {
             const RugnuxOutputs &outputs = *result.outputs;

receiver/RugnuxLiveStatus.cpp

@@ -10,6 +10,22 @@
 #include "../rugnux/Rugnux.h"
 #include "../rugnux/ReportDocument.h"
 
+namespace {
+    // A score the status carries only where it is a number: JSON has no NaN or infinity (nlohmann writes null)
+    void SetIfFinite(std::optional<double> &out, double v) {
+        if (std::isfinite(v))
+            out = v;
+    }
+
+    // The validation's shells are the merge's, labelled by the same d_min (report_shell_d_min)
+    void AddShellRFree(std::vector<RugnuxLiveShell> &shells, const ModelValidationResult &validation) {
+        for (const auto &m : validation.cc_model_shells)
+            for (auto &sh : shells)
+                if (static_cast<float>(sh.d_min) == m.d_min && m.n_free > 0)
+                    SetIfFinite(sh.r_free, m.r_free);
+    }
+}
+
 double RugnuxLiveStatus::Since(std::chrono::steady_clock::time_point t) const {
     return std::chrono::duration<double>(t - start).count();
 }
@@ -143,6 +159,18 @@ void RugnuxLiveStatus::Stills(const RugnuxLiveStills &stills) {
     snapshot.stills = stills;
 }
 
+void RugnuxLiveStatus::Shells(const std::string &merge, const std::vector<RugnuxLiveShell> &shells) {
+    std::unique_lock ul(status_mutex);
+    snapshot.shells_merge = merge;
+    snapshot.shells = shells;
+}
+
+void RugnuxLiveStatus::ShellRFree(const std::string &merge, const ModelValidationResult &validation) {
+    std::unique_lock ul(status_mutex);
+    if (snapshot.shells_merge == merge)
+        AddShellRFree(snapshot.shells, validation);
+}
+
 void RugnuxLiveStatus::Latency(double last_image_to_written_s) {
     std::unique_lock ul(status_mutex);
     snapshot.last_image_to_written_s = last_image_to_written_s;
@@ -207,12 +235,32 @@ RugnuxLiveSnapshot RugnuxLiveStatus::Get() const {
     return ret;
 }
 
-namespace {
-    // A score the status carries only where it is a number: JSON has no NaN or infinity (nlohmann writes null)
-    void SetIfFinite(std::optional<double> &out, double v) {
-        if (std::isfinite(v))
-            out = v;
+std::vector<RugnuxLiveShell> LiveShells(const MergeStatistics &statistics, const ModelValidationResult *validation) {
+    std::vector<RugnuxLiveShell> ret;
+    // The rows of the report's table: a shell with no reflection is not printed
+    for (const auto &in : statistics.shells) {
+        if (in.unique_reflections == 0)
+            continue;
+        RugnuxLiveShell sh;
+        sh.d_min = in.d_min;
+        sh.d_max = in.d_max;
+        sh.n_obs = in.total_observations;
+        sh.n_unique = in.unique_reflections;
+        if (in.possible_unique_reflections > 0)
+            sh.completeness = static_cast<double>(in.unique_reflections) / in.possible_unique_reflections;
+        sh.multiplicity = static_cast<double>(in.total_observations) / in.unique_reflections;
+        SetIfFinite(sh.i_over_sigma, in.mean_i_over_sigma);
+        if (!statistics.stills)
+            SetIfFinite(sh.r_meas, in.r_meas);
+        SetIfFinite(sh.cc_half, in.cc_half);
+        SetIfFinite(sh.cc_ref, in.cc_ref);
+        if (in.n_split_pairs > 0)
+            SetIfFinite(sh.r_split, in.r_split);
+        ret.push_back(sh);
     }
+    if (validation)
+        AddShellRFree(ret, *validation);
+    return ret;
 }
 
 std::optional<RugnuxLiveModel> LiveModel(const ProcessResult &result) {

receiver/RugnuxLiveStatus.h

@@ -14,6 +14,8 @@
 class RugnuxStructure;   // RugnuxStructure.h
 struct ProcessResult;     // rugnux/Rugnux.h
 struct ReportDocument;    // rugnux/ReportDocument.h
+struct MergeStatistics;   // image_analysis/scale_merge/Merge.h
+struct ModelValidationResult;   // image_analysis/structure_refinement/ModelValidation.h
 
 // Where the live rugnux run of the last collection is.
 // Times come from the image puller (when a message came off the socket) and are in seconds since the first image arrived.
@@ -90,6 +92,20 @@ struct RugnuxLiveTriage {
     std::string detail;                   // the warnings that fired, one per line
 };
 
+// A resolution shell of a merge: a row of the shell table of the report (operator<< of MergeStatistics), with
+// R_split from its serial table and R-free from MODEL VALIDATION's table on the same shells.
+// Fractions, not percent; unset: not measured for this shell or this run.
+struct RugnuxLiveShell {
+    double d_min = 0, d_max = 0;
+    int64_t n_obs = 0, n_unique = 0;
+    std::optional<double> completeness, multiplicity, i_over_sigma;
+    std::optional<double> r_meas;          // rotation only, as the report
+    std::optional<double> cc_half;
+    std::optional<double> cc_ref;          // against a reference
+    std::optional<double> r_split;         // stills only
+    std::optional<double> r_free;          // of the placed model, where a model was given
+};
+
 // Serial frames analysed on the GPU after a pre-pass (StillsGPUAnalysis)
 enum class RugnuxLivePrePass { Holding, Running, Done, Failed };
 
@@ -141,6 +157,9 @@ struct RugnuxLiveSnapshot {
     std::shared_ptr<const RugnuxStructure> structure;
     std::shared_ptr<const RugnuxStructure> refined_structure;
     std::optional<RugnuxLiveStills> stills;
+    // The shell table of the latest merge: the merge of the collection, or on stills a provisional one
+    std::string shells_merge;             // which merge, e.g. "the collection" or "50% of the images"
+    std::vector<RugnuxLiveShell> shells;
 };
 
 class RugnuxLiveStatus {
@@ -177,6 +196,9 @@ public:
     void Structure(std::shared_ptr<const RugnuxStructure> structure,
                    std::shared_ptr<const RugnuxStructure> refined_structure = nullptr);
     void Stills(const RugnuxLiveStills &stills);
+    void Shells(const std::string &merge, const std::vector<RugnuxLiveShell> &shells);
+    // The R-free by shell of a model placed on a merge, if that merge's table is the one shown
+    void ShellRFree(const std::string &merge, const ModelValidationResult &validation);
     // Back to Idle with nothing of the last run, as before any collection.
     void Clear();
     RugnuxLiveSnapshot Get() const;
@@ -187,3 +209,7 @@ std::vector<RugnuxLiveTriage> LiveTriage(const ReportDocument &doc);
 
 // What the run made of the model it was given, for the live status; none where no model was given.
 std::optional<RugnuxLiveModel> LiveModel(const ProcessResult &result);
+
+// The shell table of a merge as the report prints it, with the R-free by shell of the model validated on it
+std::vector<RugnuxLiveShell> LiveShells(const MergeStatistics &statistics,
+                                        const ModelValidationResult *validation = nullptr);

receiver/StillsGPUAnalysis.cpp

@@ -447,6 +447,7 @@ void StillsGPUAnalysis::PublishMerge(const StillsMergeOutput &out, const std::st
                                          overall.mean_i_over_sigma, out.seconds);
     logger.Info("{}", text);
     status.Note(text);
+    status.Shells(title, LiveShells(out.merge.statistics));
 }
 
 void StillsGPUAnalysis::SubmitModel(StillsMergeOutput &out, const std::string &title, bool final) {
@@ -560,6 +561,8 @@ void StillsGPUAnalysis::PublishModel(const ModelStepResult &result, const std::s
     status.Note(text);
     if (const auto live_model = LiveModel(ModelStepReport(result)))
         status.Model(*live_model);
+    if (result.validation)
+        status.ShellRFree(title, *result.validation);
     if (!result.placed_model)
         return;
     std::optional<PlacedModelOutput> refined;

tests/JFJochReceiverRugnuxTest.cpp

@@ -201,6 +201,14 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
     CHECK(triage_line("Completeness")->severity != RugnuxLiveTriageSeverity::NotMeasured);
     REQUIRE(triage_line("Powder") != live.triage.end());
     CHECK(triage_line("Powder")->severity == RugnuxLiveTriageSeverity::NotMeasured);
+
+    // The shell table of the merge of the collection, without the rotation-only R_meas
+    CHECK(live.shells_merge == "the collection");
+    REQUIRE(!live.shells.empty());
+    for (const auto &sh : live.shells) {
+        CHECK(sh.n_unique > 0);
+        CHECK(!sh.r_meas);
+    }
     REQUIRE(live.stills);
     CHECK(live.stills->prepass == RugnuxLivePrePass::Done);
     CHECK(live.stills->images_analysed == images);

tests/RugnuxLiveStatusTest.cpp

@@ -3,7 +3,10 @@
 
 #include <catch2/catch_all.hpp>
 
+#include <cctype>
 #include <cmath>
+#include <map>
+#include <sstream>
 
 #include "../receiver/RugnuxLiveStatus.h"
 #include "../receiver/RugnuxLiveStatus.h"
@@ -351,3 +354,149 @@ TEST_CASE("RugnuxLiveStatus_StepsAfterDeadline", "[RugnuxLive]") {
     status.Cut("deadline");
     CHECK(status.Get().steps[0].title == "Integration, 1000 images");
 }
+
+namespace {
+    MergeStatisticsShell TestShell(float d_max, float d_min, int n_obs, int n_unique, int n_possible) {
+        MergeStatisticsShell sh;
+        sh.d_max = d_max;
+        sh.d_min = d_min;
+        sh.total_observations = n_obs;
+        sh.unique_reflections = n_unique;
+        sh.possible_unique_reflections = n_possible;
+        sh.mean_i_over_sigma = 2000.0 * d_min / n_obs;
+        sh.cc_half = 0.5 + 0.1 * d_min;
+        sh.r_merge = 0.02 / d_min;
+        sh.r_meas = 0.03 / d_min;
+        sh.r_pim = 0.01 / d_min;
+        return sh;
+    }
+
+    // The rows of the report's shell table (operator<< of MergeStatistics, which the report prints) by their
+    // d_min label, and the rows of its serial table
+    std::map<std::string, std::vector<std::string>> ReportRows(const MergeStatistics &ms, bool serial) {
+        std::ostringstream text;
+        text << ms;
+        std::istringstream lines(text.str());
+        std::map<std::string, std::vector<std::string>> rows;
+        bool in_serial = false;
+        for (std::string line; std::getline(lines, line);) {
+            if (line.find("Serial statistics") != std::string::npos)
+                in_serial = true;
+            std::istringstream words(line);
+            std::vector<std::string> row;
+            for (std::string w; words >> w;)
+                row.push_back(w);
+            if (in_serial == serial && !row.empty() && std::isdigit(row[0][0]))
+                rows[row[0]] = row;
+        }
+        return rows;
+    }
+
+    std::string Percent(const std::optional<double> &v) {
+        return v ? fmt::format("{:.1f}%", 100.0 * v.value()) : "-";
+    }
+
+    std::string Plain(const std::optional<double> &v) {
+        return v ? fmt::format("{:.1f}", v.value()) : "-";
+    }
+}
+
+TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") {
+    MergeStatistics ms;
+    ms.shells = {TestShell(50.0, 4.0, 4000, 1000, 1010), TestShell(4.0, 3.0, 3000, 1000, 1050),
+                 TestShell(3.0, 2.5, 0, 0, 900),   // empty: no row in the report, none here
+                 TestShell(2.5, 2.2, 900, 600, 1000)};
+    ms.shells[1].cc_ref = 0.93;
+    ms.shells[3].mean_i_over_sigma = NAN;
+
+    SECTION("rotation") {
+        const auto shells = LiveShells(ms);
+        const auto rows = ReportRows(ms, false);
+        REQUIRE(shells.size() == 3);
+        REQUIRE(rows.size() == 3);
+        for (const auto &sh : shells) {
+            const auto &row = rows.at(fmt::format("{:.2f}", sh.d_min));
+            // d_min N_obs N_uniq N_possib Compl Mult <I/sig> R_merge R_meas R_pim CC1/2 CC* CCref CCanom SigAno
+            REQUIRE(row.size() == 15);
+            CHECK(row[1] == std::to_string(sh.n_obs));
+            CHECK(row[2] == std::to_string(sh.n_unique));
+            CHECK(row[4] == Percent(sh.completeness));
+            CHECK(row[5] == Plain(sh.multiplicity));
+            CHECK(row[6] == Plain(sh.i_over_sigma));
+            CHECK(row[8] == Percent(sh.r_meas));
+            CHECK(row[10] == Percent(sh.cc_half));
+            CHECK(row[12] == Percent(sh.cc_ref));
+            CHECK(!sh.r_split);
+            CHECK(!sh.r_free);
+        }
+        CHECK(shells[0].d_max == Catch::Approx(50.0));
+        CHECK(shells[1].cc_ref);
+        CHECK(!shells[2].i_over_sigma);
+    }
+
+    SECTION("stills") {
+        ms.stills = true;
+        ms.shells[0].n_split_pairs = 800;
+        ms.shells[0].r_split = 0.12;
+        ms.shells[0].cc_half_split = 0.9;
+        ms.overall.n_split_pairs = 800;
+        const auto shells = LiveShells(ms);
+        const auto rows = ReportRows(ms, false);
+        const auto serial = ReportRows(ms, true);
+        REQUIRE(shells.size() == 3);
+        REQUIRE(rows.size() == 3);
+        for (const auto &sh : shells) {
+            const auto &row = rows.at(fmt::format("{:.2f}", sh.d_min));
+            // No R_merge / R_meas / R_pim on stills
+            REQUIRE(row.size() == 12);
+            CHECK(row[1] == std::to_string(sh.n_obs));
+            CHECK(row[6] == Plain(sh.i_over_sigma));
+            CHECK(row[7] == Percent(sh.cc_half));
+            CHECK(row[9] == Percent(sh.cc_ref));
+            CHECK(!sh.r_meas);
+            // d_min N_pairs CC1/2 CC* R_split, for the shells both halves measured
+            const auto it = serial.find(fmt::format("{:.2f}", sh.d_min));
+            CHECK((it != serial.end()) == sh.r_split.has_value());
+            if (it != serial.end())
+                CHECK(it->second[4] == Percent(sh.r_split));
+        }
+    }
+
+    SECTION("R-free of the model validated on the merge") {
+        ModelValidationResult v;
+        v.cc_model_shells = {{4.0f, 0.9, 900, NAN, NAN, NAN, 0.21, 50},
+                             {3.0f, 0.8, 900, NAN, NAN, NAN, 0.27, 45},
+                             {2.5f, NAN, 0, NAN, NAN, NAN, NAN, 0},
+                             {2.2f, 0.4, 500, NAN, NAN, NAN, NAN, 0}};   // no free reflection
+        const auto shells = LiveShells(ms, &v);
+        REQUIRE(shells.size() == 3);
+        CHECK(shells[0].r_free.value() == Catch::Approx(0.21));
+        CHECK(shells[1].r_free.value() == Catch::Approx(0.27));
+        CHECK(!shells[2].r_free);
+
+        RugnuxLiveStatus status;
+        status.Start(10, 1.0, std::chrono::steady_clock::now());
+        status.Shells("50% of the images (provisional)", LiveShells(ms));
+        // A model of another merge does not label this one
+        status.ShellRFree("the collection", v);
+        CHECK(!status.Get().shells[0].r_free);
+        status.ShellRFree("50% of the images (provisional)", v);
+        CHECK(status.Get().shells[0].r_free.value() == Catch::Approx(0.21));
+
+        nlohmann::json j = Convert(status.Get());
+        CHECK(j["shells_merge"] == "50% of the images (provisional)");
+        REQUIRE(j["shells"].size() == 3);
+        CHECK(j["shells"][0]["d_min"].get<double>() == Catch::Approx(4.0));
+        CHECK(j["shells"][0]["d_max"].get<double>() == Catch::Approx(50.0));
+        CHECK(j["shells"][0]["r_free"].get<double>() == Catch::Approx(0.21));
+        CHECK(j["shells"][0]["r_meas"].get<double>() == Catch::Approx(0.03 / 4.0));
+        CHECK(!j["shells"][0].contains("cc_ref"));
+        CHECK(!j["shells"][0].contains("r_split"));
+        CHECK(!j["shells"][2].contains("i_over_sigma"));
+
+        status.Clear();
+        CHECK(status.Get().shells.empty());
+        j = Convert(status.Get());
+        CHECK(!j.contains("shells"));
+    }
+}