diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index e5a86ff62..1b7e567c3 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -1355,27 +1355,31 @@ org::openapitools::server::model::Rugnux_live_status Convert(const RugnuxLiveSna stills.setAnalysisRateHz(st.analysis_rate_hz.value()); ret.setStills(stills); } + const auto convert_shell = [](const RugnuxLiveShell &in) { + org::openapitools::server::model::Rugnux_live_shell sh; + sh.setDMin(in.d_min); + sh.setDMax(in.d_max); + sh.setNObs(in.n_obs); + sh.setNUnique(in.n_unique); + if (in.completeness) sh.setCompleteness(in.completeness.value()); + if (in.multiplicity) sh.setMultiplicity(in.multiplicity.value()); + if (in.i_over_sigma) sh.setIOverSigma(in.i_over_sigma.value()); + if (in.r_meas) sh.setRMeas(in.r_meas.value()); + if (in.cc_half) sh.setCcHalf(in.cc_half.value()); + if (in.cc_ref) sh.setCcRef(in.cc_ref.value()); + if (in.r_split) sh.setRSplit(in.r_split.value()); + if (in.r_free) sh.setRFree(in.r_free.value()); + return sh; + }; if (!input.shells.empty()) { ret.setShellsMerge(input.shells_merge); std::vector shells; - for (const auto &in: input.shells) { - org::openapitools::server::model::Rugnux_live_shell sh; - sh.setDMin(in.d_min); - sh.setDMax(in.d_max); - sh.setNObs(in.n_obs); - sh.setNUnique(in.n_unique); - if (in.completeness) sh.setCompleteness(in.completeness.value()); - if (in.multiplicity) sh.setMultiplicity(in.multiplicity.value()); - if (in.i_over_sigma) sh.setIOverSigma(in.i_over_sigma.value()); - if (in.r_meas) sh.setRMeas(in.r_meas.value()); - if (in.cc_half) sh.setCcHalf(in.cc_half.value()); - if (in.cc_ref) sh.setCcRef(in.cc_ref.value()); - if (in.r_split) sh.setRSplit(in.r_split.value()); - if (in.r_free) sh.setRFree(in.r_free.value()); - shells.push_back(sh); - } + for (const auto &in: input.shells) + shells.push_back(convert_shell(in)); ret.setShells(shells); } + if (input.overall) + ret.setOverall(convert_shell(input.overall.value())); if (input.model) { const RugnuxLiveModel &m = input.model.value(); org::openapitools::server::model::Rugnux_live_model model; diff --git a/broker/gen/model/Rugnux_live_status.cpp b/broker/gen/model/Rugnux_live_status.cpp index 91b8bfabb..c5d1e8c7f 100644 --- a/broker/gen/model/Rugnux_live_status.cpp +++ b/broker/gen/model/Rugnux_live_status.cpp @@ -31,6 +31,7 @@ Rugnux_live_status::Rugnux_live_status() m_Shells_merge = ""; m_Shells_mergeIsSet = false; m_ShellsIsSet = false; + m_OverallIsSet = false; m_Space_group = ""; m_Space_groupIsSet = false; m_D_min = 0.0; @@ -104,7 +105,7 @@ bool Rugnux_live_status::validate(std::stringstream& msg, const std::string& pat } } - + if (stepProgressIsSet()) { const float& value = m_Step_progress; @@ -244,6 +245,9 @@ bool Rugnux_live_status::operator==(const Rugnux_live_status& rhs) const ((!shellsIsSet() && !rhs.shellsIsSet()) || (shellsIsSet() && rhs.shellsIsSet() && getShells() == rhs.getShells())) && + ((!overallIsSet() && !rhs.overallIsSet()) || (overallIsSet() && rhs.overallIsSet() && getOverall() == rhs.getOverall())) && + + ((!spaceGroupIsSet() && !rhs.spaceGroupIsSet()) || (spaceGroupIsSet() && rhs.spaceGroupIsSet() && getSpaceGroup() == rhs.getSpaceGroup())) && @@ -333,6 +337,8 @@ void to_json(nlohmann::json& j, const Rugnux_live_status& o) j["shells_merge"] = o.m_Shells_merge; if(o.shellsIsSet() || !o.m_Shells.empty()) j["shells"] = o.m_Shells; + if(o.overallIsSet()) + j["overall"] = o.m_Overall; if(o.spaceGroupIsSet()) j["space_group"] = o.m_Space_group; if(o.dMinIsSet()) @@ -397,6 +403,11 @@ void from_json(const nlohmann::json& j, Rugnux_live_status& o) j.at("shells").get_to(o.m_Shells); o.m_ShellsIsSet = true; } + if(j.find("overall") != j.end()) + { + j.at("overall").get_to(o.m_Overall); + o.m_OverallIsSet = true; + } if(j.find("space_group") != j.end()) { j.at("space_group").get_to(o.m_Space_group); @@ -578,6 +589,23 @@ void Rugnux_live_status::unsetShells() { m_ShellsIsSet = false; } +org::openapitools::server::model::Rugnux_live_shell Rugnux_live_status::getOverall() const +{ + return m_Overall; +} +void Rugnux_live_status::setOverall(org::openapitools::server::model::Rugnux_live_shell const& value) +{ + m_Overall = value; + m_OverallIsSet = true; +} +bool Rugnux_live_status::overallIsSet() const +{ + return m_OverallIsSet; +} +void Rugnux_live_status::unsetOverall() +{ + m_OverallIsSet = false; +} std::string Rugnux_live_status::getSpaceGroup() const { return m_Space_group; diff --git a/broker/gen/model/Rugnux_live_status.h b/broker/gen/model/Rugnux_live_status.h index 004c01a1f..3cf7590d1 100644 --- a/broker/gen/model/Rugnux_live_status.h +++ b/broker/gen/model/Rugnux_live_status.h @@ -112,13 +112,20 @@ public: bool shellsMergeIsSet() const; void unsetShells_merge(); /// - /// Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. + /// Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. `overall` is the same table's \"Overall\" row (whole-merge completeness, multiplicity, I/sigma, CC1/2, R_meas or R_split, CCref, and the overall R-free of the model validated on it), present with `shells`. /// std::vector getShells() const; void setShells(std::vector const& value); bool shellsIsSet() const; void unsetShells(); /// + /// + /// + org::openapitools::server::model::Rugnux_live_shell getOverall() const; + void setOverall(org::openapitools::server::model::Rugnux_live_shell const& value); + bool overallIsSet() const; + void unsetOverall(); + /// /// Space group of the result (the summary's \"Space group\" row); absent until the run is over /// std::string getSpaceGroup() const; @@ -274,6 +281,8 @@ protected: bool m_Shells_mergeIsSet; std::vector m_Shells; bool m_ShellsIsSet; + org::openapitools::server::model::Rugnux_live_shell m_Overall; + bool m_OverallIsSet; std::string m_Space_group; bool m_Space_groupIsSet; double m_D_min; diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 6e6dabb05..5e3e0fa5e 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -3283,8 +3283,12 @@ components: description: | Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. + `overall` is the same table's "Overall" row (whole-merge completeness, multiplicity, I/sigma, CC1/2, + R_meas or R_split, CCref, and the overall R-free of the model validated on it), present with `shells`. items: $ref: '#/components/schemas/rugnux_live_shell' + overall: + $ref: '#/components/schemas/rugnux_live_shell' space_group: type: string description: Space group of the result (the summary's "Space group" row); absent until the run is over diff --git a/docs/python_client/docs/RugnuxLiveStatus.md b/docs/python_client/docs/RugnuxLiveStatus.md index 36d58bc86..ca117597c 100644 --- a/docs/python_client/docs/RugnuxLiveStatus.md +++ b/docs/python_client/docs/RugnuxLiveStatus.md @@ -14,7 +14,8 @@ Name | Type | Description | Notes **model** | [**RugnuxLiveModel**](RugnuxLiveModel.md) | | [optional] **stills** | [**RugnuxLiveStills**](RugnuxLiveStills.md) | | [optional] **shells_merge** | **str** | Which merge `shells` are of - \"the collection\", or on serial data a provisional merge such as \"50% of the images (provisional)\"; absent until there is a merge | [optional] -**shells** | [**List[RugnuxLiveShell]**](RugnuxLiveShell.md) | Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. | [optional] +**shells** | [**List[RugnuxLiveShell]**](RugnuxLiveShell.md) | Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. `overall` is the same table's \"Overall\" row (whole-merge completeness, multiplicity, I/sigma, CC1/2, R_meas or R_split, CCref, and the overall R-free of the model validated on it), present with `shells`. | [optional] +**overall** | [**RugnuxLiveShell**](RugnuxLiveShell.md) | | [optional] **space_group** | **str** | Space group of the result (the summary's \"Space group\" row); absent until the run is over | [optional] **d_min** | **float** | High-resolution limit of the merge, in Angstrom; absent until the run is over, or without a merge | [optional] **isa** | **float** | ISa of the error model; absent until the run is over, or where it was not determined | [optional] diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 1ccf50f25..43ab1f52c 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -2402,9 +2402,12 @@ export type rugnux_live_status = { /** * Merging statistics by resolution shell of the latest merge, coarse to fine, as the shell table of * `_report.txt`. Rotation: once the run is over; serial: at each provisional merge and at the end. + * `overall` is the same table's "Overall" row (whole-merge completeness, multiplicity, I/sigma, CC1/2, + * R_meas or R_split, CCref, and the overall R-free of the model validated on it), present with `shells`. * */ shells?: Array; + overall?: rugnux_live_shell; /** * Space group of the result (the summary's "Space group" row); absent until the run is over */ diff --git a/frontend/src/client/zod.gen.ts b/frontend/src/client/zod.gen.ts index 092a0d55e..e7645e880 100644 --- a/frontend/src/client/zod.gen.ts +++ b/frontend/src/client/zod.gen.ts @@ -1043,6 +1043,7 @@ export const zRugnuxLiveStatus = z.object({ stills: zRugnuxLiveStills.optional(), shells_merge: z.string().optional(), shells: z.array(zRugnuxLiveShell).optional(), + overall: zRugnuxLiveShell.optional(), space_group: z.string().optional(), d_min: z.number().optional(), isa: z.number().optional(), diff --git a/receiver/JFJochReceiverRugnuxRotation.cpp b/receiver/JFJochReceiverRugnuxRotation.cpp index 0497fc9f1..6028f3a55 100644 --- a/receiver/JFJochReceiverRugnuxRotation.cpp +++ b/receiver/JFJochReceiverRugnuxRotation.cpp @@ -300,9 +300,11 @@ void JFJochReceiverRugnuxRotation::RugnuxThread() { if (!result.merge_statistics.stills && result.error_model_isa_resolved && result.error_model_isa > 0.0) isa = result.error_model_isa; live_status.Headline(result.space_group ? result.space_group->xhm() : "", d_min, isa); - if (result.has_merge_statistics) - live_status.Shells("the collection", LiveShells(result.merge_statistics, - result.model_validation ? &*result.model_validation : nullptr)); + if (result.has_merge_statistics) { + const ModelValidationResult *validation = result.model_validation ? &*result.model_validation : nullptr; + live_status.Shells("the collection", LiveShells(result.merge_statistics, validation), + LiveOverall(result.merge_statistics, validation)); + } // For the broker's viewer routes; a copy of the map coefficients, as the end message takes the originals if (result.outputs && result.outputs->placed_model) { const RugnuxOutputs &outputs = *result.outputs; diff --git a/receiver/RugnuxLiveStatus.cpp b/receiver/RugnuxLiveStatus.cpp index b272a5df7..403834bfd 100644 --- a/receiver/RugnuxLiveStatus.cpp +++ b/receiver/RugnuxLiveStatus.cpp @@ -17,12 +17,37 @@ namespace { out = v; } - // The validation's shells are the merge's, labelled by the same d_min (report_shell_d_min) - void AddShellRFree(std::vector &shells, const ModelValidationResult &validation) { + // The validation's shells are the merge's, labelled by the same d_min (report_shell_d_min); its overall R-free + // is the overall row's + void AddShellRFree(std::vector &shells, std::optional &overall, + const ModelValidationResult &validation) { for (const auto &m : validation.cc_model_shells) for (auto &sh : shells) if (static_cast(sh.d_min) == m.d_min && m.n_free > 0) SetIfFinite(sh.r_free, m.r_free); + if (overall && validation.ok) + SetIfFinite(overall->r_free, validation.r_free); + } + + // A row of the report's shell table - a shell, or the overall row + RugnuxLiveShell LiveShell(const MergeStatisticsShell &in, bool stills) { + RugnuxLiveShell sh; + sh.d_min = in.d_min; + sh.d_max = in.d_max; + sh.n_obs = in.total_observations; + sh.n_unique = in.unique_reflections; + if (in.possible_unique_reflections > 0) + sh.completeness = static_cast(in.unique_reflections) / in.possible_unique_reflections; + if (in.unique_reflections > 0) + sh.multiplicity = static_cast(in.total_observations) / in.unique_reflections; + SetIfFinite(sh.i_over_sigma, in.mean_i_over_sigma); + if (!stills) + SetIfFinite(sh.r_meas, in.r_meas); + SetIfFinite(sh.cc_half, in.cc_half); + SetIfFinite(sh.cc_ref, in.cc_ref); + if (in.n_split_pairs > 0) + SetIfFinite(sh.r_split, in.r_split); + return sh; } } @@ -159,16 +184,18 @@ void RugnuxLiveStatus::Stills(const RugnuxLiveStills &stills) { snapshot.stills = stills; } -void RugnuxLiveStatus::Shells(const std::string &merge, const std::vector &shells) { +void RugnuxLiveStatus::Shells(const std::string &merge, const std::vector &shells, + const std::optional &overall) { std::unique_lock ul(status_mutex); snapshot.shells_merge = merge; snapshot.shells = shells; + snapshot.overall = overall; } void RugnuxLiveStatus::ShellRFree(const std::string &merge, const ModelValidationResult &validation) { std::unique_lock ul(status_mutex); if (snapshot.shells_merge == merge) - AddShellRFree(snapshot.shells, validation); + AddShellRFree(snapshot.shells, snapshot.overall, validation); } void RugnuxLiveStatus::Latency(double last_image_to_written_s) { @@ -238,31 +265,23 @@ RugnuxLiveSnapshot RugnuxLiveStatus::Get() const { std::vector LiveShells(const MergeStatistics &statistics, const ModelValidationResult *validation) { std::vector ret; // The rows of the report's table: a shell with no reflection is not printed - for (const auto &in : statistics.shells) { - if (in.unique_reflections == 0) - continue; - RugnuxLiveShell sh; - sh.d_min = in.d_min; - sh.d_max = in.d_max; - sh.n_obs = in.total_observations; - sh.n_unique = in.unique_reflections; - if (in.possible_unique_reflections > 0) - sh.completeness = static_cast(in.unique_reflections) / in.possible_unique_reflections; - sh.multiplicity = static_cast(in.total_observations) / in.unique_reflections; - SetIfFinite(sh.i_over_sigma, in.mean_i_over_sigma); - if (!statistics.stills) - SetIfFinite(sh.r_meas, in.r_meas); - SetIfFinite(sh.cc_half, in.cc_half); - SetIfFinite(sh.cc_ref, in.cc_ref); - if (in.n_split_pairs > 0) - SetIfFinite(sh.r_split, in.r_split); - ret.push_back(sh); - } + for (const auto &in : statistics.shells) + if (in.unique_reflections > 0) + ret.push_back(LiveShell(in, statistics.stills)); + std::optional no_overall; if (validation) - AddShellRFree(ret, *validation); + AddShellRFree(ret, no_overall, *validation); return ret; } +RugnuxLiveShell LiveOverall(const MergeStatistics &statistics, const ModelValidationResult *validation) { + std::vector no_shells; + std::optional ret = LiveShell(statistics.overall, statistics.stills); + if (validation) + AddShellRFree(no_shells, ret, *validation); + return *ret; +} + std::optional LiveModel(const ProcessResult &result) { if (!result.model_validation && !result.model_prior) return std::nullopt; diff --git a/receiver/RugnuxLiveStatus.h b/receiver/RugnuxLiveStatus.h index efbb217cb..2f0b9710f 100644 --- a/receiver/RugnuxLiveStatus.h +++ b/receiver/RugnuxLiveStatus.h @@ -160,6 +160,7 @@ struct RugnuxLiveSnapshot { // The shell table of the latest merge: the merge of the collection, or on stills a provisional one std::string shells_merge; // which merge, e.g. "the collection" or "50% of the images" std::vector shells; + std::optional overall; // the same merge's overall row }; class RugnuxLiveStatus { @@ -196,7 +197,8 @@ public: void Structure(std::shared_ptr structure, std::shared_ptr refined_structure = nullptr); void Stills(const RugnuxLiveStills &stills); - void Shells(const std::string &merge, const std::vector &shells); + void Shells(const std::string &merge, const std::vector &shells, + const std::optional &overall); // The R-free by shell of a model placed on a merge, if that merge's table is the one shown void ShellRFree(const std::string &merge, const ModelValidationResult &validation); // Back to Idle with nothing of the last run, as before any collection. @@ -213,3 +215,6 @@ std::optional LiveModel(const ProcessResult &result); // The shell table of a merge as the report prints it, with the R-free by shell of the model validated on it std::vector LiveShells(const MergeStatistics &statistics, const ModelValidationResult *validation = nullptr); + +// The overall row of the same table, with the overall R-free of the model validated on it +RugnuxLiveShell LiveOverall(const MergeStatistics &statistics, const ModelValidationResult *validation = nullptr); diff --git a/receiver/StillsGPUAnalysis.cpp b/receiver/StillsGPUAnalysis.cpp index 86a312411..65721f92c 100644 --- a/receiver/StillsGPUAnalysis.cpp +++ b/receiver/StillsGPUAnalysis.cpp @@ -447,7 +447,7 @@ void StillsGPUAnalysis::PublishMerge(const StillsMergeOutput &out, const std::st overall.mean_i_over_sigma, out.seconds); logger.Info("{}", text); status.Note(text); - status.Shells(title, LiveShells(out.merge.statistics)); + status.Shells(title, LiveShells(out.merge.statistics), LiveOverall(out.merge.statistics)); } void StillsGPUAnalysis::SubmitModel(StillsMergeOutput &out, const std::string &title, bool final) { diff --git a/tests/RugnuxLiveStatusTest.cpp b/tests/RugnuxLiveStatusTest.cpp index d015f41cc..bd3b33483 100644 --- a/tests/RugnuxLiveStatusTest.cpp +++ b/tests/RugnuxLiveStatusTest.cpp @@ -386,7 +386,7 @@ namespace { std::vector row; for (std::string w; words >> w;) row.push_back(w); - if (in_serial == serial && !row.empty() && std::isdigit(row[0][0])) + if (in_serial == serial && !row.empty() && (std::isdigit(row[0][0]) || row[0] == "Overall")) rows[row[0]] = row; } return rows; @@ -408,12 +408,13 @@ TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") { TestShell(2.5, 2.2, 900, 600, 1000)}; ms.shells[1].cc_ref = 0.93; ms.shells[3].mean_i_over_sigma = NAN; + ms.overall = TestShell(50.0, 2.2, 7900, 2600, 3960); SECTION("rotation") { const auto shells = LiveShells(ms); const auto rows = ReportRows(ms, false); REQUIRE(shells.size() == 3); - REQUIRE(rows.size() == 3); + REQUIRE(rows.size() == 4); // three shells and the Overall row for (const auto &sh : shells) { const auto &row = rows.at(fmt::format("{:.2f}", sh.d_min)); // d_min N_obs N_uniq N_possib Compl Mult R_merge R_meas R_pim CC1/2 CC* CCref CCanom SigAno @@ -432,6 +433,19 @@ TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") { CHECK(shells[0].d_max == Catch::Approx(50.0)); CHECK(shells[1].cc_ref); CHECK(!shells[2].i_over_sigma); + + // The overall row of the same table + const auto overall = LiveOverall(ms); + const auto &row = rows.at("Overall"); + REQUIRE(row.size() == 15); + CHECK(row[1] == std::to_string(overall.n_obs)); + CHECK(row[2] == std::to_string(overall.n_unique)); + CHECK(row[4] == Percent(overall.completeness)); + CHECK(row[5] == Plain(overall.multiplicity)); + CHECK(row[6] == Plain(overall.i_over_sigma)); + CHECK(row[8] == Percent(overall.r_meas)); + CHECK(row[10] == Percent(overall.cc_half)); + CHECK(overall.d_min == Catch::Approx(2.2)); } SECTION("stills") { @@ -444,7 +458,7 @@ TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") { const auto rows = ReportRows(ms, false); const auto serial = ReportRows(ms, true); REQUIRE(shells.size() == 3); - REQUIRE(rows.size() == 3); + REQUIRE(rows.size() == 4); // three shells and the Overall row for (const auto &sh : shells) { const auto &row = rows.at(fmt::format("{:.2f}", sh.d_min)); // No R_merge / R_meas / R_pim on stills @@ -474,14 +488,20 @@ TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") { CHECK(shells[1].r_free.value() == Catch::Approx(0.27)); CHECK(!shells[2].r_free); + v.ok = true; + v.r_free = 0.25; + CHECK(LiveOverall(ms, &v).r_free.value() == Catch::Approx(0.25)); + RugnuxLiveStatus status; status.Start(10, 1.0, std::chrono::steady_clock::now()); - status.Shells("50% of the images (provisional)", LiveShells(ms)); + status.Shells("50% of the images (provisional)", LiveShells(ms), LiveOverall(ms)); // A model of another merge does not label this one status.ShellRFree("the collection", v); CHECK(!status.Get().shells[0].r_free); + CHECK(!status.Get().overall->r_free); status.ShellRFree("50% of the images (provisional)", v); CHECK(status.Get().shells[0].r_free.value() == Catch::Approx(0.21)); + CHECK(status.Get().overall->r_free.value() == Catch::Approx(0.25)); nlohmann::json j = Convert(status.Get()); CHECK(j["shells_merge"] == "50% of the images (provisional)"); @@ -493,10 +513,16 @@ TEST_CASE("RugnuxLiveShells_AreTheReportTable", "[RugnuxLive]") { CHECK(!j["shells"][0].contains("cc_ref")); CHECK(!j["shells"][0].contains("r_split")); CHECK(!j["shells"][2].contains("i_over_sigma")); + REQUIRE(j.contains("overall")); + CHECK(j["overall"]["n_unique"].get() == 2600); + CHECK(j["overall"]["cc_half"].get() == Catch::Approx(0.5 + 0.1 * 2.2f)); + CHECK(j["overall"]["r_free"].get() == Catch::Approx(0.25)); status.Clear(); CHECK(status.Get().shells.empty()); + CHECK(!status.Get().overall); j = Convert(status.Get()); CHECK(!j.contains("shells")); + CHECK(!j.contains("overall")); } }