Files
Jungfraujoch/tests/RugnuxLiveTriageTest.cpp
leonarski_f aecd48b9d3
Build Packages / Build documentation (push) Successful in 48s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 15m47s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 23m19s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 18m47s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m54s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 11m27s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 26m59s
Build Packages / Unit tests (push) Successful in 1h50m25s
Build Packages / Validation Battery (push) Successful in 46m38s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 32m52s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 32m32s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 28m34s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 29m19s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 30m11s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 33m58s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 31m26s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 34m4s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 32m17s
Build Packages / Generate python client (push) Successful in 32s
Build Packages / Create release (push) Successful in 9s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 31m50s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 19m38s
v1.0.0-rc.186 (#97)
* jfjoch_broker (experimental): Live Rugnux covers serial collections from DECTRIS detectors; a rotation collection too large for the image buffer falls back to standard per-image processing with a warning, and a stream that ends without its end message finishes the collection instead of hanging it.
* jfjoch_broker (experimental): Live Rugnux settings to build the solvent during refinement and to process a rotation collection as stills, a Clear action for the last result, a `_report.txt` for live rotation runs, R-free reported for the refined model, and ice-ring detection on by default for rotation as in the command line.
* jfjoch_broker (experimental): More complete and reproducible live serial merges, analysed for twinning, pseudo-translation and anisotropy, with merging statistics by resolution shell in the live status.
* jfjoch_writer: Live results with large models are written reliably; update jfjoch_writer together with jfjoch_broker.
* Rugnux: Several masters can be processed together with `--force-still`, including grid scans; more robust and faster merging of several sweeps; improved twinning, pseudo-translation, anisotropy, anomalous-signal and space-group decisions.
* Rugnux: Molecular replacement places the copies of a multi-copy solution together; the pseudo-translation warning quotes the Patterson peak and is not raised once a model is placed; the report advises on extending the rotation or reorienting the crystal where the coverage would gain from it.
* Web frontend: Light and dark modes, a phone layout, a hideable page list, a redesigned Rugnux live page with step-through of the Fo-Fc peaks in the structure view.
* jfjoch_viewer: A broker URL given on the command line follows the running collection.

Reviewed-on: #97
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-11-13 20:13:43 +01:00

238 lines
11 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../broker/OpenAPIConvert.h"
#include "../receiver/RugnuxLiveStatus.h"
#include "../rugnux/ResultReport.h"
#include "../rugnux/StillsLiveMerge.h"
namespace {
const RugnuxLiveTriage *Find(const std::vector<RugnuxLiveTriage> &triage, const std::string &label) {
for (const auto &t: triage)
if (t.label == label)
return &t;
return nullptr;
}
// A merge to 2 A whose shells inside the fitted resolution are far from complete
ProcessResult ThinMerge() {
ProcessResult result;
result.images_processed = 100;
result.indexing_rate = 0.9f;
result.consensus_cell = UnitCell{.a = 50.0f, .b = 60.0f, .c = 70.0f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
result.has_merge_statistics = true;
auto &o = result.merge_statistics.overall;
o.d_max = 50.0f;
o.d_min = 2.00f;
o.unique_reflections = 1000;
o.possible_unique_reflections = 1100;
o.total_observations = 4000;
o.mean_i_over_sigma = 12.0;
o.cc_half = 0.98;
result.resolution_fit_A = 2.0;
MergeStatisticsShell shell;
shell.d_max = 50.0f;
shell.d_min = 2.0f;
shell.cc_half = shell.cc_half_internal = 0.98;
shell.unique_reflections = 300;
shell.possible_unique_reflections = 2000;
result.merge_statistics.shells.push_back(shell);
return result;
}
}
// The live list is the report's checklist line for line: same labels, codes and words, and its colour read
// off the same flags - so the live view and _report.txt cannot disagree.
TEST_CASE("RugnuxLiveTriage_SameAsReport", "[RugnuxLive]") {
DiffractionExperiment x(DetJF(1));
const ProcessResult result = ThinMerge();
const ReportDocument doc = BuildReportDocument("p", "in.h5", x, result);
const auto triage = LiveTriage(doc);
REQUIRE(triage.size() == doc.checks.size());
REQUIRE(!triage.empty());
for (size_t i = 0; i < triage.size(); i++) {
CHECK(triage[i].label == doc.checks[i].label);
CHECK(triage[i].codes == doc.checks[i].codes);
CHECK(triage[i].text == doc.checks[i].text);
CHECK(triage[i].detail == doc.checks[i].detail);
switch (doc.checks[i].status) {
case ReportCheck::Status::Absent:
CHECK(triage[i].severity == RugnuxLiveTriageSeverity::Ok);
break;
case ReportCheck::Status::Present:
CHECK((triage[i].severity == RugnuxLiveTriageSeverity::Attention
|| triage[i].severity == RugnuxLiveTriageSeverity::Serious));
break;
case ReportCheck::Status::Undetermined:
CHECK(triage[i].severity == RugnuxLiveTriageSeverity::NotMeasured);
break;
}
}
// Every flag of the verdict line is a coloured entry
REQUIRE(!doc.pathology_flags.empty());
for (const auto &code: doc.pathology_flags) {
bool found = false;
for (const auto &t: triage)
if (std::ranges::find(t.codes, code) != t.codes.end())
found = found || t.severity == RugnuxLiveTriageSeverity::Attention
|| t.severity == RugnuxLiveTriageSeverity::Serious;
CHECK(found);
}
const auto *completeness = Find(triage, "Completeness");
REQUIRE(completeness);
CHECK(completeness->severity == RugnuxLiveTriageSeverity::Attention);
CHECK(!completeness->detail.empty());
const auto *twinning = Find(triage, "Twinning");
REQUIRE(twinning);
CHECK(twinning->severity == RugnuxLiveTriageSeverity::NotMeasured);
}
// A merge without signal leaves no result to use: the one red entry
TEST_CASE("RugnuxLiveTriage_UnusableMergeIsSerious", "[RugnuxLive]") {
DiffractionExperiment x(DetJF(1));
ProcessResult result = ThinMerge();
result.merge_statistics.overall.cc_half = -0.005;
const ReportDocument doc = BuildReportDocument("p", "in.h5", x, result);
REQUIRE(doc.verdict == "UNUSABLE");
const auto triage = LiveTriage(doc);
const auto *unusable = Find(triage, "Unusable merge");
REQUIRE(unusable);
CHECK(unusable->severity == RugnuxLiveTriageSeverity::Serious);
CHECK(Find(triage, "Completeness")->severity == RugnuxLiveTriageSeverity::Attention);
}
// The live serial merge has no pre-scan: powder, ice and further lattices are not measured, and the report says
// so rather than "no rings detected". Its own analyses - twinning, pseudo-translation, anisotropy - are not
// measured either until the merge was analysed (AnalyseStillsMerge).
TEST_CASE("RugnuxLiveTriage_StillsMergeClaimsOnlyWhatItMeasured", "[RugnuxLive]") {
DiffractionExperiment x(DetJF(1));
x.SetUnitCell(UnitCell{.a = 50.0f, .b = 60.0f, .c = 70.0f, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f});
StillsMergeOutput out;
out.merge.statistics = ThinMerge().merge_statistics;
out.merge.statistics.stills = true;
out.merge.resolution_fit_A = 2.0;
out.images = 90;
const ProcessResult result = StillsLiveResult(x, out, 100, 90);
const auto triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
for (const char *label: {"Powder", "Ice", "Twinning", "Pseudo-symmetry", "Multiple lattices", "Anisotropy"}) {
const auto *t = Find(triage, label);
REQUIRE(t);
CHECK(t->severity == RugnuxLiveTriageSeverity::NotMeasured);
}
CHECK(!Find(triage, "Indexing ambiguity"));
CHECK(Find(triage, "Completeness")->severity == RugnuxLiveTriageSeverity::Attention);
}
// An analysed stills merge carries its twinning and pseudo-translation verdicts and the indexing ambiguity's
// resolution into the checklist, as the report of a `rugnux` run would; the twin row of a merge that reads
// twinned points at the ambiguity, since crystals left mis-indexed read the same.
TEST_CASE("RugnuxLiveTriage_StillsMergeCarriesItsAnalyses", "[RugnuxLive]") {
DiffractionExperiment x(DetJF(1));
x.SetUnitCell(UnitCell{.a = 50.0f, .b = 50.0f, .c = 70.0f, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f});
x.SetSpaceGroup(*gemmi::find_spacegroup_by_name("P 4"));
StillsMergeOutput out;
out.merge.statistics = ThinMerge().merge_statistics;
out.merge.statistics.stills = true;
out.merge.resolution_fit_A = 2.0;
out.images = 90;
out.ambiguity.crystals = 80;
out.ambiguity.reindexed = 35;
out.ambiguity.median_sigma = 4.2;
out.ambiguity.decided_fraction = 0.7;
out.tncs.measurable = true;
out.tncs.n_reflections = 2000;
out.twinning.l_test_pairs = 5000;
out.twinning.mean_abs_l = 0.47;
ProcessResult result = StillsLiveResult(x, out, 100, 90);
CHECK(result.ambiguity_reference_free);
CHECK(result.ambiguity_crystals == 80);
auto triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
CHECK(Find(triage, "Twinning")->severity == RugnuxLiveTriageSeverity::Ok);
CHECK(Find(triage, "Pseudo-symmetry")->severity == RugnuxLiveTriageSeverity::Ok);
const auto *ambiguity = Find(triage, "Indexing ambiguity");
REQUIRE(ambiguity);
CHECK(ambiguity->severity == RugnuxLiveTriageSeverity::Ok);
CHECK(ambiguity->text == "resolved from the data: 35 of 80 crystals reindexed; the chosen indexing ahead of the "
"best other by 4.2 sigma (median), 70% of the crystals decided at over 3 sigma");
CHECK(Find(triage, "Anisotropy")->severity == RugnuxLiveTriageSeverity::NotMeasured);
out.twinning.mean_abs_l = 0.42;
out.twinning.merohedral_twinning_possible = true;
out.twinning.twinning_suspected = true;
result = StillsLiveResult(x, out, 100, 90);
result.warnings.push_back({PathologyCode::TWINNING, "twinned"});
triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
const auto *twin = Find(triage, "Twinning");
CHECK(twin->severity == RugnuxLiveTriageSeverity::Attention);
CHECK(twin->text.find("see Indexing ambiguity") != std::string::npos);
}
TEST_CASE("RugnuxLiveTriage_OpenAPI", "[RugnuxLive]") {
RugnuxLiveStatus status;
status.Start(10, std::nullopt, std::chrono::steady_clock::now());
CHECK(!Convert(status.Get()).triageIsSet());
status.Triage({
{.label = "Twinning", .codes = {"TWINNING"}, .severity = RugnuxLiveTriageSeverity::Ok, .text = "no indication"},
{.label = "Completeness", .codes = {"LOW_COMPLETENESS"}, .severity = RugnuxLiveTriageSeverity::Attention,
.text = "40.0 %", .detail = "only 40 % of reciprocal space"},
{.label = "Unusable merge", .codes = {"UNUSABLE_MERGE"}, .severity = RugnuxLiveTriageSeverity::Serious},
{.label = "Ice", .codes = {"ICE_RINGS"}, .severity = RugnuxLiveTriageSeverity::NotMeasured,
.text = "not measured"}
});
const auto api = Convert(status.Get());
REQUIRE(api.triageIsSet());
const auto items = api.getTriage();
REQUIRE(items.size() == 4);
CHECK(items[0].getLabel() == "Twinning");
CHECK(items[0].getCodes() == std::vector<std::string>{"TWINNING"});
CHECK(items[0].getSeverity() == "ok");
CHECK(!items[0].detailIsSet());
CHECK(items[1].getSeverity() == "attention");
CHECK(items[1].getDetail() == "only 40 % of reciprocal space");
CHECK(items[2].getSeverity() == "serious");
CHECK(items[3].getSeverity() == "not_measured");
}
TEST_CASE("RugnuxLiveFrames_OpenAPI", "[RugnuxLive]") {
using D = FrameDisposition;
const auto d = [](D x) { return static_cast<uint8_t>(x); };
SweepQuality sweep;
sweep.frame_disposition = {d(D::Merged), d(D::Merged), d(D::Downgraded), d(D::Downgraded), d(D::Rejected),
d(D::Rejected), d(D::Rejected), d(D::Merged), d(D::Merged), d(D::Rejected)};
sweep.ranges = {{.first_image = 2, .last_image = 6, .reason = SweepQualityReason::CrystalOutOfBeam},
{.first_image = 9, .last_image = 9, .reason = SweepQualityReason::RadiationDamage}};
const auto ranges = LiveFrameRanges(sweep);
REQUIRE(ranges.size() == 3);
CHECK(ranges[0].first_image == 2);
CHECK(ranges[0].last_image == 3);
CHECK(ranges[0].disposition == "downgraded");
CHECK(ranges[0].reason == SweepQualityReasonText(SweepQualityReason::CrystalOutOfBeam));
CHECK(ranges[1].first_image == 4);
CHECK(ranges[1].last_image == 6);
CHECK(ranges[1].disposition == "rejected");
CHECK(ranges[2].first_image == 9);
CHECK(ranges[2].reason == SweepQualityReasonText(SweepQualityReason::RadiationDamage));
CHECK(LiveFrameRanges(SweepQuality{}).empty());
RugnuxLiveStatus status;
status.Start(10, std::nullopt, std::chrono::steady_clock::now());
CHECK(!Convert(status.Get()).frameScalesIsSet());
status.Frames({1.0f, NAN, 0.5f}, ranges);
const auto api = Convert(status.Get());
REQUIRE(api.frameScalesIsSet());
CHECK(api.getFrameScales() == std::vector<float>{1.0f, 0.0f, 0.5f});
REQUIRE(api.getFrameRanges().size() == 3);
CHECK(api.getFrameRanges()[1].getFirstImage() == 4);
CHECK(api.getFrameRanges()[1].getDisposition() == "rejected");
CHECK(api.getFrameRanges()[0].getReason() == ranges[0].reason);
}