Build Packages / Build documentation (push) Successful in 48s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 15m47s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 23m19s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 18m47s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m54s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 11m27s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 26m59s
Build Packages / Unit tests (push) Successful in 1h50m25s
Build Packages / Validation Battery (push) Successful in 46m38s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 32m52s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 32m32s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 28m34s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 29m19s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 30m11s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 33m58s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 31m26s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 34m4s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 32m17s
Build Packages / Generate python client (push) Successful in 32s
Build Packages / Create release (push) Successful in 9s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 31m50s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 19m38s
* jfjoch_broker (experimental): Live Rugnux covers serial collections from DECTRIS detectors; a rotation collection too large for the image buffer falls back to standard per-image processing with a warning, and a stream that ends without its end message finishes the collection instead of hanging it. * jfjoch_broker (experimental): Live Rugnux settings to build the solvent during refinement and to process a rotation collection as stills, a Clear action for the last result, a `_report.txt` for live rotation runs, R-free reported for the refined model, and ice-ring detection on by default for rotation as in the command line. * jfjoch_broker (experimental): More complete and reproducible live serial merges, analysed for twinning, pseudo-translation and anisotropy, with merging statistics by resolution shell in the live status. * jfjoch_writer: Live results with large models are written reliably; update jfjoch_writer together with jfjoch_broker. * Rugnux: Several masters can be processed together with `--force-still`, including grid scans; more robust and faster merging of several sweeps; improved twinning, pseudo-translation, anisotropy, anomalous-signal and space-group decisions. * Rugnux: Molecular replacement places the copies of a multi-copy solution together; the pseudo-translation warning quotes the Patterson peak and is not raised once a model is placed; the report advises on extending the rotation or reorienting the crystal where the coverage would gain from it. * Web frontend: Light and dark modes, a phone layout, a hideable page list, a redesigned Rugnux live page with step-through of the Fo-Fc peaks in the structure view. * jfjoch_viewer: A broker URL given on the command line follows the running collection. Reviewed-on: #97 Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
238 lines
11 KiB
C++
238 lines
11 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
|
|
#include "../broker/OpenAPIConvert.h"
|
|
#include "../receiver/RugnuxLiveStatus.h"
|
|
#include "../rugnux/ResultReport.h"
|
|
#include "../rugnux/StillsLiveMerge.h"
|
|
|
|
namespace {
|
|
const RugnuxLiveTriage *Find(const std::vector<RugnuxLiveTriage> &triage, const std::string &label) {
|
|
for (const auto &t: triage)
|
|
if (t.label == label)
|
|
return &t;
|
|
return nullptr;
|
|
}
|
|
|
|
// A merge to 2 A whose shells inside the fitted resolution are far from complete
|
|
ProcessResult ThinMerge() {
|
|
ProcessResult result;
|
|
result.images_processed = 100;
|
|
result.indexing_rate = 0.9f;
|
|
result.consensus_cell = UnitCell{.a = 50.0f, .b = 60.0f, .c = 70.0f,
|
|
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
|
|
result.has_merge_statistics = true;
|
|
auto &o = result.merge_statistics.overall;
|
|
o.d_max = 50.0f;
|
|
o.d_min = 2.00f;
|
|
o.unique_reflections = 1000;
|
|
o.possible_unique_reflections = 1100;
|
|
o.total_observations = 4000;
|
|
o.mean_i_over_sigma = 12.0;
|
|
o.cc_half = 0.98;
|
|
result.resolution_fit_A = 2.0;
|
|
MergeStatisticsShell shell;
|
|
shell.d_max = 50.0f;
|
|
shell.d_min = 2.0f;
|
|
shell.cc_half = shell.cc_half_internal = 0.98;
|
|
shell.unique_reflections = 300;
|
|
shell.possible_unique_reflections = 2000;
|
|
result.merge_statistics.shells.push_back(shell);
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// The live list is the report's checklist line for line: same labels, codes and words, and its colour read
|
|
// off the same flags - so the live view and _report.txt cannot disagree.
|
|
TEST_CASE("RugnuxLiveTriage_SameAsReport", "[RugnuxLive]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
const ProcessResult result = ThinMerge();
|
|
const ReportDocument doc = BuildReportDocument("p", "in.h5", x, result);
|
|
const auto triage = LiveTriage(doc);
|
|
|
|
REQUIRE(triage.size() == doc.checks.size());
|
|
REQUIRE(!triage.empty());
|
|
for (size_t i = 0; i < triage.size(); i++) {
|
|
CHECK(triage[i].label == doc.checks[i].label);
|
|
CHECK(triage[i].codes == doc.checks[i].codes);
|
|
CHECK(triage[i].text == doc.checks[i].text);
|
|
CHECK(triage[i].detail == doc.checks[i].detail);
|
|
switch (doc.checks[i].status) {
|
|
case ReportCheck::Status::Absent:
|
|
CHECK(triage[i].severity == RugnuxLiveTriageSeverity::Ok);
|
|
break;
|
|
case ReportCheck::Status::Present:
|
|
CHECK((triage[i].severity == RugnuxLiveTriageSeverity::Attention
|
|
|| triage[i].severity == RugnuxLiveTriageSeverity::Serious));
|
|
break;
|
|
case ReportCheck::Status::Undetermined:
|
|
CHECK(triage[i].severity == RugnuxLiveTriageSeverity::NotMeasured);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Every flag of the verdict line is a coloured entry
|
|
REQUIRE(!doc.pathology_flags.empty());
|
|
for (const auto &code: doc.pathology_flags) {
|
|
bool found = false;
|
|
for (const auto &t: triage)
|
|
if (std::ranges::find(t.codes, code) != t.codes.end())
|
|
found = found || t.severity == RugnuxLiveTriageSeverity::Attention
|
|
|| t.severity == RugnuxLiveTriageSeverity::Serious;
|
|
CHECK(found);
|
|
}
|
|
|
|
const auto *completeness = Find(triage, "Completeness");
|
|
REQUIRE(completeness);
|
|
CHECK(completeness->severity == RugnuxLiveTriageSeverity::Attention);
|
|
CHECK(!completeness->detail.empty());
|
|
const auto *twinning = Find(triage, "Twinning");
|
|
REQUIRE(twinning);
|
|
CHECK(twinning->severity == RugnuxLiveTriageSeverity::NotMeasured);
|
|
}
|
|
|
|
// A merge without signal leaves no result to use: the one red entry
|
|
TEST_CASE("RugnuxLiveTriage_UnusableMergeIsSerious", "[RugnuxLive]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
ProcessResult result = ThinMerge();
|
|
result.merge_statistics.overall.cc_half = -0.005;
|
|
const ReportDocument doc = BuildReportDocument("p", "in.h5", x, result);
|
|
REQUIRE(doc.verdict == "UNUSABLE");
|
|
const auto triage = LiveTriage(doc);
|
|
const auto *unusable = Find(triage, "Unusable merge");
|
|
REQUIRE(unusable);
|
|
CHECK(unusable->severity == RugnuxLiveTriageSeverity::Serious);
|
|
CHECK(Find(triage, "Completeness")->severity == RugnuxLiveTriageSeverity::Attention);
|
|
}
|
|
|
|
// The live serial merge has no pre-scan: powder, ice and further lattices are not measured, and the report says
|
|
// so rather than "no rings detected". Its own analyses - twinning, pseudo-translation, anisotropy - are not
|
|
// measured either until the merge was analysed (AnalyseStillsMerge).
|
|
TEST_CASE("RugnuxLiveTriage_StillsMergeClaimsOnlyWhatItMeasured", "[RugnuxLive]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
x.SetUnitCell(UnitCell{.a = 50.0f, .b = 60.0f, .c = 70.0f, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f});
|
|
StillsMergeOutput out;
|
|
out.merge.statistics = ThinMerge().merge_statistics;
|
|
out.merge.statistics.stills = true;
|
|
out.merge.resolution_fit_A = 2.0;
|
|
out.images = 90;
|
|
|
|
const ProcessResult result = StillsLiveResult(x, out, 100, 90);
|
|
const auto triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
|
|
for (const char *label: {"Powder", "Ice", "Twinning", "Pseudo-symmetry", "Multiple lattices", "Anisotropy"}) {
|
|
const auto *t = Find(triage, label);
|
|
REQUIRE(t);
|
|
CHECK(t->severity == RugnuxLiveTriageSeverity::NotMeasured);
|
|
}
|
|
CHECK(!Find(triage, "Indexing ambiguity"));
|
|
CHECK(Find(triage, "Completeness")->severity == RugnuxLiveTriageSeverity::Attention);
|
|
}
|
|
|
|
// An analysed stills merge carries its twinning and pseudo-translation verdicts and the indexing ambiguity's
|
|
// resolution into the checklist, as the report of a `rugnux` run would; the twin row of a merge that reads
|
|
// twinned points at the ambiguity, since crystals left mis-indexed read the same.
|
|
TEST_CASE("RugnuxLiveTriage_StillsMergeCarriesItsAnalyses", "[RugnuxLive]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
x.SetUnitCell(UnitCell{.a = 50.0f, .b = 50.0f, .c = 70.0f, .alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f});
|
|
x.SetSpaceGroup(*gemmi::find_spacegroup_by_name("P 4"));
|
|
StillsMergeOutput out;
|
|
out.merge.statistics = ThinMerge().merge_statistics;
|
|
out.merge.statistics.stills = true;
|
|
out.merge.resolution_fit_A = 2.0;
|
|
out.images = 90;
|
|
out.ambiguity.crystals = 80;
|
|
out.ambiguity.reindexed = 35;
|
|
out.ambiguity.median_sigma = 4.2;
|
|
out.ambiguity.decided_fraction = 0.7;
|
|
out.tncs.measurable = true;
|
|
out.tncs.n_reflections = 2000;
|
|
out.twinning.l_test_pairs = 5000;
|
|
out.twinning.mean_abs_l = 0.47;
|
|
|
|
ProcessResult result = StillsLiveResult(x, out, 100, 90);
|
|
CHECK(result.ambiguity_reference_free);
|
|
CHECK(result.ambiguity_crystals == 80);
|
|
auto triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
|
|
CHECK(Find(triage, "Twinning")->severity == RugnuxLiveTriageSeverity::Ok);
|
|
CHECK(Find(triage, "Pseudo-symmetry")->severity == RugnuxLiveTriageSeverity::Ok);
|
|
const auto *ambiguity = Find(triage, "Indexing ambiguity");
|
|
REQUIRE(ambiguity);
|
|
CHECK(ambiguity->severity == RugnuxLiveTriageSeverity::Ok);
|
|
CHECK(ambiguity->text == "resolved from the data: 35 of 80 crystals reindexed; the chosen indexing ahead of the "
|
|
"best other by 4.2 sigma (median), 70% of the crystals decided at over 3 sigma");
|
|
CHECK(Find(triage, "Anisotropy")->severity == RugnuxLiveTriageSeverity::NotMeasured);
|
|
|
|
out.twinning.mean_abs_l = 0.42;
|
|
out.twinning.merohedral_twinning_possible = true;
|
|
out.twinning.twinning_suspected = true;
|
|
result = StillsLiveResult(x, out, 100, 90);
|
|
result.warnings.push_back({PathologyCode::TWINNING, "twinned"});
|
|
triage = LiveTriage(BuildReportDocument("p", "in.h5", x, result));
|
|
const auto *twin = Find(triage, "Twinning");
|
|
CHECK(twin->severity == RugnuxLiveTriageSeverity::Attention);
|
|
CHECK(twin->text.find("see Indexing ambiguity") != std::string::npos);
|
|
}
|
|
|
|
TEST_CASE("RugnuxLiveTriage_OpenAPI", "[RugnuxLive]") {
|
|
RugnuxLiveStatus status;
|
|
status.Start(10, std::nullopt, std::chrono::steady_clock::now());
|
|
CHECK(!Convert(status.Get()).triageIsSet());
|
|
|
|
status.Triage({
|
|
{.label = "Twinning", .codes = {"TWINNING"}, .severity = RugnuxLiveTriageSeverity::Ok, .text = "no indication"},
|
|
{.label = "Completeness", .codes = {"LOW_COMPLETENESS"}, .severity = RugnuxLiveTriageSeverity::Attention,
|
|
.text = "40.0 %", .detail = "only 40 % of reciprocal space"},
|
|
{.label = "Unusable merge", .codes = {"UNUSABLE_MERGE"}, .severity = RugnuxLiveTriageSeverity::Serious},
|
|
{.label = "Ice", .codes = {"ICE_RINGS"}, .severity = RugnuxLiveTriageSeverity::NotMeasured,
|
|
.text = "not measured"}
|
|
});
|
|
const auto api = Convert(status.Get());
|
|
REQUIRE(api.triageIsSet());
|
|
const auto items = api.getTriage();
|
|
REQUIRE(items.size() == 4);
|
|
CHECK(items[0].getLabel() == "Twinning");
|
|
CHECK(items[0].getCodes() == std::vector<std::string>{"TWINNING"});
|
|
CHECK(items[0].getSeverity() == "ok");
|
|
CHECK(!items[0].detailIsSet());
|
|
CHECK(items[1].getSeverity() == "attention");
|
|
CHECK(items[1].getDetail() == "only 40 % of reciprocal space");
|
|
CHECK(items[2].getSeverity() == "serious");
|
|
CHECK(items[3].getSeverity() == "not_measured");
|
|
}
|
|
|
|
TEST_CASE("RugnuxLiveFrames_OpenAPI", "[RugnuxLive]") {
|
|
using D = FrameDisposition;
|
|
const auto d = [](D x) { return static_cast<uint8_t>(x); };
|
|
SweepQuality sweep;
|
|
sweep.frame_disposition = {d(D::Merged), d(D::Merged), d(D::Downgraded), d(D::Downgraded), d(D::Rejected),
|
|
d(D::Rejected), d(D::Rejected), d(D::Merged), d(D::Merged), d(D::Rejected)};
|
|
sweep.ranges = {{.first_image = 2, .last_image = 6, .reason = SweepQualityReason::CrystalOutOfBeam},
|
|
{.first_image = 9, .last_image = 9, .reason = SweepQualityReason::RadiationDamage}};
|
|
const auto ranges = LiveFrameRanges(sweep);
|
|
REQUIRE(ranges.size() == 3);
|
|
CHECK(ranges[0].first_image == 2);
|
|
CHECK(ranges[0].last_image == 3);
|
|
CHECK(ranges[0].disposition == "downgraded");
|
|
CHECK(ranges[0].reason == SweepQualityReasonText(SweepQualityReason::CrystalOutOfBeam));
|
|
CHECK(ranges[1].first_image == 4);
|
|
CHECK(ranges[1].last_image == 6);
|
|
CHECK(ranges[1].disposition == "rejected");
|
|
CHECK(ranges[2].first_image == 9);
|
|
CHECK(ranges[2].reason == SweepQualityReasonText(SweepQualityReason::RadiationDamage));
|
|
CHECK(LiveFrameRanges(SweepQuality{}).empty());
|
|
|
|
RugnuxLiveStatus status;
|
|
status.Start(10, std::nullopt, std::chrono::steady_clock::now());
|
|
CHECK(!Convert(status.Get()).frameScalesIsSet());
|
|
status.Frames({1.0f, NAN, 0.5f}, ranges);
|
|
const auto api = Convert(status.Get());
|
|
REQUIRE(api.frameScalesIsSet());
|
|
CHECK(api.getFrameScales() == std::vector<float>{1.0f, 0.0f, 0.5f});
|
|
REQUIRE(api.getFrameRanges().size() == 3);
|
|
CHECK(api.getFrameRanges()[1].getFirstImage() == 4);
|
|
CHECK(api.getFrameRanges()[1].getDisposition() == "rejected");
|
|
CHECK(api.getFrameRanges()[0].getReason() == ranges[0].reason);
|
|
}
|