mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 73 Wiki Activity
Files
0903a8eee33a47c6c6208ae4112d38bb5916fa0f
Jungfraujoch/tests/ResultReportTest.cpp
T
leonarski_fandClaude Opus 5 4af23e9b27 model: build the null only where the model claims something 
The null costs about 12 s - five replicates, each a full fit and a rigid-body
placement - and it gates exactly two decisions: the enantiomorph label and the
merohedral indexing. A model that claims neither, already in the group the data
were merged in on a crystal with no indexing ambiguity, has nothing for the null
to arbitrate, and paying for it there is 12 s spent gating a decision nobody is
making. That is the isomorphous fragment-screening run - hundreds of crystals of
one form against one apo model, a map wanted seconds after the last image - and
it is the case that has to be fast.

So the null runs only when a decision is pending: an enantiomorph candidate
exists, or the indexing probe returned a winner that is not the identity. The
probe is a handful of scaling fits and had to run first anyway, so the question
is answered before the expensive part starts.

Measured on a rotation dataset, --mode scale --model, interleaved on a loaded
machine: the isomorphous case is 3.6-4.7 s before this whole line of work and
3.8-4.7 s with the gate, against 16.5 s while the null ran unconditionally. The
case that still pays is unchanged at 17 s, and the three wrong-model cases give
bit-identical verdicts and bit-identical files to before.

The rigid-body refinement of the replicates stays: 3 of 5 commit a placement, so
without it the null would score a weaker procedure than the one it judges.

MODEL_FIT gains a third value, NOT_TESTED, alongside ACCEPTED and REJECTED - the
question was never put, which is not the data answering it badly - and the
MODEL_FIT_NULL_* and MODEL_FIT_SIGMA keys are absent with it rather than
reporting a null that was never built. REPORT_VERSION stays 6: MODEL_FIT is
itself new in this release, so its vocabulary has never shipped and a consumer
cannot be relying on the two-value form.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EFEJG6WBQv8th4UJFNe53N
2026-09-02 14:06:29 +02:00

334 lines
16 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include "../common/DiffractionExperiment.h"
#include "../image_analysis/scale_merge/Merge.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../rugnux/ResultReport.h"
#include "../common/CUDAWrapper.h"
#include "../writer/FileWriter.h"
// Two synthetic ranges, one of each shape the report has to handle.
namespace {
SweepQuality TestSweepQuality() {
SweepQuality out;
out.measured = true;
out.sweep_deg = 60.0f;
out.ranges.push_back(SweepQualityRange{.first_image = 100, .last_image = 149,
.reason = SweepQualityReason::CrystalOutOfBeam,
.severity = 0.83f, .rotation_deg = 5.0f,
.mean_relative_scale = 0.12f, .mean_relative_cc = 0.30f,
.indexed_fraction = 0.015f});
out.ranges.push_back(SweepQualityRange{.first_image = 400, .last_image = 499,
.reason = SweepQualityReason::RadiationDamage,
.severity = 0.41f, .rotation_deg = 10.0f,
.mean_relative_scale = 0.55f, .mean_relative_cc = 0.80f,
.indexed_fraction = 0.62f});
return out;
}
std::vector<std::string> ReasonVocabulary() {
std::vector<std::string> out;
for (int r = 0; r <= static_cast<int>(SweepQualityReason::RadiationDamage); ++r)
out.emplace_back(SweepQualityReasonCode(static_cast<SweepQualityReason>(r)));
return out;
}
}
TEST_CASE("SweepQuality_ReasonVocabulary", "[Diagnostics]") {
// The codes are an interface - they are written verbatim into <prefix>_report.txt and into HDF5,
// where the per-image codes index this list from 1. A rename or a reorder silently breaks every
// consumer, so pin both the spelling and the order here.
const auto codes = ReasonVocabulary();
REQUIRE(codes.size() == 5);
CHECK(codes[0] == "no_diffraction");
CHECK(codes[1] == "crystal_out_of_beam");
CHECK(codes[2] == "weak_diffraction");
CHECK(codes[3] == "loss_of_centring");
CHECK(codes[4] == "radiation_damage");
}
TEST_CASE("ResultReport_Render", "[Diagnostics]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(600);
ProcessResult result;
result.images_processed = 600;
result.indexing_rate = 0.87f;
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
result.space_group = *gemmi::find_spacegroup_by_number(96);
result.used_beam_x_pxl = 766.62f;
result.used_beam_y_pxl = 846.87f;
result.used_distance_mm = 243.53f;
result.has_merge_statistics = true;
result.merge_statistics.sweep_quality = TestSweepQuality();
const auto text = RenderResultReport("prefix", "in.h5", x, result);
// The stable keys a consumer greps for.
CHECK(text.find("\nREPORT_VERSION= 6\n") != std::string::npos);
CHECK(text.find("\nOUTPUT_PREFIX= prefix\n") != std::string::npos);
CHECK(text.find("\nINDEXING_RATE= 0.8700\n") != std::string::npos);
CHECK(text.find("\nSPACE_GROUP_NUMBER= 96\n") != std::string::npos);
// What the data could NOT decide, beside what they did. The group here was given rather
// than searched for, and it is one of the 22 that come in enantiomorphic pairs.
CHECK(text.find("\nSPACE_GROUP_ALTERNATIVES= NONE\n") != std::string::npos);
CHECK(text.find("\nSPACE_GROUP_ENANTIOMORPH= GIVEN\n") != std::string::npos);
CHECK(text.find("\nSPACE_GROUP_REFUSED_POINT_GROUP= NONE\n") != std::string::npos);
CHECK(text.find("\nSWEEP_QUALITY_STATUS= COMPUTED\n") != std::string::npos);
CHECK(text.find("\nSWEEP_QUALITY_COUNT= 2\n") != std::string::npos);
CHECK(text.find("\nSWEEP_QUALITY_REASONS= no_diffraction crystal_out_of_beam weak_diffraction "
"loss_of_centring radiation_damage\n") != std::string::npos);
// One table row per range, with the reason code verbatim.
CHECK(text.find(" 100 149 50 5.0 crystal_out_of_beam ")
!= std::string::npos);
CHECK(text.find(" 400 499 100 10.0 radiation_damage ")
!= std::string::npos);
// ... and one plain-English WARNING line per range, greppable by the marker alone.
CHECK(text.find("\nWARNING_COUNT= 2\n") != std::string::npos);
CHECK(text.find("\nWARNING: Frames 100-149 out of beam (5.0 deg,") != std::string::npos);
CHECK(text.find("\nWARNING: Frames 400-499 radiation damage (10.0 deg,") != std::string::npos);
}
TEST_CASE("ResultReport_RenderEmpty", "[Diagnostics]") {
// A clean run and a run that never looked must be distinguishable: both have a count of 0, and
// only the STATUS key separates them. This is the property a consumer relies on.
DiffractionExperiment x(DetJF(1));
ProcessResult clean;
clean.has_merge_statistics = true;
clean.merge_statistics.sweep_quality.measured = true;
const auto clean_text = RenderResultReport("p", "in.h5", x, clean);
CHECK(clean_text.find("\nSWEEP_QUALITY_STATUS= COMPUTED\n") != std::string::npos);
CHECK(clean_text.find("\nSWEEP_QUALITY_COUNT= 0\n") != std::string::npos);
CHECK(clean_text.find("FIRST_IMAGE LAST_IMAGE") != std::string::npos);
ProcessResult not_merged;
const auto not_merged_text = RenderResultReport("p", "in.h5", x, not_merged);
CHECK(not_merged_text.find("\nSWEEP_QUALITY_STATUS= NOT_COMPUTED\n") != std::string::npos);
CHECK(not_merged_text.find("\nMERGE= NOT_PERFORMED\n") != std::string::npos);
CHECK(not_merged_text.find("FIRST_IMAGE LAST_IMAGE") != std::string::npos);
}
TEST_CASE("ResultReport_RadiationDamage", "[Diagnostics]") {
// RADIATION_DAMAGE_RELATIVE_B is a number only when there is one. A curve that was measured but
// that no straight line describes, and a monitor that could not run at all, are different answers,
// and a consumer has to be able to tell them apart - and both from a measured zero.
DiffractionExperiment x(DetJF(1));
ProcessResult result;
result.has_merge_statistics = true;
result.radiation_damage_text = "per-batch relative-B";
result.merge_statistics.radiation_damage_batch_deg = 10.0;
result.merge_statistics.radiation_damage_b_batch = {0.0f, 4.0f, NAN};
result.merge_statistics.radiation_damage_delta_b = 8.25;
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nRADIATION_DAMAGE_RELATIVE_B= 8.25\n")
!= std::string::npos);
result.merge_statistics.radiation_damage_delta_b = NAN;
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nRADIATION_DAMAGE_RELATIVE_B= NOT_A_TREND\n")
!= std::string::npos);
result.merge_statistics.radiation_damage_b_batch.clear();
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nRADIATION_DAMAGE_RELATIVE_B= NOT_MEASURED\n")
!= std::string::npos);
}
TEST_CASE("SweepQuality_HDF5RoundTrip", "[HDF5][Full][Diagnostics]") {
// The per-image codes have to survive the writer and come back out of the reader. Without an
// assertion here the field can ship as all-zeros without anyone noticing.
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(6).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("sweep_quality_roundtrip");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
// Images 2-3 out of beam, image 5 dead from radiation damage; the rest in no flagged range.
const std::vector<uint8_t> expected{0, 0,
static_cast<uint8_t>(SweepQualityReason::CrystalOutOfBeam) + 1,
static_cast<uint8_t>(SweepQualityReason::CrystalOutOfBeam) + 1,
0,
static_cast<uint8_t>(SweepQualityReason::RadiationDamage) + 1};
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
end_message.sweep_quality = expected;
end_message.sweep_quality_reasons = ReasonVocabulary();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 42);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
JFJochHDF5Reader reader;
REQUIRE_NOTHROW(reader.ReadFile("sweep_quality_roundtrip_master.h5"));
auto dataset = reader.GetDataset();
CHECK(dataset->sweep_quality == expected);
// The vocabulary travels with the codes, so a consumer can name them without this source.
CHECK(dataset->sweep_quality_reasons == ReasonVocabulary());
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("sweep_quality_roundtrip_master.h5");
}
// The command line, the wall time and the GPUs say how the result was produced, what it cost and what
// it ran on, so the report can be read on its own once the shell history is gone. They come from the
// CLI, which is the only caller that knows them; a caller that does not - the library, the viewer -
// must get a report without them rather than one claiming the run was invoked by nobody, took no
// time, and saw no GPU.
TEST_CASE("ResultReport_ProvenanceKeys", "[Diagnostics]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(600);
ProcessResult result;
result.images_processed = 600;
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
RunProvenance provenance;
provenance.command_line = "rugnux -o prefix in.h5";
provenance.wall_time_s = 262.409;
provenance.gpu_count = 4;
provenance.gpu_description = "4x NVIDIA A100-SXM4-80GB";
const auto text = RenderResultReport("prefix", "in.h5", x, result, provenance);
CHECK(text.find("\nCOMMAND_LINE= rugnux -o prefix in.h5\n") != std::string::npos);
CHECK(text.find("\nWALL_TIME= 262.41\n") != std::string::npos);
CHECK(text.find("\nGPU_COUNT= 4\n") != std::string::npos);
CHECK(text.find("\nGPU= 4x NVIDIA A100-SXM4-80GB\n") != std::string::npos);
// A machine with no GPU says so - GPU_COUNT= 0 is a statement about why the run took as long as
// it did, and only the name list has nothing to report.
RunProvenance cpu_only;
cpu_only.gpu_count = 0;
const auto cpu_text = RenderResultReport("prefix", "in.h5", x, result, cpu_only);
CHECK(cpu_text.find("\nGPU_COUNT= 0\n") != std::string::npos);
CHECK(cpu_text.find("\nGPU= ") == std::string::npos);
const auto without = RenderResultReport("prefix", "in.h5", x, result);
CHECK(without.find("COMMAND_LINE=") == std::string::npos);
CHECK(without.find("WALL_TIME=") == std::string::npos);
CHECK(without.find("GPU_COUNT=") == std::string::npos);
}
// A model that was asked for gets a section either way: the R-factors when it worked, and why it did
// not when it did not. A run that never asked for one has no section at all.
TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(600);
ProcessResult result;
result.images_processed = 600;
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
CHECK(RenderResultReport("p", "in.h5", x, result).find("MODEL VALIDATION") == std::string::npos);
ModelValidationResult good;
good.ok = true;
good.model_path = "model.cif";
good.model_space_group_number = 96;
good.r_work = 0.1823;
good.r_free = 0.2145;
good.n_work = 16682;
good.n_free = 928;
good.mean_atom_density_sigma = 1.42;
good.fit_tested = true;
good.model_fits = true;
good.null_replicates = 5;
good.null_r_work_mean = 0.6031;
good.null_r_work_sd = 0.0094;
good.r_work_sigma = 4.12;
good.model_enantiomorph_candidate = true;
good.adopted_model_enantiomorph = true;
result.model_validation = good;
result.space_group = *gemmi::find_spacegroup_by_number(96);
const auto text = RenderResultReport("p", "in.h5", x, result);
CHECK(text.find("10. MODEL VALIDATION") != std::string::npos);
CHECK(text.find("\nMODEL_FILE= model.cif\n") != std::string::npos);
CHECK(text.find("\nR_FREE= 0.2145\n") != std::string::npos);
CHECK(text.find("\nR_FREE_REFLECTIONS= 928\n") != std::string::npos);
CHECK(text.find("\nMODEL_VALIDATION= PERFORMED\n") != std::string::npos);
CHECK(text.find("MODEL_VALIDATION= NOT_PERFORMED") == std::string::npos);
CHECK(text.find("\nMODEL_FIT= ACCEPTED\n") != std::string::npos);
CHECK(text.find("\nMODEL_FIT_STATISTIC= R_WORK\n") != std::string::npos);
CHECK(text.find("\nMODEL_FIT_NULL_REPLICATES= 5\n") != std::string::npos);
CHECK(text.find("\nMODEL_FIT_SIGMA= +4.12\n") != std::string::npos);
CHECK(text.find("\nMODEL_DECISIONS_TAKEN= ENANTIOMORPH\n") != std::string::npos);
// The hand is ASSUMED from the model, never determined: merged intensities cannot see it.
CHECK(text.find("\nSPACE_GROUP_ENANTIOMORPH= ASSUMED_FROM_MODEL\n") != std::string::npos);
// The same model, not accepted by the data: it still has R-factors and maps, but it decides
// nothing - the reflections stay in the group and the indexing they were merged in.
ModelValidationResult rejected = good;
rejected.model_fits = false;
rejected.r_work_sigma = 0.59;
rejected.adopted_model_enantiomorph = false;
result.model_validation = rejected;
const auto rejected_text = RenderResultReport("p", "in.h5", x, result);
CHECK(rejected_text.find("\nMODEL_FIT= REJECTED\n") != std::string::npos);
CHECK(rejected_text.find("\nMODEL_DECISIONS_TAKEN= NONE\n") != std::string::npos);
CHECK(rejected_text.find("\nR_WORK= 0.1823\n") != std::string::npos);
CHECK(rejected_text.find("\nSPACE_GROUP_ENANTIOMORPH= ASSUMED_FROM_MODEL\n") == std::string::npos);
CHECK(rejected_text.find("byte for byte") != std::string::npos);
// A model that claims nothing - already in the data's group, no merohedral ambiguity - is never
// put to the null, and NOT_TESTED has to be distinguishable from a model the data refused.
ModelValidationResult untested = good;
untested.fit_tested = false;
untested.model_fits = false;
untested.model_enantiomorph_candidate = false;
untested.adopted_model_enantiomorph = false;
result.model_validation = untested;
const auto untested_text = RenderResultReport("p", "in.h5", x, result);
CHECK(untested_text.find("\nMODEL_FIT= NOT_TESTED\n") != std::string::npos);
CHECK(untested_text.find("\nMODEL_DECISIONS_TAKEN= NONE\n") != std::string::npos);
CHECK(untested_text.find("\nR_WORK= 0.1823\n") != std::string::npos);
CHECK(untested_text.find("MODEL_FIT_NULL_MEAN=") == std::string::npos);
CHECK(untested_text.find("was tried and REJECTED") == std::string::npos);
result.model_validation = good;
ModelValidationResult failed;
failed.model_path = "broken.pdb";
failed.failure_reason = "model broken.pdb has no atoms or no unit cell";
result.model_validation = failed;
const auto failed_text = RenderResultReport("p", "in.h5", x, result);
CHECK(failed_text.find("\nMODEL_VALIDATION= NOT_PERFORMED\n") != std::string::npos);
CHECK(failed_text.find("has no atoms") != std::string::npos);
CHECK(failed_text.find("R_FREE=") == std::string::npos);
}
// get_gpu_description collapses repeats, so a four-card machine reads as one line rather than the
// same name four times. Build-independent: without CUDA there are no names and it is empty.
TEST_CASE("ResultReport_GpuDescription", "[Diagnostics]") {
const auto names = get_gpu_names();
CHECK(names.size() == static_cast<size_t>(std::max(0, get_gpu_count())));
const auto description = get_gpu_description();
if (names.empty())
CHECK(description.empty());
else
CHECK(description.find(names.front()) != std::string::npos);
}
Reference in New Issue View Git Blame Copy Permalink