mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 79 Wiki Activity
Files
508fcfb400aad5b01aee9f0af5e3ef95cbaffbd3
Jungfraujoch/rugnux/ResultReport.cpp
T

2045 lines
132 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <cmath>
#include <algorithm>
#include <fstream>
#include <sstream>
#include <spdlog/fmt/fmt.h>
#include "../common/JFJochMath.h" // PI - M_PI is not portable to MSVC
#include "../common/GitInfo.h"
#include "../common/time_utc.h"
#include "../image_analysis/SensorAbsorption.h"
#include "../image_analysis/scale_merge/AnisotropyAnalysis.h"
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/scale_merge/TranslationalNCS.h"
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
#include "ResultReport.h"
namespace {
// The version of this file format. Bumped when a key is renamed or removed, a table column moves,
// or a reason code changes meaning - a consumer can gate on it.
// 8: SUMMARY section with VERDICT / PATHOLOGY_FLAGS; sections renumbered 1-5 with no holes;
// pipeline-internal keys and the long explanatory blocks moved behind --developer.
// 9: FLIGHT_PATH / FLIGHT_PATH_WILSON_B in section 1 - the medium the beam flew through, which
// no file states and the run therefore assumes, and what assuming it was worth.
// 10: the TNCS_* block in section 4 - translational pseudo-symmetry, which nothing measured
// before, and the UNDECLARED_LATTICE_TRANSLATION it reports instead where the data are
// exactly invariant under the vector it found.
// 11: L_TEST_VS_TNCS beside <|L|> - whether the pseudo-symmetry above biased the L-test, and
// whether choosing the partner reflections differently could repair it.
// 13: CC_MODEL_* in section 5 - the correlation of the merged intensities with the placed model,
// by resolution shell, which only a run given a model can report.
// 8: every line that is not `KEY= value` and not blank starts with `#`, warnings included
// (`# WARNING:`); the header gains RUGNUX_DOWNLOAD and BUILD_CXX_FLAGS, and RUGNUX_GIT is
// the full build-time hash.
//
// The entries above are the format's history; the constant counts RELEASES, not entries, and
// is bumped ONCE PER RELEASE BRANCH. A reader only ever meets what was built from main, so a
// second bump inside an rc branch numbers a format no one received. Several entries above were
// written on branches that shipped together, which is why there are more of them than there
// are versions - that is the convention working, not the constant lagging.
constexpr int REPORT_VERSION = 8;
const char *BANNER = " ******************************************************************************";
std::string CellString(const UnitCell &c) {
return fmt::format("{:.3f} {:.3f} {:.3f} {:.3f} {:.3f} {:.3f}", c.a, c.b, c.c,
c.alpha, c.beta, c.gamma);
}
// Which reciprocal axis a principal anisotropy direction lies along. The tensor is fitted on
// s = frac.mat * (h,k,l), so in that Cartesian frame a*, b*, c* are the rows of frac.mat and the
// eigenvector is named by whichever it makes the smallest angle with. The label is exact in every
// Laue class the tensor has a free direction in except triclinic; the cosine says how well it fits.
std::string ReciprocalAxisLabel(const UnitCell &cell, const double v[3]) {
static const char *NAME[3] = {"a*", "b*", "c*"};
const gemmi::UnitCell gc = cell;
int best = 0;
double best_cos = -1.0;
for (int i = 0; i < 3; ++i) {
const gemmi::Vec3 axis = gc.frac.mat.left_multiply(
gemmi::Vec3(i == 0 ? 1.0 : 0.0, i == 1 ? 1.0 : 0.0, i == 2 ? 1.0 : 0.0));
const double len = axis.length();
if (!(len > 0.0))
continue;
const double c = std::fabs((axis.x * v[0] + axis.y * v[1] + axis.z * v[2]) / len);
if (c > best_cos) {
best_cos = c;
best = i;
}
}
return fmt::format("{} (cos {:.2f})", NAME[best], best_cos);
}
// ------------------------------------------------------------------ document builders
// Small constructors, so the body below reads as a list of what the report SAYS rather than as a
// list of how it is punctuated. Each fills the typed field beside the text, so a rendering that
// is not text does not have to parse the string back.
ReportEntry KeyText(const char *key, std::string value, bool dev = false) {
ReportEntry e;
e.key = key;
e.value.type = ReportValue::Type::Text;
e.value.text = std::move(value);
e.developer_only = dev;
return e;
}
ReportEntry KeyEnum(const char *key, const std::string &value,
std::vector<std::string> domain, bool dev = false) {
ReportEntry e = KeyText(key, value, dev);
e.value.type = ReportValue::Type::Enumerated;
e.value.domain = std::move(domain);
return e;
}
ReportEntry KeyBool(const char *key, bool value, bool dev = false) {
ReportEntry e = KeyEnum(key, value ? "TRUE" : "FALSE", {"TRUE", "FALSE"}, dev);
e.value.type = ReportValue::Type::Boolean;
e.value.boolean = value;
return e;
}
ReportEntry KeyInt(const char *key, int64_t value, bool dev = false) {
ReportEntry e;
e.key = key;
e.value.type = ReportValue::Type::Integer;
e.value.integer = value;
e.value.text = std::to_string(value);
e.developer_only = dev;
return e;
}
// The format is passed in because the spelling of a number is part of the interface: a consumer
// that greps R_MEAS gets four decimals whatever a renderer would rather do with it.
ReportEntry KeyReal(const char *key, double value, const char *format, bool dev = false) {
ReportEntry e;
e.key = key;
e.value.type = ReportValue::Type::Real;
e.value.real = value;
e.value.text = fmt::format(fmt::runtime(format), value);
e.developer_only = dev;
return e;
}
ReportEntry Prose(std::string text, bool dev = false) {
ReportEntry e;
e.kind = ReportEntry::Kind::Prose;
e.prose = std::move(text);
e.developer_only = dev;
return e;
}
// A paragraph that exists only because the long version of the same point moved to --developer.
ReportEntry ProseDefaultOnly(std::string text) {
ReportEntry e = Prose(std::move(text));
e.default_only = true;
return e;
}
ReportEntry Blank() {
ReportEntry e;
e.kind = ReportEntry::Kind::Blank;
return e;
}
void Add(ReportSection &s, ReportEntry e) { s.entries.push_back(std::move(e)); }
void Warn(ReportDocument &doc, const char *code, std::string text, bool dev = false) {
doc.warnings.push_back({code, std::move(text), dev});
}
// Completeness over the shells the fit says carry signal, which is not the same number as the
// overall completeness: the resolution cut is deliberately generous, so the outer shells dilute
// the denominator with reciprocal space the detector only reaches in its corners. Users compare
// the overall figure against another program's and conclude data were lost.
struct CompletenessInRange {
bool measured = false;
double percent = 0.0;
double d_min = 0.0;
};
CompletenessInRange CompletenessTo(const MergeStatistics &ms, double d_min_limit) {
CompletenessInRange out;
int64_t unique = 0, possible = 0;
for (const auto &sh : ms.shells) {
if (sh.d_min < d_min_limit)
continue;
unique += sh.unique_reflections;
possible += sh.possible_unique_reflections;
out.d_min = sh.d_min;
}
if (possible > 0) {
out.measured = true;
out.percent = 100.0 * static_cast<double>(unique) / static_cast<double>(possible);
}
return out;
}
// The reference-range table of --report-resolution, with the columns of the run's own. A shell
// finer than the run's own limit holds nothing BY THE RUN'S DECISION, and says so in words rather
// than as a row of zeros, so a reader cannot take it for a measured total failure.
std::string ReferenceShellTable(const MergeStatistics &rs, double own_d_min) {
std::ostringstream o;
o << "\n" << ShellTableHeader() << "\n" << ShellTableRule() << "\n";
for (const auto &sh : rs.shells) {
if (sh.unique_reflections == 0) {
if (sh.d_max <= own_d_min)
o << fmt::format(" {:8.2f} past the run's own limit of {:.2f} A: not merged ({} possible)\n",
sh.d_min, own_d_min, sh.possible_unique_reflections);
continue;
}
o << fmt::format(" {:8.2f} ", sh.d_min) << sh;
if (sh.d_min < own_d_min)
o << fmt::format(" <- the run's own limit, {:.2f} A, is inside this shell", own_d_min);
o << "\n";
}
o << ShellTableRule() << "\n";
o << fmt::format(" {:>8s} ", "Overall") << rs.overall << "\n";
return o.str();
}
// The finest shell whose CC1/2 still reaches the cut's own target: where the table says the signal
// is, for the completeness quoted beside the written range.
double FinestSupportedShell(const MergeStatistics &ms, double target) {
double finest = 0.0;
for (const auto &sh : ms.shells)
if (std::isfinite(sh.cc_half) && sh.cc_half >= target && (finest == 0.0 || sh.d_min < finest))
finest = sh.d_min;
return finest; // 0.0 = no shell reaches the target at all
}
// The finest resolution the shell table supports a fitted crossing at. A shell's CC1/2 is an
// average over the shell, so the crossing sits INSIDE the first shell whose average falls below the
// target, between its d_max and its d_min: a fit anywhere down to that shell's d_min is what the
// table shows, and only a fit finer than it is one the table refutes. Comparing against the last
// shell ABOVE the target instead withheld the number on 68 of 100 battery datasets whose fit sat
// 0.01-0.02 A inside the first sub-target shell. 0.0 when no shell reaches the target; the finest
// shell when none falls below it.
double CrossingShellLimit(const MergeStatistics &ms, double target) {
double finest = 0.0;
for (const auto &sh : ms.shells) {
if (!std::isfinite(sh.cc_half))
continue;
if (sh.cc_half < target)
return finest == 0.0 ? 0.0 : sh.d_min;
finest = sh.d_min;
}
return finest;
}
// Where CC1/2 climbs back above the cut's target after it has already fallen below it. Below the
// target a shell carries no signal the cut would keep, so a shell past that point which
// correlates again is noise correlating with noise: the curve is not a fall-off, and no single
// resolution can be read off it. Returns that shell's d_min, 0.0 when the curve never climbs
// back. A wobble that stays ABOVE the target is not this - a fall-off with a noisy shell in it
// is still a fall-off.
double CCHalfClimbsBackAboveTarget(const MergeStatistics &ms, double target) {
bool below = false;
for (const auto &sh : ms.shells) {
if (!std::isfinite(sh.cc_half))
continue;
if (sh.cc_half < target)
below = true;
else if (below)
return sh.d_min;
}
return 0.0;
}
}
// ---------------------------------------------------------------------------- the emitter
// Populates the document. Nothing here writes to a stream: the order of the FILE is the renderer's
// business, which is what lets the SUMMARY sit above the sections whose warnings it reports.
ReportDocument BuildReportDocument(const std::string &output_prefix,
const std::string &input_file,
const DiffractionExperiment &experiment,
const ProcessResult &result,
const RunProvenance &provenance) {
ReportDocument doc;
const bool rotation = experiment.IsRotationIndexing();
const bool merged = result.has_merge_statistics;
doc.warnings = result.warnings;
// ------------------------------------------------------------------ header block
{
ReportSection h;
Add(h, KeyInt("REPORT_VERSION", REPORT_VERSION));
Add(h, KeyText("RUGNUX_VERSION", jfjoch_version()));
// The release page of exactly this version; the tag on a release is the version string.
Add(h, KeyText("RUGNUX_DOWNLOAD",
"https://gitea.psi.ch/mx/Jungfraujoch/releases/tag/" + jfjoch_version()));
// The commit the binary was built from - stamped at BUILD time, not configure time, and a
// "-dirty" suffix marks uncommitted changes - and the compiler flags it was built with.
// Both are provenance the version string does not carry: two builds of one commit can
// differ by flags alone, and -march moves the CPU-bound results.
if (!jfjoch_git_sha1().empty())
Add(h, KeyText("RUGNUX_GIT", jfjoch_git_sha1() + " " + jfjoch_git_date()));
Add(h, KeyText("BUILD_CXX_FLAGS",
jfjoch_build_cxx_flags().empty() ? "NONE" : jfjoch_build_cxx_flags()));
Add(h, KeyText("DATE", time_UTC(std::chrono::system_clock::now())));
Add(h, KeyText("INPUT_FILE", input_file));
Add(h, KeyText("OUTPUT_PREFIX", output_prefix));
// How the result was produced, what it cost and what it ran on, so the report stands on its
// own once the shell history it came from is gone. Absent rather than zero where the caller
// does not know them - the library and the viewer have no command line and no run to time.
if (!provenance.command_line.empty())
Add(h, KeyText("COMMAND_LINE", provenance.command_line));
if (provenance.wall_time_s > 0.0)
Add(h, KeyReal("WALL_TIME", provenance.wall_time_s, "{:.2f}"));
if (provenance.gpu_count >= 0) {
// GPU_COUNT= 0 with no GPU= line is the CPU-only case, and saying so is the point:
// whether the GPUs were there is the first question about how long the run took.
Add(h, KeyInt("GPU_COUNT", provenance.gpu_count));
if (!provenance.gpu_description.empty())
Add(h, KeyText("GPU", provenance.gpu_description));
}
doc.sections.push_back(std::move(h));
}
// ------------------------------------------------- 1. DATA SET AND GEOMETRY
{
ReportSection s;
s.title = "1. DATA SET AND GEOMETRY";
Add(s, KeyEnum("EXPERIMENT_TYPE", rotation ? "ROTATION" : "STILLS", {"ROTATION", "STILLS"}));
Add(s, KeyInt("IMAGES_PROCESSED", static_cast<int64_t>(result.images_processed)));
Add(s, KeyReal("WAVELENGTH", experiment.GetWavelength_A(), "{:.5f}"));
if (const auto gonio = experiment.GetGoniometer()) {
Add(s, KeyReal("OSCILLATION_RANGE", gonio->GetIncrement_deg(), "{:.4f}"));
Add(s, KeyReal("STARTING_ANGLE", gonio->GetStart_deg(), "{:.3f}"));
const auto ax = gonio->GetAxis();
Add(s, KeyText("ROTATION_AXIS", fmt::format("{:.6f} {:.6f} {:.6f}", ax.x, ax.y, ax.z)));
}
Add(s, KeyReal("DETECTOR_DISTANCE", result.used_distance_mm, "{:.3f}"));
Add(s, KeyText("BEAM_CENTRE", fmt::format("{:.2f} {:.2f}", result.used_beam_x_pxl,
result.used_beam_y_pxl)));
Add(s, KeyText("DETECTOR_TILT", fmt::format("{:.4f} {:.4f} {:.4f}",
result.used_detector_tilt_deg[0],
result.used_detector_tilt_deg[1],
result.used_detector_tilt_deg[2])));
Add(s, KeyText("DIRECT_BEAM", fmt::format("{:.2f} {:.2f}", result.used_direct_beam_x_pxl,
result.used_direct_beam_y_pxl)));
if (result.refined_detector_tilt_deg)
Add(s, KeyText("REFINED_DETECTOR_TILT",
fmt::format("{:.4f} {:.4f}", (*result.refined_detector_tilt_deg)[0],
(*result.refined_detector_tilt_deg)[1])));
// What the beam flew through, and what assuming it was worth. NOTHING in any file rugnux
// reads states the medium, so this is an ASSUMPTION the run made on the user's behalf and
// the report is the only place it becomes visible. Its worth is quoted as the shift it puts
// into the Wilson B, because on an untilted detector the correction is a pure function of
// resolution: it cancels within a resolution shell, so a change of Wilson-plot slope is its
// whole effect on merged data, and it is the number a user would otherwise be puzzled by.
{
const auto medium = experiment.GetBraggIntegrationSettings().GetFlightPath();
const auto fp = sensor_absorption::FlightPathAttenuation::Build(
medium, result.used_distance_mm, experiment.GetWavelength_A());
// Over the range the Wilson B is actually fitted on: from as far as the detector
// reaches in to the 4 A convention CalcGlobalWilsonB uses, since below that the Wilson
// plot is not a line. Quoting it over the whole detector instead would average the
// slope against a low-resolution stretch where the correction is flat and understate it.
const double dB = fp.WilsonBShift_A2(experiment.GetDetectorMaxResolution_A(), 4.0,
experiment.GetWavelength_A());
Add(s, KeyEnum("FLIGHT_PATH", medium == FlightPathMedium::Air ? "AIR"
: (medium == FlightPathMedium::Helium ? "HELIUM" : "VACUUM"),
{"AIR", "HELIUM", "VACUUM"}));
Add(s, KeyReal("FLIGHT_PATH_WILSON_B", dB, "{:.2f}"));
// Worth a warning only where it is worth something: below a couple of A^2 the assumption
// cannot be read off the merged data at all, and every hard-X-ray geometry in use is far
// under that. This gates a SENTENCE, never a number - the correction itself is applied
// the same way whatever this says.
if (medium == FlightPathMedium::Air && std::fabs(dB) >= 2.0)
Warn(doc, PathologyCode::FLIGHT_PATH, fmt::format(
"an AIR flight path was assumed, which is worth {:.1f} A^2 of Wilson B here; "
"no file states the medium, so a station with a helium cone or an evacuated "
"flight tube needs a re-run with --flight-path helium (or vacuum)", dB));
}
// The same geometry once more as the object jfjoch_broker takes it in - the four required
// properties of dataset_settings in broker/jfjoch_api.yaml, spelled the way the API spells
// them. One line, valid JSON, so a script can lift it with a grep and POST it back.
{
constexpr double RAD = PI / 180.0;
const auto &t = result.used_detector_tilt_deg;
std::string rot;
// The rotations belong here whenever they are not zero: a block that omits them describes
// a FLAT detector, which is a different geometry from the one this run used.
if (t[0] != 0.0 || t[1] != 0.0 || t[2] != 0.0)
rot = fmt::format(R"(, "poni_rot1_rad": {:.6f}, "poni_rot2_rad": {:.6f}, )"
R"("poni_rot3_rad": {:.6f})",
t[0] * RAD, t[1] * RAD, t[2] * RAD);
Add(s, KeyText("JFJOCH_DATASET_SETTINGS",
fmt::format(R"({{"beam_x_pxl": {:.2f}, "beam_y_pxl": {:.2f}, )"
R"("detector_distance_mm": {:.3f}, "incident_energy_keV": {:.4f}{}}})",
result.used_beam_x_pxl, result.used_beam_y_pxl,
result.used_distance_mm,
experiment.GetDatasetSettings().GetPhotonEnergy_keV(), rot)));
}
if (result.spot_resolution_estimate_A.has_value())
Add(s, KeyReal("SPOT_RESOLUTION_ESTIMATE", *result.spot_resolution_estimate_A, "{:.2f}"));
if (result.pass_count > 1) {
Add(s, KeyText("PASS", fmt::format("{} of {}", result.pass_number, result.pass_count)));
Add(s, KeyText("PASS_DECISION", result.pass_decision, true));
if (result.geometry_not_converged)
Warn(doc, PathologyCode::GEOMETRY_NOT_CONVERGED,
"the detector geometry post-refinement was still moving when the run stopped walking "
"it: every round lowered the held-out residual, so the geometry this result was "
"integrated at is the best one reached, not a settled one (see PASS_DECISION)");
}
Add(s, Blank());
// Where the geometry came from. This is the whole provenance story a user needs, because a
// user acts on exactly one geometry: the one the result was produced with.
Add(s, Prose(fmt::format(
" This is the geometry the result was integrated at{}. BEAM_CENTRE is the PONI - the\n"
" foot of the perpendicular from the sample - and DIRECT_BEAM is where the beam actually\n"
" lands; on a tilted detector they are different points. DETECTOR_TILT is rot1/rot2/rot3\n"
" in degrees. JFJOCH_DATASET_SETTINGS is the same geometry as jfjoch_broker takes it,\n"
" ready to POST back for the next collection.",
result.pass_count > 1 ? ", which on a rotation run is the post-refined\n"
" geometry rather than the values in the input file"
: "")));
if (result.spot_resolution_estimate_A.has_value())
Add(s, Prose(" SPOT_RESOLUTION_ESTIMATE is how far the data were expected to reach, read off the\n"
" found spots alone - a prediction to about 0.2 A, not what the run achieved."));
// The long-form geometry material: correct, and none of it changes what a user does next.
if (result.refined_detector_tilt_deg)
Add(s, Prose(" REFINED_DETECTOR_TILT is rot1/rot2 in degrees as THIS pass's rotation indexing fitted\n"
" them, and is not what this pass integrated at: on a two-pass run the tilt above is the\n"
" FIRST pass's fit, and this is the second pass re-fitting it on its own better geometry.\n"
" Where the two agree the fit has settled; where they do not, that component is still\n"
" travelling and neither number is the end of it.\n"
" How much one sweep determines depends on how far its 2theta reaches. The tilt moves the\n"
" direct beam exactly as the beam centre does, so at small 2theta what a sweep determines\n"
" is the DIRECT_BEAM above and not the split between the two; what survives that alias\n"
" grows as the square of the scattering angle, and on a short-distance long-wavelength\n"
" sweep it is large enough that the fit reproduces a separately measured tilt to a few\n"
" hundredths of a degree. Where 2theta is ordinary, the MEDIAN of this value over\n"
" several crystals collected on the same detector - not one run's - does track\n"
" a calibration well enough to show up a placeholder or a stale tilt in the file, and it\n"
" does not replace the calibration.\n"
" rot3 is omitted because a rotation about the beam is an exact null of this experiment\n"
" and the fit cannot move it.", true));
if (result.pass_count > 1)
Add(s, Prose(fmt::format(
" A rotation run integrates twice: once at the geometry in the input file, then again at\n"
" the post-refined geometry. Every number in this report describes the pass named above,\n"
" whose files are {}.*; the header-geometry pass is not written.", output_prefix), true));
doc.sections.push_back(std::move(s));
}
// ------------------------------------------------------------- 2. CRYSTAL
{
ReportSection s;
s.title = "2. CRYSTAL";
if (result.indexing_rate.has_value())
Add(s, KeyReal("INDEXING_RATE", *result.indexing_rate, "{:.4f}"));
Add(s, KeyBool("LATTICE_FOUND", result.consensus_cell.has_value()));
if (result.consensus_cell.has_value())
Add(s, KeyText("UNIT_CELL_CONSTANTS", CellString(*result.consensus_cell)));
if (result.space_group.has_value()) {
Add(s, KeyInt("SPACE_GROUP_NUMBER", result.space_group->number));
// The number alone does not name the setting, and the search can return one that is not
// gemmi's reference setting - P 1 1 2_1 and P 1 2_1 1 are both number 4, different axes.
Add(s, KeyText("SPACE_GROUP_NAME", result.space_group->xhm()));
}
// The best Sohncke group for the same data, written on every run and not only where it differs.
// A crystal built from chiral molecules - any protein - has no glide plane and no inversion
// centre, so where the search names a group carrying one, this is the answer that crystal could
// actually have; where it does not, this repeats SPACE_GROUP_NAME. Both cases write the key,
// because a reader who knows their sample is a protein has to be able to grep one name rather
// than decide first whether the run found a glide.
if (result.space_group_search.has_value()
&& result.space_group_search->sohncke_space_group.has_value())
Add(s, KeyText("SOHNCKE_SPACE_GROUP",
result.space_group_search->sohncke_space_group->xhm()));
// This fired on every --mode scale run, falsely: that path never fills indexing_rate, because
// the lattice comes from the input file, and a missing optional read as a measured zero.
if (result.indexing_rate.has_value() && *result.indexing_rate <= 0.0f)
Warn(doc, PathologyCode::NO_LATTICE,
"No image indexed - no crystal lattice was determined from this dataset");
// What the data could NOT decide, beside what they did. SPACE_GROUP_NAME is a scalar and reads
// like a determination; where several groups predict the same absences it is one of them,
// chosen by convention, so the ambiguity has to survive the same grep.
std::string alts;
if (result.space_group_search.has_value() && result.space_group.has_value())
for (const auto &alt : result.space_group_search->alternatives)
if (alt.number != result.space_group->number)
alts += (alts.empty() ? "" : " | ") + alt.xhm();
Add(s, KeyText("SPACE_GROUP_ALTERNATIVES", alts.empty() ? "NONE" : alts));
// ...and WHICH axis is open, which the list of names above does not say. A screw the data
// could not test is not a near miss between two groups - it is a question nobody asked, and
// the group that gets written is the one claiming no screw, by convention. Say the axis, say
// why, and leave SPACE_GROUP_NAME and the written files exactly as they are: a reflection
// file cannot hold "maybe a screw".
std::string undet;
if (result.space_group_search.has_value())
for (const auto &u : result.space_group_search->undetermined_screws)
undet += (undet.empty() ? "" : " ") + std::string(1, u.axis);
Add(s, KeyText("SPACE_GROUP_SCREW_UNDETERMINED", undet.empty() ? "NONE" : undet,
undet.empty()));
if (!undet.empty() && result.space_group_search.has_value()) {
std::string why;
for (const auto &u : result.space_group_search->undetermined_screws)
why += fmt::format(
"\n {}: the {} row {}", u.axis, u.row_label,
u.n_observed == 0
? "holds no reflections in this merge at all, so a screw along this axis\n"
" was never tested - the row was not recorded (it can lie in the "
"spindle's blind\n cone, or outside the resolution range)."
: u.n_control == 0
? fmt::format("holds {} reflection(s) but no control class to judge their\n"
" strength against, so how weak they are says nothing.",
u.n_observed)
: fmt::format("does not separate the candidates ({} absent, {} control).",
u.n_absent, u.n_control));
Add(s, Prose(fmt::format(
" SPACE_GROUP_SCREW_UNDETERMINED names the axes whose SCREW these data could not\n"
" decide at all:{}\n"
" On those axes SPACE_GROUP_NAME is not an answer - the answer is it or any of\n"
" SPACE_GROUP_ALTERNATIVES, and no measurement here chooses. The lowest-numbered\n"
" group, the one claiming no screw, is what the .mtz/.cif/.hkl carry, because a\n"
" reflection file must name one group. To settle it, collect the missing row: a\n"
" different crystal orientation, or a sweep that reaches it.", why)));
}
// The hand is a separate question from the alternatives, decidable from the 22 groups' numbers
// alone. Written only where the group HAS an enantiomorphic partner: on every other group the
// answer is NOT_APPLICABLE, which answers a question nobody asked.
const bool enantiomorphic = result.space_group.has_value()
&& result.space_group->is_enantiomorphic();
if (!enantiomorphic)
Add(s, KeyEnum("SPACE_GROUP_ENANTIOMORPH", "NOT_APPLICABLE",
{"NOT_APPLICABLE", "UNDETERMINED", "GIVEN", "ASSUMED_FROM_MODEL"}, true));
if (enantiomorphic) {
const char *enantiomorph = "UNDETERMINED";
const auto &mv = result.model_validation;
// ASSUMED, not determined: nothing here measured the hand. A model that fits carries prior
// chemical knowledge these intensities do not - and cannot, since |Fcalc| is invariant
// under the change of hand - so taking its group is an assertion, not a measurement.
if (mv.has_value() && mv->ok && mv->model_fits
&& (mv->adopted_model_enantiomorph
|| mv->model_space_group_number == result.space_group->number))
enantiomorph = "ASSUMED_FROM_MODEL";
else if (!result.space_group_search.has_value())
enantiomorph = "GIVEN"; // -S, or a reference MTZ: the user's assertion
Add(s, KeyEnum("SPACE_GROUP_ENANTIOMORPH", enantiomorph,
{"UNDETERMINED", "GIVEN", "ASSUMED_FROM_MODEL"}));
}
// A higher point group whose operators the intensities confirmed and whose promotion was
// refused - written only when there was one, because a refusal is exactly the case where a
// user might want to try the higher group as well.
const bool refused = result.space_group_search.has_value()
&& !result.space_group_search->refused_point_group_hm.empty();
if (refused) {
Add(s, KeyText("SPACE_GROUP_REFUSED_POINT_GROUP",
result.space_group_search->refused_point_group_hm));
Add(s, KeyText("SPACE_GROUP_REFUSED_REASON", result.space_group_search->refused_reason));
} else {
Add(s, KeyText("SPACE_GROUP_REFUSED_POINT_GROUP", "NONE", true));
}
if (result.post_refine.has_value()) {
const auto &pr = *result.post_refine;
Add(s, KeyInt("POSTREFINE_EVENTS_USED", pr.events_used));
Add(s, KeyInt("POSTREFINE_OBS_USED", pr.obs_used));
Add(s, KeyBool("POSTREFINE_CELL_COMMITTED", pr.cell_refined));
Add(s, KeyBool("POSTREFINE_DETECTOR_COMMITTED", pr.detector_refined));
Add(s, KeyText("POSTREFINE_DISTANCE", fmt::format("{:.3f} -> {:.3f}", pr.distance_before_mm,
pr.distance_after_mm)));
// TRUE where the fit left the distance at the header because freeing it did not lower the
// held-out residual beyond the noise of that comparison - the data do not determine it.
Add(s, KeyBool("POSTREFINE_DISTANCE_HELD", pr.distance_held));
Add(s, KeyText("POSTREFINE_BEAM_CENTRE",
fmt::format("{:.2f} {:.2f} -> {:.2f} {:.2f}", pr.beam_x_before_px,
pr.beam_y_before_px, pr.beam_x_after_px, pr.beam_y_after_px)));
// Which test refused the fit, where one did. Every refusal leaves the same geometry - the
// header's - so without this the report cannot tell a fit that was thrown away from one that
// never wanted to move, and a refusal is the one case where the number worth reading is the
// one that was thrown away.
if (!pr.refused_reason.empty())
Add(s, KeyText("POSTREFINE_REFUSED", pr.refused_reason));
Add(s, KeyReal("GONIOMETER_ROTATION_SCALE", pr.rotation_scale, "{:.5f}"));
Add(s, KeyBool("GONIOMETER_ROTATION_SCALE_SUSPECT", pr.rotation_scale_suspect));
if (pr.rotation_scale_suspect)
Warn(doc, PathologyCode::GONIO_SCALE, fmt::format(
"The goniometer turned by a factor {:.5f} of the angles stored in the file - the "
"stage rotation looks mis-calibrated by {:+.2f}%. The correction was applied to "
"this run, but the fault is in the hardware and should be fixed there",
pr.rotation_scale, 100.0 * (pr.rotation_scale - 1.0)));
}
Add(s, KeyEnum("SPACE_GROUP_SEARCH",
result.space_group_search.has_value() ? "DE_NOVO"
: result.space_group.has_value() ? "FIXED" : "NONE",
{"DE_NOVO", "FIXED", "NONE"}));
Add(s, Blank());
if (result.space_group_search.has_value()) {
const auto &search = *result.space_group_search;
// The compact form, composed here from the search result rather than taken from its text
// helper, so the default report says what was decided and on what margin without the
// operator and candidate tables that justify it.
std::string line = fmt::format(" Point group {} from the intensity correlations",
search.point_group_hm);
// A bound the adopted group EXCEEDS is a gate that fired and was overridden - the point group
// was taken from the other search arm - and must not read as the margin of a pass.
if (std::isfinite(search.h_ratio))
line += search.h_ratio > search.h_ratio_bound
? fmt::format(";\n twin-law H ratio {:.2f} EXCEEDS its bound of {:.2f} - that gate fired "
"and was overridden", search.h_ratio, search.h_ratio_bound)
: fmt::format(";\n twin-law H ratio {:.2f} against a bound of {:.2f}",
search.h_ratio, search.h_ratio_bound);
// The best-operator R gate is the one that applies once two operators are confirmed, i.e.
// above order 2 (the first step out of P1 is judged on the noise floor instead).
if (search.point_group_order > 2 && std::isfinite(search.r_over_best)
&& search.r_over_best_bound > 0.0 && search.r_over_best > search.r_over_best_bound)
line += fmt::format(";\n added-operator R {:.2f}x the best operator EXCEEDS its bound of {:.2f}x - "
"that gate fired and was overridden", search.r_over_best,
search.r_over_best_bound);
line += ".\n The space group follows from the systematic absences.";
if (!alts.empty())
line += fmt::format("\n These data cannot separate it from {} - both are consistent with"
" every absence\n measured here.", alts);
Add(s, Prose(line));
if (enantiomorphic && !alts.empty() && alts.find('|') == std::string::npos)
Add(s, Prose(" The pair differs only in the hand of the screw axis, which merged intensities\n"
" cannot name. Molecular replacement settles it: one of the two refines and the\n"
" other does not - the normal outcome for a chiral group, not a failure of the\n"
" run."));
if (!result.space_group_arm_override.empty()) {
Add(s, KeyText("SPACE_GROUP_ARM_OVERRIDE", result.space_group_arm_override));
Add(s, Prose(fmt::format(" The two search merges disagreed and the higher point group was taken:\n {}.",
result.space_group_arm_override)));
}
if (!result.twin_zone_decision.empty()) {
Add(s, KeyText("TWIN_ZONE_DECISION", result.twin_zone_decision));
Add(s, Prose(" A twin gate refused this promotion, and the twin-immune zone of the operators it\n"
" adds decided it (reflections a twin law cannot touch: centric if the operators\n"
" are real, acentric if they are a twin law or a pseudo-symmetry; read less what\n"
" the acentric control shows the normalisation to add; margin 20 nats):\n"
" " + result.twin_zone_decision + "."));
}
if (refused)
Add(s, Prose(fmt::format(
" A promotion to {} was refused: {}.\n"
" If the higher symmetry is expected here, it is worth re-running with -S to force\n"
" it and comparing.", search.refused_point_group_hm, search.refused_reason)));
// The full evidence - operator correlations, the ranked candidate table, the screw
// conditions zone by zone - is what a person diagnosing a symmetry call needs and nothing
// a person deciding whether to recollect can act on.
Add(s, Prose("\n" + SearchSpaceGroupResultToText(search), true));
// A centering the data could not test must not read like one they confirmed. The group may
// still be right - the lattice metric says so - but nothing in these intensities backs it.
if (search.best_space_group.has_value())
for (const auto &c : search.candidates)
if (c.space_group.number == search.best_space_group->number && c.centering_untested)
Warn(doc, PathologyCode::CENTERING_UNTESTED, fmt::format(
"The {} centering of {} was NOT confirmed from these data: the crystal was "
"indexed and integrated on the primitive sub-cell, so the reflections a "
"{}-centred lattice extinguishes are not in this merge at all. It comes "
"from the lattice metric. The point group is confirmed from the "
"intensities; the centering is not",
search.best_space_group->centring_type(),
search.best_space_group->short_name(),
search.best_space_group->centring_type()));
} else if (result.space_group.has_value()) {
Add(s, Prose(" The space group was given, not determined here."));
} else {
Add(s, Prose(" No space group was determined."));
}
doc.sections.push_back(std::move(s));
}
// -------------------------------------------------------- 3. MERGED DATA
// Facts the SUMMARY needs, measured while this section is built.
bool unusable_merge = false;
bool completeness_narrower = false; // the signal range is coarser than what was written
CompletenessInRange completeness_fit;
std::optional<double> fitted_resolution;
{
ReportSection s;
s.title = "3. MERGED DATA";
if (!merged) {
Add(s, KeyEnum("MERGE", "NOT_PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
Add(s, Blank());
Add(s, Prose(fmt::format(
" No scaling or merging was performed on this run, so there are no merging statistics,\n"
" no error model and no sweep-quality diagnosis. The integrated reflections are in\n"
" {}_process.h5.", output_prefix)));
} else {
const auto &ms = result.merge_statistics;
const auto &o = ms.overall;
const double cc_target = experiment.GetScalingSettings().GetResolutionCCTarget();
Add(s, KeyEnum("MERGE", "PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
Add(s, KeyText("INCLUDE_RESOLUTION_RANGE", fmt::format("{:.3f} {:.3f}", o.d_max, o.d_min)));
// FITTED_RESOLUTION is presented by this report as "the number to quote", so it must not
// be quotable when the curve it was read off never behaved like a fall-off. A fit finer
// than the finest shell that still reaches the target asserts a resolution the shell
// table on this very page refutes.
const double finest_supported = FinestSupportedShell(ms, cc_target);
const double crossing_limit = CrossingShellLimit(ms, cc_target);
const double climbs_back = CCHalfClimbsBackAboveTarget(ms, cc_target);
std::string fit_suppressed;
// Only where there are shells to judge the fit against; with no table there is no
// evidence either way, and suppressing on no evidence is its own false claim.
if (!ms.shells.empty()) {
if (climbs_back > 0.0)
fit_suppressed = fmt::format(
"CC1/2 falls below the cut's target of {:.2f} and then climbs back above it"
" at {:.2f} A, so this curve is not a fall-off and no one resolution can be"
" read off it", cc_target, climbs_back);
else if (result.resolution_fit_A) {
if (crossing_limit == 0.0)
fit_suppressed = fmt::format(
"CC1/2 never reaches the cut's target of {:.2f} in any shell, so there is"
" no fall-off to fit", cc_target);
else if (*result.resolution_fit_A < crossing_limit - 1e-6)
fit_suppressed = fmt::format(
"the fit reads {:.2f} A, finer than the shell where CC1/2 falls below"
" {:.2f} ({:.2f} A), so the curve it was read off is not a fall-off",
*result.resolution_fit_A, cc_target, crossing_limit);
else
fitted_resolution = *result.resolution_fit_A;
}
}
if (fitted_resolution)
Add(s, KeyReal("FITTED_RESOLUTION", *fitted_resolution, "{:.2f}"));
Add(s, KeyBool("FRIEDELS_LAW", experiment.GetScalingSettings().GetMergeFriedel()));
Add(s, KeyInt("UNIQUE_REFLECTIONS", o.unique_reflections));
Add(s, KeyInt("TOTAL_OBSERVATIONS", o.total_observations));
// Outlier rejection drops observations from the merge AND from R_meas and the CC1/2
// half-sets, so a run that rejects too much scores BETTER on every other number here.
Add(s, KeyInt("OBSERVATIONS_REJECTED", ms.n_observations_rejected));
// One rule for every quantity: a run that did not measure it writes NO key, rather than
// the word "nan" or a zero that reads as a measured absence.
if (o.possible_unique_reflections > 0)
Add(s, KeyReal("COMPLETENESS",
100.0 * o.unique_reflections / o.possible_unique_reflections, "{:.1f}"));
if (o.unique_reflections > 0)
Add(s, KeyReal("MULTIPLICITY",
static_cast<double>(o.total_observations) / o.unique_reflections, "{:.2f}"));
if (std::isfinite(o.mean_i_over_sigma))
Add(s, KeyReal("I_OVER_SIGMA", o.mean_i_over_sigma, "{:.2f}"));
if (std::isfinite(o.r_meas))
Add(s, KeyReal("R_MEAS", o.r_meas, "{:.4f}"));
if (std::isfinite(o.cc_half))
Add(s, KeyReal("CC_HALF", o.cc_half, "{:.4f}"));
if (std::isfinite(o.abs_diff_over_sigma_anomalous))
Add(s, KeyReal("SIGANO", o.abs_diff_over_sigma_anomalous, "{:.3f}"));
if (std::isfinite(o.cc_anom))
Add(s, KeyReal("CC_ANOM", o.cc_anom, "{:.4f}"));
if (std::isfinite(ms.wilson_b))
Add(s, KeyReal("WILSON_B", ms.wilson_b, "{:.2f}"));
// The error model in XDS's convention, so the numbers are directly comparable with a
// CORRECT.LP. Kept in the default report because beamline monitoring scripts read them.
Add(s, KeyReal("ERROR_MODEL_A", result.error_model_a, "{:.4f}"));
Add(s, KeyReal("ERROR_MODEL_B", result.error_model_b, "{:.4e}"));
// A b that is zero within its error gives a 1/b that is the noise in b, not an I/sigma.
Add(s, KeyText("SCALING_ITERATIONS",
fmt::format("{} partials, {} fulls{}", result.scaling_iterations_partials,
result.scaling_iterations_fulls,
result.scaling_converged ? "" : " (NOT settled)"), true));
if (!result.scaling_converged || !result.search_scaling_converged)
Warn(doc, PathologyCode::SCALING_NOT_CONVERGED, fmt::format(
"the per-frame scales of {} were still moving when the iteration cap stopped "
"them, so everything read off that merge is read off an unsettled state "
"(--scaling-iterations raises the cap)",
!result.scaling_converged && !result.search_scaling_converged
? "the space-group search merge and the final merge"
: !result.scaling_converged ? "the final merge"
: "the space-group search merge"));
if (result.error_model_isa_resolved)
Add(s, KeyReal("ISA", result.error_model_isa, "{:.2f}"));
else
Add(s, KeyText("ISA", "undetermined"));
if (result.error_model_isa_resolved && result.error_model_isa_asymptotic > 0.0)
Add(s, KeyReal("ISA_ASYMPTOTIC", result.error_model_isa_asymptotic, "{:.2f}", true));
// Only when TRUE: FALSE is the ordinary case and says nothing.
if (result.has_reference)
Add(s, KeyBool("REFERENCE_DATA_USED", true));
else
Add(s, KeyBool("REFERENCE_DATA_USED", false, true));
const double signal_limit = fitted_resolution.value_or(
finest_supported > 0.0 ? finest_supported : o.d_min);
completeness_fit = CompletenessTo(ms, signal_limit);
// Worth PRINTING twice only when the two ranges are actually different. On a
// detector-limited crystal every shell still carries signal, so the range where the
// signal is IS the written range, and "82.4 % to 1.22 A; 82.4 % over the full written
// range" says one thing twice. The warning below is a different question and must
// still fire on a thin data set whose signal reaches the edge of what was written.
completeness_narrower = completeness_fit.measured && signal_limit > o.d_min + 1e-3;
Add(s, Blank());
if (!fit_suppressed.empty())
Add(s, Prose(fmt::format(
" No FITTED_RESOLUTION is given: {}.\n"
" INCLUDE_RESOLUTION_RANGE says what was WRITTEN, and the shell table below says\n"
" where the signal actually stops.", fit_suppressed)));
else if (fitted_resolution)
Add(s, Prose(fmt::format(
" INCLUDE_RESOLUTION_RANGE is the range the reflections were WRITTEN to;"
" FITTED_RESOLUTION\n is where the CC1/2 fall-off crosses {:.2f}, and is the number"
" to quote. The data are kept\n one shell past it on purpose: an included shell can"
" still be downweighted by refinement,\n while a truncated one cannot be put back.",
cc_target)));
if (completeness_narrower && o.possible_unique_reflections > 0)
Add(s, Prose(fmt::format(
" COMPLETENESS is over that whole written range. Within the fitted resolution it is\n"
" {:.1f}% to {:.2f} A - the lower overall figure is the generous cut diluting the\n"
" denominator with reciprocal space the detector only reaches in its corners.",
completeness_fit.percent, completeness_fit.d_min)));
if (std::isfinite(o.cc_anom))
Add(s, ProseDefaultOnly(
" CC_ANOM is the anomalous difference measured twice and correlated between the two\n"
" halves; it is what says whether there is a signal to phase on. A negative value is\n"
" a measurement, not an error. XDS's `Anomal Corr` is NOT this quantity."));
Add(s, Prose(" CC_ANOM is the anomalous difference measured twice - I(+)-I(-) from one half of the\n"
" observations against the same difference from the other half - and correlated over the\n"
" acentric pairs where both hands were measured at least twice. It is what says whether\n"
" there is an anomalous signal to phase on, and unlike SIGANO it is not a ratio against\n"
" the error model, so an optimistic sigma cannot inflate it. It agrees with AIMLESS's\n"
" CCanom and phenix.merging_statistics' cc_anom. XDS's CORRECT.LP has a column named\n"
" `Anomal Corr` which is NOT this quantity and reads considerably higher at low\n"
" resolution, so the two are not comparable.\n"
" A negative value is a measurement, not an error: on data with little anomalous signal\n"
" and around two observations per Bijvoet mate the statistic is unstable and goes\n"
" negative, in this program and in the others alike. Where no pair could be split in\n"
" both hands the quantity does not exist: the key is then absent here and the column is\n"
" a dash in the table below, which is not the same claim as a signal measured to be\n"
" zero.\n\n"
" ERROR_MODEL_A / ERROR_MODEL_B are in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2),\n"
" so ISA = 1/sqrt(a*b) means what CORRECT.LP's ISa means. ISA_ASYMPTOTIC, where present,\n"
" is the strong-reflection tier only.", true));
{
std::ostringstream shells;
shells << ms;
Add(s, Blank());
Add(s, Prose(shells.str()));
}
// The reference-range table (--report-resolution): the same kept reflections binned over
// a range given from outside, so the run can be read against another program's table at
// that program's range without having been RUN at it - which moves the cut, the symmetry
// decision and everything after them. Nothing past the run's own limit is in it: those
// shells are empty by the run's decision, REFRES_COMPLETENESS counts their reflections as
// missing, and REFRES_SHELLS_PAST_LIMIT says how many there are, so a consumer can tell
// "not merged" from a measured zero.
if (result.reference_merge_statistics) {
const auto &rs = *result.reference_merge_statistics;
const auto &ro = rs.overall;
const auto range = experiment.GetScalingSettings().GetReportResolutionRange_A();
const double d_max_req = range ? range->d_max : rs.shells.front().d_max;
const double d_min_req = range ? range->d_min : rs.shells.back().d_min;
int past = 0;
for (const auto &sh : rs.shells)
if (sh.d_max <= o.d_min)
++past;
Add(s, Blank());
Add(s, KeyText("REFRES_RANGE", fmt::format(
"{} {:.3f}", std::isfinite(d_max_req) ? fmt::format("{:.3f}", d_max_req) : "INF",
d_min_req)));
if (ro.d_min > 0.0f)
Add(s, KeyText("REFRES_MEASURED_RANGE", fmt::format("{:.3f} {:.3f}", ro.d_max, ro.d_min)));
Add(s, KeyInt("REFRES_SHELLS_PAST_LIMIT", past));
Add(s, KeyInt("REFRES_UNIQUE_REFLECTIONS", ro.unique_reflections));
Add(s, KeyInt("REFRES_TOTAL_OBSERVATIONS", ro.total_observations));
if (ro.possible_unique_reflections > 0)
Add(s, KeyReal("REFRES_COMPLETENESS",
100.0 * ro.unique_reflections / ro.possible_unique_reflections, "{:.1f}"));
if (ro.unique_reflections > 0) {
Add(s, KeyReal("REFRES_MULTIPLICITY",
static_cast<double>(ro.total_observations) / ro.unique_reflections, "{:.2f}"));
Add(s, KeyReal("REFRES_I_OVER_SIGMA", ro.mean_i_over_sigma, "{:.2f}"));
}
if (std::isfinite(ro.r_meas))
Add(s, KeyReal("REFRES_R_MEAS", ro.r_meas, "{:.4f}"));
if (std::isfinite(ro.cc_half))
Add(s, KeyReal("REFRES_CC_HALF", ro.cc_half, "{:.4f}"));
if (std::isfinite(ro.abs_diff_over_sigma_anomalous))
Add(s, KeyReal("REFRES_SIGANO", ro.abs_diff_over_sigma_anomalous, "{:.3f}"));
if (std::isfinite(ro.cc_anom))
Add(s, KeyReal("REFRES_CC_ANOM", ro.cc_anom, "{:.4f}"));
if (result.reference_isa > 0.0)
Add(s, KeyReal("REFRES_ISA", result.reference_isa, "{:.2f}"));
Add(s, Blank());
Add(s, Prose(fmt::format(
" REFRES_* is the same merged data binned over REFRES_RANGE, the reference range given\n"
" with --report-resolution, so it can be read against another program's table at that\n"
" program's range; nothing was processed differently for it. It holds only what the run\n"
" kept: {}{}",
past > 0
? fmt::format("the run's own limit is {:.2f} A (INCLUDE_RESOLUTION_RANGE), the {} shell{}\n"
" finer than it hold{} nothing because the run judged them to carry no signal and did\n"
" not merge them, REFRES_COMPLETENESS counts their reflections as missing, and every\n"
" other REFRES_ number is over the shells the run reached.",
o.d_min, past, past == 1 ? "" : "s", past == 1 ? "s" : "")
: std::string("the range lies within the run's own, so this is a subset of the table\n"
" above."),
result.reference_isa > 0.0
? "\n REFRES_ISA is the error model refitted on the reflections of this table alone, in\n"
" XDS's convention."
: "")));
Add(s, Prose(ReferenceShellTable(rs, o.d_min)));
}
// A merge with no signal in it. Neither of these is caught anywhere else: CC1/2 near zero
// beside a healthy I/sigma is the signature of observations of DIFFERENT reflections being
// averaged together (mixed indexing hands, a wrong point group), and a rejection count
// above the kept count is a scaling that threw the data away. Both leave every other
// number in this section looking ordinary.
const bool no_correlation = std::isfinite(o.cc_half) && o.cc_half < 0.5
&& std::isfinite(o.mean_i_over_sigma) && o.mean_i_over_sigma > 3.0;
const bool over_rejected = ms.n_observations_rejected > o.total_observations;
if (no_correlation || over_rejected) {
unusable_merge = true;
const std::string why = no_correlation
? fmt::format("CC1/2 is {:.3f} while I/sigma is {:.1f} - the observations correlate"
" with each other far worse than their own errors allow, which means"
" reflections that are not equivalent were averaged together",
o.cc_half, o.mean_i_over_sigma)
: fmt::format("{} observations were rejected against {} kept - scaling threw away"
" more than it merged", ms.n_observations_rejected,
o.total_observations);
Warn(doc, PathologyCode::UNUSABLE_MERGE,
"These merged data are not usable as they stand: " + why
+ ". The written reflection files carry the same problem, and refinement against "
"them is meaningless until the cause is established");
}
// Completeness measured where the signal is, not over the generous cut - so a
// corner-limited detector on good data does not fire it and a genuinely thin dataset does.
constexpr double MIN_COMPLETENESS_PERCENT = 80.0;
if (completeness_fit.measured && completeness_fit.percent < MIN_COMPLETENESS_PERCENT)
Warn(doc, PathologyCode::LOW_COMPLETENESS, fmt::format(
"Only {:.1f}% of the unique reflections to {:.2f} A were measured - the merge is "
"missing a large part of reciprocal space, which no amount of multiplicity in what "
"was measured makes up for. A wider sweep, or a second one about a different "
"axis, is what fills the gap", completeness_fit.percent, completeness_fit.d_min));
if (!fit_suppressed.empty() && !unusable_merge)
Warn(doc, PathologyCode::RESOLUTION_FIT,
"No resolution could be fitted: " + fit_suppressed
+ ". The shell table, not a single number, says where the signal stops");
}
doc.sections.push_back(std::move(s));
}
// -------------------------------------------------------- 4. DIAGNOSTICS
{
ReportSection s;
s.title = "4. DIAGNOSTICS";
// ---- higher-order contamination of the beam
// Read off the images rather than off the merge, so it is reported whether or not anything
// merged. Report-only: nothing downstream reads it, and no spot was treated differently for
// it. See HarmonicContamination.h for the test and for the one thing it cannot separate a
// contaminated beam from - a crystal whose true cell really is three times the one indexed.
{
const auto &hc = result.harmonic;
if (hc.measurable) {
Add(s, KeyEnum("HARMONIC_CONTAMINATION", hc.detected ? "TRUE" : "FALSE",
{"TRUE", "FALSE", "NOT_MEASURED"}));
Add(s, KeyReal("HARMONIC_SPOT_PCT", hc.fraction, "{:.1f}"));
Add(s, KeyReal("HARMONIC_NULL_PCT", hc.null_fraction, "{:.1f}"));
Add(s, KeyInt("HARMONIC_SPOTS_TESTED", hc.tested, true));
if (hc.detected)
Add(s, KeyReal("HARMONIC_MEAN_D", hc.mean_d_A, "{:.1f}"));
} else {
Add(s, KeyEnum("HARMONIC_CONTAMINATION", "NOT_MEASURED",
{"TRUE", "FALSE", "NOT_MEASURED"}));
}
Add(s, Blank());
if (hc.detected) {
Add(s, Prose(fmt::format(
" Higher-order contamination: INDICATED. Of the {} spots this run found and could\n"
" not put on the lattice, {:.0f}% are on that same lattice at one third of the stated\n"
" wavelength, against {:.1f}% for the same test at a scale no monochromator passes.\n"
" A Si(111) monochromator passes lambda/3 with lambda, so the crystal writes a second\n"
" diffraction pattern; it sits at {:.1f} A apparent resolution against {:.1f} A for the\n"
" indexed spots, which is where a harmonic puts it. Nothing here was excluded for it -\n"
" a spot on the second pattern that falls near a real reflection is integrated with it.",
hc.tested, hc.fraction, hc.null_fraction, hc.mean_d_A,
hc.mean_indexed_d_A)));
Warn(doc, PathologyCode::HARMONIC_CONTAMINATION,
fmt::format("The beam carries a higher harmonic: {:.0f}% of the unindexed spots are "
"lattice points at lambda/3, mostly around {:.1f} A apparent resolution",
hc.fraction, hc.mean_d_A));
}
}
// ---- powder contamination
// A crystalline phase other than the crystal, diffracting as rings among its reflections.
// Measured in the pre-scan on every run, from the spots found there for the width and the
// beam centre, and reported whatever it says - a user whose crystal sat in a powder should
// be told so whether or not it cost this run anything. Only hexagonal ice has rings that can
// be named in advance; POWDER_RINGS_A is what THIS sample showed.
{
const auto &pw = result.powder;
Add(s, KeyEnum("POWDER_RINGS_DETECTED", pw.rings_q_recipA.empty() ? "FALSE" : "TRUE",
{"TRUE", "FALSE"}));
if (!pw.rings_q_recipA.empty()) {
Add(s, KeyInt("POWDER_RING_COUNT", static_cast<int64_t>(pw.rings_q_recipA.size())));
Add(s, KeyReal("POWDER_SPOT_FRACTION", pw.spot_fraction, "{:.3f}"));
if (pw.resolved_to_d_A)
Add(s, KeyReal("POWDER_RINGS_SEPARABLE_TO", *pw.resolved_to_d_A, "{:.2f}"));
std::string ds;
for (const float q : pw.rings_q_recipA)
ds += fmt::format("{}{:.3f}", ds.empty() ? "" : " ", 6.283185307f / q);
Add(s, KeyText("POWDER_RINGS_A", ds));
Add(s, KeyEnum("POWDER_EXCLUDED_FROM_INDEXING",
result.powder_excluded_from_indexing ? "TRUE" : "FALSE",
{"TRUE", "FALSE"}));
if (result.powder_indexing_d_min_A)
Add(s, KeyReal("POWDER_INDEXING_D_MIN", *result.powder_indexing_d_min_A, "{:.2f}"));
Add(s, Blank());
Add(s, Prose(fmt::format(
" Powder contamination: {} rings, holding {:.0f}% of the spots the pre-scan found\n"
" above the smooth fall-off of spot density. That is a crystalline phase other than\n"
" this crystal - hexagonal ice, a shower of microcrystals, a salt out of the\n"
" cryoprotectant - and its reflections are recorded and integrated alongside the\n"
" crystal's own. {}",
pw.rings_q_recipA.size(), 100.0f * pw.spot_fraction,
result.powder_excluded_from_indexing
? "They were left out of indexing on this run."
: "Indexing did not need them left out, so nothing was excluded for them.")));
}
Add(s, Blank());
}
// ---- translational pseudo-symmetry
// Printed immediately before the twinning block, and in the same section, because the two
// interact: a pseudo-translation that is not a half-integer vector biases <|L|> upwards and
// so, left unstated, silently overrules the twin verdict below. They are deliberately NOT
// combined into one statistic - measured over a corpus of merged datasets the correlation
// between the Patterson peak and <I^2>/<I>^2 is 0.03, and the second moment in this pipeline
// runs above 2.0 for unrelated reasons.
if (merged) {
const auto &t = result.tncs;
if (!t.measurable) {
Add(s, KeyEnum("TNCS_DETECTED", "NOT_MEASURED",
{"TRUE", "FALSE", "INCONCLUSIVE", "NOT_MEASURED"}));
Add(s, Blank());
Add(s, Prose(fmt::format(
" Translational pseudo-symmetry: NOT MEASURED{}{}. A refusal is not a\n"
" statement that this crystal has none - the question was not answered.",
t.refusal.empty() ? "" : " - ", t.refusal)));
} else {
Add(s, KeyEnum("TNCS_DETECTED",
t.detected ? "TRUE"
: t.modulation_measured ? "FALSE"
: "INCONCLUSIVE",
{"TRUE", "FALSE", "INCONCLUSIVE", "NOT_MEASURED"}));
// A different finding from a pseudo-symmetry, and reported as one: the data are
// EXACTLY invariant under these, so the centring or the cell is wrong rather than the
// molecules being packed unusually.
for (const auto &v : t.undeclared_lattice_translations)
Add(s, KeyText("UNDECLARED_LATTICE_TRANSLATION",
fmt::format("{:.3f} {:.3f} {:.3f}", v[0], v[1], v[2])));
Add(s, KeyReal("TNCS_PATTERSON_PEAK_PCT", t.peak_percent, "{:.1f}"));
Add(s, KeyReal("TNCS_PATTERSON_PEAK_Z", t.peak_z, "{:.1f}"));
Add(s, KeyText("TNCS_PATTERSON_NULL_PCT",
fmt::format("{:.1f} +- {:.2f}", t.null_mean, t.null_sd)));
if (t.modulation_measured) {
Add(s, KeyText("TNCS_VECTOR",
fmt::format("{:.3f} {:.3f} {:.3f}", t.vector_frac[0],
t.vector_frac[1], t.vector_frac[2])));
Add(s, KeyReal("TNCS_VECTOR_LENGTH", t.vector_length_A, "{:.1f}"));
Add(s, KeyReal("TNCS_MODULATION", t.modulation, "{:.2f}"));
Add(s, KeyReal("TNCS_MODULATION_NULL", t.modulation_null, "{:.2f}"));
}
// Two different claims, deliberately not merged. PSEUDO_CENTRED says the vector is a
// rational translation of a cell that is not itself in doubt - the suppressed class is
// weak rather than absent, and a smaller cell would contradict it. SUBLATTICE is the
// one that says the cell may be a supercell, and only a near-extinct class earns it.
if (t.detected && t.commensurate)
Add(s, KeyText("TNCS_PSEUDO_CENTRED",
fmt::format("1/{}", t.commensurate_denominator)));
if (t.detected && t.near_extinct_class)
Add(s, KeyEnum("TNCS_SUBLATTICE", "NEAR_EXTINCT_CLASS", {"NEAR_EXTINCT_CLASS"}));
Add(s, KeyInt("TNCS_REFLECTIONS", t.n_reflections, true));
Add(s, Blank());
if (t.detected)
Add(s, Prose(fmt::format(
" Translational pseudo-symmetry: INDICATED. The native Patterson has an off-origin peak\n"
" at {:.1f}% of the origin at ({:.3f}, {:.3f}, {:.3f}), {:.1f} A, and the intensities\n"
" modulate {:.1f}x along that vector. The molecular-replacement program needs the vector\n"
" declared: the modulation is not in the search model, and without it the search can\n"
" fail on data that are otherwise good.",
t.peak_percent, t.vector_frac[0], t.vector_frac[1], t.vector_frac[2],
t.vector_length_A, t.modulation)));
else if (t.modulation_measured)
Add(s, Prose(fmt::format(
" Translational pseudo-symmetry: no indication. The largest off-origin Patterson peak is\n"
" {:.1f}% of the origin, against a permutation null of {:.1f} +- {:.2f}% for these data.",
t.peak_percent, t.null_mean, t.null_sd)));
else
Add(s, Prose(fmt::format(
" Translational pseudo-symmetry: INCONCLUSIVE. The Patterson was measured ({:.1f}% of the\n"
" origin, z = {:.1f}) but too few acentric reflections remain to test whether that vector\n"
" modulates the intensities, so only half the evidence exists. This is not a statement\n"
" that the crystal has no pseudo-symmetry.",
t.peak_percent, t.peak_z)));
if (t.detected && t.near_extinct_class)
Add(s, ProseDefaultOnly(
"\n The class this translation suppresses is not weak but very nearly EXTINCT, which is what\n"
" a lattice translation of a smaller cell looks like when it is measured in a larger one.\n"
" The reported cell is therefore a supercell candidate rather than a settled result,\n"
" and the rival hypothesis is a cell with this translation as a lattice vector."));
if (!t.undeclared_lattice_translations.empty()) {
const auto &v = t.undeclared_lattice_translations.front();
Add(s, ProseDefaultOnly(fmt::format(
"\n The merged data are EXACTLY invariant under ({:.3f}, {:.3f}, {:.3f}), which the space\n"
" group in use does not declare. A translation the data are exactly invariant under is a\n"
" lattice vector by definition, so it is the centring or the cell that is wrong, not the\n"
" packing. The pseudo-symmetry search continues underneath it, so the numbers above still\n"
" describe the crystal and not the missing centring.", v[0], v[1], v[2])));
}
Add(s, Prose("\n" + TranslationalNCSToText(t), true));
if (t.modulation_measured && !t.detected)
Add(s, Prose(
"\n TNCS_DETECTED= FALSE means the two tests disagreed with each other or neither fired, not\n"
" that a Patterson peak was absent: a peak is called only when it stands above THIS\n"
" dataset's own permutation noise floor (TNCS_PATTERSON_NULL_PCT, which runs from ~1% on a\n"
" large merge to ~18% on a small one) AND the vector it names is shown to modulate the\n"
" intensities. Either test alone over-calls by about a factor of two on a corpus of merged\n"
" datasets.", true));
if (t.detected)
Warn(doc, PathologyCode::PSEUDO_TRANSLATION, fmt::format(
"Translational pseudo-symmetry: Patterson off-origin peak {:.1f}% of the origin at "
"({:.3f}, {:.3f}, {:.3f}), {:.1f} A - the molecular-replacement program needs the "
"vector declared, or it will be searching against a modulation it does not model",
t.peak_percent, t.vector_frac[0], t.vector_frac[1], t.vector_frac[2],
t.vector_length_A));
if (!t.undeclared_lattice_translations.empty())
Warn(doc, PathologyCode::LATTICE_TRANSLATION, fmt::format(
"The merged data are exactly invariant under the translation ({:.3f}, {:.3f}, {:.3f}), "
"which the space group in use does not declare - the lattice centring or the unit cell "
"is wrong",
t.undeclared_lattice_translations[0][0],
t.undeclared_lattice_translations[0][1],
t.undeclared_lattice_translations[0][2]));
if (t.detected && t.near_extinct_class)
Warn(doc, PathologyCode::LATTICE_TRANSLATION, fmt::format(
"The pseudo-translation at {:.1f} A suppresses its phase class almost completely, so the "
"reported cell may be a supercell - it is not settled until the indexing has been re-examined",
t.vector_length_A));
}
}
// ---- twinning
if (merged && result.twinning.l_test_pairs > 0) {
const auto &tw = result.twinning;
Add(s, KeyBool("TWINNING_SUSPECTED", tw.twinning_suspected));
Add(s, KeyReal("L_TEST_MEAN_ABS_L", tw.mean_abs_l, "{:.4f}"));
// How the pseudo-symmetry above and this L-test interact, beside the number it qualifies:
// REPAIRED when partners sharing the pseudo-translation's class were available, UNREADABLE
// when none were and <|L|> is therefore not read in either direction.
Add(s, KeyEnum("L_TEST_VS_TNCS",
tw.l_test_contaminated_by_tncs ? "UNREADABLE"
: tw.l_test_tncs_step_restricted ? "REPAIRED"
: "UNAFFECTED",
{"UNAFFECTED", "REPAIRED", "UNREADABLE"}));
Add(s, KeyReal("SECOND_MOMENT_I", tw.second_moment, "{:.4f}"));
// One verdict, in the words every other place uses (TwinningVerdictLine): INDICATED is a twin
// in a class that admits one; SYMMETRY_SUSPECT is the same low reading in a holohedral class,
// where it says an adopted operator averages unequal intensities.
Add(s, KeyEnum("TWINNING_VERDICT",
tw.adopted_operators_suspect ? "SYMMETRY_SUSPECT"
: tw.twinning_suspected ? "INDICATED"
: "NO_INDICATION",
{"NO_INDICATION", "INDICATED", "SYMMETRY_SUSPECT"}));
// The fraction comes from the statistic that carries the verdict, and the key beside it says
// which. None is quoted where it cannot be measured: merged under a false operator, or from a
// second moment a pseudo-translation inflates.
if (tw.twin_fraction_source != TwinFractionSource::None)
Add(s, KeyReal("ESTIMATED_TWIN_FRACTION", tw.estimated_twin_fraction, "{:.3f}"));
Add(s, KeyEnum("ESTIMATED_TWIN_FRACTION_FROM",
tw.twin_fraction_source == TwinFractionSource::LTest ? "L_TEST"
: tw.twin_fraction_source == TwinFractionSource::SecondMoment ? "SECOND_MOMENT"
: "NONE",
{"SECOND_MOMENT", "L_TEST", "NONE"}));
Add(s, KeyReal("L_TEST_MEAN_L_SQUARED", tw.mean_l_squared, "{:.4f}", true));
if (result.pre_promotion_twinning) {
const auto &pre = *result.pre_promotion_twinning;
Add(s, KeyReal("L_TEST_MEAN_ABS_L_BEFORE_SEARCH", pre.mean_abs_l, "{:.4f}"));
Add(s, KeyReal("SECOND_MOMENT_I_BEFORE_SEARCH", pre.second_moment, "{:.4f}"));
Add(s, KeyEnum("L_TEST_BEFORE_SEARCH_VS_TNCS",
pre.l_test_contaminated_by_tncs ? "UNREADABLE"
: pre.l_test_tncs_step_restricted ? "REPAIRED"
: result.pre_promotion_tncs_measured ? "UNAFFECTED"
: "NOT_MEASURED",
{"UNAFFECTED", "REPAIRED", "UNREADABLE", "NOT_MEASURED"}));
}
Add(s, Blank());
Add(s, Prose(" Twinning: " + TwinningVerdictLine(tw) + "."));
if (tw.adopted_operators_suspect)
Add(s, Prose(fmt::format(
" The Laue class is holohedral, so no twin law exists within it - but merging under a\n"
" false operator gives exactly this distribution, whatever the twin fraction. {} Either\n"
" the crystal has lower symmetry or it is a twin whose law the point group absorbed;\n"
" refinement belongs in a subgroup.",
tw.laue_class_was_chosen_by_promotion
? "The class was chosen by the space-group search, so that promotion is suspect."
: "The space group in use is higher than these intensities support.")));
else if (tw.twinning_suspected)
Add(s, Prose(" Refinement against these data needs a twin law."));
else if (tw.l_test_contaminated_by_tncs)
// <|L|> is the number this sentence would otherwise rest on, and on this crystal it is
// biased by the pseudo-translation above.
Add(s, Prose(" The L-test, normally the stronger of the two statistics, could not be read here:\n"
" this crystal's translational pseudo-symmetry biases <|L|> and no partner reflection\n"
" available to the test escapes that bias. A weaker statement than usual."));
Add(s, Prose("\n" + TwinningAnalysisToText(tw), true));
if (result.pre_promotion_twinning) {
const auto &pre = *result.pre_promotion_twinning;
Add(s, Prose(fmt::format(
" Before the space-group search (P1 merge{}): {}.",
result.pre_promotion_tncs_measured ? "" : ", not corrected for a pseudo-translation",
TwinningVerdictLine(pre))));
Add(s, Prose(" The _BEFORE_SEARCH pair was measured on the merge the space-group search was given,\n"
" before any point group was adopted. The statistics above are computed in the Laue\n"
" class this run ADOPTED; where the search promoted the point group, a twin law it\n"
" absorbed can only show there as an operator averaging unequal intensities. These\n"
" numbers are not circular in that way.", true));
}
if (!result.twinning.twin_law_operator.empty()) {
Add(s, KeyText("TWIN_LAW", fmt::format("{} fraction {:.2f}", result.twinning.twin_law_operator,
result.twinning.twin_law_fraction)));
Add(s, Prose(fmt::format(" Twin law identified: operator {} at a fraction of about {:.2f} (a lower bound from\n"
" the operator's own disagreement). The mmCIF carries it as _pdbx_reflns_twin; refine\n"
" against these data in this space group with that law.",
result.twinning.twin_law_operator, result.twinning.twin_law_fraction)));
}
for (size_t i = 0; i < result.twin_law_evidence.size(); ++i)
Add(s, KeyText(fmt::format("TWIN_LAW_{}", i + 1).c_str(), result.twin_law_evidence[i]));
if (result.twin_immune_zones && !result.twin_immune_zones->zones.empty()) {
// Report-only evidence, one row per class of 2-folds of the adopted group. Not a decision:
// the space group above was chosen without it.
const auto &tz = *result.twin_immune_zones;
for (size_t i = 0; i < tz.zones.size(); ++i) {
const auto &z = tz.zones[i];
Add(s, KeyText(fmt::format("TWIN_ZONE_{}", i + 1).c_str(),
fmt::format("{}; n={} <|E2-1|>={:.3f}+-{:.3f} evidence={:+.1f} nats", z.operators,
z.n, z.mean_abs_e2_minus_1, z.standard_error, z.evidence_nats)));
}
Add(s, KeyText("TWIN_ZONE_CONTROL",
fmt::format("n={} <|E2-1|>={:.3f}+-{:.3f}", tz.control.n,
tz.control.mean_abs_e2_minus_1, tz.control.standard_error)));
Add(s, Prose("\n" + TwinImmuneZonesToText(tz)));
}
if (tw.adopted_operators_suspect)
Warn(doc, PathologyCode::TWINNING, fmt::format(
"An adopted symmetry operator relates unequal intensities (<|L|> = {:.3f} in a holohedral "
"Laue class, where a genuine operator reads about 0.5 and a false one 0.375): the "
"space group may be too high, or the crystal is a twin whose law the point group "
"absorbed - refine in a subgroup",
tw.mean_abs_l));
else if (tw.twinning_suspected)
Warn(doc, PathologyCode::TWINNING, fmt::format(
"Twinning is indicated ({}) - refinement against the merged data needs a twin law",
TwinningVerdictLine(tw)));
}
// ---- radiation damage
if (!result.radiation_damage_text.empty()) {
const double db = result.merge_statistics.radiation_damage_delta_b;
// A number, or a word saying why there is none: NOT_A_TREND where the per-batch curve was
// measured but no straight line describes it, NOT_MEASURED where the monitor could not run.
if (std::isfinite(db))
Add(s, KeyReal("RADIATION_DAMAGE_RELATIVE_B", db, "{:.2f}"));
else
Add(s, KeyEnum("RADIATION_DAMAGE_RELATIVE_B",
result.merge_statistics.radiation_damage_b_batch.empty()
? "NOT_MEASURED" : "NOT_A_TREND",
{"NOT_MEASURED", "NOT_A_TREND"}));
Add(s, Blank());
if (std::isfinite(db))
Add(s, Prose(fmt::format(
" Radiation damage: the relative B changes by {:+.2f} A^2 over the sweep{}.",
db, db < 2.0 ? ", which is negligible" : "")));
Add(s, Prose("\n" + result.radiation_damage_text, true));
}
// ---- sweep quality
{
const auto &sq = result.merge_statistics.sweep_quality;
const int n_sweep_frames = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected;
auto pct = [&](int n) { return n_sweep_frames > 0 ? 100.0 * n / n_sweep_frames : 0.0; };
// A delta-CC1/2 or its standard error that could not be measured prints as a dash - a
// range too thin to judge is not a range that was judged and came out at zero.
auto cc_cell = [](float v) {
return std::isfinite(v) ? fmt::format("{:+.4f}", v) : std::string("-");
};
auto se_cell = [](float v) {
return std::isfinite(v) ? fmt::format("{:.4f}", v) : std::string("-");
};
Add(s, KeyInt("SWEEP_QUALITY_COUNT", static_cast<int64_t>(sq.ranges.size())));
Add(s, KeyEnum("SWEEP_QUALITY_STATUS", sq.measured ? "COMPUTED" : "NOT_COMPUTED",
{"COMPUTED", "NOT_COMPUTED"}, true));
{
std::string codes;
for (int r = 0; r <= static_cast<int>(SweepQualityReason::InconsistentWithMerge); ++r)
codes += (codes.empty() ? "" : " ")
+ std::string(SweepQualityReasonCode(static_cast<SweepQualityReason>(r)));
Add(s, KeyText("SWEEP_QUALITY_REASONS", codes, true));
std::string dispositions;
for (int d = 0; d <= static_cast<int>(FrameDisposition::Rejected); ++d)
dispositions += (dispositions.empty() ? "" : " ")
+ std::string(FrameDispositionCode(static_cast<FrameDisposition>(d)));
Add(s, KeyText("SWEEP_DISPOSITIONS", dispositions, true));
}
if (sq.measured) {
Add(s, KeyInt("FRAMES_MERGED", sq.frames_merged));
Add(s, KeyInt("FRAMES_DOWNGRADED", sq.frames_downgraded));
Add(s, KeyInt("FRAMES_REJECTED", sq.frames_rejected));
Add(s, KeyReal("FRAMES_REJECTED_PCT", pct(sq.frames_rejected), "{:.2f}"));
Add(s, KeyReal("ROTATION_REJECTED_DEG", sq.rejected_deg, "{:.1f}"));
Add(s, KeyReal("SWEEP_ROTATION", sq.sweep_deg, "{:.1f}", true));
Add(s, KeyReal("FLUX_PEAK_TO_TROUGH", sq.flux_peak_to_trough, "{:.2f}", true));
Add(s, KeyReal("SCALE_MODULATION_PEAK_TO_TROUGH", sq.modulation_peak_to_trough,
"{:.2f}", true));
}
Add(s, Blank());
if (sq.measured)
// Frames AND degrees, because a percentage of frames can be moved by re-slicing the
// same experiment - 5% of 0.05 deg frames is not 5% of the experiment - and a percentage
// of the rotation cannot.
Add(s, Prose(fmt::format(
" Sweep: {} frames / {:.1f} deg; {} merged ({:.1f}%), {} downgraded ({:.1f}%),\n"
" {} rejected ({:.1f}% of frames, {:.1f} deg).",
n_sweep_frames, sq.sweep_deg,
sq.frames_merged, pct(sq.frames_merged),
sq.frames_downgraded, pct(sq.frames_downgraded),
sq.frames_rejected, pct(sq.frames_rejected), sq.rejected_deg)));
if (sq.ranges.empty()) {
// The empty table used to print its header and two rules around nothing on every
// clean run - a table that says "no problem" by being blank.
Add(s, Prose(sq.measured
? fmt::format(" Sweep: no degraded ranges over {:.1f} deg - the crystal delivered evenly.",
sq.sweep_deg)
: std::string(" Sweep quality was not measured on this run.")));
} else {
Add(s, Prose(" Stretches of the sweep over which the crystal delivered much less than the rest of\n"
" the run. SEVERITY is the fraction of the run's typical diffracting power missing over\n"
" the range; SCALE and CC are relative to the run median; INDEXED is the fraction of the\n"
" range's frames that were scaled at all. DELTA_CC_HALF is what keeping the range costs\n"
" the merged intensities - the overall CC1/2 with it minus the CC1/2 without it, over the\n"
" reflections it touches - beside the standard error of a CC1/2 on that many reflections.\n"
" A range is REJECTED only where that cost puts it well below the rest of this run's own\n"
" batches, is several standard errors below zero, AND every one of its frames is one the\n"
" SCALE and CC channels call worse than a typical frame of the run; everything else is\n"
" DOWNGRADED, which means it stays in the merge at the reduced weight its own scale and\n"
" sigmas give it. Weak but consistent data belong in the merge.\n"));
ReportEntry e;
e.kind = ReportEntry::Kind::Table;
e.table.columns = {"FIRST_IMAGE", "LAST_IMAGE", "N_IMAGES", "ROTATION", "REASON",
"SEVERITY", "SCALE", "CC", "INDEXED", "DISPOSITION",
"DELTA_CC_HALF", "DELTA_CC_HALF_SE"};
e.table.text_header =
" FIRST_IMAGE LAST_IMAGE N_IMAGES ROTATION REASON SEVERITY SCALE CC INDEXED DISPOSITION DELTA_CC_HALF DELTA_CC_HALF_SE\n"
" ----------- ----------- --------- -------- -------------------- -------- ------ ------ -------- ----------- ------------- ----------------";
for (const auto &r : sq.ranges) {
e.table.text_rows.push_back(fmt::format(
" {:11d} {:11d} {:9d} {:8.1f} {:<20} {:8.2f} {:6.2f} {:6.2f} {:8.2f} {:<11} {:>13} {:>16}",
r.first_image, r.last_image, r.last_image - r.first_image + 1,
r.rotation_deg, SweepQualityReasonCode(r.reason), r.severity,
r.mean_relative_scale, r.mean_relative_cc, r.indexed_fraction,
FrameDispositionCode(r.disposition), cc_cell(r.delta_cc_half),
se_cell(r.delta_cc_half_se)));
std::vector<ReportValue> row;
row.push_back(KeyInt("", r.first_image).value);
row.push_back(KeyInt("", r.last_image).value);
row.push_back(KeyInt("", r.last_image - r.first_image + 1).value);
row.push_back(KeyReal("", r.rotation_deg, "{:.1f}").value);
row.push_back(KeyText("", SweepQualityReasonCode(r.reason)).value);
row.push_back(KeyReal("", r.severity, "{:.2f}").value);
row.push_back(KeyReal("", r.mean_relative_scale, "{:.2f}").value);
row.push_back(KeyReal("", r.mean_relative_cc, "{:.2f}").value);
row.push_back(KeyReal("", r.indexed_fraction, "{:.2f}").value);
row.push_back(KeyText("", FrameDispositionCode(r.disposition)).value);
row.push_back(KeyText("", cc_cell(r.delta_cc_half)).value);
row.push_back(KeyText("", se_cell(r.delta_cc_half_se)).value);
e.table.cells.push_back(std::move(row));
const bool rejected = r.disposition == FrameDisposition::Rejected;
std::string cost;
if (!std::isfinite(r.delta_cc_half))
cost = rejected ? " - REJECTED"
: " - kept at their own reduced weight; too few reflections to "
"measure what they cost";
else if (rejected)
cost = fmt::format(" - REJECTED: keeping them lowered CC1/2 by {:.4f} +/- {:.4f} "
"over the reflections they touch",
-r.delta_cc_half, r.delta_cc_half_se);
else
cost = fmt::format(" - kept at their own reduced weight; removing them would change "
"CC1/2 by {:+.4f} +/- {:.4f} over the reflections they touch",
-r.delta_cc_half, r.delta_cc_half_se);
Warn(doc, PathologyCode::SWEEP_GAPS, fmt::format(
"Frames {}-{} {} ({:.1f} deg, scale {:.2f} and CC {:.2f} of the run, {:.0f}% scaled){}",
r.first_image, r.last_image, SweepQualityReasonText(r.reason),
r.rotation_deg, r.mean_relative_scale, r.mean_relative_cc,
100.0 * r.indexed_fraction, cost));
}
Add(s, std::move(e));
}
}
// ---- how the crystal sat on the spindle
if (result.spindle_lost_unique_fraction.has_value())
Add(s, KeyReal("SPINDLE_LOST_UNIQUE_FRACTION", *result.spindle_lost_unique_fraction, "{:.4f}"));
if (result.spindle_symmetry_axis_deg.has_value()) {
Add(s, KeyReal("SPINDLE_SYMMETRY_AXIS_ANGLE_DEG", *result.spindle_symmetry_axis_deg,
"{:.1f}", true));
Add(s, KeyInt("SPINDLE_SYMMETRY_AXIS_ORDER", result.spindle_symmetry_axis_order, true));
}
if (result.spindle_lost_unique_fraction.has_value()) {
Add(s, Blank());
Add(s, Prose(*result.spindle_lost_unique_fraction > 0.0
? fmt::format(" Mounting: the sweep's blind cone about the spindle costs {:.1f}% of the unique\n"
" reflections, which the measured point group cannot recover however long the\n"
" run.", 100.0 * *result.spindle_lost_unique_fraction)
: std::string(" Mounting: the crystal's symmetry axis was far enough off the spindle that the\n"
" blind cone costs no unique reflections.")));
}
// ---- anisotropy
const auto &an = result.merge_statistics.anisotropy;
if (merged && an.n_reflections > 0) {
Add(s, KeyEnum("ANISOTROPY_VERDICT", AnisotropyVerdictCode(an.verdict),
{"DETECTED", "NOT_DETECTED", "CANNOT_DETERMINE"}));
Add(s, KeyReal("ANISOTROPY_DELTA_B", an.delta_b, "{:.2f}"));
// The one rule again: a direction the fit could not limit prints "-", and a triplet
// with no measured direction at all writes no key - never the word "nan".
if (std::isfinite(an.d_min_axis[0]) || std::isfinite(an.d_min_axis[1])
|| std::isfinite(an.d_min_axis[2])) {
std::string principal;
for (int i = 0; i < 3; ++i)
principal += (i ? " " : "") + (std::isfinite(an.d_min_axis[i])
? fmt::format("{:.2f}", an.d_min_axis[i]) : std::string("-"));
Add(s, KeyText("ANISOTROPY_D_MIN_PRINCIPAL", principal));
}
if (std::isfinite(an.d_min_spread))
Add(s, KeyReal("ANISOTROPY_D_MIN_SPREAD", an.d_min_spread, "{:.2f}"));
// The gate's internals. Correct, and none of it changes what a user does next; the verdict
// key above already carries the decision they were computed to make.
Add(s, KeyInt("ANISOTROPY_FREE_DIRECTIONS", an.n_free_parameters, true));
Add(s, KeyReal("ANISOTROPY_DELTA_B_LINEAR", an.delta_b_linear, "{:.2f}", true));
Add(s, KeyText("ANISOTROPY_PRINCIPAL_B",
fmt::format("{:.2f} {:.2f} {:.2f}", an.eigenvalue[0] - an.eigenvalue[2],
an.eigenvalue[1] - an.eigenvalue[2], 0.0), true));
// Capped, because the uncapped value is the Debye-Waller factor of the fitted deltaB
// evaluated at the GLOBAL d_min - far beyond where the weak direction has any data at all
// - and printing six figures of it states something unphysical.
Add(s, KeyText("ANISOTROPY_FOLD_WEAKENING",
an.fold_weakening > 1000.0 ? std::string("> 1000")
: fmt::format("{:.1f}", an.fold_weakening), true));
Add(s, KeyText("ANISOTROPY_D_MIN_CENSORED",
fmt::format("{} {} {}", an.d_min_censored[0] ? 1 : 0,
an.d_min_censored[1] ? 1 : 0, an.d_min_censored[2] ? 1 : 0), true));
// Skipping the non-finite entries: a cone too sparse to cross the threshold leaves its
// limit NaN, and every comparison against NaN is false, so a bare min_element returns
// element 0 and prints nan even where the other two are measured.
const double *best_axis = nullptr;
for (const double *p = an.d_min_axis; p != an.d_min_axis + 3; ++p)
if (std::isfinite(*p) && (!best_axis || *p < *best_axis))
best_axis = p;
if (best_axis)
Add(s, KeyReal("ANISOTROPY_D_MIN_BEST", *best_axis, "{:.2f}", true));
Add(s, KeyText("ANISOTROPY_SHAPE", AnisotropyShapeCode(an.shape), true));
Add(s, KeyReal("ANISOTROPY_SHAPE_INTERCEPT", an.shape_intercept, "{:.3f}", true));
Add(s, KeyReal("ANISOTROPY_SHAPE_INTERCEPT_Z", an.shape_intercept_z, "{:.1f}", true));
Add(s, KeyReal("ANISOTROPY_SHAPE_SLOPE", an.shape_slope, "{:.2f}", true));
Add(s, KeyReal("ANISOTROPY_SHAPE_RESIDUAL", an.shape_residual, "{:.1f}", true));
Add(s, KeyInt("ANISOTROPY_N_OBSERVATIONS", an.n_observations, true));
Add(s, KeyReal("ANISOTROPY_SIGMA_SYSTEMATIC", an.sigma_systematic, "{:.3f}", true));
Add(s, KeyReal("ANISOTROPY_FORBIDDEN_Z", an.forbidden_z, "{:.1f}", true));
Add(s, KeyReal("ANISOTROPY_FLOOR", an.floor, "{:.3f}", true));
Add(s, KeyReal("ANISOTROPY_SIGNIFICANCE", an.significance, "{:.2f}", true));
Add(s, KeyReal("ANISOTROPY_DETECTION_LIMIT", an.detection_limit, "{:.2f}", true));
const double *worst = nullptr, *best = nullptr;
for (const double *p = an.d_min_axis; p != an.d_min_axis + 3; ++p) {
if (!std::isfinite(*p))
continue;
if (!worst || *p > *worst) worst = p;
if (!best || *p < *best) best = p;
}
const auto along = [&](const double *it) {
const int n = static_cast<int>(it - an.d_min_axis);
return result.consensus_cell
? ReciprocalAxisLabel(*result.consensus_cell, an.eigenvector[n])
: fmt::format("principal direction {}", n + 1);
};
// Severity in the one unit a user feels - how far apart the diffraction limits are - and
// never in the confidence of the detection. "DETECTED (strong)" meant statistically
// established and was read as severely anisotropic on every second dataset in the corpus.
const char *severity = an.d_min_spread < 0.25 ? "measurable but small"
: an.d_min_spread < 0.5 ? "noticeable"
: "STRONG";
Add(s, Blank());
if (an.verdict == AnisotropyVerdict::Detected && worst && best && worst != best) {
std::string p = fmt::format(
" Anisotropy: {}. Diffraction reaches {:.2f} A along {} and {:.2f} A along {}\n"
" (deltaB {:.1f} A^2).", severity, *best, along(best), *worst, along(worst),
an.delta_b);
// "Censored" is survival-analysis jargon for good news, and it read as an accusation.
// The whole content of the flag, in plain words:
const int worst_i = static_cast<int>(worst - an.d_min_axis);
if (an.d_min_censored[worst_i])
p += fmt::format("\n Along {} the crystal reaches AT LEAST {:.2f} A - the measured data"
" end before\n the signal does.", along(worst), *worst);
p += "\n Nothing was corrected or removed: the merged data and the written files do not\n"
" depend on direction at all.";
Add(s, Prose(p));
} else if (an.verdict == AnisotropyVerdict::Detected) {
Add(s, Prose(fmt::format(
" Anisotropy: {} (deltaB {:.1f} A^2). Nothing was corrected or removed: the merged\n"
" data do not depend on direction.", severity, an.delta_b)));
} else if (an.verdict == AnisotropyVerdict::CannotDetermine) {
Add(s, Prose(" Anisotropy: cannot be determined in this Laue class - there is no\n"
" symmetry-forbidden direction to calibrate the fit's own noise against."));
} else {
Add(s, Prose(" Anisotropy: below this data set's own noise floor. No directional statement can\n"
" be made, and the merged data do not depend on direction."));
}
Add(s, Prose("\n How much the fall-off depends on direction, and whether that is established above this\n"
" data set's own systematic error. ANISOTROPY_DELTA_B is the range of the principal\n"
" components of the anisotropy tensor, on the ordinary crystallographic B scale (the same\n"
" scale as phenix.xtriage's B_cart and ctruncate's anisotropic B), fitted on intensities\n"
" with nothing dropped; ANISOTROPY_SIGNIFICANCE gates ANISOTROPY_DELTA_B_LINEAR, the part\n"
" of it that follows exp(-1/2 s^T B s), which is not the same number. A 1 in\n"
" ANISOTROPY_D_MIN_CENSORED marks a direction whose limit is the edge of the measured\n"
" data rather than the crystal's own. ANISOTROPY_FOLD_WEAKENING is evaluated at the\n"
" GLOBAL d_min, which extrapolates past the weak direction's own limit, and is capped\n"
" at 1000 here for that reason.", true));
if (fitted_resolution && best_axis)
Add(s, Prose(fmt::format(
" Three resolutions, three questions. INCLUDE_RESOLUTION_RANGE ({:.2f} A) is what was\n"
" WRITTEN, deliberately one shell past the fit. FITTED_RESOLUTION ({:.2f} A) is where the\n"
" isotropic CC1/2 fall-off crosses its target. ANISOTROPY_D_MIN_BEST ({:.2f} A) is how far\n"
" the crystal reaches along its strongest direction. None of the three is wrong; quoting\n"
" one without saying which it is, is.",
result.merge_statistics.overall.d_min, *fitted_resolution, *best_axis), true));
if (an.verdict == AnisotropyVerdict::Detected && an.d_min_spread > 0.5) {
if (worst && best && worst != best)
Warn(doc, PathologyCode::ANISOTROPY, fmt::format(
"Diffraction is anisotropic (deltaB {:.1f} A^2; the diffraction limit is {:.2f} A "
"along {} and {:.2f} A along {}) - refinement and map interpretation should allow "
"for it; no intensity has been corrected for it here",
an.delta_b, *worst, along(worst), *best, along(best)));
else
Warn(doc, PathologyCode::ANISOTROPY, fmt::format(
"Diffraction is anisotropic (deltaB {:.1f} A^2) - refinement and map "
"interpretation should allow for it; no intensity has been corrected for it here",
an.delta_b));
}
}
doc.sections.push_back(std::move(s));
}
// -------------------------------------------------- 5. MODEL VALIDATION
if (result.model_validation.has_value()) {
const auto &mv = *result.model_validation;
ReportSection s;
s.title = "5. MODEL VALIDATION";
Add(s, KeyText("MODEL_FILE", mv.model_path));
if (!mv.ok) {
Add(s, KeyEnum("MODEL_VALIDATION", "NOT_PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
Add(s, KeyText("MODEL_VALIDATION_REASON", mv.failure_reason));
Add(s, Blank());
Add(s, Prose(" A model was given but could not be used, so there are no R-factors and no maps.\n"
" Everything else in this report is unaffected: the merge does not depend on the model."));
} else {
Add(s, KeyEnum("MODEL_VALIDATION", "PERFORMED", {"PERFORMED", "NOT_PERFORMED"}));
Add(s, KeyInt("MODEL_SPACE_GROUP_NUMBER", mv.model_space_group_number));
Add(s, KeyReal("R_WORK", mv.r_work, "{:.4f}"));
Add(s, KeyReal("R_FREE", mv.r_free, "{:.4f}"));
Add(s, KeyInt("R_WORK_REFLECTIONS", mv.n_work));
Add(s, KeyInt("R_FREE_REFLECTIONS", mv.n_free));
Add(s, KeyReal("BULK_SOLVENT_K_SOL", mv.k_sol, "{:.3f}", true));
Add(s, KeyReal("BULK_SOLVENT_B_SOL", mv.b_sol, "{:.1f}", true));
Add(s, KeyReal("SCALE_OVERALL", mv.k_overall, "{:.4f}", true));
// NOT_TESTED is a third answer and not a missing one: the model claimed nothing that
// needed arbitrating, so the null was not built.
Add(s, KeyEnum("MODEL_FIT",
!mv.fit_tested ? "NOT_TESTED" : (mv.model_fits ? "ACCEPTED" : "REJECTED"),
{"ACCEPTED", "REJECTED", "NOT_TESTED"}));
if (mv.fit_tested) {
Add(s, KeyText("MODEL_FIT_STATISTIC", "R_WORK", true));
Add(s, KeyReal("MODEL_FIT_VALUE", mv.r_work, "{:.4f}", true));
Add(s, KeyReal("MODEL_FIT_NULL_MEAN", mv.null_r_work_mean, "{:.4f}", true));
Add(s, KeyReal("MODEL_FIT_NULL_SD", mv.null_r_work_sd, "{:.4f}", true));
Add(s, KeyInt("MODEL_FIT_NULL_REPLICATES", mv.null_replicates, true));
Add(s, KeyReal("MODEL_FIT_SIGMA", mv.r_work_sigma, "{:+.2f}"));
}
Add(s, KeyEnum("RIGID_BODY", mv.rigid_body_applied ? "APPLIED" : "NOT_APPLIED",
{"APPLIED", "NOT_APPLIED"}));
if (mv.rigid_body_applied) {
Add(s, KeyReal("RIGID_BODY_ROTATION_DEG", mv.rigid_body_angle_deg, "{:.3f}"));
Add(s, KeyReal("RIGID_BODY_SHIFT_A", mv.rigid_body_shift_A, "{:.3f}"));
Add(s, KeyReal("R_FREE_BEFORE_RIGID_BODY", mv.r_free_before_rigid_body, "{:.4f}"));
}
Add(s, KeyText("MAP_COEFFICIENTS", "2mFo-DFc / mFo-DFc"));
Add(s, KeyInt("MAP_SIGMA_A_SHELLS", mv.sigma_a_shells, true));
Add(s, KeyReal("MAP_MEAN_FOM", mv.mean_fom, "{:.3f}"));
Add(s, KeyReal("MEAN_ATOM_DENSITY_SIGMA", mv.mean_atom_density_sigma, "{:.2f}"));
if (!mv.cc_model_shells.empty()) {
Add(s, KeyReal("CC_MODEL_OVERALL", mv.cc_model_overall, "{:.4f}"));
Add(s, KeyInt("CC_MODEL_REFLECTIONS", mv.cc_model_n));
// The finest shell in which the model confirms signal, at the significance XDS calls a
// shell's CC1/2 established at. A LOWER BOUND on the useful resolution and nothing else:
// shells past it are not thereby empty, only unconfirmed by this model.
constexpr double CC_MODEL_SIGNIFICANT = 3.0;
float confirmed = 0.0f;
for (const auto &sh : mv.cc_model_shells)
if (sh.sigma >= CC_MODEL_SIGNIFICANT)
confirmed = sh.d_min;
if (confirmed > 0.0f)
Add(s, KeyReal("CC_MODEL_CONFIRMED_TO_D_MIN", confirmed, "{:.2f}"));
else
Add(s, KeyEnum("CC_MODEL_CONFIRMED_TO_D_MIN", "NONE", {"NONE"}));
}
if (!mv.anomalous_sites.empty()) {
Add(s, KeyInt("ANOMALOUS_BIJVOET_PAIRS", mv.anomalous_pairs));
for (size_t i = 0; i < mv.anomalous_sites.size(); i++)
// Two digits so the ten keys are the same width and the values line up.
Add(s, KeyText(fmt::format("ANOMALOUS_SITE_{:02}", i + 1).c_str(),
fmt::format("{:<18} {:6.2f} sigma", mv.anomalous_sites[i].label,
mv.anomalous_sites[i].sigma)));
}
const bool took_indexing = !(mv.indexing_op == gemmi::Op::identity());
Add(s, KeyEnum("MODEL_DECISIONS_TAKEN",
mv.adopted_model_enantiomorph
? (took_indexing ? "ENANTIOMORPH+INDEXING" : "ENANTIOMORPH")
: (took_indexing ? "INDEXING" : "NONE"),
{"NONE", "INDEXING", "ENANTIOMORPH", "ENANTIOMORPH+INDEXING"}));
Add(s, KeyBool("MODEL_ENANTIOMORPH_ADOPTED", mv.adopted_model_enantiomorph));
Add(s, KeyText("MODEL_INDEXING_OPERATOR", mv.indexing_op.triplet()));
if (!(mv.change_of_basis_op == gemmi::Op::identity())) {
Add(s, KeyText("MODEL_CHANGE_OF_BASIS", mv.change_of_basis_op.triplet()));
Add(s, KeyText("MODEL_SETTING_AS_READ", mv.setting_as_read));
}
if (mv.indexing_probed) {
Add(s, KeyReal("MODEL_INDEXING_MARGIN", mv.indexing_margin, "{:.4f}", true));
Add(s, KeyText("MODEL_INDEXING_MARGIN_NULL",
fmt::format("{:.4f} +- {:.4f}", mv.indexing_margin_null_mean,
mv.indexing_margin_null_sd), true));
Add(s, KeyReal("MODEL_INDEXING_MARGIN_SIGMA", mv.indexing_margin_sigma, "{:+.2f}", true));
}
if (!mv.maps_prefix.empty())
Add(s, KeyText("MAPS_PREFIX", mv.maps_prefix));
if (!mv.cc_model_shells.empty()) {
Add(s, Blank());
Add(s, Prose(" CC(model, data): the correlation of the merged intensities with the placed, scaled\n"
" model's |F_model|^2, on the same shells the merge table above reports, so a row here\n"
" can be read straight across from that shell's CC1/2. SIGMA is how far the correlation\n"
" sits above zero (Fisher's transform on N reflections).\n"));
ReportEntry e;
e.kind = ReportEntry::Kind::Table;
e.table.columns = {"D_MIN", "CC_MODEL", "N", "SIGMA"};
e.table.text_header = " D_MIN CC_MODEL N SIGMA\n"
" -------- -------- --------- --------";
for (const auto &sh : mv.cc_model_shells) {
if (sh.n == 0)
continue;
e.table.text_rows.push_back(fmt::format(
" {:8.2f} {:8.4f} {:9d} {:>8s}", sh.d_min, sh.cc, sh.n,
std::isfinite(sh.sigma) ? fmt::format("{:+.1f}", sh.sigma) : std::string("-")));
std::vector<ReportValue> row;
row.push_back(KeyReal("", sh.d_min, "{:.2f}").value);
row.push_back(KeyReal("", sh.cc, "{:.4f}").value);
row.push_back(KeyInt("", sh.n).value);
row.push_back(KeyReal("", sh.sigma, "{:+.1f}").value);
e.table.cells.push_back(std::move(row));
}
Add(s, std::move(e));
Add(s, Prose("\n"
" The table reads in ONE direction only. A shell whose correlation is significantly\n"
" above zero carries signal, because a model cannot agree by accident with measurements\n"
" it was never fitted to, so CC_MODEL_CONFIRMED_TO_D_MIN is a lower bound on the useful\n"
" resolution and an argument for keeping MORE data. A shell whose correlation is near\n"
" zero says nothing about the data: the model may be incomplete, in the wrong hand or\n"
" simply wrong for this crystal, and cutting data on it would be cutting because the\n"
" model is poor. Nothing in the pipeline acts on these numbers."));
}
Add(s, Blank());
Add(s, Prose(" R-free here measures the merged intensities against an external structure, which is what\n"
" CC1/2 and R_meas cannot do - they only measure the data against themselves. The model is\n"
" not refined, only scaled and placed, so these R-factors are higher than a refined\n"
" structure's and are a data-quality reading, not a refinement result. The scale and the\n"
" placement are both fitted on the working reflections alone, so the free set is held out\n"
" of the fit and not only out of the sum."));
if (mv.fit_tested)
Add(s, Prose(fmt::format(
" MODEL_FIT says whether these data accept the model: the same model was refitted from\n"
" {} random orientations, and MODEL_FIT_SIGMA ({:+.2f}) is how far the real fit sits\n"
" above that null.", mv.null_replicates, mv.r_work_sigma)));
else
Add(s, Prose(" MODEL_FIT= NOT_TESTED is not a failed test: the model asserted neither another hand\n"
" nor another indexing, so there was nothing to arbitrate and the null was not built.\n"
" The R-factors, the maps and the rigid-body placement are exactly what they would\n"
" have been."));
if (mv.fit_tested && !mv.model_fits)
Add(s, Prose(fmt::format(
" The model was tried and REJECTED - R-work {:.4f} against {:.4f} +- {:.4f} for the same\n"
" model in random orientations, which is no better than chance. Nothing downstream moved:\n"
" the reflection files are byte for byte what a run with no model would have written.",
mv.r_work, mv.null_r_work_mean, mv.null_r_work_sd)));
if (!mv.anomalous_sites.empty())
Add(s, Prose(" The anomalous sites are the highest peaks of the anomalous difference map, read at\n"
" the model's own atom centres. A dataset with no anomalous signal still lists ten\n"
" sites; it is their height, a few sigma at most, that says so."));
if (took_indexing)
Add(s, Prose(" The written reflections were reindexed into the model's frame - the operator above\n"
" says how - so the reflection files, the R-factors and the maps all describe one\n"
" indexing."));
if (mv.adopted_model_enantiomorph)
Add(s, Prose(" The written reflections carry the model's enantiomorph as their space group. That is\n"
" a change of label and nothing else: the two groups of an enantiomorphic pair have\n"
" the same rotation operations, so no reflection moved."));
else if (mv.model_enantiomorph_candidate)
Add(s, Prose(" The model asserts the other enantiomorph of the written space group, and that\n"
" assertion was NOT taken up: merged intensities cannot check it, and the anomalous\n"
" difference map is the only measurement here sensitive to the hand."));
if (mv.anomalous_hands_disagree)
Warn(doc, PathologyCode::MODEL_HAND, fmt::format(
"The anomalous density at the model's atoms is inverted ({} reads {:.1f} sigma, deeper "
"than the highest peak) - the data and the model are in opposite hands. Either the model "
"is the wrong enantiomorph for this crystal or the data were indexed in the wrong hand; "
"the reflections have not been reindexed, which would have hidden which of the two it is",
mv.anomalous_deepest_site, mv.anomalous_deepest_sigma));
}
doc.sections.push_back(std::move(s));
}
if (result.cancelled)
Warn(doc, PathologyCode::CANCELLED,
fmt::format("Processing was cancelled after {} images - this report describes an "
"incomplete run", result.images_processed));
// ---------------------------------------------------------------- SUMMARY
// Composed last and written first. Everything it says was decided by the sections above; this is
// the decide-late step a writer that printed as it went could not take at all.
{
for (const auto &w : doc.warnings)
if (!w.developer_only
&& std::find(doc.pathology_flags.begin(), doc.pathology_flags.end(), w.code)
== doc.pathology_flags.end())
doc.pathology_flags.push_back(w.code);
const bool user_warnings = std::any_of(doc.warnings.begin(), doc.warnings.end(),
[](const ReportWarning &w) { return !w.developer_only; });
const bool no_lattice = !result.consensus_cell.has_value();
if (no_lattice || result.cancelled)
doc.verdict = "FAILED";
else if (unusable_merge)
doc.verdict = "UNUSABLE";
else if (user_warnings)
doc.verdict = "WARNINGS";
else
doc.verdict = "OK";
const auto &o = result.merge_statistics.overall;
const std::string group = result.space_group ? result.space_group->xhm() : std::string("P 1");
if (no_lattice)
doc.verdict_text = fmt::format(
"No crystal lattice was determined from {} images; nothing was integrated or merged.",
result.images_processed);
else if (result.cancelled)
doc.verdict_text = fmt::format(
"The run was cancelled after {} images - these results describe an incomplete sweep.",
result.images_processed);
else if (!merged)
doc.verdict_text = fmt::format(
"Integrated in {}; no merge was performed, so the reflections are in {}_process.h5.",
group, output_prefix);
else {
doc.verdict_text = fmt::format("Merged to {:.2f} A in {}", o.d_min, group);
std::string alt;
if (result.space_group_search.has_value() && result.space_group.has_value())
for (const auto &a : result.space_group_search->alternatives)
if (a.number != result.space_group->number)
alt += (alt.empty() ? "" : " or ") + a.xhm();
if (!alt.empty())
doc.verdict_text += fmt::format(" (or {}, which these data cannot separate from it)", alt);
doc.verdict_text += ".";
if (unusable_merge)
doc.verdict_text += " The merged data carry no usable signal - the warnings say why,"
" and the written files carry the same problem.";
else if (!doc.pathology_flags.empty()) {
std::string f;
for (const auto &c : doc.pathology_flags)
f += (f.empty() ? "" : ", ") + c;
doc.verdict_text += fmt::format(" {} condition{} attention: {}.",
doc.pathology_flags.size(),
doc.pathology_flags.size() == 1 ? " needs" : "s need",
f);
} else {
doc.verdict_text += " No warnings.";
}
}
ReportSection s;
s.title = "SUMMARY";
Add(s, KeyEnum("VERDICT", doc.verdict, {"OK", "WARNINGS", "UNUSABLE", "FAILED"}));
Add(s, KeyText("VERDICT_TEXT", doc.verdict_text));
Add(s, Blank());
// The facts a reader needs before deciding whether to read further, one line each.
{
std::string f;
const auto row = [&](const char *label, const std::string &value) {
if (!value.empty())
f += fmt::format(" {:<18} {}\n", label, value);
};
if (result.space_group)
row("Space group", group);
if (result.consensus_cell)
row("Unit cell", CellString(*result.consensus_cell));
if (result.harmonic.detected)
row("Beam", fmt::format("higher harmonic INDICATED ({:.0f} % of the unindexed spots "
"are lattice points at lambda/3)", result.harmonic.fraction));
if (merged) {
std::string res = fmt::format("{:.2f} A written", o.d_min);
if (fitted_resolution)
res += fmt::format("; the CC1/2 fall-off is fitted at {:.2f} A", *fitted_resolution);
row("Resolution", res);
if (o.possible_unique_reflections > 0) {
std::string c = fmt::format("{:.1f} % over the full written range",
100.0 * o.unique_reflections / o.possible_unique_reflections);
if (completeness_narrower)
c = fmt::format("{:.1f} % to {:.2f} A; {}", completeness_fit.percent,
completeness_fit.d_min, c);
row("Completeness", c);
}
std::string sig;
if (std::isfinite(o.mean_i_over_sigma))
sig += fmt::format("I/sigma {:.1f}", o.mean_i_over_sigma);
if (std::isfinite(o.r_meas))
sig += fmt::format(" R_meas {:.1f} %", 100.0 * o.r_meas);
if (std::isfinite(o.cc_half))
sig += fmt::format(" CC1/2 {:.3f}", o.cc_half);
if (!result.error_model_isa_resolved)
sig += " ISa undetermined";
else if (result.error_model_isa > 0.0)
sig += fmt::format(" ISa {:.1f}", result.error_model_isa);
row("Signal", sig);
if (std::isfinite(o.cc_anom))
row("Anomalous", o.cc_anom > 0.15
? fmt::format("signal present (CC_anom {:.2f})", o.cc_anom)
: fmt::format("no usable signal (CC_anom {:.2f})", o.cc_anom));
const auto &an = result.merge_statistics.anisotropy;
if (an.n_reflections > 0) {
if (an.verdict != AnisotropyVerdict::Detected)
row("Anisotropy", "not established above this data set's own noise");
else {
const double *lo = nullptr, *hi = nullptr;
for (const double *p = an.d_min_axis; p != an.d_min_axis + 3; ++p) {
if (!std::isfinite(*p))
continue;
if (!lo || *p < *lo) lo = p;
if (!hi || *p > *hi) hi = p;
}
const char *sev = an.d_min_spread < 0.25 ? "small"
: an.d_min_spread < 0.5 ? "noticeable" : "STRONG";
row("Anisotropy", lo && hi
? fmt::format("{}: deltaB {:.1f} A^2, d_min {:.2f}-{:.2f} A by direction",
sev, an.delta_b, *lo, *hi)
: fmt::format("{}: deltaB {:.1f} A^2", sev, an.delta_b));
}
}
if (result.tncs.measurable)
row("Pseudo-symmetry", result.tncs.detected
? fmt::format("INDICATED (Patterson peak {:.1f}% of origin, {:.1f} A)",
result.tncs.peak_percent, result.tncs.vector_length_A)
: std::string("no indication"));
if (result.twinning.l_test_pairs > 0)
row("Twinning", TwinningVerdictLine(result.twinning));
const double db = result.merge_statistics.radiation_damage_delta_b;
if (std::isfinite(db))
row("Radiation damage", fmt::format("relative B {:+.2f} A^2 over the sweep", db));
const auto &sq = result.merge_statistics.sweep_quality;
if (sq.measured) {
const int n_sweep = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected;
row("Sweep", sq.frames_rejected > 0
? fmt::format("{} of {} frames rejected ({:.1f} deg of {:.1f}), "
"{} degraded range(s)", sq.frames_rejected, n_sweep,
sq.rejected_deg, sq.sweep_deg, sq.ranges.size())
: (sq.ranges.empty()
? fmt::format("no degraded ranges in {:.1f} deg", sq.sweep_deg)
: fmt::format("{} degraded range(s) in {:.1f} deg, none rejected",
sq.ranges.size(), sq.sweep_deg)));
}
}
if (!f.empty()) {
f.pop_back();
Add(s, Prose(f));
Add(s, Blank());
}
}
Add(s, KeyInt("WARNING_COUNT", static_cast<int64_t>(
std::count_if(doc.warnings.begin(), doc.warnings.end(),
[](const ReportWarning &w) { return !w.developer_only; }))));
{
std::string flags;
for (const auto &c : doc.pathology_flags)
flags += (flags.empty() ? "" : " ") + c;
Add(s, KeyText("PATHOLOGY_FLAGS", flags.empty() ? "NONE" : flags));
}
doc.sections.insert(doc.sections.begin() + 1, std::move(s));
}
return doc;
}
// -------------------------------------------------------------------------- the text rendering
// Reads only from the document. Nothing here decides what the report SAYS; it decides only how the
// text file looks, which is what makes a second rendering a second function rather than a fork.
std::string RenderReportText(const ReportDocument &doc, bool developer) {
std::ostringstream os;
// Everything that is not a `KEY= value` line and not blank is written as a `#` comment, so a
// consumer that drops the `#` lines is left with the data alone. Applied here, on whole
// rendered blocks, so no emitter can forget it.
const auto comment = [&os](const std::string &text) {
size_t start = 0;
while (true) {
const size_t end = text.find('\n', start);
const std::string line = text.substr(
start, end == std::string::npos ? std::string::npos : end - start);
if (line.empty())
os << "\n";
else
os << (line[0] == ' ' ? "#" : "# ") << line << "\n";
if (end == std::string::npos)
break;
start = end + 1;
}
};
comment(BANNER);
comment(" RUGNUX PROCESSING REPORT");
comment(BANNER);
os << "\n";
// Authorship, licence and the acknowledgement open the report rather than close it: a reader
// who stops after the first screen has still seen who wrote this, what they may do with it,
// and whose methods it implements.
comment(RUGNUX_AUTHORSHIP);
os << "\n";
comment(RUGNUX_ACKNOWLEDGEMENT);
os << "\n";
for (const auto &section : doc.sections) {
if (section.developer_only && !developer)
continue;
// A section whose every entry is developer-only would otherwise print an empty banner.
const bool any = std::any_of(section.entries.begin(), section.entries.end(),
[&](const ReportEntry &e) {
return developer ? !e.default_only : !e.developer_only;
});
if (!any)
continue;
if (!section.title.empty()) {
os << "\n";
comment(BANNER);
comment(" " + section.title);
comment(BANNER);
os << "\n";
}
for (const auto &e : section.entries) {
if ((e.developer_only && !developer) || (e.default_only && developer))
continue;
switch (e.kind) {
case ReportEntry::Kind::Key:
os << e.key << "= " << e.value.text << "\n";
break;
case ReportEntry::Kind::Prose:
comment(e.prose);
break;
case ReportEntry::Kind::Blank:
os << "\n";
break;
case ReportEntry::Kind::Table:
comment(e.table.text_header);
for (const auto &r : e.table.text_rows)
comment(r);
break;
}
}
// The warnings belong to the SUMMARY, which is where a reader looks first.
if (section.title == "SUMMARY") {
os << "\n";
size_t shown = 0;
for (const auto &w : doc.warnings) {
if (w.developer_only && !developer)
continue;
comment("WARNING: " + w.text);
++shown;
}
if (shown == 0)
comment(" (none)");
}
}
os << "\n";
comment(BANNER);
comment(" END OF REPORT");
comment(BANNER);
return os.str();
}
std::string RenderResultReport(const std::string &output_prefix,
const std::string &input_file,
const DiffractionExperiment &experiment,
const ProcessResult &result,
const RunProvenance &provenance) {
return RenderReportText(
BuildReportDocument(output_prefix, input_file, experiment, result, provenance),
provenance.developer);
}
void WriteResultReport(const std::string &output_prefix,
const std::string &input_file,
const DiffractionExperiment &experiment,
const ProcessResult &result,
Logger &logger,
const RunProvenance &provenance) {
if (output_prefix.empty())
return; // "compute the statistics, persist nothing"
const std::string filename = output_prefix + "_report.txt";
// The report is unconditional, so it must never be the reason a run fails: a run that produced a
// good .mtz must survive an unwritable path or a full disk. Report the failure and carry on.
try {
std::ofstream file(filename);
file.exceptions(std::ios::failbit | std::ios::badbit);
file << RenderResultReport(output_prefix, input_file, experiment, result, provenance);
} catch (const std::exception &e) {
logger.Warning("Could not write the results report {}: {}", filename, e.what());
}
}
Reference in New Issue View Git Blame Copy Permalink