Model validation: R-free by resolution shell on the merge table's shells
The placed, scaled model's R-free is now also formed per shell of the merge statistics (report_shell_d_min), over the free reflections of each shell and with the same Fmodel the overall R-free uses. The report's CC(model, data) table gains an R_FREE column (a dash where a shell has no free reflection). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
This commit is contained in:
1 parent
f4ee70ea18
commit
172235e06c
6 files changed
+42
-8
No files matched your search
@@ -1247,7 +1247,7 @@ Both sides are divided by the reflection's symmetry enhancement ε, which they s
|
||||
|---|---|
|
||||
| `CC_MODEL_OVERALL`, `CC_MODEL_REFLECTIONS` | The correlation over every reflection in the table, and how many — the `N` column sums to it. Like any overall correlation it is shell-weighted and can take any value between the best shell and the worst; the table is what to read |
|
||||
| `CC_MODEL_CONFIRMED_TO_D_MIN` | The finest shell that, together with the shell coarser than it, reaches 3 σ, or `NONE`. A **lower bound** on the useful resolution |
|
||||
| the `D_MIN / CC_MODEL / N / CC_NULL / SIGMA` table | Per shell: the correlation, the reflections it was formed on, its null, and how far above the null it sits |
|
||||
| the `D_MIN / CC_MODEL / N / CC_NULL / SIGMA / I/I_MODEL / R_FREE` table | Per shell: the correlation, the reflections it was formed on, its null, how far above the null it sits, the mean intensity over the scaled model's, and the shell's part of `R_FREE` - the same placed, scaled model over the free reflections in the shell (`-` where the shell has none) |
|
||||
|
||||
**The null.** A correlation above zero is not evidence on its own. The data and a scaled model share
|
||||
everything that is smooth in reciprocal space — the falloff across a shell, the anisotropic B (fitted
|
||||
|
||||
@@ -1682,6 +1682,20 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
|
||||
overall_cc.Add(I, Ic / eps);
|
||||
++overall_n;
|
||||
}
|
||||
// R-free by shell, on the reflections and the scaled model of r_free above
|
||||
std::vector<double> shell_free_num(report_shell_d_min.size(), 0.0), shell_free_den(report_shell_d_min.size(), 0.0);
|
||||
std::vector<int> shell_n_free(report_shell_d_min.size(), 0);
|
||||
for (const auto &hv : fmodel.v) {
|
||||
const auto it = obs_by_hkl.find(hkl_key(hv.hkl));
|
||||
if (it == obs_by_hkl.end() || !it->second.free)
|
||||
continue;
|
||||
const size_t bin = shell_of(it->second.d);
|
||||
if (bin >= shell_n_free.size())
|
||||
continue;
|
||||
shell_free_num[bin] += std::fabs(it->second.F - std::abs(hv.value));
|
||||
shell_free_den[bin] += it->second.F;
|
||||
++shell_n_free[bin];
|
||||
}
|
||||
result.cc_model_shells.reserve(report_shell_d_min.size());
|
||||
for (size_t i = 0; i < report_shell_d_min.size(); i++) {
|
||||
const double cc = shell_cc[i].GetCC();
|
||||
@@ -1700,7 +1714,9 @@ ModelValidationResult Validate(const std::vector<MergedReflection> &merged,
|
||||
sigma = (std::atanh(cc) - z_mean) / std::max(z_sd, 1.0 / std::sqrt(shell_n[i] - 3.0));
|
||||
}
|
||||
result.cc_model_shells.push_back({report_shell_d_min[i], cc, shell_n[i], sigma, cc_null,
|
||||
shell_sum_Im[i] > 0.0 ? shell_sum_I[i] / shell_sum_Im[i] : NAN});
|
||||
shell_sum_Im[i] > 0.0 ? shell_sum_I[i] / shell_sum_Im[i] : NAN,
|
||||
shell_free_den[i] > 0.0 ? shell_free_num[i] / shell_free_den[i] : NAN,
|
||||
shell_n_free[i]});
|
||||
}
|
||||
result.cc_model_overall = overall_cc.GetCC();
|
||||
result.cc_model_n = overall_n;
|
||||
|
||||
@@ -136,6 +136,9 @@ struct ModelValidationResult {
|
||||
// run away from 1.0 means one of the two radial profiles is wrong. It cannot say which: a
|
||||
// real crystal can out-diffract or under-diffract its deposited model.
|
||||
double i_over_model = NAN;
|
||||
// R-free of the placed, scaled model over the shell's free reflections, as r_free over all of them
|
||||
double r_free = NAN;
|
||||
int n_free = 0;
|
||||
};
|
||||
std::vector<ModelDataCCShell> cc_model_shells;
|
||||
double cc_model_overall = NAN;
|
||||
|
||||
@@ -3099,21 +3099,23 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
" overall scale, anisotropic B and bulk solvent - so it reads 1.000 where the two\n"
|
||||
" agree on how the data fall off with resolution, and drifts away from 1.000 where\n"
|
||||
" they do not. It is the only number here that can see a scale shared by a whole\n"
|
||||
" set of symmetry equivalents; R_meas, CC1/2 and chi^2 all cancel one.\n"));
|
||||
" set of symmetry equivalents; R_meas, CC1/2 and chi^2 all cancel one. R_FREE is the\n"
|
||||
" shell's part of R_FREE above: the same scaled model over the free reflections in it.\n"));
|
||||
ReportEntry e;
|
||||
e.kind = ReportEntry::Kind::Table;
|
||||
e.table.columns = {"D_MIN", "CC_MODEL", "N", "CC_NULL", "SIGMA", "I_OVER_MODEL"};
|
||||
e.table.text_header = " D_MIN CC_MODEL N CC_NULL SIGMA I/I_MODEL\n"
|
||||
" -------- -------- --------- -------- -------- ---------";
|
||||
e.table.columns = {"D_MIN", "CC_MODEL", "N", "CC_NULL", "SIGMA", "I_OVER_MODEL", "R_FREE"};
|
||||
e.table.text_header = " D_MIN CC_MODEL N CC_NULL SIGMA I/I_MODEL R_FREE\n"
|
||||
" -------- -------- --------- -------- -------- --------- --------";
|
||||
for (const auto &sh : mv.cc_model_shells) {
|
||||
if (sh.n == 0)
|
||||
continue;
|
||||
e.table.text_rows.push_back(fmt::format(
|
||||
" {:8.2f} {:8.4f} {:9d} {:>8s} {:>8s} {:>9s}", sh.d_min, sh.cc, sh.n,
|
||||
" {:8.2f} {:8.4f} {:9d} {:>8s} {:>8s} {:>9s} {:>8s}", sh.d_min, sh.cc, sh.n,
|
||||
std::isfinite(sh.cc_null) ? fmt::format("{:.4f}", sh.cc_null) : std::string("-"),
|
||||
std::isfinite(sh.sigma) ? fmt::format("{:+.1f}", sh.sigma) : std::string("-"),
|
||||
std::isfinite(sh.i_over_model) ? fmt::format("{:.3f}", sh.i_over_model)
|
||||
: std::string("-")));
|
||||
: std::string("-"),
|
||||
std::isfinite(sh.r_free) ? fmt::format("{:.4f}", sh.r_free) : std::string("-")));
|
||||
std::vector<ReportValue> row;
|
||||
row.push_back(KeyReal("", sh.d_min, "{:.2f}").value);
|
||||
row.push_back(KeyReal("", sh.cc, "{:.4f}").value);
|
||||
@@ -3121,6 +3123,7 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
row.push_back(KeyReal("", sh.cc_null, "{:.4f}").value);
|
||||
row.push_back(KeyReal("", sh.sigma, "{:+.1f}").value);
|
||||
row.push_back(KeyReal("", sh.i_over_model, "{:.3f}").value);
|
||||
row.push_back(KeyReal("", sh.r_free, "{:.4f}").value);
|
||||
e.table.cells.push_back(std::move(row));
|
||||
}
|
||||
Add(s, std::move(e));
|
||||
|
||||
@@ -914,6 +914,13 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
|
||||
n_total += sh.n;
|
||||
}
|
||||
CHECK(n_total == result.cc_model_n);
|
||||
// R-free by shell covers the free set of R-free, and follows the noise
|
||||
int n_free = 0;
|
||||
for (const auto &sh : result.cc_model_shells)
|
||||
n_free += sh.n_free;
|
||||
CHECK(n_free == result.n_free);
|
||||
CHECK(result.cc_model_shells[0].r_free < result.cc_model_shells[1].r_free);
|
||||
CHECK(result.cc_model_shells[1].r_free < result.cc_model_shells[2].r_free);
|
||||
// Essentially noiseless: the model is the data, so the correlation is high and hugely significant.
|
||||
CHECK(result.cc_model_shells[0].cc > 0.9);
|
||||
CHECK(result.cc_model_shells[0].sigma > 10.0);
|
||||
|
||||
@@ -541,6 +541,8 @@ TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") {
|
||||
ModelValidationResult with_cc = good;
|
||||
with_cc.cc_model_shells = {{3.20f, 0.9412, 4210, 41.4, 0.1211}, {2.10f, 0.5533, 3980, 20.2, 0.0832},
|
||||
{1.80f, 0.0412, 2110, 1.9, 0.0105}};
|
||||
with_cc.cc_model_shells[0].r_free = 0.1934;
|
||||
with_cc.cc_model_shells[0].n_free = 210;
|
||||
with_cc.cc_model_overall = 0.8123;
|
||||
with_cc.cc_model_n = 10300;
|
||||
result.model_validation = with_cc;
|
||||
@@ -550,6 +552,9 @@ TEST_CASE("ResultReport_ModelValidationSection", "[Diagnostics]") {
|
||||
CHECK(cc_text.find("\nCC_MODEL_CONFIRMED_TO_D_MIN= 2.10\n") != std::string::npos);
|
||||
CHECK(cc_text.find("D_MIN CC_MODEL N CC_NULL SIGMA") != std::string::npos);
|
||||
CHECK(cc_text.find(" 1.80 0.0412 2110 0.0105 +1.9") != std::string::npos);
|
||||
// R-free by shell where the shell has free reflections, a dash where it has none
|
||||
CHECK(cc_text.find(" 3.20 0.9412 4210 0.1211 +41.4 - 0.1934") != std::string::npos);
|
||||
CHECK(cc_text.find(" 1.80 0.0412 2110 0.0105 +1.9 - -") != std::string::npos);
|
||||
CHECK(cc_text.find("The table reads in ONE direction only") != std::string::npos);
|
||||
|
||||
// The comparable R rides on the same shells. R_WORK and R_FREE are what they always were - the
|
||||
|
||||
Reference in new issue
Block a user