From adde6d5a5c0998fe30643bcabca7813a103b21cc Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:32:50 +0100 Subject: [PATCH] Refinement: R-free compared paired; the model as given kept only where better R-free in FreeSetPrefers is now compared as a paired difference: each trial keeps R-free as one term per free twin group (FreeRTerms, the grouping TwinNull's paired z uses, now shared as TwinGroups), and a candidate may be higher than the incumbent by no more than the standard error of the sum of the groups' differences. The single-model standard error R_free / sqrt(2 N_free) it replaces was several times wider than the scatter of two trials from one start on the same reflections, which left -LL free deciding almost alone. The as-given guard now asks whether the model as given displaces the refinement on the free set (rather than whether the refinement displaces it), and never keeps it where the refinement is better on both R-work and R-free: on one guard set the refinement lowered R-free 0.2408 -> 0.2373 and R-work 0.242 -> 0.215 but raised -LL free by 7, and the model as given came out. That -LL free is higher there however D and the variance are fitted (on the free set, on the work set, on both, or crosswise), so it is not a consequence of fitting the variance on the free set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- .../structure_refinement/Refinement.cpp | 97 ++++++++++++++----- .../structure_refinement/Refinement.h | 1 + 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 643a306d9..71d66f645 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -93,11 +93,21 @@ constexpr double MAX_RMSD_BONDS = 0.02, MAX_RMSD_ANGLES = 2.0; constexpr int SOLVENT_ITERATIONS = 10; // The free set decides the weights and the displacement model alone, and only where both of its measures agree: a // candidate is preferred to the incumbent where its -ln L of the free reflections is lower and its R-free not higher -// by more than the standard error of the incumbent's, R_free / sqrt(2 N_free). -// Following Tickle, Laskowski & Moss (2000) Acta Cryst. D56, 442-450 (the standard error of R-free), as PDB-REDO -// decides on more than one free measure, Joosten, Long, Murshudov & Perrakis (2014) IUCrJ 1, 213-220 -bool FreeSetPrefers(double r_free, double nll_free, double r_free_b, double nll_free_b, int n_free) { - return nll_free < nll_free_b && r_free < r_free_b + r_free_b / std::sqrt(2.0 * n_free); +// than the incumbent's by more than the standard error of the difference. The two are scored on the same free +// reflections, so the difference is paired: R-free is a sum of one term per free twin group (FreeRTerms), and the +// standard error is that of the sum of the groups' differences. +// As PDB-REDO decides on more than one free measure, Joosten, Long, Murshudov & Perrakis (2014) IUCrJ 1, 213-220 +bool FreeSetPrefers(const std::vector &r_terms, double nll_free, const std::vector &r_terms_b, + double nll_free_b) { + const size_t n = r_terms.size(); + double sum = 0, sum2 = 0; + for (size_t k = 0; k < n; k++) { + const double d = r_terms[k] - r_terms_b[k]; + sum += d; + sum2 += d * d; + } + const double se = n > 1 ? std::sqrt(std::max(sum2 - sum * sum / n, 0.0) * n / (n - 1)) : 0.0; + return nll_free < nll_free_b && sum < se; } // Every other step is taken on the work-set target (WorkTarget) where it lowers it by more than this, relative. constexpr double TARGET_TOLERANCE = 1e-4; @@ -1909,6 +1919,39 @@ std::string DomainText(const RefinementReflections &refl, const XrayRowModel &ro // As REFMAC5 drops twin domains of small fraction, Murshudov et al. (2011) Acta Cryst. D67, 355-367 constexpr double TWIN_DOMAIN_MIN = 0.05; +// Each observed reflection's twin group, named by its lowest observed row (itself where there is no twin law). +std::vector TwinGroups(const RefinementReflections &refl) { + const size_t n = refl.Size(); + std::vector group(n); + for (size_t m = 0; m < n; m++) { + group[m] = static_cast(m); + for (size_t d = 0; d < refl.Laws(); d++) + if (refl.mates[d][m] < static_cast(n)) + group[m] = std::min(group[m], refl.mates[d][m]); + } + return group; +} + +// R-free of the model as the last Fcalc left it, as one term per free twin group, in the order of the groups (the +// members of a group share their F_calc, so their terms are summed into one, as TwinNull sums them): the terms +// FreeSetPrefers pairs. +std::vector FreeRTerms(const Context &ctx) { + const std::vector> fmodel = TwinnedModel(ctx.refl, ctx.rows, ctx.fcalc); + const std::vector group = TwinGroups(ctx.refl); + double fo = 0; + std::vector sum(ctx.refl.Size(), 0.0); + for (size_t m = 0; m < ctx.refl.Size(); m++) + if (ctx.refl.free[m]) { + fo += ctx.refl.fo[m]; + sum[group[m]] += std::fabs(ctx.refl.fo[m] - std::abs(fmodel[m])); + } + std::vector terms; + for (size_t m = 0; m < ctx.refl.Size(); m++) + if (ctx.refl.free[m] && group[m] == static_cast(m)) + terms.push_back(static_cast(sum[m] / fo)); + return terms; +} + // The null of a twinned refinement, for the model as the last Refit() left it: its R-free at the same fraction with // every reflection's mate replaced by an unrelated observed one of the same resolution and the same set (free with // free, work with work, as a law's twin pairs are). The reflections of each set are put in order of 1/d^2 with the two @@ -1932,14 +1975,7 @@ constexpr double TWIN_STOP_Z = 0.5; void TwinNull(const Context &ctx, RefinementResult &result, size_t first = 0) { RefinementReflections null = ctx.refl; const size_t n = null.Size(); - // each observed reflection's twin group, named by its lowest observed row - std::vector group(n); - for (size_t m = 0; m < n; m++) { - group[m] = static_cast(m); - for (size_t d = 0; d < ctx.refl.Laws(); d++) - if (ctx.refl.mates[d][m] < static_cast(n)) - group[m] = std::min(group[m], ctx.refl.mates[d][m]); - } + const std::vector group = TwinGroups(ctx.refl); for (size_t d = first; d < null.Laws(); d++) std::iota(null.mates[d].begin(), null.mates[d].end(), 0); for (const bool free : {false, true}) { @@ -2265,6 +2301,7 @@ void MeasureWorkTarget(Context &ctx, const RefinementTopology &topo, const NcsCo const RefinementScore sc = ctx.Score(); t.nll_work = sc.nll_work; t.nll_free = sc.nll_free; + t.free_r_terms = FreeRTerms(ctx); const std::vector> xyzb = ctx.XyzB(); t.e_geometry = EvaluateRestraints(RestraintsNow(ctx, topo, ncs, ncs_sigma), xyzb, nullptr); t.e_b = EvaluateRestraints(BRestraintsNow(ctx, topo), xyzb, nullptr); @@ -2938,6 +2975,7 @@ RFactors ReadInputDisplacements(Context &ctx, const std::vector &tls, L int best = -1; std::vector rf; std::vector nll_free; + std::vector> r_terms; for (size_t k = 0; k < readings.size(); k++) { ctx.SetB(readings[k].b); ctx.SetAniso(readings[k].aniso); @@ -2947,9 +2985,10 @@ RFactors ReadInputDisplacements(Context &ctx, const std::vector &tls, L ctx.ForgetStarts(); rf.push_back(ctx.Refit()); nll_free.push_back(ctx.Score().nll_free); + r_terms.push_back(FreeRTerms(ctx)); logger.Info("Refinement: the model's {} TLS groups ({} atoms) read as {}: R-work {:.4f}, R-free {:.4f}, -LL free " "{:.1f}", tls.size(), tls_u.size(), readings[k].name, rf[k].r_work, rf[k].r_free, nll_free[k]); - if (best < 0 || FreeSetPrefers(rf[k].r_free, nll_free[k], rf[best].r_free, nll_free[best], rf[k].n_free)) + if (best < 0 || FreeSetPrefers(r_terms[k], nll_free[k], r_terms[best], nll_free[best])) best = static_cast(k); } ctx.SetB(readings[best].b); @@ -3194,6 +3233,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector start_r_terms = FreeRTerms(ctx); logger.Info("Refinement{}: {} atoms ({} riding hydrogens), {} reflections to {:.2f} A ({} free), on the {}; as " "given R-work {:.4f}, R-free {:.4f}; k_sol {:.3f}, b_sol {:.1f}, overall B {:.1f}, Wilson B {:.0f}, B " "ceiling {:.0f}, mask grid {:.2f} A", @@ -3211,6 +3251,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector own_tls = state.tls; @@ -4451,12 +4498,10 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector free_r_terms; // R-free as one term per free twin group (FreeRTerms), for paired comparisons GeometryStatistics geometry; bool within_bounds = false; // the geometry bounds the choice is made inside std::string adp; // the displacement model: "isotropic", "TLS ..." or "anisotropic"