From 18b6638cb95b707be5ee9997fa77ba506385353d Mon Sep 17 00:00:00 2001
From: Filip Leonarski <filip.leonarski@psi.ch>
Date: Thu, 29 Oct 2026 03:25:45 +0100
Subject: [PATCH] RESULT-CHANGING: Refinement: displacement trials refine the
 positions where the geometry is out of bounds

A trial that holds the positions keeps the geometry where the weights ended; where that is out of bounds, no
held trial can be within bounds and none can displace it - while a trial that refines the positions can bring
the geometry back (rmsd bonds 0.036 -> 0.019 on the synthetic twin of Refinement_twin_decided), and did before
365e496c0. There the out-of-bounds refinement then lost to the model as given, and the twin test with it
(null z 5.7 -> 2.6). So the positions are held only where the geometry is within bounds where the weights
ended; elsewhere each trial is as it was (positions refined, a macro-cycle longer than a step) and no step
follows.

None of the 50 refinements of the battery subset had its geometry out of bounds there, so its comparison
stands. The [Refinement] tag passes (26 cases, CPU).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37
---
 docs/CPU_DATA_ANALYSIS_REFINEMENT.md               |  3 ++-
 image_analysis/structure_refinement/Refinement.cpp | 12 ++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md
index 02ddc62b9..85bccdd9f 100644
--- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md
+++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md
@@ -310,7 +310,8 @@ are:
 4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U
    held), each trial as long as a step on the positions where the weights ended, held - only the displacement
    parameters refine, so the trials differ in the displacement model alone; the model chosen then takes a step with
-   the positions refined, kept where it displaces its trial: TLS groups with a residual B - the model's own, one per chain
+   the positions refined, kept where it displaces its trial (where the geometry is out of bounds there, which held
+   positions cannot mend, each trial refines the positions too, a macro-cycle longer than a step): TLS groups with a residual B - the model's own, one per chain
    of at least 50 refinable atoms, and the chains cut into 2, 4, 8 and 16 contiguous segments of at least 10 residues
    as TLSMD cuts them (by dynamic programming over the least-squares fit of an isotropic TLS model, a quadratic B(r),
    to the refined B factors), each cut where it gives more groups than the one before; and a U per atom, at the U
diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp
index 475426df9..d1983a38b 100644
--- a/image_analysis/structure_refinement/Refinement.cpp
+++ b/image_analysis/structure_refinement/Refinement.cpp
@@ -2895,7 +2895,9 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
         //    ended, held, so that the trials differ in the displacement model alone; the model chosen then takes a
         //    step with the positions refined, kept where it displaces its trial (against trials a macro-cycle longer
         //    that refined the positions too, on 50 refinements of the battery R-free 0.001 lower on average and none
-        //    higher by more than 0.0015, at 78% of the evaluations). One displaces the one before it (the first
+        //    higher by more than 0.0015, at 78% of the evaluations). Where the geometry is out of bounds there, which
+        //    held positions cannot mend, each trial refines them too, a macro-cycle longer than a step, and no step
+        //    follows. One displaces the one before it (the first
         //    where the weights ended, the B factors as they are) only where its geometry is within bounds and both its
         //    R-free (beyond the tie) and the -ln L of the free reflections are lower: the free set is the test of the
         //    parameters it adds.
@@ -2967,6 +2969,9 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
             if (u_quarter >= 0)
                 later[u_quarter + 1] = 1;
             outcomes.assign(trials.size(), TrialOutcome());
+            // the positions held only where the geometry is within its bounds where the weights ended: a trial that
+            // holds them cannot bring it there, one that refines them can (and is then the whole trial, no step after)
+            const bool hold = state.trial.within_bounds;
             auto run = [&](const std::vector<char> &these) {
                 std::vector<TrialWeights> t;
                 std::vector<const AdpHypothesis *> h;
@@ -2978,7 +2983,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
                 if (t.empty())
                     return;
                 std::vector<TrialOutcome> ran = RunTrials(trial_slots, t, refinable, state.pos, state.b, state.aniso,
-                                                          &state.fit, settings.macro_cycles, settings, ncs_p, h, true);
+                                                          &state.fit, settings.macro_cycles + (hold ? 0 : 1), settings,
+                                                          ncs_p, h, hold);
                 for (size_t k = 0, m = 0; k < trials.size(); k++)
                     if (these[k])
                         outcomes[k] = std::move(ran[m++]);
@@ -3029,6 +3035,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
             if (best >= 0) {
                 result.adp_model = outcomes[best].trial.adp;
                 state = std::move(outcomes[best]);
+            }
+            if (best >= 0 && hold) {
                 // the model chosen refined on with the positions, a step from where its trial ended, kept where it
                 // displaces that
                 AdpHypothesis chosen = *which[best];