From 344912139adcc3cba77d91a3c9c6806a4209ed08 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 13 Nov 2026 23:33:46 +0100 Subject: [PATCH 1/4] Refinement twin null: each unrelated mate on its own anisotropic scale The twin null replaces a law's mates by unrelated reflections of the same resolution and asks what averaging alone buys. TwinnedModel puts a mate on the scale k_total of the reflection it is the mate of - right for a twin law, whose domain reflection is measured at the same spot - but an unrelated mate lies in another direction, where the anisotropic scale can differ many times over. On a trigonal crystal with a pseudo-translation, refined from a model whose atomic U carried the anisotropy opposite to k_total, a weak reflection's random mate was put on a scale 22 times its own (amplitude, at 2 A): the null R-free read 0.466 against an untwinned 0.406, and a law at fraction 0.04 was adopted 7.9 standard errors ahead of it. With each mate on its own scale the null reads 0.400 (z 0.8, untwinned); the same crystal's experimental model, adopted twinned at z 3.1 in rc186, reads z 0.6. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- .../structure_refinement/Refinement.cpp | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 07f0e089c..cb1e3406e 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1450,7 +1450,29 @@ void TwinNull(const Context &ctx, RefinementResult &result, size_t first = 0) { for (size_t d = first; d < a.size(); d++) result.twin_fraction_null += a[d]; } - const std::vector> f_null = TwinnedModel(null, rows, ctx.fcalc); + // TwinnedModel with each unrelated mate on its own scale k_total: TwinnedModel puts a mate on the scale of the + // reflection it is the mate of, as the twin domain's reflection at the same spot is measured, but an unrelated + // mate lies in another direction, where the anisotropic scale can differ many times over (22 times in amplitude + // at 2 A, on a model whose U carried the anisotropy opposite to k_total), and on the reflection's scale it put + // the null 0.06 in R-free above the untwinned refinement - a law "ahead" of its null by 7.9 standard errors at a + // fraction of 0.04. + std::vector> f_null(n); + double own_fraction = 1.0; + for (float a : rows.twin_fractions) + own_fraction -= a; + for (size_t m = 0; m < n; m++) { + const std::complex u = std::complex(ctx.fcalc[m]) + std::complex(rows.fmask[m]); + double i_model = own_fraction * rows.k_total[m] * rows.k_total[m] * std::norm(u); + for (size_t d = 0; d < null.Laws(); d++) { + const int j = null.mates[d][m]; + const double k = d < first ? rows.k_total[m] : rows.k_total[j]; + i_model += rows.twin_fractions[d] * k * k * + std::norm(std::complex(ctx.fcalc[j]) + std::complex(rows.fmask[j])); + } + const double amplitude = std::sqrt(std::max(i_model, 0.0)); + f_null[m] = std::norm(u) > 0 ? std::complex(u * (amplitude / std::abs(u))) + : std::complex(static_cast(amplitude), 0); + } result.twin_r_free_null = ComputeRFactors(null, f_null).r_free; // How far the laws beat their null on the free reflections, paired: the mean of |Fo - |F_null|| - |Fo - |F_twin|| // over its standard error. The members of a free twin group share their F_calc, so their terms are summed into From fabcb0e4d3a1a284213791ba07e6ad713bbaee14 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 13 Nov 2026 23:51:02 +0100 Subject: [PATCH 2/4] Refinement: run again from the refined model while R-free falls (3 rounds at most) One refinement chooses its X-ray weight and NCS on the B factors as given and its displacement model (TLS / U) only after them. Where the displacements change much - a placed prediction, whose B factors are its pLDDT - the weights stand on B factors the model no longer has. Running the refinement again from its own result (as REFMAC users do by hand) lets the weight ladder and the NCS choice see the refined displacements. Cytochrome C, three copies placed by MR from the AlphaFold model, 1.73 A, live merge: round 2 chose an eight times lower X-ray weight without NCS, R-work 0.367 -> 0.361 at bonds 0.002 -> 0.001 A, R-free 0.418 -> 0.405; with --refine-solvent 0.427 -> 0.409 -> 0.401. REFMAC5 from the same model reaches 0.41-0.43. On 22 battery merges with their placed models no R-free rose; lyso_x06da_ref -0.003, cytc_x10sa -0.005, 9khr -0.003, 6hv2 -0.002, the rest unchanged. End to end from the AlphaFold DB models of the timing benchmark (MR, rugnux mx): myob_x10sa 0.341 -> 0.312, cytc_x10sa 0.419 -> 0.410, 9zlo 0.289 -> 0.283, 9s02 and 9gjx unchanged. Cost: refinement time x1.3-4 (cytochrome C 6 -> 12 s on the GPU; 9s02 22 -> 30 s), the second round being run to be judged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- .../structure_refinement/Refinement.cpp | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index cb1e3406e..7e8c81502 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3332,6 +3332,57 @@ double FinalRWork(const RefinementResult &r) { return r.restrained ? r.r_work : r.r_work_placement; } +// The refinement run again from the model it ended with, while that lowers R-free by more than the tie, up to +// REFINEMENT_ROUNDS in all. One refinement chooses its weights on the B factors as given and its displacement model +// only after them, so where the displacements change much - a placed prediction, whose B factors are its pLDDT - the +// weights were chosen for B factors it no longer has: on a placed prediction of three copies the second round chose +// an eight times lower X-ray weight, R-work and R-free both fell and R-free went 0.418 -> 0.405 (with the solvent +// build 0.427 -> 0.409 -> 0.401), where REFMAC5 reached 0.41-0.43 from the same model. The first round's view of +// the model as given is kept; the solvent build is the first round's. +constexpr int REFINEMENT_ROUNDS = 3; + +RefinementResult RefineRounds(const gemmi::Structure &placed, const std::vector &merged, + const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double wavelength_A, + Logger &logger, const RefinementSettings &settings, + const std::vector &twin_laws = {}, size_t null_first = 0, + CardHeld *held = nullptr) { + RefinementResult result = RefineOnce(placed, merged, cell, sg, wavelength_A, logger, settings, twin_laws, + null_first, held); + RefinementSettings again = settings; + again.solvent = false; + for (int round = 2; round <= REFINEMENT_ROUNDS; round++) { + if (!result.ok || !result.restrained || result.as_given || result.cancelled || settings.score_only || + result.model == nullptr) + break; + logger.Info("Refinement, round {}: from the model the round before ended with (R-free {:.4f})", round, + result.r_free); + RefinementResult next = RefineOnce(*result.model, merged, cell, sg, wavelength_A, logger, again, twin_laws, + null_first); + if (!next.ok || !next.restrained || next.as_given || !(next.r_free < result.r_free - R_FREE_TIE)) { + logger.Info("Refinement, round {}: not kept (R-free {:.4f})", round, FinalRFree(next)); + result.seconds += next.seconds; + break; + } + next.seconds += result.seconds; + next.r_work_start = result.r_work_start; + next.r_free_start = result.r_free_start; + next.score_start = result.score_start; + next.adp_input = result.adp_input; + next.adp_input_aniso_atoms = result.adp_input_aniso_atoms; + next.rigid_bodies = result.rigid_bodies; + next.group_b = result.group_b; + next.r_work_placement = result.r_work_placement; + next.r_free_placement = result.r_free_placement; + next.geometry_start = result.geometry_start; + next.ncs_rmsd_start = result.ncs_rmsd_start; + next.solvent_build = result.solvent_build; + if (next.adp_model == "the model's U held") // the round before's displacement model, refined on + next.adp_model = result.adp_model; + result = std::move(next); + } + return result; +} + // The twin hypotheses beside the untwinned refinement `result`, as a ladder. First one law: the placed model refined // again, solvent build and displacement model included, against the data as twinned by a law of the lattice, and the // twinned refinement taken (into `result`) where the law beats its own null (TwinNull) on the free reflections by @@ -3407,7 +3458,7 @@ void DecideTwin(const gemmi::Structure &placed, const std::vector kept{law}; @@ -3590,7 +3641,7 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector Date: Sat, 14 Nov 2026 03:24:01 +0100 Subject: [PATCH 3/4] Refinement rounds: no placement in rounds 2-3, no maps for a round not kept A further round starts from the model the round before refined, so the rigid-body and per-residue-B placement is skipped there; and a round whose R-free does not beat the one before is returned before its maps, twin null and blobs (it is discarded anyway). The placement skip changes the path of the later rounds, not the rule: on 22 battery merges with their placed models R-free equal to the rounds without it except lyso_x06da_ref 0.2413 -> 0.2430 and cytc_x06da_1 0.3143 -> 0.3128; from the timing benchmark's AlphaFold models (MR, mx): cytc_x10sa 0.410 -> 0.399, 9zlo 0.283 -> 0.284, myob_x10sa 0.312 -> 0.312, 9s02/9gjx equal; cytochrome C live merge, MR model 0.405 -> 0.400. Saves about 1-2 s of the 7-8 s the rounds add on cytochrome C (wall 15.5 -> 13.6 s). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- image_analysis/structure_refinement/Refinement.cpp | 11 ++++++++++- image_analysis/structure_refinement/Refinement.h | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 7e8c81502..9db12da54 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -2563,7 +2563,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector(std::chrono::steady_clock::now() - start).count(); + return result; + } finish(); logger.Info("Refinement: done in {:.1f} s, {} target evaluations{}", result.seconds, ctx.evaluations, @@ -3350,12 +3357,14 @@ RefinementResult RefineRounds(const gemmi::Structure &placed, const std::vector< null_first, held); RefinementSettings again = settings; again.solvent = false; + again.placement = false; // the chains and the B per residue were placed by the first round for (int round = 2; round <= REFINEMENT_ROUNDS; round++) { if (!result.ok || !result.restrained || result.as_given || result.cancelled || settings.score_only || result.model == nullptr) break; logger.Info("Refinement, round {}: from the model the round before ended with (R-free {:.4f})", round, result.r_free); + again.keep_below = result.r_free - R_FREE_TIE; RefinementResult next = RefineOnce(*result.model, merged, cell, sg, wavelength_A, logger, again, twin_laws, null_first); if (!next.ok || !next.restrained || next.as_given || !(next.r_free < result.r_free - R_FREE_TIE)) { diff --git a/image_analysis/structure_refinement/Refinement.h b/image_analysis/structure_refinement/Refinement.h index c10e6dd84..c5b46c787 100644 --- a/image_analysis/structure_refinement/Refinement.h +++ b/image_analysis/structure_refinement/Refinement.h @@ -84,6 +84,10 @@ struct RefinementSettings { // statistic - false where the model has most likely been refined against the free reflections too (a deposited // model of this crystal scored on a free set drawn here). bool as_given_competes = true; + // For a further round of the refinement (RefineRounds): the placement stage skipped, and the maps, the twin null + // and the blobs made only where R-free ends below keep_below - the round is not kept otherwise. + bool placement = true; + double keep_below = INFINITY; bool gpu = true; size_t nthreads = 1; size_t gpu_resident_bytes = 0; // held on the card by what runs beside From f10829dabc17f6e849c63c475325ced417c2a307 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 09:18:09 +0100 Subject: [PATCH 4/4] Refinement: rounds 2 and 3 as the weights and the displacement model decided again on the refined model The rounds (fabcb0e4d, a64d3e3c4) ran the whole refinement again from its written result - setup, placement, ladder, trajectory, final ladder, displacement hypotheses - and a round not kept had cost a full refinement. What a second round changed, read off the logs and by instrumenting the balanced weights and the TLS trials: the ceiling of B (twice the model's largest B: 100 on a placed prediction given with B factors of 6 A^2 under an overall B of 45, which refines at B near 50-70 - the TLS bodies and the U per atom were held at the ceiling, and the TLS hypothesis lost to a U per atom; 200 from the written model), the B factors' weight (balanced on the B factors as given, 2500 against 23 on the refined model: the B similarity's gradient goes as 1/B), the rung (the middle one with NCS on flat B factors, the tightest without NCS once the B factors were refined) and the TLS bodies fitted from refined B factors instead of flat ones. None of this can be decided in the first round - it is the alternation of positions and displacements - so the rounds stay, reduced to what changes. A round is now inside RefineOnce, after the displacement model: the ceiling and the balanced weights from the model as it is (the engines made again where the ceiling rises), the ladder from it with its displacements as they are, the trajectory, and the hypotheses from there; kept where it lowers R-free beyond the tie and undone otherwise, three rounds in all. Where the ladder and the trajectory leave the model where it was, the hypotheses would run identical trials, so the round ends there - an exact rule: a model refined before pays one ladder and one step. No round where the refinement does not displace the model as given. The first round is the one refinement as before, so a model the rounds do not improve comes out as it did. RefineRounds and the placement and keep_below settings go. On the cytochrome C live merge and MR model, R-free 0.4183 (one round) -> 0.4028 (0.3999 with the full rounds, which also re-placed the chains and re-ran the solvent build each round). Tried and rejected: the trajectory continued from the chosen displacement model with positions and U together (no gain: the U per atom overfits), B refined ahead of B held in the ladder's trust (worse), the B weight balanced with the overall B moved in in the first round (fixes the prediction, halves the weight on models with an overall B of 15 and costs them 0.002), and the ceiling raised with the overall B at setup (changes the first round: one AF-start run 0.006 worse). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 18 +- .../structure_refinement/Refinement.cpp | 190 ++++++++++-------- .../structure_refinement/Refinement.h | 4 - 3 files changed, 125 insertions(+), 87 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index b0a12e309..90257354f 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -28,7 +28,7 @@ statistics and its map coefficients go out; the restraint dictionary is compiled The model is taken as the refinement can use it: the first model of the file only, its hydrogens removed, and every B clamped to the range the refinement allows (2 Ų up to twice the model's largest B, within -100-300 Ų). An atom's anisotropic U (ANISOU, `_atom_site_anisotrop`) is kept. Where the model has TLS +100-300 Ų; a later round of the weights (§15.1, 4) raises the ceiling from the refined model). An atom's anisotropic U (ANISOU, `_atom_site_anisotrop`) is kept. Where the model has TLS groups (`_pdbx_refine_tls*`, PDB `REMARK 3`) whose atoms bring no U of their own, their U is made from the groups, and the B column is read either as the total B (as phenix.refine writes it) or as the residual to the groups (as REFMAC writes it) - whichever of the two, and of the isotropic B as it stands, scores the lowest @@ -309,9 +309,9 @@ are: was never kept on 55 refinements, and taking it away changed no choice). 4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U held), each trial as long as a step on the positions where the weights ended, held - only the displacement - parameters refine, so the trials differ in the displacement model alone; the model chosen then takes a step with - the positions refined, kept where it displaces its trial (where the geometry is out of bounds there, which held - positions cannot mend, each trial refines the positions too, a macro-cycle longer than a step): TLS groups with a residual B - the model's own, one per chain + parameters refine, so the trials differ in the displacement model alone (where the geometry is out of bounds + there, which held positions cannot mend, each trial refines the positions too, a macro-cycle longer than a + step): TLS groups with a residual B - the model's own, one per chain of at least 50 refinable atoms, and the chains cut into 4, 8 and 16 contiguous segments of at least 10 residues as TLSMD cuts them (by dynamic programming over the least-squares fit of an isotropic TLS model, a quadratic B(r), to the refined B factors), each cut where it gives more groups than the one before; and a U per atom, at the U @@ -327,6 +327,16 @@ are: `REFINE_ANISO_ATOMS`, `REFINE_TLS_GROUPS`). It is decided after the solvent build (§15.10), on the model with its solvent: a water a round put in refines a U of its own in the per-atom trial as every refinable atom does, and no TLS group holds a water. + Then the weights and the displacement model are **decided again** from the model as it now is, up to twice: the + B ceiling and the balanced weights from its B factors, the ladder from it with its displacements as they are, the + trajectory, and the displacement hypotheses from where that ends. The first choice took its weights with the + B factors as given and its displacement model after them; on a placed prediction, whose B factors are its pLDDT, + the B factors' weight balanced on them is a hundred times too stiff (the B similarity's gradient goes as $1/B$), + the middle rung wins where the tightest wins once the B factors are refined, and the TLS bodies fitted from the + refined B factors come 0.02 lower in R-free than from the B factors as given. A round is taken while it lowers + R-free beyond the tie and undone otherwise; where its ladder and trajectory leave the model where it was, the + hypotheses would repeat the same trials, and it ends there - from a model refined before, a round is one ladder + and one step. No round runs where the refinement does not displace the model as given. 5. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the rule every trial is held to (its geometry within bounds and R-free lower beyond the tie), the model is written as it came, scaled to the data, and the solvent build's edits are undone (`REFINE_AS_GIVEN`). What comes out is never diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 9db12da54..e20e9c584 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -63,6 +63,9 @@ constexpr double MAX_RMSD_BONDS = 0.02, MAX_RMSD_ANGLES = 2.0; constexpr int SOLVENT_ITERATIONS = 10; // R-free differences smaller than this do not decide between two weights: the tighter geometry is taken. constexpr double R_FREE_TIE = 0.001; +// The rounds of RefineOnce's weight and displacement-model choice (4b.): the first from the model as given, each +// after it from the model the one before ended with. +constexpr int REFINEMENT_ROUNDS = 3; // The minimiser (RefinementMinimiser): a solve stops where an iteration lowers the target by less than this, // relative; the most a position (A) or a B (A^2) moves in the first trial of a line search. constexpr double MINIMISER_F_TOLERANCE = 1e-7; @@ -2563,8 +2566,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector aniso0 = ctx.aniso; + std::vector outcomes; + // The ladder from `from` (positions, B, anisotropy; `fit` what was fitted there, null to fit it), at the + // balanced weights as they stand: the trial kept, or one that did not run where none did. + auto ladder_choice = [&](const std::vector &from_pos, const std::vector &from_b, + const std::vector &from_aniso, const Context::Fit *fit) { std::vector ladder; for (double rung : settings.weight_ladder) for (double adp : settings.adp_ladder) { @@ -2695,14 +2703,13 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector aniso0 = ctx.aniso; - std::vector outcomes(ladder.size()); + outcomes.assign(ladder.size(), TrialOutcome()); auto run_ladder = [&](const std::function &these) { std::vector t; for (const TrialWeights &l : ladder) if (these(l)) t.push_back(l); - std::vector ran = RunTrials(trial_slots, t, refinable, pos0, b0, aniso0, nullptr, + std::vector ran = RunTrials(trial_slots, t, refinable, from_pos, from_b, from_aniso, fit, settings.macro_cycles, settings, ncs_p); for (size_t i = 0, m = 0; i < ladder.size(); i++) if (these(ladder[i])) @@ -2743,10 +2750,10 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector= 0) - state = std::move(outcomes[k]); + const int k = choose(outcomes, "ladder"); + return k >= 0 ? std::move(outcomes[k]) : TrialOutcome(); + }; + TrialOutcome state = ladder_choice(pos0, b0, aniso0, nullptr); // 2. The trajectory: steps of a few macro-cycles from where the last one ended, each trying the weights as they // are and, in turn, the positions' X-ray weight halved and doubled (halved only in the first step, from the @@ -2761,7 +2768,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector from_best; // `fresh`: the first step fits the scale, the solvent and the likelihood anew (the model changed since). - auto trajectory = [&](TrialOutcome state, bool fresh, std::vector &from_best) { + // `label`: the steps' name in the log. + auto trajectory = [&](TrialOutcome state, bool fresh, std::vector &from_best, const char *label) { TrialOutcome best = state; from_best.clear(); bool state_is_best = true; @@ -2786,7 +2794,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector= 0 && displaces(outcomes[k].trial, state.trial)) state = std::move(outcomes[k]); } @@ -3110,7 +3118,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector &own_tls) { const RefinementStage adp_stage("adp_hypotheses"); ctx.SetPositions(state.pos); ctx.SetB(state.b); @@ -3120,7 +3129,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector hyps = AdpHypotheses(ctx, topo, input_tls); + std::vector hyps = AdpHypotheses(ctx, topo, own_tls); const double w = state.trial.weight, adp = state.trial.adp_weight > 0 ? state.trial.adp_weight : adp_w0; // the U weight balanced on the model with every refinable atom given a U double u_w0 = adp; @@ -3255,6 +3264,89 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector own_tls = state.tls; + ctx.SetPositions(state.pos); + ctx.SetB(state.b); + ctx.SetAniso(state.aniso); + ctx.Restore(state.fit); + PlaceRiders(ctx, topo); + // the ceiling of B from the model as it now is, as from the model as given: a placed prediction comes + // with B factors of 6 A^2 under an overall B of 45 (its ceiling 100) and refines at B near 50-70, where + // the TLS bodies and the U per atom were held at the ceiling; where it rises the engines are made again + // (the gather is reserved for it) + double b_max = 0; + for (size_t i = 0; i < ctx.atoms.size(); i++) + b_max = std::max(b_max, ctx.atoms[i]->b_iso); + const double ceiling = std::clamp(2 * b_max, B_CEILING_MIN, B_CEILING_MAX); + if (ceiling > ctx.b_ceiling) { + const RefinementStage stage("engine"); + for (std::unique_ptr &o : own) + if (o) + o->ctx.engine.reset(); + ctx.b_ceiling = ceiling; + MakeEngine(ctx, settings.gpu, settings, logger, restraint_records, restraint_slots); + ctx.engine->SetRestraints(0, RestraintsNow(ctx, topo)); + ctx.engine->SetRestraints(1, topo.adp); + for (size_t k = 0; k < own.size(); k++) + if (own[k]) { + Context &o = own[k]->ctx; + o.b_ceiling = ceiling; +#ifdef JFJOCH_USE_CUDA + if (ctx.engine->OnGPU()) + o.engine = MakeRefinementEngineGPU(*ctx.engine, devices[k].device, o.nthreads, devices[k].own_lane); +#endif + if (!o.engine) + o.engine = MakeRefinementEngineCPU(o.cell, *o.sg, o.d_min, o.refl, o.nthreads); + o.engine->SetAniso(&o.aniso); + o.engine->SetRestraints(1, own[k]->topo.adp); + } + ctx.Restore(state.fit); + } + { + const RefinementStage stage("balanced_weight"); + w0 = BalancedWeight(ctx, refinable); + adp_w0 = BalancedADPWeight(ctx, BRestraintsNow(ctx, topo), refinable); + } + logger.Info("Refinement, round {}: balanced X-ray weights {:.3g} (positions), {:.3g} (B), B ceiling {:.0f}", round, + w0, adp_w0, ctx.b_ceiling); + TrialOutcome rung = ladder_choice(state.pos, state.b, state.aniso, &state.fit); + if (rung.ran && displaces(rung.trial, state.trial)) + state = std::move(rung); + std::vector unused; + const std::string step = fmt::format("round {} step", round); + state = trajectory(std::move(state), false, unused, step.c_str()); + if (state.trial.r_work == before.trial.r_work && state.trial.r_free == before.trial.r_free) { + logger.Info("Refinement, round {}: the weights leave the model where it was", round); + break; + } + displacements(own_tls.empty() ? input_tls : own_tls); + const bool kept = state.trial.r_free < before.trial.r_free - R_FREE_TIE; + logger.Info("Refinement, round {}: R-free {:.4f} -> {:.4f}{}", round, before.trial.r_free, state.trial.r_free, + kept ? "" : ", not kept"); + if (!kept) { + state = before; + break; + } } // 5. The model as given, first in order of trust: the refinement is kept where it displaces it by the rule every // trial is held to, else the model is given back as it came (the solvent build's edits undone), so that what @@ -3315,13 +3407,6 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector(std::chrono::steady_clock::now() - start).count(); - return result; - } - finish(); logger.Info("Refinement: done in {:.1f} s, {} target evaluations{}", result.seconds, ctx.evaluations, result.restrained ? fmt::format("; R-work {:.4f}, R-free {:.4f}, weight {:.3g}", result.r_work, @@ -3339,59 +3424,6 @@ double FinalRWork(const RefinementResult &r) { return r.restrained ? r.r_work : r.r_work_placement; } -// The refinement run again from the model it ended with, while that lowers R-free by more than the tie, up to -// REFINEMENT_ROUNDS in all. One refinement chooses its weights on the B factors as given and its displacement model -// only after them, so where the displacements change much - a placed prediction, whose B factors are its pLDDT - the -// weights were chosen for B factors it no longer has: on a placed prediction of three copies the second round chose -// an eight times lower X-ray weight, R-work and R-free both fell and R-free went 0.418 -> 0.405 (with the solvent -// build 0.427 -> 0.409 -> 0.401), where REFMAC5 reached 0.41-0.43 from the same model. The first round's view of -// the model as given is kept; the solvent build is the first round's. -constexpr int REFINEMENT_ROUNDS = 3; - -RefinementResult RefineRounds(const gemmi::Structure &placed, const std::vector &merged, - const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double wavelength_A, - Logger &logger, const RefinementSettings &settings, - const std::vector &twin_laws = {}, size_t null_first = 0, - CardHeld *held = nullptr) { - RefinementResult result = RefineOnce(placed, merged, cell, sg, wavelength_A, logger, settings, twin_laws, - null_first, held); - RefinementSettings again = settings; - again.solvent = false; - again.placement = false; // the chains and the B per residue were placed by the first round - for (int round = 2; round <= REFINEMENT_ROUNDS; round++) { - if (!result.ok || !result.restrained || result.as_given || result.cancelled || settings.score_only || - result.model == nullptr) - break; - logger.Info("Refinement, round {}: from the model the round before ended with (R-free {:.4f})", round, - result.r_free); - again.keep_below = result.r_free - R_FREE_TIE; - RefinementResult next = RefineOnce(*result.model, merged, cell, sg, wavelength_A, logger, again, twin_laws, - null_first); - if (!next.ok || !next.restrained || next.as_given || !(next.r_free < result.r_free - R_FREE_TIE)) { - logger.Info("Refinement, round {}: not kept (R-free {:.4f})", round, FinalRFree(next)); - result.seconds += next.seconds; - break; - } - next.seconds += result.seconds; - next.r_work_start = result.r_work_start; - next.r_free_start = result.r_free_start; - next.score_start = result.score_start; - next.adp_input = result.adp_input; - next.adp_input_aniso_atoms = result.adp_input_aniso_atoms; - next.rigid_bodies = result.rigid_bodies; - next.group_b = result.group_b; - next.r_work_placement = result.r_work_placement; - next.r_free_placement = result.r_free_placement; - next.geometry_start = result.geometry_start; - next.ncs_rmsd_start = result.ncs_rmsd_start; - next.solvent_build = result.solvent_build; - if (next.adp_model == "the model's U held") // the round before's displacement model, refined on - next.adp_model = result.adp_model; - result = std::move(next); - } - return result; -} - // The twin hypotheses beside the untwinned refinement `result`, as a ladder. First one law: the placed model refined // again, solvent build and displacement model included, against the data as twinned by a law of the lattice, and the // twinned refinement taken (into `result`) where the law beats its own null (TwinNull) on the free reflections by @@ -3467,7 +3499,7 @@ void DecideTwin(const gemmi::Structure &placed, const std::vector kept{law}; @@ -3650,7 +3682,7 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector