From 28bf8e3aeb6fbd5a5fbebdca9a551229077e7c86 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 08:08:58 +0100 Subject: [PATCH 01/12] PROTOTYPE: damped Newton steps for the positions and the B factors in refinement (RUGNUX_NEWTON) Behind environment switches, off by default; nothing changes without them. - RefinementNewton (RefinementMinimiser.h): Levenberg-Marquardt steps, (H + lambda D) d = -g by preconditioned CG, accepted on the target. - H = the X-ray curvature per atom as a multiple of I (XrayXyzCurvatureShape, the B shape's counterpart, calibrated once per trial by central differences: 2 counted evaluations) plus the restraints' Hessian: by differences of their gradient (RUGNUX_NEWTON=1) or Gauss-Newton (RUGNUX_NEWTON_GN=1, RestraintGaussNewton: J = g / sqrt(2E) per record, planes per atom). - RUGNUX_NEWTON_XYZ / _B iterations per macro-cycle (10 / 6), _CGMAX, _CGTOL, _CLIP, _RELSTOP, _TRACE; RUGNUX_NEWTON=2 an L-BFGS with the Newton H as H0 (worse, kept for the record). - RUGNUX_REFINE_STATS: MINSTAT/TRIALSTAT/TRIALSSET/WEIGHTSTAT lines on stderr (evaluation budget). - EvaluateRestraints: the per-record terms moved into RecordTerm (same operations, same order). Measured (refine-only, CPU one round, 6 sets): evaluations -40..-63% at equal R-free on average; on the GPU protocol R-free worse on 4 of 6 sets. Not for production as is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 294 +++++++++++++++++- .../RefinementMinimiser.h | 176 ++++++++++- .../RefinementRestraints.cpp | 136 ++++++-- .../RefinementRestraints.h | 9 + 4 files changed, 571 insertions(+), 44 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 924ed0803..9581d46da 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -97,6 +99,12 @@ constexpr double MAX_STEP_XYZ = 0.3, MAX_STEP_B = 10.0; // The loosest NCS restraint a trajectory step tries (A); looser than that, the copies are refined independently. constexpr double NCS_SIGMA_MAX = 0.4; +// PROTOTYPE instrumentation (lane k): MINSTAT/TRIALSTAT/... lines on stderr where RUGNUX_REFINE_STATS is set. +bool RefineStats() { + static const bool on = getenv("RUGNUX_REFINE_STATS") != nullptr; + return on; +} + // The figure of merit of the likelihood, , at X = 2 Fo D Fc / var: I1(X)/I0(X) for an acentric // reflection, tanh(X / 2) for a centric one. // Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 @@ -607,7 +615,13 @@ struct Context { // The X-ray target's curvature in each atom's B, up to one factor (CalibrateBCurvature): the likelihood's // curvature in |F_model| of the cycle, 2 (k_total D)^2 / var, summed in resolution shells, against the atom's own // part of it, (s^2 / 4)^2 f(s)^2 exp(-B s^2 / 2). A hydrogen's is its parent's. - std::vector XrayBCurvatureShape() const { + std::vector XrayBCurvatureShape() const { return XrayCurvatureShape(rows, false); } + // The same for each of an atom's x, y and z: its own part of the likelihood's curvature in |F_model| is then + // (2 pi)^2 s^2 / 3 f(s)^2 exp(-B s^2 / 2), the reflections taken as spread evenly over directions - the X-ray + // block of the normal matrix as a multiple of I per atom (Murshudov, Vagin & Dodson 1997 keep the X-ray blocks + // of atoms on the diagonal). `r`: the row model the engine has (blurred or not). + std::vector XrayXyzCurvatureShape(const XrayRowModel &r) const { return XrayCurvatureShape(r, true); } + std::vector XrayCurvatureShape(const XrayRowModel &rows, bool xyz) const { constexpr int SHELLS = 32; double s2_max = 0; for (double s2 : refl.inv_d2) @@ -638,7 +652,10 @@ struct Context { for (int k = 0; k < SHELLS; k++) if (n[k] > 0) { const double s2 = s2_sum[k] / n[k]; - shape[i] += w[k] * f[k] * a.occ * a.occ * std::exp(-a.b_iso * s2 / 2.0) * s2 * s2 / 16.0; + if (xyz) + shape[i] += w[k] * f[k] * a.occ * a.occ * std::exp(-a.b_iso * s2 / 2.0) * s2 * (4.0 * M_PI * M_PI / 3.0); + else + shape[i] += w[k] * f[k] * a.occ * a.occ * std::exp(-a.b_iso * s2 / 2.0) * s2 * s2 / 16.0; } } for (size_t i = 0; i < rider_parent.size(); i++) @@ -675,6 +692,37 @@ struct Context { return den > 0 ? std::max(num / den, 0.0) : 0.0; } + // The factor that turns XrayXyzCurvatureShape() into the X-ray target's curvature, as CalibrateBCurvature() does + // for B: along a random +-1 direction of the positions of `params`, by central differences of the gradient. + // Two evaluations, counted. + double CalibrateXyzCurvature(const std::vector ¶ms, const std::vector &shape) { + constexpr double EPS = 0.01; // A + const std::vector p0 = Positions(); + std::mt19937 rng(7); + std::vector v(params.size()); + for (gemmi::Vec3 &x : v) + x = gemmi::Vec3(rng() % 2 == 0 ? 1.0 : -1.0, rng() % 2 == 0 ? 1.0 : -1.0, rng() % 2 == 0 ? 1.0 : -1.0); + std::array>, 2> g; + for (int side = 0; side < 2; side++) { + for (size_t k = 0; k < params.size(); k++) + atoms[params[k]]->pos = p0[params[k]] + gemmi::Position(v[k] * (side == 0 ? EPS : -EPS)); + MoveRiders(); + engine->Evaluate(Model(), &g[side]); + evaluations++; + FoldRiders(g[side]); + } + SetPositions(p0); + MoveRiders(); + double num = 0, den = 0; + for (size_t k = 0; k < params.size(); k++) { + const int i = params[k]; + num += (v[k].x * (g[0][i][0] - g[1][i][0]) + v[k].y * (g[0][i][1] - g[1][i][1]) + + v[k].z * (g[0][i][2] - g[1][i][2])) / (2 * EPS); + den += 3 * shape[i]; + } + return den > 0 ? std::max(num / den, 0.0) : 0.0; + } + // F_model with the cycle's scale and solvent, for the atoms as they are now. std::vector> FModel() { engine->Fcalc(Model(), fcalc); @@ -792,6 +840,29 @@ public: return std::isfinite(*cost); } + // The restraints' gradient alone at p, as Evaluate() makes it: for products with their Hessian by differences. + void RestraintGradient(const double *p, double *gradient) const { + Set(p); + std::vector> &gr = gr_; + ctx_.engine->EvaluateRestraints(restraint_set_, ctx_.XyzB(), &gr); + const int n = static_cast(param_atoms_.size()); + for (int k = 0; k < n; k++) { + const int i = param_atoms_[k]; + if (mode_ == Mode::Xyz) { + gemmi::Vec3 g(gr[i][0], gr[i][1], gr[i][2]); + if (const SpecialPosition *sp = Special(i)) + g = sp->projector.multiply(g); + gradient[3 * k] = g.x; + gradient[3 * k + 1] = g.y; + gradient[3 * k + 2] = g.z; + } else + gradient[k] = gr[i][3]; + } + } + Mode GetMode() const { return mode_; } + double Weight() const { return weight_; } + const std::vector &Atoms() const { return param_atoms_; } + private: const SpecialPosition *Special(int atom) const { const int k = topo_.special_index.empty() ? -1 : topo_.special_index[atom]; @@ -956,16 +1027,151 @@ std::vector BDiagonal(const std::vector &xray, const std::vector // `minimiser` carries its memory from the call before; `diagonal` is the curvature it scales the steps by (none: 1). template void Minimise(Context &ctx, const Target &target, int iterations, RefinementMinimiser &minimiser, - std::vector diagonal = {}) { + std::vector diagonal = {}, const char *tag = "?") { std::vector p = target.Get(); if (p.empty()) return; minimiser.SetDiagonal(diagonal.empty() ? std::vector(p.size(), 1.0) : std::move(diagonal)); - minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { + const RefinementMinimiser::Outcome o = minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); target.Set(p.data()); // where the minimiser ended, not where its last line search probed + if (RefineStats()) fprintf(stderr, "MINSTAT lbfgs.%s it=%d ev=%d lim=%d n=%zu f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, + iterations, p.size(), o.f_start, o.f_end); +} + +bool NewtonHybrid(); +// PROTOTYPE (RUGNUX_NEWTON=1): the positions or the B factors by damped Newton steps (RefinementNewton). H is the +// X-ray target's curvature per parameter (`xray`: per atom, already times the weight) plus the restraints' exact +// Hessian, its products by forward differences of their gradient; D is H's diagonal, the restraints' part of it +// probed by differences on atoms far apart in the list at a time (the atoms of one restraint are near in it). +void MinimiseNewton(Context &ctx, const AtomTarget &target, int iterations, RefinementNewton &newton, + const std::vector &xray, RefinementMinimiser &lbfgs, const RestraintSet &set) { + std::vector p = target.Get(); + if (p.empty()) + return; + const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; + const int per = xyz ? 3 : 1; + const size_t n = p.size(); + const double eps = xyz ? 1e-4 : 0.1; // the largest change of a parameter in a difference (A, A^2) + std::vector h(n); + for (size_t k = 0; k < n; k++) + h[k] = xray[target.Atoms()[k / per]]; + std::vector x_at, g_at(n), gp(n), xp(n); + auto restraint_hv = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (x_at != x) { + x_at = x; + target.RestraintGradient(x.data(), g_at.data()); + } + double v_max = 0; + for (double e : v) + v_max = std::max(v_max, std::fabs(e)); + hv.assign(n, 0.0); + if (v_max == 0) + return; + const double t = eps / v_max; + for (size_t k = 0; k < n; k++) + xp[k] = x[k] + t * v[k]; + target.RestraintGradient(xp.data(), gp.data()); + for (size_t k = 0; k < n; k++) + hv[k] = (gp[k] - g_at[k]) / t; + }; + // RUGNUX_NEWTON_GN=1: the restraints' Gauss-Newton Hessian (RestraintGaussNewton) instead of the exact one by + // differences - positive semi-definite, and its diagonal exact + static const bool gn = getenv("RUGNUX_NEWTON_GN") != nullptr; + std::vector param_of(ctx.atoms.size(), -1); + for (size_t k = 0; k < target.Atoms().size(); k++) + param_of[target.Atoms()[k]] = static_cast(k); + const int c0 = xyz ? 0 : 3; + RestraintJacobian jac; + std::vector x_jac; + auto gn_hv = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (x_jac != x) { + x_jac = x; + target.Set(x.data()); + jac = RestraintGaussNewton(set, ctx.XyzB()); + } + hv.assign(n, 0.0); + for (size_t r = 0; r + 1 < jac.start.size(); r++) { + double dot = 0; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + dot += jac.j[e][c0 + c] * v[per * k + c]; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + hv[per * k + c] += dot * jac.j[e][c0 + c]; + } + }; + auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (gn) + gn_hv(x, v, hv); + else + restraint_hv(x, v, hv); + for (size_t k = 0; k < n; k++) + hv[k] += h[k] * v[k]; + }; + // the restraints' diagonal: every COLOURS-th atom at a time, one direction at a time + constexpr int COLOURS = 24; + std::vector diagonal(n, 0.0), v(n), hv; + if (gn) { + target.Set(p.data()); + x_jac = p; + jac = RestraintGaussNewton(set, ctx.XyzB()); + for (size_t e = 0; e < jac.atom.size(); e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + diagonal[per * k + c] += jac.j[e][c0 + c] * jac.j[e][c0 + c]; + } + for (int c = 0; c < (gn ? 0 : COLOURS); c++) + for (int dir = 0; dir < per; dir++) { + std::fill(v.begin(), v.end(), 0.0); + for (size_t k = static_cast(per * c + dir); k < n; k += static_cast(per * COLOURS)) + v[k] = 1.0; + restraint_hv(p, v, hv); + for (size_t k = static_cast(per * c + dir); k < n; k += static_cast(per * COLOURS)) + diagonal[k] = hv[k]; + } + std::vector sorted; + for (size_t k = 0; k < n; k++) + sorted.push_back(h[k] + std::max(diagonal[k], 0.0)); + std::nth_element(sorted.begin(), sorted.begin() + n / 2, sorted.end()); + const double floor = std::max(1e-3 * sorted[n / 2], 1e-12); + for (size_t k = 0; k < n; k++) + diagonal[k] = std::max(h[k] + std::max(diagonal[k], 0.0), floor); + if (const char *e = getenv("RUGNUX_NEWTON_CGMAX")) + newton.cg_max = std::atoi(e); + newton.clip_step = getenv("RUGNUX_NEWTON_CLIP") != nullptr; + newton.trace = getenv("RUGNUX_NEWTON_TRACE") != nullptr; + if (const char *e = getenv("RUGNUX_NEWTON_RELSTOP")) + newton.relative_stop = std::atof(e); + if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) + newton.cg_tolerance = std::atof(e); + const int cg_before = newton.cg_iterations; + if (NewtonHybrid()) { + lbfgs.SetPreconditioner([&](const std::vector &x, std::vector &q) { + std::vector z; + RefinementNewton::Solve(product, diagonal, 0.0, x, q, z, newton.cg_max, newton.cg_tolerance, + newton.cg_iterations); + q.swap(z); + }); + const RefinementMinimiser::Outcome o = lbfgs.Minimise( + [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, + p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); + lbfgs.SetPreconditioner(nullptr); + target.Set(p.data()); + if (RefineStats()) fprintf(stderr, "MINSTAT hybrid.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", xyz ? "xyz" : "b", o.iterations, + o.evaluations, iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); + return; + } + const RefinementMinimiser::Outcome o = newton.Minimise( + [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, + product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); + target.Set(p.data()); + if (RefineStats()) fprintf(stderr, "MINSTAT newton.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", xyz ? "xyz" : "b", o.iterations, + o.evaluations, iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis @@ -1144,6 +1350,18 @@ double BalancedWeight(Context &ctx, const std::vector &refinable) { std::vector> gx, gr; ctx.engine->Evaluate(ctx.Model(), &gx); std::vector> xyzb = ctx.XyzB(); + { // MEASUREMENT (lane k): the ratio of the gradients' norms at the model as given, unshaken + std::vector> g0; + ctx.engine->EvaluateRestraints(0, xyzb, &g0); + double a = 0, b = 0; + for (int i : refinable) + for (int c = 0; c < 3; c++) { + a += gx[i][c] * gx[i][c]; + b += g0[i][c] * g0[i][c]; + } + if (RefineStats()) fprintf(stderr, "WEIGHTSTAT unshaken_ratio=%.4g |gx|=%.4g |gr0|=%.4g\n", a > 0 ? std::sqrt(b / a) : 0.0, + std::sqrt(a), std::sqrt(b)); + } std::mt19937 rng(1); std::normal_distribution shake(0.0, 0.1); for (std::array &a : xyzb) @@ -1788,6 +2006,35 @@ double XrayGradientNorm(Context &ctx, const std::vector &refinable) { return n; } +// PROTOTYPE switches: RUGNUX_NEWTON=1 the positions and the B factors by MinimiseNewton(), with +// RUGNUX_NEWTON_XYZ / RUGNUX_NEWTON_B iterations per macro-cycle. +bool NewtonOn() { + static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) != "0"; + return on; +} +// RUGNUX_NEWTON=2: L-BFGS with the Newton H as its initial matrix instead of the damped Newton steps. +bool NewtonHybrid() { + static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) == "2"; + return on; +} +int NewtonIterations(bool xyz) { + const char *e = getenv(xyz ? "RUGNUX_NEWTON_XYZ" : "RUGNUX_NEWTON_B"); + return e != nullptr ? std::atoi(e) : (xyz ? 10 : 6); +} +// The X-ray curvature per atom in each of x, y, z at weight w, for the row model `rows` the engine has; `scale` +// calibrated at the first call of the trial (negative: not yet). +std::vector NewtonXyzCurvature(Context &ctx, const std::vector &refinable, const XrayRowModel &rows, + double w, double &scale) { + std::vector shape = ctx.XrayXyzCurvatureShape(rows); + if (scale < 0) { + scale = ctx.CalibrateXyzCurvature(refinable, shape); + if (RefineStats()) fprintf(stderr, "NEWTONCAL xyz_scale=%.4g\n", scale); + } + for (double &x : shape) + x *= w * scale; + return shape; +} + TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const std::vector &pos0, const std::vector &b0, const std::vector &aniso0, const Context::Fit *fit, const TrialWeights &tw, int cycles, const RefinementSettings &settings, const NcsCopies *ncs, @@ -1838,7 +2085,8 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); - double xray_scale = 0; + RefinementNewton xyz_newton, b_newton; + double xray_scale = 0, xyz_scale = -1; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { PlaceRiders(ctx, topo); if (cycle == 0 && fit != nullptr) @@ -1852,10 +2100,13 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s if (adp_w > 0) ctx.MoveOverallB(); if (!hold_positions) { + RestraintSet xyz_set; { const RestraintSet set = RestraintsNow(ctx, topo, ncs, ncs_sigma); const RefinementStage s1("set_restraints"); ctx.engine->SetRestraints(0, set); + if (NewtonOn()) + xyz_set = set; } const RefinementStage s2("minimise.xyz"); const double b_h = cycle < static_cast(tw.blur.size()) ? tw.blur[cycle] : 0.0; @@ -1870,15 +2121,22 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s ctx.engine->SetRowModel(blurred); const double nh = XrayGradientNorm(ctx, refinable); const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); - Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser); + if (NewtonOn()) + MinimiseNewton(ctx, xyz_h, NewtonIterations(true), xyz_newton, + NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), xyz_minimiser, xyz_set); + else + Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser, {}, "xyz_blur"); ctx.engine->SetRowModel(ctx.rows); - } else - Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser); + } else if (NewtonOn()) + MinimiseNewton(ctx, xyz, NewtonIterations(true), xyz_newton, + NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), xyz_minimiser, xyz_set); + else + Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser, {}, "xyz"); } if (kind == AdpHypothesis::Kind::Tls) { ctx.ApplyTls(); // the bodies' U at the atoms as they are const RefinementStage s4("minimise.tls"); - Minimise(ctx, tls_target, settings.b_iterations, tls_minimiser); + Minimise(ctx, tls_target, settings.b_iterations, tls_minimiser, {}, "tls"); } if (adp_w > 0 && kind == AdpHypothesis::Kind::Anisotropic) { // each U's six elements take a third of the curvature of its B @@ -1903,7 +2161,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double h : BDiagonal(xray, restraint, u_atoms, adp_w)) diagonal.insert(diagonal.end(), 6, h); const RefinementStage s3("minimise.u"); - Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal); + Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal, "u"); } else if (adp_w > 0) { const RestraintSet b_set = BRestraintsNow(ctx, topo); ctx.engine->SetRestraints(1, b_set); @@ -1913,8 +2171,13 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double &x : xray) x *= xray_scale; const RefinementStage s3("minimise.b"); - Minimise(ctx, b, settings.b_iterations, b_minimiser, - BDiagonal(xray, BRestraintCurvature(b_set, ctx.XyzB()), refinable, adp_w)); + if (NewtonOn()) { + for (double &x : xray) + x *= adp_w; + MinimiseNewton(ctx, b, NewtonIterations(false), b_newton, xray, b_minimiser, b_set); + } else + Minimise(ctx, b, settings.b_iterations, b_minimiser, + BDiagonal(xray, BRestraintCurvature(b_set, ctx.XyzB()), refinable, adp_w), "b"); } } PlaceRiders(ctx, topo); @@ -1959,6 +2222,8 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s out.fit = ctx.Fitted(); out.ran = true; out.evaluations = ctx.evaluations - evaluations_before; + if (RefineStats()) fprintf(stderr, "TRIALSTAT kind=%d adp_w=%.3g w=%.3g cycles=%d ev=%d rfree=%.4f\n", static_cast(kind), adp_w, w, cycles, + out.evaluations, t.r_free); return out; } @@ -2024,6 +2289,11 @@ std::vector RunTrials(const std::vector &trial_slots, for (const std::exception_ptr &e : errors) if (e) std::rethrow_exception(e); + int ev = 0; + for (const TrialOutcome &o : out) + ev += o.evaluations; + if (RefineStats()) fprintf(stderr, "TRIALSSET t=%.3f n=%zu cycles=%d ev=%d\n", + std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(), out.size(), cycles, ev); return out; } diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 1a19b2677..521a3a6fc 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include @@ -34,6 +35,9 @@ public: } // The estimated diagonal of the Hessian; every element > 0. void SetDiagonal(std::vector h) { h_ = std::move(h); } + // PROTOTYPE: H0 as an operator instead of the diagonal, q = H0 q at x (null: the diagonal). + using Preconditioner = std::function &x, std::vector &q)>; + void SetPreconditioner(Preconditioner p) { precondition_ = std::move(p); } struct Outcome { int iterations = 0; @@ -59,11 +63,11 @@ public: for (int it = 0; it < iterations; it++) { if (stop && stop()) break; - Direction(g, d); + Direction(x, g, d); double gd = Dot(g, d); if (!(gd < 0)) { // not a descent direction: the memory no longer fits, start again from the diagonal Reset(); - Direction(g, d); + Direction(x, g, d); gd = Dot(g, d); if (!(gd < 0)) break; @@ -145,7 +149,11 @@ private: // additions, so each sweep over the parameters takes the next pair's dot product while it applies the // previous pair's update to q - the same operations on each element in the same order as a sweep for // each would do, with the update's memory traffic hidden under the chain. - void Direction(const std::vector &g, std::vector &d) const { + void Direction(const std::vector &x, const std::vector &g, std::vector &d) const { + if (precondition_) { + DirectionPreconditioned(x, g, d); + return; + } const size_t n = g.size(), m = s_.size(); std::vector &q = q_; // kept from call to call: made anew, its pages fault in every time q.assign(g.begin(), g.end()); @@ -202,6 +210,27 @@ private: d[k] = -q[k]; } + // The two-loop recursion plainly, with H0 the preconditioner (its own scale: no Oren-Shanno factor). + void DirectionPreconditioned(const std::vector &x, const std::vector &g, std::vector &d) const { + const size_t m = s_.size(); + std::vector q = g, a(m); + for (size_t i = m; i-- > 0;) { + a[i] = Dot(s_[i], q) / sy_[i]; + for (size_t k = 0; k < q.size(); k++) + q[k] -= a[i] * y_[i][k]; + } + precondition_(x, q); + for (size_t i = 0; i < m; i++) { + const double b = Dot(y_[i], q) / sy_[i]; + for (size_t k = 0; k < q.size(); k++) + q[k] += s_[i][k] * (a[i] - b); + } + d.resize(q.size()); + for (size_t k = 0; k < q.size(); k++) + d[k] = -q[k]; + } + + Preconditioner precondition_; size_t memory_, nthreads_; std::deque> s_, y_; std::deque sy_; // s.y of each pair, as taken when it was kept @@ -209,3 +238,144 @@ private: mutable std::vector q_; // Direction()'s working vector std::vector h_; }; + +// Damped Newton steps (Levenberg-Marquardt): each iteration solves (H + lambda D) d = -g by conjugate gradients +// preconditioned with (1 + lambda) D, where H v is the caller's product with an approximation of the Hessian at the +// point of the last accepted evaluation and D an estimate of its diagonal, and takes d where it lowers f. lambda falls +// after a step the quadratic model predicted well and rises after one it did not; a step that does not lower f is +// solved again with the larger lambda, without a new gradient. Every iteration is one evaluation of the target where +// its step is taken. +// Following Levenberg (1944) Q. Appl. Math. 2, 164-168 and Marquardt (1963) J. SIAM 11, 431-441 +class RefinementNewton { +public: + using Target = RefinementMinimiser::Target; + // hv = H v at x, the point of the last accepted evaluation. + using Product = std::function &x, const std::vector &v, std::vector &hv)>; + using Outcome = RefinementMinimiser::Outcome; + + int cg_iterations = 0; // summed over the calls, for the log + int cg_max = 50; + double cg_tolerance = 0.1; + bool clip_step = false; + bool trace = false; + double relative_stop = 0; // PROTOTYPE: stop where an iteration lowers f by less than this of all the calls' decrease so far // PROTOTYPE: each parameter's step clipped to max_step, not the whole step scaled + + Outcome Minimise(const Target &target, const Product &product, const std::vector &diagonal, + std::vector &x, int iterations, double f_tolerance, double max_step, + const std::function &stop = nullptr) { + const size_t n = x.size(); + Outcome out; + double f = 0; + std::vector g(n), d(n), x_new(n), g_new(n), hd(n), rhs(n); + if (!target(x, f, g)) + return out; + out.evaluations++; + out.f_start = out.f_end = f; + for (int it = 0; it < iterations; it++) { + if (stop && stop()) + break; + bool accepted = false; + double f_new = 0; + for (int attempt = 0; attempt < 6 && !accepted; attempt++) { + for (size_t k = 0; k < n; k++) + rhs[k] = -g[k]; + Solve(product, diagonal, lambda_, x, rhs, d, cg_max, cg_tolerance, cg_iterations); + double d_max = 0; + for (double v : d) + d_max = std::max(d_max, std::fabs(v)); + if (clip_step) + for (double &v : d) + v = std::clamp(v, -max_step, max_step); + else if (d_max > max_step) + for (double &v : d) + v *= max_step / d_max; + product(x, d, hd); + const double gd = Dot(g, d), predicted = -(gd + 0.5 * Dot(d, hd)); + if (!(gd < 0)) + break; + for (size_t k = 0; k < n; k++) + x_new[k] = x[k] + d[k]; + const bool ok = target(x_new, f_new, g_new); + out.evaluations++; + const double rho = ok ? (f - f_new) / predicted : -1.0; + if (trace) + fprintf(stderr, "NTRACE it=%d att=%d dmax=%.3g lambda=%.3g pred=%.4g act=%.4g rho=%.3f\n", it, attempt, + d_max, lambda_, predicted, f - f_new, rho); + if (ok && f_new <= f + 1e-4 * gd) { + accepted = true; + if (rho > 0.75) + lambda_ = std::max(lambda_ / 3.0, 1e-4); + else if (rho < 0.25) + lambda_ *= 2.0; + } else + lambda_ = std::max(4.0 * lambda_, 0.1); + } + if (!accepted) + break; + const double decrease = f - f_new; + x.swap(x_new); + g.swap(g_new); + f = f_new; + out.iterations++; + out.f_end = f; + if (decrease <= f_tolerance * std::max(std::fabs(f), 1.0)) + break; + if (relative_stop > 0 && decrease < relative_stop * (out.f_start - f)) + break; + } + return out; + } + + // d with (H + lambda D) d = rhs to `tolerance` of |rhs|, by conjugate gradients preconditioned with + // (1 + lambda) D; where a direction of negative curvature turns up, d as far as it got (the preconditioned rhs at + // first). + static void Solve(const Product &product, const std::vector &diagonal, double lambda_, + const std::vector &x, const std::vector &rhs, std::vector &d, int CG_MAX, + double tolerance, int &cg_iterations) { + const size_t n = rhs.size(); + d.assign(n, 0.0); + std::vector r(n), z(n), p(n), hp(n); + for (size_t k = 0; k < n; k++) { + r[k] = rhs[k]; + z[k] = r[k] / ((1 + lambda_) * diagonal[k]); + } + p = z; + double rz = Dot(r, z); + const double r0 = std::sqrt(Dot(r, r)); + for (int k = 0; k < CG_MAX; k++) { + cg_iterations++; + product(x, p, hp); + for (size_t j = 0; j < n; j++) + hp[j] += lambda_ * diagonal[j] * p[j]; + const double php = Dot(p, hp); + if (!(php > 0)) { + if (k == 0) + d = z; + break; + } + const double a = rz / php; + for (size_t j = 0; j < n; j++) { + d[j] += a * p[j]; + r[j] -= a * hp[j]; + } + if (std::sqrt(Dot(r, r)) < tolerance * r0) + break; + for (size_t j = 0; j < n; j++) + z[j] = r[j] / ((1 + lambda_) * diagonal[j]); + const double rz_new = Dot(r, z); + for (size_t j = 0; j < n; j++) + p[j] = z[j] + rz_new / rz * p[j]; + rz = rz_new; + } + } + +private: + static double Dot(const std::vector &a, const std::vector &b) { + double s = 0; + for (size_t k = 0; k < a.size(); k++) + s += a[k] * b[k]; + return s; + } + + double lambda_ = 0.01; // kept from call to call, as the L-BFGS memory is +}; diff --git a/image_analysis/structure_refinement/RefinementRestraints.cpp b/image_analysis/structure_refinement/RefinementRestraints.cpp index d35c89910..7d0813183 100644 --- a/image_analysis/structure_refinement/RefinementRestraints.cpp +++ b/image_analysis/structure_refinement/RefinementRestraints.cpp @@ -397,6 +397,48 @@ double RestraintValue(const RestraintSet &set, const RestraintRecord &r, const s return NAN; } +namespace { +// E of a restraint of the kinds over up to four atoms (not a plane, not a B similarity), its gradient in g[0..n). +double RecordTerm(const RestraintRecord &r, const std::vector> &xyzb, V3 g[4], int &n) { + const double ideal = r.ideal, sigma = r.sigma; + double e = 0; + n = 0; + switch (static_cast(r.kind)) { + case RestraintKind::Bond: + e = refine_math::BondTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), ideal, sigma, g); + n = 2; + break; + case RestraintKind::Angle: + e = refine_math::AngleTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), ideal, sigma, g); + n = 3; + break; + case RestraintKind::Torsion: + e = refine_math::TorsionTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), + At(xyzb, r.atom[3]), ideal, sigma, r.period, g); + n = 4; + break; + case RestraintKind::Chiral: + e = refine_math::ChiralTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), + At(xyzb, r.atom[3]), ideal, sigma, g); + n = 4; + break; + case RestraintKind::NcsDistance: + e = refine_math::NcsDistanceTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), + At(xyzb, r.atom[3]), sigma, ideal, g); + n = 4; + break; + case RestraintKind::Repulsion: + e = refine_math::RepulsionTerm(At(xyzb, r.atom[0]), Image(r, At(xyzb, r.atom[1])), ideal, sigma, g); + g[1] = FromImage(r, g[1]); + n = 2; + break; + default: + break; + } + return e; +} +} // namespace + double EvaluateRestraints(const RestraintSet &set, const std::vector> &xyzb, std::vector> *grad) { double sum = 0; @@ -405,36 +447,8 @@ double EvaluateRestraints(const RestraintSet &set, const std::vector g[4] = {}; int n = 0; double e = 0; - const double ideal = r.ideal, sigma = r.sigma; + const double sigma = r.sigma; switch (static_cast(r.kind)) { - case RestraintKind::Bond: - e = refine_math::BondTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), ideal, sigma, g); - n = 2; - break; - case RestraintKind::Angle: - e = refine_math::AngleTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), ideal, sigma, g); - n = 3; - break; - case RestraintKind::Torsion: - e = refine_math::TorsionTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), - At(xyzb, r.atom[3]), ideal, sigma, r.period, g); - n = 4; - break; - case RestraintKind::Chiral: - e = refine_math::ChiralTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), - At(xyzb, r.atom[3]), ideal, sigma, g); - n = 4; - break; - case RestraintKind::NcsDistance: - e = refine_math::NcsDistanceTerm(At(xyzb, r.atom[0]), At(xyzb, r.atom[1]), At(xyzb, r.atom[2]), - At(xyzb, r.atom[3]), sigma, ideal, g); - n = 4; - break; - case RestraintKind::Repulsion: - e = refine_math::RepulsionTerm(At(xyzb, r.atom[0]), Image(r, At(xyzb, r.atom[1])), ideal, sigma, g); - g[1] = FromImage(r, g[1]); - n = 2; - break; case RestraintKind::Plane: { const int m = r.atom[1]; px.resize(m); @@ -460,6 +474,8 @@ double EvaluateRestraints(const RestraintSet &set, const std::vector> &xyzb) { + RestraintJacobian jac; + jac.start.push_back(0); + auto row = [&](int atom, double x, double y, double z, double b) { + jac.atom.push_back(atom); + jac.j.push_back({x, y, z, b}); + }; + std::vector> px, pg; + for (const RestraintRecord &r : set.records) { + const RestraintKind kind = static_cast(r.kind); + if (kind == RestraintKind::Plane) { + // each atom's distance from the plane a residual of its own, d r_i / d x_j = (delta_ij - 1/n) normal / sigma + // (the normal held), so one row per atom of the plane over all of them + const int m = r.atom[1]; + px.resize(m); + pg.resize(m); + for (int k = 0; k < m; k++) + px[k] = At(xyzb, set.plane_atoms[r.atom[0] + k]); + refine_math::PlaneTerm(px.data(), m, static_cast(r.sigma), pg.data()); + const double c = std::sqrt(2.0) / r.sigma; + V3 nv{0, 0, 0}; + for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n) + if (refine_math::Norm(pg[k]) > refine_math::Norm(nv)) + nv = pg[k]; + if (refine_math::Norm(nv) == 0) + continue; + nv = (1.0 / refine_math::Norm(nv)) * nv; + for (int i = 0; i < m; i++) { + for (int k = 0; k < m; k++) { + const double f = c * ((i == k ? 1.0 : 0.0) - 1.0 / m); + row(set.plane_atoms[r.atom[0] + k], f * nv.x, f * nv.y, f * nv.z, 0); + } + jac.start.push_back(static_cast(jac.atom.size())); + } + continue; + } + // E = residual^2 (or close to it): its gradient is 2 residual grad(residual), so J = g / sqrt(2 E) gives the + // Gauss-Newton term 2 grad(residual) grad(residual)^T = J J^T; nothing where E is 0 + if (kind == RestraintKind::BSimilarity) { + double gi = 0, gj = 0; + const double e = refine_math::BSimilarityTerm(xyzb[r.atom[0]][3], xyzb[r.atom[1]][3], + static_cast(r.sigma), gi, gj); + if (!(e > 0)) + continue; + const double f = 1.0 / std::sqrt(2.0 * e); + row(r.atom[0], 0, 0, 0, f * gi); + row(r.atom[1], 0, 0, 0, f * gj); + } else { + V3 g[4] = {}; + int n = 0; + const double e = RecordTerm(r, xyzb, g, n); + if (!(e > 0)) + continue; + const double f = 1.0 / std::sqrt(2.0 * e); + for (int k = 0; k < n; k++) + row(r.atom[k], f * g[k].x, f * g[k].y, f * g[k].z, 0); + } + jac.start.push_back(static_cast(jac.atom.size())); + } + return jac; +} + std::vector BRestraintCurvature(const RestraintSet &set, const std::vector> &xyzb) { std::vector h(xyzb.size(), 0.0); for (const RestraintRecord &r : set.records) diff --git a/image_analysis/structure_refinement/RefinementRestraints.h b/image_analysis/structure_refinement/RefinementRestraints.h index 0a887a7e7..23e8ff026 100644 --- a/image_analysis/structure_refinement/RefinementRestraints.h +++ b/image_analysis/structure_refinement/RefinementRestraints.h @@ -49,6 +49,15 @@ double RestraintValue(const RestraintSet &set, const RestraintRecord &r, const s double EvaluateRestraints(const RestraintSet &set, const std::vector> &xyzb, std::vector> *grad); +// PROTOTYPE: the Gauss-Newton approximation of the set's Hessian, sum over rows of J J^T: per row (a restraint's +// residual) its derivatives in (x, y, z, B) of the atoms atom[start[r] ... start[r + 1]). +struct RestraintJacobian { + std::vector start; + std::vector atom; + std::vector> j; +}; +RestraintJacobian RestraintGaussNewton(const RestraintSet &set, const std::vector> &xyzb); + // The diagonal of the Hessian of the set's B similarity restraints with respect to each atom's B, at `xyzb`. std::vector BRestraintCurvature(const RestraintSet &set, const std::vector> &xyzb); From 4dc4a47d2f0454333d17c1c4d12861cf4a18807b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 09:03:31 +0100 Subject: [PATCH 02/12] Refinement Newton (prototype): Gauss-Newton restraints only, threaded products, U, TLS, joint xyz+B, adaptive stop Still behind RUGNUX_NEWTON=1, off by default. - The restraints' Hessian is their Gauss-Newton J^T J (RestraintNormalMatrix: the Jacobian's rows and, per parameter atom, its entries, so that J v and J^T t are both gathers on the slot's threads); the finite-difference Hessian, the hybrid L-BFGS and the step clipping are gone. - U: the X-ray diagonal plus the U restraints' exact Hessian (quadratic in U); TLS: per body the dense 20 x 20 A^T C A from TlsBody::Derivatives. - RUGNUX_NEWTON_JOINT=1: positions and isotropic B in one target, X + R_geometry / w + R_B / adp_w. - A solve stops where an iteration gains less than RUGNUX_NEWTON_RELSTOP of the call's decrease, at most RUGNUX_NEWTON_XYZ / _B iterations (30 / 20); per-atom 3 x 3 block preconditioner for the positions. - RUGNUX_REFINE_PROTOCOL=,,: one protocol on the CPU and the GPU, the population not capped by the slots. - ACKNOWLEDGEMENT: Levenberg (1944), Marquardt (1963), the sparse normal matrix after Murshudov et al. (1997). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 8 + .../structure_refinement/Refinement.cpp | 613 +++++++++++++----- .../RefinementMinimiser.h | 154 ++--- 3 files changed, 531 insertions(+), 244 deletions(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index adf503c4b..dd37ce457 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -605,6 +605,14 @@ Management Sci. 20, 845-862 [doi:10.1287/mnsc.20.5.845](https://doi.org/10.1287/ D. F. Shanno and K.-H. Phua, "Matrix conditioning and nonlinear optimization" (1978), Math. Program. 14, 149-160 [doi:10.1007/BF01588962](https://doi.org/10.1007/BF01588962). +**Newton minimiser (experimental)** — damped Newton steps on a sparse normal matrix: the X-ray +target's curvature per atom on the diagonal and the restraints' Gauss-Newton matrix in full, as +REFMAC refines (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients with the +damping of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" +(1944), Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and +D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters" (1963), J. Soc. +Ind. Appl. Math. 11, 431-441 [doi:10.1137/0111030](https://doi.org/10.1137/0111030). + **Geometry restraints** — those of the CCP4 Monomer Library, a subset of which rugnux carries (see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)). A. A. Vagin, R. A. Steiner, A. A. Lebedev, L. Potterton, S. McNicholas, F. Long and G. N. Murshudov, "REFMAC5 dictionary: organization of prior diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 9581d46da..bf14d0b03 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -814,6 +814,7 @@ public: gradient != nullptr ? &gr : nullptr, restraints); } *cost = weight_ * xray + restraints; + last_xray_ = xray; if (gradient == nullptr) return std::isfinite(*cost); ctx_.FoldRiders(gx); @@ -840,26 +841,10 @@ public: return std::isfinite(*cost); } - // The restraints' gradient alone at p, as Evaluate() makes it: for products with their Hessian by differences. - void RestraintGradient(const double *p, double *gradient) const { - Set(p); - std::vector> &gr = gr_; - ctx_.engine->EvaluateRestraints(restraint_set_, ctx_.XyzB(), &gr); - const int n = static_cast(param_atoms_.size()); - for (int k = 0; k < n; k++) { - const int i = param_atoms_[k]; - if (mode_ == Mode::Xyz) { - gemmi::Vec3 g(gr[i][0], gr[i][1], gr[i][2]); - if (const SpecialPosition *sp = Special(i)) - g = sp->projector.multiply(g); - gradient[3 * k] = g.x; - gradient[3 * k + 1] = g.y; - gradient[3 * k + 2] = g.z; - } else - gradient[k] = gr[i][3]; - } - } Mode GetMode() const { return mode_; } + // The X-ray target and its gradient per atom (riders folded) at the last Evaluate(). + double LastXray() const { return last_xray_; } + const std::vector> &LastXrayGradient() const { return gx_; } double Weight() const { return weight_; } const std::vector &Atoms() const { return param_atoms_; } @@ -878,6 +863,66 @@ private: // the gradients per atom, kept from evaluation to evaluation (7 MB each on a large model: made anew, their // pages fault in every time) mutable std::vector> gx_, gr_; + mutable double last_xray_ = 0; +}; + +// PROTOTYPE (RUGNUX_NEWTON_JOINT=1): the positions and the B factors at once, against X + R_geometry / w + R_B / adp_w - +// stationary where the positions' target (w X + R_geometry) and the B factors' (adp_w X + R_B) both are, so that one +// evaluation moves both. The parameters: x y z of every atom, then its B in units of B_UNIT (one step bound for all). +class XyzBTarget { +public: + static constexpr double B_UNIT = MAX_STEP_B / MAX_STEP_XYZ; + XyzBTarget(Context &ctx, const std::vector ¶m_atoms, double w, double adp_w, const RefinementTopology &topo) + : ctx_(ctx), xyz_(ctx, AtomTarget::Mode::Xyz, param_atoms, w, 0, topo), + b_(ctx, AtomTarget::Mode::B, param_atoms, adp_w, 1, topo), n_(param_atoms.size()), w_(w), adp_w_(adp_w) {} + + std::vector Get() const { + std::vector p = xyz_.Get(); + for (double b : b_.Get()) + p.push_back(b / B_UNIT); + return p; + } + void Set(const double *p) const { + std::vector b(p + 3 * n_, p + 4 * n_); + for (double &x : b) + x *= B_UNIT; + xyz_.Set(p); + b_.Set(b.data()); + } + double MaxStep() const { return MAX_STEP_XYZ; } + + bool Evaluate(const double *p, double *cost, double *gradient) const { + // each part's own evaluation at the same point gives w X + R_geometry and adp_w X + R_B, and their gradients + std::vector g_xyz(3 * n_), g_b(n_), b(p + 3 * n_, p + 4 * n_); + for (double &x : b) + x *= B_UNIT; + b_.Set(b.data()); + double f_xyz = 0; + if (!xyz_.Evaluate(p, &f_xyz, g_xyz.data())) + return false; + // the B part at the same model: its X-ray target is the same one, only its restraints are added + std::vector> gr; + const double r_b = ctx_.engine->EvaluateRestraints(1, ctx_.XyzB(), &gr); + const double xray = xyz_.LastXray(); + *cost = xray + (f_xyz - w_ * xray) / w_ + r_b / adp_w_; + const std::vector> &gx = xyz_.LastXrayGradient(); + for (size_t k = 0; k < 3 * n_; k++) + gradient[k] = g_xyz[k] / w_; + for (size_t k = 0; k < n_; k++) { + const int i = xyz_.Atoms()[k]; + const double g = gx[i][3] + gr[i][3] / adp_w_; + const double bk = b[k]; + const bool held = (bk <= ctx_.BLow(i) && g > 0) || (bk >= ctx_.BHigh(i) && g < 0); + gradient[3 * n_ + k] = held ? 0.0 : g * B_UNIT; + } + return std::isfinite(*cost); + } + +private: + Context &ctx_; + AtomTarget xyz_, b_; + size_t n_; + double w_, adp_w_; }; // The X-ray target's derivatives in U of every atom, with what the target says of a riding hydrogen's B (its parent's @@ -945,6 +990,20 @@ public: } } double MaxStep() const { return MAX_STEP_B; } + // The parameter k of TLS body `body` over its value in the body's own units. + double ParamScale(size_t body, int k) const { return Scale(body, k); } + // Mode::U: the U restraints' gradient alone at p, as Evaluate() makes it (zero where there are none). + void RestraintGradient(const double *p, double *gradient) const { + Set(p); + std::fill(gradient, gradient + 6 * param_atoms_.size(), 0.0); + if (restraints_ == nullptr) + return; + std::vector> gr(ctx_.atoms.size(), {0, 0, 0, 0, 0, 0}); + EvaluateURestraints(*restraints_, ctx_.Positions(), ctx_.UOfAtoms(), &gr); + for (size_t k = 0; k < param_atoms_.size(); k++) + for (int c = 0; c < 6; c++) + gradient[6 * k + c] = gr[param_atoms_[k]][c] / gemmi::u_to_b(); + } bool Evaluate(const double *p, double *cost, double *gradient) const { const RefinementStage stage(mode_ == Mode::U ? "eval.u" : "eval.tls"); @@ -1041,26 +1100,297 @@ void Minimise(Context &ctx, const Target &target, int iterations, RefinementMini iterations, p.size(), o.f_start, o.f_end); } -bool NewtonHybrid(); -// PROTOTYPE (RUGNUX_NEWTON=1): the positions or the B factors by damped Newton steps (RefinementNewton). H is the -// X-ray target's curvature per parameter (`xray`: per atom, already times the weight) plus the restraints' exact -// Hessian, its products by forward differences of their gradient; D is H's diagonal, the restraints' part of it -// probed by differences on atoms far apart in the list at a time (the atoms of one restraint are near in it). -void MinimiseNewton(Context &ctx, const AtomTarget &target, int iterations, RefinementNewton &newton, - const std::vector &xray, RefinementMinimiser &lbfgs, const RestraintSet &set) { +// PROTOTYPE (RUGNUX_NEWTON=1): the minimisation by damped Newton steps (RefinementNewton) on the Gauss-Newton normal +// matrix, as REFMAC refines on its sparse normal matrix: the X-ray target's curvature per parameter on the diagonal (no +// coupling between atoms), the restraints' J^T J in full. +// Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 +struct NewtonSettings { + int xyz_iterations = 30, b_iterations = 20; // the most per macro-cycle, as the L-BFGS's + double relative_stop = 0.0; +}; +const NewtonSettings &Newton() { + static const NewtonSettings s = [] { + NewtonSettings n; + if (const char *e = getenv("RUGNUX_NEWTON_XYZ")) + n.xyz_iterations = std::atoi(e); + if (const char *e = getenv("RUGNUX_NEWTON_B")) + n.b_iterations = std::atoi(e); + if (const char *e = getenv("RUGNUX_NEWTON_RELSTOP")) + n.relative_stop = std::atof(e); + return n; + }(); + return s; +} + +// `target` by `newton` from where it is, with H v = product and D = diagonal (each element > 0). +template +void MinimiseNewton(Context &ctx, const Target &target, RefinementNewton &newton, const RefinementNewton::Product &product, + const std::vector &diagonal, int iterations, const char *tag, + const RefinementNewton::Preconditioner &precondition = nullptr) { std::vector p = target.Get(); if (p.empty()) return; + newton.relative_stop = Newton().relative_stop; + newton.trace = getenv("RUGNUX_NEWTON_TRACE") != nullptr; + if (const char *e = getenv("RUGNUX_NEWTON_CGMAX")) + newton.cg_max = std::atoi(e); + if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) + newton.cg_tolerance = std::atof(e); + const int cg_before = newton.cg_iterations; + const RefinementMinimiser::Outcome o = newton.Minimise( + [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, + product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }, + precondition); + target.Set(p.data()); + if (RefineStats()) + fprintf(stderr, "MINSTAT newton.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, + iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); +} + +// Each element at least a thousandth of the median (a parameter the target does not see still takes a finite step). +std::vector FlooredDiagonal(std::vector d) { + if (d.empty()) + return d; + std::vector sorted = d; + std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); + const double floor = std::max(1e-3 * sorted[sorted.size() / 2], 1e-12); + for (double &v : d) + v = std::max(v, floor); + return d; +} + +// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the +// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. +class RestraintNormalMatrix { +public: + RestraintNormalMatrix(const std::vector ¶m_atoms, size_t n_atoms, bool xyz, size_t nthreads) + : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(nthreads), + param_of_(n_atoms, -1) { + for (size_t k = 0; k < param_atoms.size(); k++) + param_of_[param_atoms[k]] = static_cast(k); + } + + void At(const RestraintSet &set, const std::vector> &xyzb) { + const RefinementStage stage("newton.jacobian"); + jac_ = RestraintGaussNewton(set, xyzb); + const size_t rows = jac_.start.size() - 1; + row_of_.resize(jac_.atom.size()); + param_.resize(jac_.atom.size()); + for (size_t e = 0; e < jac_.atom.size(); e++) + param_[e] = param_of_[jac_.atom[e]]; + for (size_t r = 0; r < rows; r++) + for (int e = jac_.start[r]; e < jac_.start[r + 1]; e++) + row_of_[e] = static_cast(r); + // per parameter atom its entries (CSR), for the transpose as a gather + entry_start_.assign(n_params_ + 1, 0); + for (int a : jac_.atom) + if (param_of_[a] >= 0) + entry_start_[param_of_[a] + 1]++; + for (size_t k = 0; k < n_params_; k++) + entry_start_[k + 1] += entry_start_[k]; + entry_.resize(entry_start_.back()); + std::vector fill(entry_start_.begin(), entry_start_.end() - 1); + for (size_t e = 0; e < jac_.atom.size(); e++) + if (const int k = param_of_[jac_.atom[e]]; k >= 0) + entry_[fill[k]++] = static_cast(e); + t_.resize(rows); + } + + // hv = J^T J v: J v per row, then per parameter atom the sum over its entries + void Multiply(const std::vector &v, std::vector &hv) { + const RefinementStage stage("newton.product"); + const int rows = static_cast(t_.size()); + ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) { + for (int r = lo; r < hi; r++) { + double dot = 0; + for (int e = jac_.start[r]; e < jac_.start[r + 1]; e++) + if (const int k = param_[e]; k >= 0) + for (int c = 0; c < per_; c++) + dot += jac_.j[e][c0_ + c] * v[per_ * k + c]; + t_[r] = dot; + } + }); + hv.assign(per_ * n_params_, 0.0); + const int n = static_cast(n_params_); + ParallelChunks(n, ThreadsForWork(n, nthreads_, 16384), [&](int lo, int hi) { + for (int k = lo; k < hi; k++) + for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) { + const int e = entry_[q]; + for (int c = 0; c < per_; c++) + hv[per_ * k + c] += t_[row_of_[e]] * jac_.j[e][c0_ + c]; + } + }); + } + + // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. + std::vector> Blocks() const { + std::vector> b(n_params_, {0, 0, 0, 0, 0, 0}); + for (size_t k = 0; k < n_params_; k++) + for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) { + const std::array &j = jac_.j[entry_[q]]; + b[k][0] += j[0] * j[0]; + b[k][1] += j[1] * j[1]; + b[k][2] += j[2] * j[2]; + b[k][3] += j[0] * j[1]; + b[k][4] += j[0] * j[2]; + b[k][5] += j[1] * j[2]; + } + return b; + } + + std::vector Diagonal() const { + std::vector d(per_ * n_params_, 0.0); + for (size_t k = 0; k < n_params_; k++) + for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) + for (int c = 0; c < per_; c++) + d[per_ * k + c] += jac_.j[entry_[q]][c0_ + c] * jac_.j[entry_[q]][c0_ + c]; + return d; + } + +private: + int per_, c0_; + size_t n_params_, nthreads_; + std::vector param_of_; + RestraintJacobian jac_; + std::vector row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none) + std::vector t_; +}; + +// z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. +std::array Solve3(const std::array &m, double r0, double r1, double r2) { + const double a = m[0], b = m[1], c = m[2], d = m[3], e = m[4], f = m[5]; + const double c00 = b * c - f * f, c01 = e * f - d * c, c02 = d * f - b * e; + const double c11 = a * c - e * e, c12 = d * e - a * f, c22 = a * b - d * d; + const double det = a * c00 + d * c01 + e * c02; + return {(c00 * r0 + c01 * r1 + c02 * r2) / det, (c01 * r0 + c11 * r1 + c12 * r2) / det, + (c02 * r0 + c12 * r1 + c22 * r2) / det}; +} + +// The positions' preconditioner: per atom its 3 x 3 block of H (`blocks`, the restraints' over `scale`, plus the X-ray +// curvature `h`) and lambda times the damping's diagonal `diagonal`; parameters 3k..3k+2 of atom k from `offset` on. +void PreconditionBlocks(const std::vector> &blocks, double scale, const std::vector &h, + const std::vector &diagonal, size_t offset, const std::vector &r, double lambda, + std::vector &z) { + for (size_t k = 0; k < blocks.size(); k++) { + const size_t o = offset + 3 * k; + std::array m; + for (int c = 0; c < 6; c++) + m[c] = blocks[k][c] / scale; + for (int c = 0; c < 3; c++) + m[c] += h[o + c] + lambda * diagonal[o + c]; + const std::array s = Solve3(m, r[o], r[o + 1], r[o + 2]); + z[o] = s[0]; + z[o + 1] = s[1]; + z[o + 2] = s[2]; + } +} + +// The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`. +void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewton &newton, const std::vector &xray, + const RestraintSet &set, int iterations) { const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; const int per = xyz ? 3 : 1; + std::vector p = target.Get(); const size_t n = p.size(); - const double eps = xyz ? 1e-4 : 0.1; // the largest change of a parameter in a difference (A, A^2) std::vector h(n); for (size_t k = 0; k < n; k++) h[k] = xray[target.Atoms()[k / per]]; - std::vector x_at, g_at(n), gp(n), xp(n); - auto restraint_hv = [&](const std::vector &x, const std::vector &v, std::vector &hv) { - if (x_at != x) { + RestraintNormalMatrix normal(target.Atoms(), ctx.atoms.size(), xyz, ctx.nthreads); + std::vector x_at = p; + target.Set(p.data()); + normal.At(set, ctx.XyzB()); + std::vector diagonal = normal.Diagonal(); + for (size_t k = 0; k < n; k++) + diagonal[k] += h[k]; + auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (x != x_at) { + x_at = x; + target.Set(x.data()); + normal.At(set, ctx.XyzB()); + } + normal.Multiply(v, hv); + for (size_t k = 0; k < n; k++) + hv[k] += h[k] * v[k]; + }; + diagonal = FlooredDiagonal(diagonal); + if (!xyz) { + MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "b"); + return; + } + // each atom's 3 x 3 block of H at the start, fixed for the call + const std::vector> blocks = normal.Blocks(); + MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "xyz", + [&](const std::vector &r, double lambda, std::vector &z) { + PreconditionBlocks(blocks, 1.0, h, diagonal, 0, r, lambda, z); + }); +} + +// The positions and the B factors at once (XyzBTarget): H the two blocks of MinimiseAtomsNewton(), each over its weight +// (`xray_xyz`, `xray_b`: per atom, the X-ray curvature alone), no coupling between them. +void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vector &atoms, RefinementNewton &newton, + const std::vector &xray_xyz, const std::vector &xray_b, const RestraintSet &set_xyz, + const RestraintSet &set_b, double w, double adp_w, int iterations) { + const size_t n = atoms.size(); + const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT; + RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads), + normal_b(atoms, ctx.atoms.size(), false, ctx.nthreads); + std::vector x_at = target.Get(); + auto at = [&](const std::vector &x) { + target.Set(x.data()); + const std::vector> xyzb = ctx.XyzB(); + normal_xyz.At(set_xyz, xyzb); + normal_b.At(set_b, xyzb); + }; + at(x_at); + std::vector diagonal(4 * n); + { + const std::vector dx = normal_xyz.Diagonal(), db = normal_b.Diagonal(); + for (size_t k = 0; k < n; k++) { + for (int c = 0; c < 3; c++) + diagonal[3 * k + c] = xray_xyz[atoms[k]] + dx[3 * k + c] / w; + diagonal[3 * n + k] = u2 * (xray_b[atoms[k]] + db[k] / adp_w); + } + } + std::vector vx(3 * n), vb(n), hx, hb; + auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (x != x_at) { + x_at = x; + at(x); + } + std::copy(v.begin(), v.begin() + 3 * n, vx.begin()); + std::copy(v.begin() + 3 * n, v.end(), vb.begin()); + normal_xyz.Multiply(vx, hx); + normal_b.Multiply(vb, hb); + hv.resize(4 * n); + for (size_t k = 0; k < n; k++) { + for (int c = 0; c < 3; c++) + hv[3 * k + c] = hx[3 * k + c] / w + xray_xyz[atoms[k]] * v[3 * k + c]; + hv[3 * n + k] = u2 * (hb[k] / adp_w + xray_b[atoms[k]] * v[3 * n + k]); + } + }; + diagonal = FlooredDiagonal(diagonal); + // the positions' 3 x 3 blocks of H at the start, fixed for the call; the B factors' diagonal + const std::vector> blocks = normal_xyz.Blocks(); + std::vector h(3 * n); + for (size_t k = 0; k < n; k++) + for (int c = 0; c < 3; c++) + h[3 * k + c] = xray_xyz[atoms[k]]; + MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "xyzb", + [&](const std::vector &r, double lambda, std::vector &z) { + PreconditionBlocks(blocks, w, h, diagonal, 0, r, lambda, z); + for (size_t k = 3 * n; k < 4 * n; k++) + z[k] = r[k] / ((1 + lambda) * diagonal[k]); + }); +} + +// Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their +// gradient (they are quadratic in U); D the diagonal the L-BFGS takes. +void MinimiseUNewton(Context &ctx, const AdpTarget &target, RefinementNewton &newton, const std::vector &xray, + const std::vector &diagonal, int iterations) { + const size_t n = xray.size(); + std::vector x_at, g_at(n), xp(n), gp(n); + auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + if (x != x_at) { x_at = x; target.RestraintGradient(x.data(), g_at.data()); } @@ -1068,110 +1398,56 @@ void MinimiseNewton(Context &ctx, const AtomTarget &target, int iterations, Refi for (double e : v) v_max = std::max(v_max, std::fabs(e)); hv.assign(n, 0.0); - if (v_max == 0) - return; - const double t = eps / v_max; - for (size_t k = 0; k < n; k++) - xp[k] = x[k] + t * v[k]; - target.RestraintGradient(xp.data(), gp.data()); - for (size_t k = 0; k < n; k++) - hv[k] = (gp[k] - g_at[k]) / t; - }; - // RUGNUX_NEWTON_GN=1: the restraints' Gauss-Newton Hessian (RestraintGaussNewton) instead of the exact one by - // differences - positive semi-definite, and its diagonal exact - static const bool gn = getenv("RUGNUX_NEWTON_GN") != nullptr; - std::vector param_of(ctx.atoms.size(), -1); - for (size_t k = 0; k < target.Atoms().size(); k++) - param_of[target.Atoms()[k]] = static_cast(k); - const int c0 = xyz ? 0 : 3; - RestraintJacobian jac; - std::vector x_jac; - auto gn_hv = [&](const std::vector &x, const std::vector &v, std::vector &hv) { - if (x_jac != x) { - x_jac = x; - target.Set(x.data()); - jac = RestraintGaussNewton(set, ctx.XyzB()); + if (v_max > 0) { + const double t = 1.0 / v_max; + for (size_t k = 0; k < n; k++) + xp[k] = x[k] + t * v[k]; + target.RestraintGradient(xp.data(), gp.data()); + for (size_t k = 0; k < n; k++) + hv[k] = (gp[k] - g_at[k]) / t; } - hv.assign(n, 0.0); - for (size_t r = 0; r + 1 < jac.start.size(); r++) { - double dot = 0; - for (int e = jac.start[r]; e < jac.start[r + 1]; e++) - if (const int k = param_of[jac.atom[e]]; k >= 0) - for (int c = 0; c < per; c++) - dot += jac.j[e][c0 + c] * v[per * k + c]; - for (int e = jac.start[r]; e < jac.start[r + 1]; e++) - if (const int k = param_of[jac.atom[e]]; k >= 0) - for (int c = 0; c < per; c++) - hv[per * k + c] += dot * jac.j[e][c0 + c]; - } - }; - auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { - if (gn) - gn_hv(x, v, hv); - else - restraint_hv(x, v, hv); for (size_t k = 0; k < n; k++) - hv[k] += h[k] * v[k]; + hv[k] += xray[k] * v[k]; }; - // the restraints' diagonal: every COLOURS-th atom at a time, one direction at a time - constexpr int COLOURS = 24; - std::vector diagonal(n, 0.0), v(n), hv; - if (gn) { - target.Set(p.data()); - x_jac = p; - jac = RestraintGaussNewton(set, ctx.XyzB()); - for (size_t e = 0; e < jac.atom.size(); e++) - if (const int k = param_of[jac.atom[e]]; k >= 0) - for (int c = 0; c < per; c++) - diagonal[per * k + c] += jac.j[e][c0 + c] * jac.j[e][c0 + c]; + MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "u"); +} + +// The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's +// parameters (linear in them), C the X-ray curvature of its U per element (`xray_b`: of its B, a third of it to each). +void MinimiseTlsNewton(Context &ctx, const AdpTarget &target, RefinementNewton &newton, const std::vector &xray_b, + int iterations) { + constexpr int P = TlsBody::PARAMETERS; + const size_t nb = ctx.tls.size(); + std::vector> blocks(nb); + for (size_t b = 0; b < nb; b++) { + std::array &m = blocks[b]; + m.fill(0.0); + for (int i : ctx.tls[b].atoms) { + if (!ctx.rider_parent.empty() && ctx.rider_parent[i] >= 0) + continue; // its curvature is its parent's already + const std::array, P> a = ctx.tls[b].Derivatives(ctx.atoms[i]->pos); + const double c = gemmi::u_to_b() * gemmi::u_to_b() * xray_b[i] / 3.0; + for (int k = 0; k < P; k++) + for (int l = 0; l < P; l++) { + double s = 0; + for (int e = 0; e < 6; e++) + s += a[k][e] * a[l][e]; + m[k * P + l] += c * s / (target.ParamScale(b, k) * target.ParamScale(b, l)); + } + } } - for (int c = 0; c < (gn ? 0 : COLOURS); c++) - for (int dir = 0; dir < per; dir++) { - std::fill(v.begin(), v.end(), 0.0); - for (size_t k = static_cast(per * c + dir); k < n; k += static_cast(per * COLOURS)) - v[k] = 1.0; - restraint_hv(p, v, hv); - for (size_t k = static_cast(per * c + dir); k < n; k += static_cast(per * COLOURS)) - diagonal[k] = hv[k]; - } - std::vector sorted; - for (size_t k = 0; k < n; k++) - sorted.push_back(h[k] + std::max(diagonal[k], 0.0)); - std::nth_element(sorted.begin(), sorted.begin() + n / 2, sorted.end()); - const double floor = std::max(1e-3 * sorted[n / 2], 1e-12); - for (size_t k = 0; k < n; k++) - diagonal[k] = std::max(h[k] + std::max(diagonal[k], 0.0), floor); - if (const char *e = getenv("RUGNUX_NEWTON_CGMAX")) - newton.cg_max = std::atoi(e); - newton.clip_step = getenv("RUGNUX_NEWTON_CLIP") != nullptr; - newton.trace = getenv("RUGNUX_NEWTON_TRACE") != nullptr; - if (const char *e = getenv("RUGNUX_NEWTON_RELSTOP")) - newton.relative_stop = std::atof(e); - if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) - newton.cg_tolerance = std::atof(e); - const int cg_before = newton.cg_iterations; - if (NewtonHybrid()) { - lbfgs.SetPreconditioner([&](const std::vector &x, std::vector &q) { - std::vector z; - RefinementNewton::Solve(product, diagonal, 0.0, x, q, z, newton.cg_max, newton.cg_tolerance, - newton.cg_iterations); - q.swap(z); - }); - const RefinementMinimiser::Outcome o = lbfgs.Minimise( - [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, - p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); - lbfgs.SetPreconditioner(nullptr); - target.Set(p.data()); - if (RefineStats()) fprintf(stderr, "MINSTAT hybrid.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", xyz ? "xyz" : "b", o.iterations, - o.evaluations, iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); - return; - } - const RefinementMinimiser::Outcome o = newton.Minimise( - [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, - product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); - target.Set(p.data()); - if (RefineStats()) fprintf(stderr, "MINSTAT newton.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", xyz ? "xyz" : "b", o.iterations, - o.evaluations, iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); + std::vector diagonal(nb * P); + for (size_t b = 0; b < nb; b++) + for (int k = 0; k < P; k++) + diagonal[b * P + k] = blocks[b][k * P + k]; + auto product = [&](const std::vector &, const std::vector &v, std::vector &hv) { + hv.assign(v.size(), 0.0); + for (size_t b = 0; b < nb; b++) + for (int k = 0; k < P; k++) + for (int l = 0; l < P; l++) + hv[b * P + k] += blocks[b][k * P + l] * v[b * P + l]; + }; + MinimiseNewton(ctx, target, newton, product, FlooredDiagonal(diagonal), iterations, "tls"); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis @@ -2006,21 +2282,16 @@ double XrayGradientNorm(Context &ctx, const std::vector &refinable) { return n; } -// PROTOTYPE switches: RUGNUX_NEWTON=1 the positions and the B factors by MinimiseNewton(), with -// RUGNUX_NEWTON_XYZ / RUGNUX_NEWTON_B iterations per macro-cycle. +// PROTOTYPE switch: RUGNUX_NEWTON=1 the positions, the B factors, the U and the TLS bodies by damped Newton steps. bool NewtonOn() { static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) != "0"; return on; } -// RUGNUX_NEWTON=2: L-BFGS with the Newton H as its initial matrix instead of the damped Newton steps. -bool NewtonHybrid() { - static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) == "2"; +// RUGNUX_NEWTON_JOINT=1: where the B factors are refined isotropically, they and the positions at once (XyzBTarget). +bool NewtonJoint() { + static const bool on = getenv("RUGNUX_NEWTON_JOINT") != nullptr; return on; } -int NewtonIterations(bool xyz) { - const char *e = getenv(xyz ? "RUGNUX_NEWTON_XYZ" : "RUGNUX_NEWTON_B"); - return e != nullptr ? std::atoi(e) : (xyz ? 10 : 6); -} // The X-ray curvature per atom in each of x, y, z at weight w, for the row model `rows` the engine has; `scale` // calibrated at the first call of the trial (negative: not yet). std::vector NewtonXyzCurvature(Context &ctx, const std::vector &refinable, const XrayRowModel &rows, @@ -2085,8 +2356,8 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); - RefinementNewton xyz_newton, b_newton; - double xray_scale = 0, xyz_scale = -1; + RefinementNewton xyz_newton, b_newton, tls_newton; + double xray_scale = 0, xyz_scale = -1, tls_scale = -1; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { PlaceRiders(ctx, topo); if (cycle == 0 && fit != nullptr) @@ -2099,6 +2370,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s } if (adp_w > 0) ctx.MoveOverallB(); + bool joint_done = false; // the B factors refined with the positions (MinimiseJointNewton) if (!hold_positions) { RestraintSet xyz_set; { @@ -2122,21 +2394,43 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s const double nh = XrayGradientNorm(ctx, refinable); const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); if (NewtonOn()) - MinimiseNewton(ctx, xyz_h, NewtonIterations(true), xyz_newton, - NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), xyz_minimiser, xyz_set); + MinimiseAtomsNewton(ctx, xyz_h, xyz_newton, + NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), xyz_set, + Newton().xyz_iterations); else Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser, {}, "xyz_blur"); ctx.engine->SetRowModel(ctx.rows); + } else if (NewtonOn() && NewtonJoint() && kind == AdpHypothesis::Kind::Isotropic && adp_w > 0) { + const RestraintSet b_set = BRestraintsNow(ctx, topo); + ctx.engine->SetRestraints(1, b_set); + std::vector xray_b = ctx.XrayBCurvatureShape(); + if (cycle == 0) + xray_scale = ctx.CalibrateBCurvature(refinable, xray_b); + for (double &x : xray_b) + x *= xray_scale; + const XyzBTarget xyz_b(ctx, refinable, w, adp_w, topo); + MinimiseJointNewton(ctx, xyz_b, refinable, xyz_newton, + NewtonXyzCurvature(ctx, refinable, ctx.rows, 1.0, xyz_scale), xray_b, xyz_set, b_set, w, + adp_w, Newton().xyz_iterations); + joint_done = true; } else if (NewtonOn()) - MinimiseNewton(ctx, xyz, NewtonIterations(true), xyz_newton, - NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), xyz_minimiser, xyz_set); + MinimiseAtomsNewton(ctx, xyz, xyz_newton, NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), + xyz_set, Newton().xyz_iterations); else Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser, {}, "xyz"); } if (kind == AdpHypothesis::Kind::Tls) { ctx.ApplyTls(); // the bodies' U at the atoms as they are const RefinementStage s4("minimise.tls"); - Minimise(ctx, tls_target, settings.b_iterations, tls_minimiser, {}, "tls"); + if (NewtonOn()) { + std::vector xray = ctx.XrayBCurvatureShape(); + if (tls_scale < 0) + tls_scale = ctx.CalibrateBCurvature(refinable, xray); + for (double &x : xray) + x *= tls_scale; + MinimiseTlsNewton(ctx, tls_target, tls_newton, xray, Newton().b_iterations); + } else + Minimise(ctx, tls_target, settings.b_iterations, tls_minimiser, {}, "tls"); } if (adp_w > 0 && kind == AdpHypothesis::Kind::Anisotropic) { // each U's six elements take a third of the curvature of its B @@ -2161,8 +2455,14 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double h : BDiagonal(xray, restraint, u_atoms, adp_w)) diagonal.insert(diagonal.end(), 6, h); const RefinementStage s3("minimise.u"); - Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal, "u"); - } else if (adp_w > 0) { + if (NewtonOn()) { + std::vector h; + for (int i : u_atoms) + h.insert(h.end(), 6, adp_w * xray[i]); + MinimiseUNewton(ctx, u_target, b_newton, h, diagonal, Newton().b_iterations); + } else + Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal, "u"); + } else if (adp_w > 0 && !joint_done) { const RestraintSet b_set = BRestraintsNow(ctx, topo); ctx.engine->SetRestraints(1, b_set); std::vector xray = ctx.XrayBCurvatureShape(); @@ -2174,7 +2474,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s if (NewtonOn()) { for (double &x : xray) x *= adp_w; - MinimiseNewton(ctx, b, NewtonIterations(false), b_newton, xray, b_minimiser, b_set); + MinimiseAtomsNewton(ctx, b, b_newton, xray, b_set, Newton().b_iterations); } else Minimise(ctx, b, settings.b_iterations, b_minimiser, BDiagonal(xray, BRestraintCurvature(b_set, ctx.XyzB()), refinable, adp_w), "b"); @@ -3320,7 +3620,10 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector(settings.population, trial_slots.size()); + // PROTOTYPE (lane k): under RUGNUX_REFINE_PROTOCOL the members are as many as asked for, whatever the slots + const size_t population = getenv("RUGNUX_REFINE_PROTOCOL") != nullptr + ? static_cast(std::max(settings.population, 0)) + : std::min(settings.population, trial_slots.size()); if (population > 1 && state.ran && !ctx.Cancelled()) { const RefinementStage pop_stage("population"); const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma; @@ -4493,6 +4796,16 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector,[,] on the CPU and the GPU alike + if (const char *e = getenv("RUGNUX_REFINE_PROTOCOL")) { + int v[3] = {settings.population, settings.rounds, settings.macro_cycles}; + sscanf(e, "%d,%d,%d", &v[0], &v[1], &v[2]); + settings.population = v[0]; + settings.rounds = v[1]; + settings.macro_cycles = v[2]; + logger.Info("Refinement EXPERIMENT protocol: population {}, {} rounds, {} macro-cycles", settings.population, + settings.rounds, settings.macro_cycles); + } // Each arm, untwinned and twinned, is a whole refinement - the solvent built and the displacement model chosen // against its own target - and the decision is taken on where they ended. A model only scored is scored both // ways and decided by the same rule, so the R-free it reports is not an untwinned one by default. diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 521a3a6fc..254a6474c 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -35,9 +35,6 @@ public: } // The estimated diagonal of the Hessian; every element > 0. void SetDiagonal(std::vector h) { h_ = std::move(h); } - // PROTOTYPE: H0 as an operator instead of the diagonal, q = H0 q at x (null: the diagonal). - using Preconditioner = std::function &x, std::vector &q)>; - void SetPreconditioner(Preconditioner p) { precondition_ = std::move(p); } struct Outcome { int iterations = 0; @@ -63,11 +60,11 @@ public: for (int it = 0; it < iterations; it++) { if (stop && stop()) break; - Direction(x, g, d); + Direction(g, d); double gd = Dot(g, d); if (!(gd < 0)) { // not a descent direction: the memory no longer fits, start again from the diagonal Reset(); - Direction(x, g, d); + Direction(g, d); gd = Dot(g, d); if (!(gd < 0)) break; @@ -149,11 +146,7 @@ private: // additions, so each sweep over the parameters takes the next pair's dot product while it applies the // previous pair's update to q - the same operations on each element in the same order as a sweep for // each would do, with the update's memory traffic hidden under the chain. - void Direction(const std::vector &x, const std::vector &g, std::vector &d) const { - if (precondition_) { - DirectionPreconditioned(x, g, d); - return; - } + void Direction(const std::vector &g, std::vector &d) const { const size_t n = g.size(), m = s_.size(); std::vector &q = q_; // kept from call to call: made anew, its pages fault in every time q.assign(g.begin(), g.end()); @@ -210,27 +203,6 @@ private: d[k] = -q[k]; } - // The two-loop recursion plainly, with H0 the preconditioner (its own scale: no Oren-Shanno factor). - void DirectionPreconditioned(const std::vector &x, const std::vector &g, std::vector &d) const { - const size_t m = s_.size(); - std::vector q = g, a(m); - for (size_t i = m; i-- > 0;) { - a[i] = Dot(s_[i], q) / sy_[i]; - for (size_t k = 0; k < q.size(); k++) - q[k] -= a[i] * y_[i][k]; - } - precondition_(x, q); - for (size_t i = 0; i < m; i++) { - const double b = Dot(y_[i], q) / sy_[i]; - for (size_t k = 0; k < q.size(); k++) - q[k] += s_[i][k] * (a[i] - b); - } - d.resize(q.size()); - for (size_t k = 0; k < q.size(); k++) - d[k] = -q[k]; - } - - Preconditioner precondition_; size_t memory_, nthreads_; std::deque> s_, y_; std::deque sy_; // s.y of each pair, as taken when it was kept @@ -239,12 +211,12 @@ private: std::vector h_; }; -// Damped Newton steps (Levenberg-Marquardt): each iteration solves (H + lambda D) d = -g by conjugate gradients -// preconditioned with (1 + lambda) D, where H v is the caller's product with an approximation of the Hessian at the -// point of the last accepted evaluation and D an estimate of its diagonal, and takes d where it lowers f. lambda falls -// after a step the quadratic model predicted well and rises after one it did not; a step that does not lower f is -// solved again with the larger lambda, without a new gradient. Every iteration is one evaluation of the target where -// its step is taken. +// Damped Newton steps (Levenberg-Marquardt) on a positive semi-definite approximation of the Hessian: each iteration +// solves (H + lambda D) d = -g by conjugate gradients preconditioned with (1 + lambda) D - H v the caller's product at +// the point of the last accepted evaluation, D H's diagonal - and takes d where it lowers f. lambda falls after a +// step the quadratic model predicted well and rises after one it did not; a step that does not lower f is solved again +// with the larger lambda, without a new gradient. An iteration is one evaluation of the target where its step is +// taken. A solve stops where an iteration lowers f by less than `relative_stop` of what the call has lowered it by. // Following Levenberg (1944) Q. Appl. Math. 2, 164-168 and Marquardt (1963) J. SIAM 11, 431-441 class RefinementNewton { public: @@ -256,17 +228,19 @@ public: int cg_iterations = 0; // summed over the calls, for the log int cg_max = 50; double cg_tolerance = 0.1; - bool clip_step = false; + double relative_stop = 0.0; bool trace = false; - double relative_stop = 0; // PROTOTYPE: stop where an iteration lowers f by less than this of all the calls' decrease so far // PROTOTYPE: each parameter's step clipped to max_step, not the whole step scaled + + // z = M^-1 r for M an approximation of H + lambda D that is cheap to solve (null: (1 + lambda) D). + using Preconditioner = std::function &r, double lambda, std::vector &z)>; Outcome Minimise(const Target &target, const Product &product, const std::vector &diagonal, std::vector &x, int iterations, double f_tolerance, double max_step, - const std::function &stop = nullptr) { + const std::function &stop = nullptr, const Preconditioner &precondition = nullptr) { const size_t n = x.size(); Outcome out; double f = 0; - std::vector g(n), d(n), x_new(n), g_new(n), hd(n), rhs(n); + std::vector g(n), d(n), x_new(n), g_new(n), hd(n); if (!target(x, f, g)) return out; out.evaluations++; @@ -277,16 +251,11 @@ public: bool accepted = false; double f_new = 0; for (int attempt = 0; attempt < 6 && !accepted; attempt++) { - for (size_t k = 0; k < n; k++) - rhs[k] = -g[k]; - Solve(product, diagonal, lambda_, x, rhs, d, cg_max, cg_tolerance, cg_iterations); + Solve(product, diagonal, precondition, x, g, d); double d_max = 0; for (double v : d) d_max = std::max(d_max, std::fabs(v)); - if (clip_step) - for (double &v : d) - v = std::clamp(v, -max_step, max_step); - else if (d_max > max_step) + if (d_max > max_step) for (double &v : d) v *= max_step / d_max; product(x, d, hd); @@ -318,57 +287,12 @@ public: f = f_new; out.iterations++; out.f_end = f; - if (decrease <= f_tolerance * std::max(std::fabs(f), 1.0)) - break; - if (relative_stop > 0 && decrease < relative_stop * (out.f_start - f)) + if (decrease <= f_tolerance * std::max(std::fabs(f), 1.0) || decrease < relative_stop * (out.f_start - f)) break; } return out; } - // d with (H + lambda D) d = rhs to `tolerance` of |rhs|, by conjugate gradients preconditioned with - // (1 + lambda) D; where a direction of negative curvature turns up, d as far as it got (the preconditioned rhs at - // first). - static void Solve(const Product &product, const std::vector &diagonal, double lambda_, - const std::vector &x, const std::vector &rhs, std::vector &d, int CG_MAX, - double tolerance, int &cg_iterations) { - const size_t n = rhs.size(); - d.assign(n, 0.0); - std::vector r(n), z(n), p(n), hp(n); - for (size_t k = 0; k < n; k++) { - r[k] = rhs[k]; - z[k] = r[k] / ((1 + lambda_) * diagonal[k]); - } - p = z; - double rz = Dot(r, z); - const double r0 = std::sqrt(Dot(r, r)); - for (int k = 0; k < CG_MAX; k++) { - cg_iterations++; - product(x, p, hp); - for (size_t j = 0; j < n; j++) - hp[j] += lambda_ * diagonal[j] * p[j]; - const double php = Dot(p, hp); - if (!(php > 0)) { - if (k == 0) - d = z; - break; - } - const double a = rz / php; - for (size_t j = 0; j < n; j++) { - d[j] += a * p[j]; - r[j] -= a * hp[j]; - } - if (std::sqrt(Dot(r, r)) < tolerance * r0) - break; - for (size_t j = 0; j < n; j++) - z[j] = r[j] / ((1 + lambda_) * diagonal[j]); - const double rz_new = Dot(r, z); - for (size_t j = 0; j < n; j++) - p[j] = z[j] + rz_new / rz * p[j]; - rz = rz_new; - } - } - private: static double Dot(const std::vector &a, const std::vector &b) { double s = 0; @@ -377,5 +301,47 @@ private: return s; } + // d with (H + lambda D) d = -g to `cg_tolerance` of |g|, by preconditioned conjugate gradients. + void Solve(const Product &product, const std::vector &diagonal, const Preconditioner &precondition, + const std::vector &x, const std::vector &g, std::vector &d) { + const size_t n = g.size(); + d.assign(n, 0.0); + std::vector r(n), z(n), p(n), hp(n); + auto apply = [&]() { + if (precondition) + precondition(r, lambda_, z); + else + for (size_t j = 0; j < n; j++) + z[j] = r[j] / ((1 + lambda_) * diagonal[j]); + }; + for (size_t k = 0; k < n; k++) + r[k] = -g[k]; + apply(); + p = z; + double rz = Dot(r, z); + const double r0 = std::sqrt(Dot(r, r)); + for (int k = 0; k < cg_max; k++) { + cg_iterations++; + product(x, p, hp); + for (size_t j = 0; j < n; j++) + hp[j] += lambda_ * diagonal[j] * p[j]; + const double php = Dot(p, hp); + if (!(php > 0)) + break; + const double a = rz / php; + for (size_t j = 0; j < n; j++) { + d[j] += a * p[j]; + r[j] -= a * hp[j]; + } + if (std::sqrt(Dot(r, r)) < cg_tolerance * r0) + break; + apply(); + const double rz_new = Dot(r, z); + for (size_t j = 0; j < n; j++) + p[j] = z[j] + rz_new / rz * p[j]; + rz = rz_new; + } + } + double lambda_ = 0.01; // kept from call to call, as the L-BFGS memory is }; From 5360629f4e65e17ebc449db3e175c2316e0d56aa Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 10:38:47 +0100 Subject: [PATCH 03/12] Refinement Newton: the restraints' Gauss-Newton product on the GPU With the engine on a GPU, RestraintNormalMatrix hands At/Multiply/Diagonal/Blocks to RestraintNormalMatrixGPU (RefinementNewtonGPU.{h,cpp,cu}), on the engine's card on a stream of its own at the highest priority. J is built on the device, one thread per restraint from the REFINE_HD terms in float, into a fixed layout worked out once from the set (rows the CPU drops, E = 0, are kept as zeros). A product is two gathers, J v per row and J^T t per parameter atom, without atomics, in a fixed order. Precision: J in float, v/t/hv as double-float pairs (error-free product and 2Sum per term). Plain float sums are not enough: J v is a small difference of large terms in the soft directions the X-ray term decides, and with plain float the outcomes moved beyond the CPU's own noise on a model with stiff restraints (weight choice 2/10 vs 8/10 at the CPU's value over jittered runs). Float J with exact accumulation matched the CPU's spread, as do the double-float pairs. On a large rotation set (GPU, Newton joint, relative stop 0.05): product thread-time 1008 s -> 38 s, jacobian 114 s -> 8 s, refine wall 191 s -> 32 s under load; evaluations and R-free within the spread a 1e-12 relative jitter of the CPU product gives. Interface: RestraintNormalMatrix now takes the Context (for the device) instead of n_atoms and nthreads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 3 +- .../structure_refinement/CMakeLists.txt | 3 + .../structure_refinement/Refinement.cpp | 38 +- .../RefinementNewtonGPU.cpp | 59 +++ .../RefinementNewtonGPU.cu | 355 ++++++++++++++++++ .../RefinementNewtonGPU.h | 53 +++ tests/RefinementTest.cpp | 99 +++++ 7 files changed, 602 insertions(+), 8 deletions(-) create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.cpp create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.cu create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.h diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index dd37ce457..83de168ef 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -842,7 +842,8 @@ pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472 [doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). **Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an -Apple GPU (`BraggExp.h`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the +Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU +(`RefinementNewtonGPU.cu`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the Elementary Functions" (1980), Prentice-Hall; the exact product (with Veltkamp's split) and Fast2Sum of T. J. Dekker, "A floating-point technique for extending the available precision" (1971), Numer. Math. 18, 224-242; and the 2Sum of D. E. Knuth, "The Art of Computer Programming, Vol. 2: diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index 54e66051f..07cbcd4c6 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -37,6 +37,9 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC $<$:RefinementGPU.cpp> $<$:RefinementGPU.cu> RefinementGPU.h RefinementGPUEngine.h + $<$:RefinementNewtonGPU.cpp> + $<$:RefinementNewtonGPU.cu> + RefinementNewtonGPU.h ${CMAKE_CURRENT_BINARY_DIR}/MonomerLibraryData.cpp) # The subset of the CCP4 Monomer Library that refinement restrains with (monomers/, chosen by diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index bf14d0b03..b4a2afb08 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -49,6 +49,7 @@ #ifdef JFJOCH_USE_CUDA #include "../../common/CUDAWrapper.h" #include "RefinementGPU.h" +#include "RefinementNewtonGPU.h" #endif namespace { @@ -1160,18 +1161,27 @@ std::vector FlooredDiagonal(std::vector d) { } // The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the -// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. +// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on +// a GPU, on its card (RestraintNormalMatrixGPU). class RestraintNormalMatrix { public: - RestraintNormalMatrix(const std::vector ¶m_atoms, size_t n_atoms, bool xyz, size_t nthreads) - : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(nthreads), - param_of_(n_atoms, -1) { + RestraintNormalMatrix(const Context &ctx, const std::vector ¶m_atoms, bool xyz) + : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(ctx.nthreads), + param_of_(ctx.atoms.size(), -1) { for (size_t k = 0; k < param_atoms.size(); k++) param_of_[param_atoms[k]] = static_cast(k); +#ifdef JFJOCH_USE_CUDA + if (ctx.engine->OnGPU()) + gpu_ = std::make_unique(ctx.device, param_atoms, ctx.atoms.size(), xyz); +#endif } void At(const RestraintSet &set, const std::vector> &xyzb) { const RefinementStage stage("newton.jacobian"); +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->At(set, xyzb); +#endif jac_ = RestraintGaussNewton(set, xyzb); const size_t rows = jac_.start.size() - 1; row_of_.resize(jac_.atom.size()); @@ -1199,6 +1209,10 @@ public: // hv = J^T J v: J v per row, then per parameter atom the sum over its entries void Multiply(const std::vector &v, std::vector &hv) { const RefinementStage stage("newton.product"); +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Multiply(v, hv); +#endif const int rows = static_cast(t_.size()); ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) { for (int r = lo; r < hi; r++) { @@ -1224,6 +1238,10 @@ public: // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. std::vector> Blocks() const { +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Blocks(); +#endif std::vector> b(n_params_, {0, 0, 0, 0, 0, 0}); for (size_t k = 0; k < n_params_; k++) for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) { @@ -1239,6 +1257,10 @@ public: } std::vector Diagonal() const { +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Diagonal(); +#endif std::vector d(per_ * n_params_, 0.0); for (size_t k = 0; k < n_params_; k++) for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) @@ -1254,6 +1276,9 @@ private: RestraintJacobian jac_; std::vector row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none) std::vector t_; +#ifdef JFJOCH_USE_CUDA + std::unique_ptr gpu_; +#endif }; // z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. @@ -1295,7 +1320,7 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto std::vector h(n); for (size_t k = 0; k < n; k++) h[k] = xray[target.Atoms()[k / per]]; - RestraintNormalMatrix normal(target.Atoms(), ctx.atoms.size(), xyz, ctx.nthreads); + RestraintNormalMatrix normal(ctx, target.Atoms(), xyz); std::vector x_at = p; target.Set(p.data()); normal.At(set, ctx.XyzB()); @@ -1332,8 +1357,7 @@ void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vect const RestraintSet &set_b, double w, double adp_w, int iterations) { const size_t n = atoms.size(); const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT; - RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads), - normal_b(atoms, ctx.atoms.size(), false, ctx.nthreads); + RestraintNormalMatrix normal_xyz(ctx, atoms, true), normal_b(ctx, atoms, false); std::vector x_at = target.Get(); auto at = [&](const std::vector &x) { target.Set(x.data()); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cpp b/image_analysis/structure_refinement/RefinementNewtonGPU.cpp new file mode 100644 index 000000000..990a3b692 --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cpp @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RefinementNewtonGPU.h" + +#include "RefinementRestraints.h" + +// The host half: the index of J, as RestraintGaussNewton lays out its rows, with every row kept. +void RestraintNormalMatrixGPU::At(const RestraintSet &set, const std::vector> &xyzb) { + if (!indexed_) { + std::vector record_entry, start{0}, atom; + for (const RestraintRecord &r : set.records) { + record_entry.push_back(static_cast(atom.size())); + switch (static_cast(r.kind)) { + case RestraintKind::Plane: + // one row per atom of the plane, each over all of them + for (int i = 0; i < r.atom[1]; i++) { + for (int k = 0; k < r.atom[1]; k++) + atom.push_back(set.plane_atoms[r.atom[0] + k]); + start.push_back(static_cast(atom.size())); + } + continue; + case RestraintKind::Angle: + atom.insert(atom.end(), r.atom, r.atom + 3); + break; + case RestraintKind::Torsion: + case RestraintKind::Chiral: + case RestraintKind::NcsDistance: + atom.insert(atom.end(), r.atom, r.atom + 4); + break; + default: // Bond, Repulsion, BSimilarity + atom.insert(atom.end(), r.atom, r.atom + 2); + break; + } + start.push_back(static_cast(atom.size())); + } + const size_t rows = start.size() - 1; + std::vector row_of(atom.size()), param(atom.size()); + for (size_t e = 0; e < atom.size(); e++) + param[e] = param_of_[atom[e]]; + for (size_t r = 0; r < rows; r++) + for (int e = start[r]; e < start[r + 1]; e++) + row_of[e] = static_cast(r); + std::vector entry_start(n_params_ + 1, 0); + for (int k : param) + if (k >= 0) + entry_start[k + 1]++; + for (size_t k = 0; k < n_params_; k++) + entry_start[k + 1] += entry_start[k]; + std::vector entry(entry_start.back()); + std::vector fill(entry_start.begin(), entry_start.end() - 1); + for (size_t e = 0; e < param.size(); e++) + if (param[e] >= 0) + entry[fill[param[e]]++] = static_cast(e); + Upload(set.records, set.plane_atoms, record_entry, start, param, row_of, entry_start, entry); + indexed_ = true; + } + Fill(xyzb); +} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu new file mode 100644 index 000000000..a3c61554e --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -0,0 +1,355 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per +// parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in +// float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of +// H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with +// plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise. + +#include "RefinementNewtonGPU.h" + +#include + +#include "../indexing/CUDAMemHelpers.h" +#include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS + +namespace { + +constexpr int THREADS = 256; +using V3f = refine_math::V3; + +void cuda_err(cudaError_t val) { + if (val != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); +} + +class DeviceGuard { +public: + explicit DeviceGuard(int device) { + cuda_err(cudaGetDevice(&previous_)); + cuda_err(cudaSetDevice(device)); + } + ~DeviceGuard() { cudaSetDevice(previous_); } + DeviceGuard(const DeviceGuard &) = delete; + DeviceGuard &operator=(const DeviceGuard &) = delete; + +private: + int previous_ = 0; +}; + +int GreatestStreamPriority() { + int least = 0, greatest = 0; + cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest)); + return greatest; +} + +int GridSize(int n) { + return (n + THREADS - 1) / THREADS; +} + +__device__ V3f xyz(const float4 *p, int i) { + return {p[i].x, p[i].y, p[i].z}; +} + +__device__ float Get(const float4 &j, int c) { + return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w; +} + +// sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math). +// The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2 +__device__ void AddProduct(float2 &sum, float a, float2 b) { + const float p = a * b.x; + const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly + const float s = sum.x + p; + const float v = s - sum.x; + const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly + const float lo = s_err + sum.y + p_err + a * b.y; + const float hi = s + lo; + sum = make_float2(hi, lo - (hi - s)); +} + +// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a +// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row. +__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry, + const float4 *xyzb, float4 *j) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) + return; + const RestraintRecord &r = records[i]; + float4 *out = j + record_entry[i]; + V3f g[4] = {}; + int na = 0; + float e = 0; + switch (static_cast(r.kind)) { + case RestraintKind::Bond: + e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g); + na = 2; + break; + case RestraintKind::Angle: + e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal, + r.sigma, g); + na = 3; + break; + case RestraintKind::Torsion: + e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g); + na = 4; + break; + case RestraintKind::Chiral: + e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g); + na = 4; + break; + case RestraintKind::NcsDistance: + e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g); + na = 4; + break; + case RestraintKind::Repulsion: { + const V3f q = xyz(xyzb, r.atom[1]); + const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0], + r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1], + r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]}; + e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g); + const V3f gi = g[1]; + g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z, + r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z, + r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z}; + na = 2; + break; + } + case RestraintKind::Plane: { + V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS]; + const int m = r.atom[1]; + for (int k = 0; k < m; k++) + px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]); + refine_math::PlaneTerm(px, m, r.sigma, pg); + V3f nv{0, 0, 0}; + for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n) + if (refine_math::Norm(pg[k]) > refine_math::Norm(nv)) + nv = pg[k]; + const float norm = refine_math::Norm(nv); + const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f; + for (int a = 0; a < m; a++) + for (int k = 0; k < m; k++) { + const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m); + out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f); + } + return; + } + case RestraintKind::BSimilarity: { + float gi = 0, gj = 0; + e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj); + const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; + out[0] = make_float4(0, 0, 0, f * gi); + out[1] = make_float4(0, 0, 0, f * gj); + return; + } + } + const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; + for (int k = 0; k < na; k++) + out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f); +} + +// t_r = sum over the row's entries of J_e . v_k +__global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v, + float2 *t, int rows) { + const int r = blockIdx.x * blockDim.x + threadIdx.x; + if (r >= rows) + return; + float2 dot = make_float2(0, 0); + for (int e = start[r]; e < start[r + 1]; e++) + if (const int k = param[e]; k >= 0) + for (int c = 0; c < per; c++) + AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]); + t[r] = dot; +} + +// hv_k = sum over the parameter atom's entries of t_row J_e +__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, + int c0, const float2 *t, float2 *hv, int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + for (int c = 0; c < per; c++) { + float2 sum = make_float2(0, 0); + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const int e = entry[q]; + AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); + } + hv[per * k + c] = sum; + } +} + +__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d, + int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + for (int c = 0; c < per; c++) { + float sum = 0; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float x = Get(j[entry[q]], c0 + c); + sum += x * x; + } + d[per * k + c] = sum; + } +} + +__global__ void BlocksKernel(const int *entry_start, const int *entry, const float4 *j, float *b, int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + float s[6] = {0, 0, 0, 0, 0, 0}; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float4 x = j[entry[q]]; + s[0] += x.x * x.x; + s[1] += x.y * x.y; + s[2] += x.z * x.z; + s[3] += x.x * x.y; + s[4] += x.x * x.z; + s[5] += x.y * x.z; + } + for (int c = 0; c < 6; c++) + b[6 * k + c] = s[c]; +} + +template +CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { + CudaDevicePtr d(std::max(host.size(), 1)); + cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream)); + return d; +} + +} // namespace + +struct RestraintNormalMatrixGPU::Device { + int device; + CudaStream stream; + int records = 0, rows = 0; + CudaDevicePtr record; + CudaDevicePtr plane_atoms, record_entry, start, param, row_of, entry_start, entry; + CudaDevicePtr xyzb, j; + CudaDevicePtr t, v, hv; + CudaDevicePtr diagonal; + PinnedVector host; // v and hv as float pairs + PinnedVector host_xyzb; +}; + +RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector ¶m_atoms, size_t n_atoms, + bool xyz) + : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) { + for (size_t k = 0; k < param_atoms.size(); k++) + param_of_[param_atoms[k]] = static_cast(k); + DeviceGuard guard(device); + // At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the + // X-ray evaluations of the other trials on the card. + dev_ = std::unique_ptr(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())}); + const size_t n = std::max(per_ * n_params_, 1); + dev_->xyzb = CudaDevicePtr(std::max(n_atoms, 1)); + dev_->host_xyzb.resize(n_atoms); + dev_->v = CudaDevicePtr(n); + dev_->hv = CudaDevicePtr(n); + dev_->diagonal = CudaDevicePtr(n); + dev_->host.resize(n); +} + +RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() { + DeviceGuard guard(dev_->device); + dev_.reset(); +} + +void RestraintNormalMatrixGPU::Upload(const std::vector &records, const std::vector &plane_atoms, + const std::vector &record_entry, const std::vector &start, + const std::vector ¶m, const std::vector &row_of, + const std::vector &entry_start, const std::vector &entry) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + d.records = static_cast(records.size()); + d.rows = static_cast(start.size()) - 1; + d.record = Up(records, d.stream); + d.plane_atoms = Up(plane_atoms, d.stream); + d.record_entry = Up(record_entry, d.stream); + d.start = Up(start, d.stream); + d.param = Up(param, d.stream); + d.row_of = Up(row_of, d.stream); + d.entry_start = Up(entry_start, d.stream); + d.entry = Up(entry, d.stream); + d.j = CudaDevicePtr(std::max(param.size(), 1)); + d.t = CudaDevicePtr(std::max(d.rows, 1)); + cuda_err(cudaStreamSynchronize(d.stream)); +} + +void RestraintNormalMatrixGPU::Fill(const std::vector> &xyzb) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + for (size_t a = 0; a < xyzb.size(); a++) + d.host_xyzb[a] = make_float4(static_cast(xyzb[a][0]), static_cast(xyzb[a][1]), + static_cast(xyzb[a][2]), static_cast(xyzb[a][3])); + cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice, + d.stream)); + if (d.records > 0) + JacobianKernel<<>>(d.record, d.records, d.plane_atoms, + d.record_entry, d.xyzb, d.j); + cuda_err(cudaGetLastError()); + cuda_err(cudaStreamSynchronize(d.stream)); +} + +void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vector &hv) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + hv.resize(per_ * n_params_); + if (n == 0) + return; + for (size_t i = 0; i < hv.size(); i++) { + const float hi = static_cast(v[i]); + d.host[i] = make_float2(hi, static_cast(v[i] - hi)); + } + cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); + if (d.rows > 0) + RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); + TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, + d.hv, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + for (size_t i = 0; i < hv.size(); i++) + hv[i] = static_cast(d.host[i].x) + d.host[i].y; +} + +std::vector RestraintNormalMatrixGPU::Diagonal() const { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + std::vector out(per_ * n_params_); + if (n == 0) + return out; + std::vector host(out.size()); + DiagonalKernel<<>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + std::copy(host.begin(), host.end(), out.begin()); + return out; +} + +std::vector> RestraintNormalMatrixGPU::Blocks() const { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + std::vector> out(n_params_); + if (n == 0) + return out; + CudaDevicePtr b(6 * n_params_); + std::vector host(6 * n_params_); + BlocksKernel<<>>(d.entry_start, d.entry, d.j, b, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(host.data(), b.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + for (size_t k = 0; k < n_params_; k++) + for (int c = 0; c < 6; c++) + out[k][c] = host[6 * k + c]; + return out; +} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h new file mode 100644 index 000000000..3136ed029 --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) of the Newton minimisation on a GPU (CUDA +// builds only): RestraintNormalMatrix of Refinement.cpp on the card, in single precision. J is worked out on the device, +// one thread per restraint, with the terms the engine evaluates the restraints with (RefinementMath.h); a product is two +// gathers, J v per row and J^T t per parameter atom over its entries, each summed in a fixed order without atomics, in +// double-float pairs (RefinementNewtonGPU.cu). +// +// Every restraint keeps its rows whatever its E: where RestraintGaussNewton drops a row (E = 0, a plane without a +// normal) its entries here are zero, so the index of J is worked out once, at the first At(), and the set must stay +// the same for the matrix's life. + +#include +#include +#include +#include + +#include "RefinementMath.h" // RestraintRecord + +struct RestraintSet; + +class RestraintNormalMatrixGPU { +public: + // On the x y z (`xyz`) or the B of `param_atoms` of the model's `n_atoms`, on GPU `device`. + RestraintNormalMatrixGPU(int device, const std::vector ¶m_atoms, size_t n_atoms, bool xyz); + ~RestraintNormalMatrixGPU(); + + void At(const RestraintSet &set, const std::vector> &xyzb); + // hv = J^T J v + void Multiply(const std::vector &v, std::vector &hv); + // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. + std::vector> Blocks() const; + std::vector Diagonal() const; + +private: + // The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries + // start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries. + void Upload(const std::vector &records, const std::vector &plane_atoms, + const std::vector &record_entry, const std::vector &start, const std::vector ¶m, + const std::vector &row_of, const std::vector &entry_start, const std::vector &entry); + // J at `xyzb` (x, y, z, B per atom), in float + void Fill(const std::vector> &xyzb); + + int per_, c0_; + size_t n_params_; + std::vector param_of_; + bool indexed_ = false; + struct Device; + std::unique_ptr dev_; +}; diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index 7f84cf8f4..e34653877 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -37,6 +37,7 @@ #include "../image_analysis/structure_refinement/RefinementXray.h" #ifdef JFJOCH_USE_CUDA #include "../image_analysis/structure_refinement/RefinementGPU.h" +#include "../image_analysis/structure_refinement/RefinementNewtonGPU.h" #endif namespace { @@ -986,6 +987,104 @@ TEST_CASE("RefinementXray_gradient_CPU", "[Refinement]") { } #ifdef JFJOCH_USE_CUDA +TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { + if (get_gpu_count() == 0) + return; + const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); + gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); + st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); + const RefinementTopology topo = BuildRefinementTopology(st); + RestraintSet set = topo.geometry; + set.Append(topo.adp); + set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0)); + const std::vector atoms = atoms_of(st.models[0]); + set.Finalize(atoms.size()); + std::vector> xyzb; + for (gemmi::Atom *a : atoms) + xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); + // every atom but each fifth a parameter: the rest drop out + std::vector params, param_of(atoms.size(), -1); + for (int a = 0; a < static_cast(atoms.size()); a++) + if (a % 5 != 4) { + param_of[a] = static_cast(params.size()); + params.push_back(a); + } + const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); + REQUIRE(jac.start.size() > 100); + std::mt19937 gen(3); + std::uniform_real_distribution uniform(-1, 1); + // a random v, and a smooth one - the same shift of every atom and a thousandth of it at random, J v then the small + // difference of large terms + for (const auto [xyz, smooth] : {std::pair{true, false}, {true, true}, {false, false}, {false, true}}) { + CAPTURE(xyz, smooth); + const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3; + const size_t n = per * params.size(); + std::vector v(n); + for (size_t i = 0; i < n; i++) + v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen); + // J^T J v, the diagonal and the 3 x 3 blocks on the CPU, each entry in turn + std::vector hv(n, 0.0), diagonal(n, 0.0); + std::vector> blocks(params.size(), {0, 0, 0, 0, 0, 0}); + for (size_t r = 0; r + 1 < jac.start.size(); r++) { + double t = 0; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + t += jac.j[e][c0 + c] * v[per * k + c]; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + hv[per * k + c] += t * jac.j[e][c0 + c]; + } + for (size_t e = 0; e < jac.atom.size(); e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) { + const std::array &j = jac.j[e]; + for (int c = 0; c < per; c++) + diagonal[per * k + c] += j[c0 + c] * j[c0 + c]; + blocks[k][0] += j[0] * j[0]; + blocks[k][1] += j[1] * j[1]; + blocks[k][2] += j[2] * j[2]; + blocks[k][3] += j[0] * j[1]; + blocks[k][4] += j[0] * j[2]; + blocks[k][5] += j[1] * j[2]; + } + RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); + gpu.At(set, xyzb); + std::vector hv_gpu; + gpu.Multiply(v, hv_gpu); + const std::vector diagonal_gpu = gpu.Diagonal(); + REQUIRE(hv_gpu.size() == n); + REQUIRE(diagonal_gpu.size() == n); + // In float, from float positions: within a few float epsilons of the double product in norm (in refinement + // its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself, each block + // relative to its own size. + double num = 0, den = 0, worst_diagonal = 0, worst_block = 0; + for (size_t i = 0; i < n; i++) { + num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]); + den += hv[i] * hv[i]; + if (diagonal[i] > 0) + worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]); + } + if (xyz) { + const std::vector> blocks_gpu = gpu.Blocks(); + REQUIRE(blocks_gpu.size() == params.size()); + for (size_t k = 0; k < params.size(); k++) { + double d2 = 0, b2 = 0; + for (int c = 0; c < 6; c++) { + d2 += (blocks_gpu[k][c] - blocks[k][c]) * (blocks_gpu[k][c] - blocks[k][c]); + b2 += blocks[k][c] * blocks[k][c]; + } + if (b2 > 0) + worst_block = std::max(worst_block, std::sqrt(d2 / b2)); + } + } + CAPTURE(std::sqrt(num / den), worst_diagonal, worst_block); + CHECK(std::sqrt(num / den) < 1e-5); + CHECK(worst_diagonal < 1e-5); + CHECK(worst_block < 1e-5); + } +} + TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") { if (get_gpu_count() == 0) return; From 03f8ff2d29efa3c3b8b7d35b719fb9382313da04 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 10:59:04 +0100 Subject: [PATCH 04/12] Refinement Newton (prototype): X-ray coupling between neighbouring atoms, quasi-Newton on GN, per-part switches, step scale Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 263 ++++++++++++++++-- .../RefinementMinimiser.h | 48 +++- 2 files changed, 287 insertions(+), 24 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index bf14d0b03..1fb367611 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -664,6 +664,144 @@ struct Context { return shape; } + // The X-ray normal matrix between the positions of neighbouring atoms of `params` (riders left out), in the units of + // XrayXyzCurvatureShape(): for atoms k and l a vector r apart, the sum over the shells of + // w f_k f_l exp(-(B_k + B_l) s^2 / 4) (2 pi s)^2 [(j1(x) / x) I - j2(x) r r^T / r^2], x = 2 pi s r - the overlap of + // their density derivatives, the reflections spread evenly over directions; at r = 0 the shape's own (2 pi)^2 s^2 / 3. + // Out to where it has fallen below a hundredth of its value at 0 for two carbons at the median B, and stays there. + // Per parameter index its neighbours and blocks (xx yy zz xy xz yz), both ways round. + struct XrayCoupling { + std::vector start, other; + std::vector> block; + }; + XrayCoupling XrayXyzCoupling(const XrayRowModel &rows, const std::vector ¶ms) const { + constexpr int SHELLS = 32; + double s2_max = 0; + for (double s2 : refl.inv_d2) + s2_max = std::max(s2_max, s2); + std::vector w(SHELLS, 0), s2_sum(SHELLS, 0); + std::vector n(SHELLS, 0); + for (size_t m = 0; m < refl.Size(); m++) { + if (refl.free[m]) + continue; + const int k = std::min(SHELLS - 1, static_cast(refl.inv_d2[m] / s2_max * SHELLS)); + const double kd = rows.k_total[m] * rows.d[m]; + w[k] += 2.0 * kd * kd / rows.var[m]; + s2_sum[k] += refl.inv_d2[m]; + n[k]++; + } + std::vector s(SHELLS, 0); + for (int k = 0; k < SHELLS; k++) + s[k] = n[k] > 0 ? std::sqrt(s2_sum[k] / n[k]) : 0.0; + std::map, std::vector> fs; // per element and charge, f per shell + auto form = [&](const gemmi::Atom &a) -> const std::vector & { + std::vector &f = fs[{static_cast(a.element.elem), a.charge}]; + if (f.empty()) { + const auto &coef = IonFormFactor(a.element.elem, a.charge); + f.resize(SHELLS, 0); + for (int k = 0; k < SHELLS; k++) + f[k] = coef.calculate_sf(s[k] * s[k] / 4.0); + } + return f; + }; + // a = j1(x) / x and b = j2(x) per shell at distance r, for f_k f_l exp(-(B_k + B_l) s^2 / 4) given per shell + auto kernel = [&](const std::vector &ff, double r, double &a_sum, double &b_sum) { + a_sum = b_sum = 0; + for (int k = 0; k < SHELLS; k++) { + if (n[k] == 0) + continue; + const double x = 2 * M_PI * s[k] * r, c = w[k] * ff[k] * 4 * M_PI * M_PI * s[k] * s[k]; + double a, b; + if (x < 1e-3) { + a = 1.0 / 3.0 - x * x / 30.0; + b = x * x / 15.0; + } else { + const double sx = std::sin(x), cx = std::cos(x); + a = (sx / (x * x) - cx / x) / x; + b = (3.0 / (x * x) - 1.0) * sx / x - 3.0 * cx / (x * x); + } + a_sum += c * a; + b_sum += c * b; + } + }; + // the cutoff, for two carbons at the median B + std::vector bs; + for (int i : params) + bs.push_back(atoms[i]->b_iso); + XrayCoupling out; + out.start.assign(params.size() + 1, 0); + if (bs.empty()) + return out; + std::nth_element(bs.begin(), bs.begin() + bs.size() / 2, bs.end()); + const double b_med = bs[bs.size() / 2]; + gemmi::Atom carbon; + carbon.element = gemmi::El::C; + std::vector ff(SHELLS); + const std::vector &fc = form(carbon); + for (int k = 0; k < SHELLS; k++) + ff[k] = fc[k] * fc[k] * std::exp(-b_med * s[k] * s[k] / 2.0); + double a0, b0; + kernel(ff, 0.0, a0, b0); + double cutoff = 1.0; + for (double r = 1.0; r <= 8.0; r += 0.1) { + double a, b; + kernel(ff, r, a, b); + if (std::max(std::fabs(a), std::fabs(a - b)) > 0.01 * a0) + cutoff = r + 0.1; + } + // the neighbours by a grid of cells one cutoff wide + std::map, std::vector> cells; + auto cell_of = [&](const gemmi::Position &p) { + return std::array{static_cast(std::floor(p.x / cutoff)), static_cast(std::floor(p.y / cutoff)), + static_cast(std::floor(p.z / cutoff))}; + }; + auto is_param = [&](int i) { return rider_parent.empty() || rider_parent[i] < 0; }; + for (size_t k = 0; k < params.size(); k++) + if (is_param(params[k])) + cells[cell_of(atoms[params[k]]->pos)].push_back(static_cast(k)); + std::vector>>> rows_of(params.size()); + for (size_t k = 0; k < params.size(); k++) { + const gemmi::Atom &ak = *atoms[params[k]]; + if (!is_param(params[k])) + continue; + const std::array c = cell_of(ak.pos); + const std::vector &fk = form(ak); + for (int dx = -1; dx <= 1; dx++) + for (int dy = -1; dy <= 1; dy++) + for (int dz = -1; dz <= 1; dz++) { + const auto it = cells.find({c[0] + dx, c[1] + dy, c[2] + dz}); + if (it == cells.end()) + continue; + for (int l : it->second) { + if (l == static_cast(k)) + continue; + const gemmi::Atom &al = *atoms[params[l]]; + const gemmi::Position d = al.pos - ak.pos; + const double r = d.length(); + if (r >= cutoff || r <= 0) + continue; + const std::vector &fl = form(al); + for (int q = 0; q < SHELLS; q++) + ff[q] = fk[q] * fl[q] * ak.occ * al.occ * std::exp(-(ak.b_iso + al.b_iso) * s[q] * s[q] / 4.0); + double a, b; + kernel(ff, r, a, b); + const double u[3] = {d.x / r, d.y / r, d.z / r}; + rows_of[k].push_back({l, {a - b * u[0] * u[0], a - b * u[1] * u[1], a - b * u[2] * u[2], + -b * u[0] * u[1], -b * u[0] * u[2], -b * u[1] * u[2]}}); + } + } + std::sort(rows_of[k].begin(), rows_of[k].end(), [](const auto &x, const auto &y) { return x.first < y.first; }); + } + for (size_t k = 0; k < params.size(); k++) { + out.start[k + 1] = out.start[k] + static_cast(rows_of[k].size()); + for (const auto &[l, blk] : rows_of[k]) { + out.other.push_back(l); + out.block.push_back(blk); + } + } + return out; + } + // The factor that turns XrayBCurvatureShape() into the X-ray target's curvature: its second derivative along a // random +-1 direction of the B factors of `params`, by central differences of the gradient, over what the shape // predicts for it. @@ -1107,6 +1245,7 @@ void Minimise(Context &ctx, const Target &target, int iterations, RefinementMini struct NewtonSettings { int xyz_iterations = 30, b_iterations = 20; // the most per macro-cycle, as the L-BFGS's double relative_stop = 0.0; + double step_scale = 1.0; // the step bound, times the L-BFGS's }; const NewtonSettings &Newton() { static const NewtonSettings s = [] { @@ -1117,11 +1256,14 @@ const NewtonSettings &Newton() { n.b_iterations = std::atoi(e); if (const char *e = getenv("RUGNUX_NEWTON_RELSTOP")) n.relative_stop = std::atof(e); + if (const char *e = getenv("RUGNUX_NEWTON_STEPSCALE")) + n.step_scale = std::atof(e); return n; }(); return s; } +bool NewtonQuasi(); // `target` by `newton` from where it is, with H v = product and D = diagonal (each element > 0). template void MinimiseNewton(Context &ctx, const Target &target, RefinementNewton &newton, const RefinementNewton::Product &product, @@ -1130,6 +1272,27 @@ void MinimiseNewton(Context &ctx, const Target &target, RefinementNewton &newton std::vector p = target.Get(); if (p.empty()) return; + if (NewtonQuasi() && newton.lbfgs != nullptr) { + // RUGNUX_NEWTON=2: the L-BFGS, its initial inverse Hessian the Gauss-Newton matrix solved by CG + RefinementMinimiser &m = *newton.lbfgs; + if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) + newton.cg_tolerance = std::atof(e); + const int cg_before = newton.cg_iterations; + m.SetPreconditioner([&](const std::vector &x, std::vector &q) { + std::vector z; + newton.SolveUndamped(product, diagonal, precondition, x, q, z); + q.swap(z); + }); + const RefinementMinimiser::Outcome o = m.Minimise( + [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, + p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); + m.SetPreconditioner(nullptr); + target.Set(p.data()); + if (RefineStats()) + fprintf(stderr, "MINSTAT qn.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, + iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); + return; + } newton.relative_stop = Newton().relative_stop; newton.trace = getenv("RUGNUX_NEWTON_TRACE") != nullptr; if (const char *e = getenv("RUGNUX_NEWTON_CGMAX")) @@ -1139,8 +1302,8 @@ void MinimiseNewton(Context &ctx, const Target &target, RefinementNewton &newton const int cg_before = newton.cg_iterations; const RefinementMinimiser::Outcome o = newton.Minimise( [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, - product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }, - precondition); + product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep() * Newton().step_scale, + [&]() { return ctx.Cancelled(); }, precondition); target.Set(p.data()); if (RefineStats()) fprintf(stderr, "MINSTAT newton.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, @@ -1256,6 +1419,27 @@ private: std::vector t_; }; +// hv[offset + 3k ..] += scale sum over the neighbours l of k of the X-ray block (k, l) times v[offset + 3l ..]. +void AddXrayCoupling(const Context::XrayCoupling &c, double scale, const std::vector &v, size_t offset, + std::vector &hv, size_t nthreads) { + const int n = static_cast(c.start.size()) - 1; + ParallelChunks(n, ThreadsForWork(n, nthreads, 4096), [&](int lo, int hi) { + for (int k = lo; k < hi; k++) { + double s0 = 0, s1 = 0, s2 = 0; + for (int q = c.start[k]; q < c.start[k + 1]; q++) { + const std::array &m = c.block[q]; + const double *x = &v[offset + 3 * c.other[q]]; + s0 += m[0] * x[0] + m[3] * x[1] + m[4] * x[2]; + s1 += m[3] * x[0] + m[1] * x[1] + m[5] * x[2]; + s2 += m[4] * x[0] + m[5] * x[1] + m[2] * x[2]; + } + hv[offset + 3 * k] += scale * s0; + hv[offset + 3 * k + 1] += scale * s1; + hv[offset + 3 * k + 2] += scale * s2; + } + }); +} + // z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. std::array Solve3(const std::array &m, double r0, double r1, double r2) { const double a = m[0], b = m[1], c = m[2], d = m[3], e = m[4], f = m[5]; @@ -1287,7 +1471,8 @@ void PreconditionBlocks(const std::vector> &blocks, double // The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`. void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewton &newton, const std::vector &xray, - const RestraintSet &set, int iterations) { + const RestraintSet &set, int iterations, const Context::XrayCoupling *coupling = nullptr, + double coupling_scale = 0) { const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; const int per = xyz ? 3 : 1; std::vector p = target.Get(); @@ -1311,6 +1496,8 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto normal.Multiply(v, hv); for (size_t k = 0; k < n; k++) hv[k] += h[k] * v[k]; + if (coupling != nullptr) + AddXrayCoupling(*coupling, coupling_scale, v, 0, hv, ctx.nthreads); }; diagonal = FlooredDiagonal(diagonal); if (!xyz) { @@ -1329,7 +1516,8 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto // (`xray_xyz`, `xray_b`: per atom, the X-ray curvature alone), no coupling between them. void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vector &atoms, RefinementNewton &newton, const std::vector &xray_xyz, const std::vector &xray_b, const RestraintSet &set_xyz, - const RestraintSet &set_b, double w, double adp_w, int iterations) { + const RestraintSet &set_b, double w, double adp_w, int iterations, + const Context::XrayCoupling *coupling = nullptr, double coupling_scale = 0) { const size_t n = atoms.size(); const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT; RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads), @@ -1367,6 +1555,8 @@ void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vect hv[3 * k + c] = hx[3 * k + c] / w + xray_xyz[atoms[k]] * v[3 * k + c]; hv[3 * n + k] = u2 * (hb[k] / adp_w + xray_b[atoms[k]] * v[3 * n + k]); } + if (coupling != nullptr) + AddXrayCoupling(*coupling, coupling_scale, v, 0, hv, ctx.nthreads); }; diagonal = FlooredDiagonal(diagonal); // the positions' 3 x 3 blocks of H at the start, fixed for the call; the B factors' diagonal @@ -2287,6 +2477,21 @@ bool NewtonOn() { static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) != "0"; return on; } +// RUGNUX_NEWTON_PARTS (default "xyz,b,tls,u"): which of the minimisations take the Newton steps. +bool NewtonOn(const char *part) { + static const std::string parts = getenv("RUGNUX_NEWTON_PARTS") != nullptr ? getenv("RUGNUX_NEWTON_PARTS") : "xyz,b,tls,u"; + return NewtonOn() && parts.find(part) != std::string::npos; +} +// RUGNUX_NEWTON=2: the L-BFGS on top of the Gauss-Newton matrix (MinimiseNewton). +bool NewtonQuasi() { + static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) == "2"; + return on; +} +// RUGNUX_NEWTON_COUPLING=1: the X-ray normal matrix between neighbouring atoms' positions in H (XrayXyzCoupling). +bool NewtonCoupling() { + static const bool on = getenv("RUGNUX_NEWTON_COUPLING") != nullptr; + return on; +} // RUGNUX_NEWTON_JOINT=1: where the B factors are refined isotropically, they and the positions at once (XyzBTarget). bool NewtonJoint() { static const bool on = getenv("RUGNUX_NEWTON_JOINT") != nullptr; @@ -2356,7 +2561,11 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); - RefinementNewton xyz_newton, b_newton, tls_newton; + RefinementNewton xyz_newton, b_newton, tls_newton, u_newton; + xyz_newton.lbfgs = &xyz_minimiser; + b_newton.lbfgs = &b_minimiser; + tls_newton.lbfgs = &tls_minimiser; + u_newton.lbfgs = &b_minimiser; double xray_scale = 0, xyz_scale = -1, tls_scale = -1; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { PlaceRiders(ctx, topo); @@ -2393,10 +2602,13 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s ctx.engine->SetRowModel(blurred); const double nh = XrayGradientNorm(ctx, refinable); const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); - if (NewtonOn()) - MinimiseAtomsNewton(ctx, xyz_h, xyz_newton, - NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), xyz_set, - Newton().xyz_iterations); + if (NewtonOn("xyz")) { + const std::vector h = NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale); + const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(blurred, refinable) + : Context::XrayCoupling(); + MinimiseAtomsNewton(ctx, xyz_h, xyz_newton, h, xyz_set, Newton().xyz_iterations, + NewtonCoupling() ? &c : nullptr, xyz_h.Weight() * xyz_scale); + } else Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser, {}, "xyz_blur"); ctx.engine->SetRowModel(ctx.rows); @@ -2409,20 +2621,26 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double &x : xray_b) x *= xray_scale; const XyzBTarget xyz_b(ctx, refinable, w, adp_w, topo); - MinimiseJointNewton(ctx, xyz_b, refinable, xyz_newton, - NewtonXyzCurvature(ctx, refinable, ctx.rows, 1.0, xyz_scale), xray_b, xyz_set, b_set, w, - adp_w, Newton().xyz_iterations); + const std::vector h = NewtonXyzCurvature(ctx, refinable, ctx.rows, 1.0, xyz_scale); + const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(ctx.rows, refinable) + : Context::XrayCoupling(); + MinimiseJointNewton(ctx, xyz_b, refinable, xyz_newton, h, xray_b, xyz_set, b_set, w, adp_w, + Newton().xyz_iterations, NewtonCoupling() ? &c : nullptr, xyz_scale); joint_done = true; - } else if (NewtonOn()) - MinimiseAtomsNewton(ctx, xyz, xyz_newton, NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), - xyz_set, Newton().xyz_iterations); + } else if (NewtonOn("xyz")) { + const std::vector h = NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale); + const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(ctx.rows, refinable) + : Context::XrayCoupling(); + MinimiseAtomsNewton(ctx, xyz, xyz_newton, h, xyz_set, Newton().xyz_iterations, + NewtonCoupling() ? &c : nullptr, w * xyz_scale); + } else Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser, {}, "xyz"); } if (kind == AdpHypothesis::Kind::Tls) { ctx.ApplyTls(); // the bodies' U at the atoms as they are const RefinementStage s4("minimise.tls"); - if (NewtonOn()) { + if (NewtonOn("tls")) { std::vector xray = ctx.XrayBCurvatureShape(); if (tls_scale < 0) tls_scale = ctx.CalibrateBCurvature(refinable, xray); @@ -2455,11 +2673,11 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double h : BDiagonal(xray, restraint, u_atoms, adp_w)) diagonal.insert(diagonal.end(), 6, h); const RefinementStage s3("minimise.u"); - if (NewtonOn()) { + if (NewtonOn("u")) { std::vector h; for (int i : u_atoms) h.insert(h.end(), 6, adp_w * xray[i]); - MinimiseUNewton(ctx, u_target, b_newton, h, diagonal, Newton().b_iterations); + MinimiseUNewton(ctx, u_target, u_newton, h, diagonal, Newton().b_iterations); } else Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal, "u"); } else if (adp_w > 0 && !joint_done) { @@ -2471,7 +2689,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s for (double &x : xray) x *= xray_scale; const RefinementStage s3("minimise.b"); - if (NewtonOn()) { + if (NewtonOn("b")) { for (double &x : xray) x *= adp_w; MinimiseAtomsNewton(ctx, b, b_newton, xray, b_set, Newton().b_iterations); @@ -4798,11 +5016,14 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector,[,] on the CPU and the GPU alike if (const char *e = getenv("RUGNUX_REFINE_PROTOCOL")) { - int v[3] = {settings.population, settings.rounds, settings.macro_cycles}; - sscanf(e, "%d,%d,%d", &v[0], &v[1], &v[2]); + int v[5] = {settings.population, settings.rounds, settings.macro_cycles, settings.xyz_iterations, + settings.b_iterations}; + sscanf(e, "%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4]); settings.population = v[0]; settings.rounds = v[1]; settings.macro_cycles = v[2]; + settings.xyz_iterations = v[3]; + settings.b_iterations = v[4]; logger.Info("Refinement EXPERIMENT protocol: population {}, {} rounds, {} macro-cycles", settings.population, settings.rounds, settings.macro_cycles); } diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 254a6474c..eeb4e85be 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -35,6 +35,10 @@ public: } // The estimated diagonal of the Hessian; every element > 0. void SetDiagonal(std::vector h) { h_ = std::move(h); } + // The initial inverse Hessian as an operator instead of the diagonal: q = H0 q at x (null: the diagonal). It is + // taken as it is, without the Oren-Shanno factor. + using Preconditioner = std::function &x, std::vector &q)>; + void SetPreconditioner(Preconditioner p) { precondition_ = std::move(p); } struct Outcome { int iterations = 0; @@ -60,11 +64,11 @@ public: for (int it = 0; it < iterations; it++) { if (stop && stop()) break; - Direction(g, d); + Direction(x, g, d); double gd = Dot(g, d); if (!(gd < 0)) { // not a descent direction: the memory no longer fits, start again from the diagonal Reset(); - Direction(g, d); + Direction(x, g, d); gd = Dot(g, d); if (!(gd < 0)) break; @@ -146,7 +150,11 @@ private: // additions, so each sweep over the parameters takes the next pair's dot product while it applies the // previous pair's update to q - the same operations on each element in the same order as a sweep for // each would do, with the update's memory traffic hidden under the chain. - void Direction(const std::vector &g, std::vector &d) const { + void Direction(const std::vector &x, const std::vector &g, std::vector &d) const { + if (precondition_) { + DirectionPreconditioned(x, g, d); + return; + } const size_t n = g.size(), m = s_.size(); std::vector &q = q_; // kept from call to call: made anew, its pages fault in every time q.assign(g.begin(), g.end()); @@ -203,6 +211,27 @@ private: d[k] = -q[k]; } + // The two-loop recursion with H0 the preconditioner. + void DirectionPreconditioned(const std::vector &x, const std::vector &g, std::vector &d) const { + const size_t m = s_.size(), n = g.size(); + std::vector q = g, a(m); + for (size_t i = m; i-- > 0;) { + a[i] = Dot(s_[i], q) / sy_[i]; + for (size_t k = 0; k < n; k++) + q[k] -= a[i] * y_[i][k]; + } + precondition_(x, q); + for (size_t i = 0; i < m; i++) { + const double b = Dot(y_[i], q) / sy_[i]; + for (size_t k = 0; k < n; k++) + q[k] += s_[i][k] * (a[i] - b); + } + d.resize(n); + for (size_t k = 0; k < n; k++) + d[k] = -q[k]; + } + + Preconditioner precondition_; size_t memory_, nthreads_; std::deque> s_, y_; std::deque sy_; // s.y of each pair, as taken when it was kept @@ -226,6 +255,7 @@ public: using Outcome = RefinementMinimiser::Outcome; int cg_iterations = 0; // summed over the calls, for the log + RefinementMinimiser *lbfgs = nullptr; // PROTOTYPE (RUGNUX_NEWTON=2): the L-BFGS the GN matrix preconditions int cg_max = 50; double cg_tolerance = 0.1; double relative_stop = 0.0; @@ -293,6 +323,18 @@ public: return out; } + // z = H^-1 q (lambda 0), by the same conjugate gradients: H0 for the L-BFGS (RefinementMinimiser::SetPreconditioner). + void SolveUndamped(const Product &product, const std::vector &diagonal, const Preconditioner &precondition, + const std::vector &x, const std::vector &q, std::vector &z) { + const double l = lambda_; + lambda_ = 0; + std::vector minus(q.size()); + for (size_t k = 0; k < q.size(); k++) + minus[k] = -q[k]; + Solve(product, diagonal, precondition, x, minus, z); + lambda_ = l; + } + private: static double Dot(const std::vector &a, const std::vector &b) { double s = 0; From 959f8ae739b3877381c13a8a28af532febcde15c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 13:03:29 +0100 Subject: [PATCH 05/12] Refinement: L-BFGS on the Gauss-Newton normal matrix, one protocol on the CPU and the GPU The minimiser's initial inverse Hessian is the target's sparse normal matrix, solved by conjugate gradients: the restraints' Gauss-Newton J^T J in full (RestraintNormalMatrix, on the slot's threads or its GPU), the X-ray target's curvature per atom on the diagonal (calibrated once per trial), the U restraints' exact Hessian, and per TLS body a dense 20 x 20 block. The L-BFGS pairs learn what the matrix leaves out (the X-ray coupling of neighbouring atoms). 10 iterations for the positions and 7 for B, U and TLS per macro-cycle, instead of 30 and 20. The protocol is the same on both paths: no population (kept as a setting, off by default), two rounds; lean_on_cpu is gone. Measured, refine-only, 19 sets on the GPU against the former GPU protocol (population, 2 rounds, plain L-BFGS): R-free -0.0005 mean, -0.0012 median, worse by > 0.002 on 2 and better on 6, -LL free lower on 11; 158k -> 49k evaluations. The same protocol with the plain L-BFGS on the CPU's former protocol took 76k. Quiet single runs at 32 threads: cytc CPU 113 s (was 114 s with one round), GPU 6.1 s (10.9 s); myob CPU 41.5 s (53.1 s), GPU 4.1 s (5.1 s). The damped Newton steps, the joint xyz+B target, the X-ray coupling term, the adaptive stop and the experiment switches of the prototype are removed: pure Gauss-Newton steps converged each trial further than the protocol's decisions were made for, R-free +0.0005 to +0.0022 in the mean of the 19 sets. BRestraintCurvature and the L-BFGS's diagonal and Oren-Shanno scaling are gone with them. Tests: RefinementMinimiser_quadratic on the new interface, RefinementMinimiser_conjugate_gradients, RefinementRestraints_gauss_newton (J^T J against the Hessian near the ideal geometry). Docs: CPU_DATA_ANALYSIS_REFINEMENT 15.6 and the population, references, ACKNOWLEDGEMENT (Hestenes & Stiefel 1952, the normal matrix after Murshudov, Vagin & Dodson 1997). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 20 +- docs/CPU_DATA_ANALYSIS.md | 4 +- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 60 +- .../structure_refinement/Refinement.cpp | 722 +++--------------- .../structure_refinement/Refinement.h | 16 +- .../RefinementMinimiser.h | 315 ++------ .../RefinementRestraints.cpp | 12 - .../RefinementRestraints.h | 5 +- tests/RefinementTest.cpp | 102 ++- 9 files changed, 329 insertions(+), 927 deletions(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 83de168ef..0c86fbbd6 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -598,20 +598,12 @@ Cryst. A34, 791-809 [doi:10.1107/S0567739478001618](https://doi.org/10.1107/S056 memory BFGS method for large scale optimization" (1989), Math. Program. 45, 503-528 [doi:10.1007/BF01589116](https://doi.org/10.1007/BF01589116); its two-loop recursion: J. Nocedal, "Updating quasi-Newton matrices with limited storage" (1980), Math. Comp. 35, 773-782 -[doi:10.1090/S0025-5718-1980-0572855-7](https://doi.org/10.1090/S0025-5718-1980-0572855-7). The -initial inverse Hessian is rescaled at each iteration by the latest pair, the self-scaling of -S. S. Oren and D. G. Luenberger, "Self-scaling variable metric (SSVM) algorithms. Part I" (1974), -Management Sci. 20, 845-862 [doi:10.1287/mnsc.20.5.845](https://doi.org/10.1287/mnsc.20.5.845), and -D. F. Shanno and K.-H. Phua, "Matrix conditioning and nonlinear optimization" (1978), Math. Program. -14, 149-160 [doi:10.1007/BF01588962](https://doi.org/10.1007/BF01588962). - -**Newton minimiser (experimental)** — damped Newton steps on a sparse normal matrix: the X-ray -target's curvature per atom on the diagonal and the restraints' Gauss-Newton matrix in full, as -REFMAC refines (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients with the -damping of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" -(1944), Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and -D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters" (1963), J. Soc. -Ind. Appl. Math. 11, 431-441 [doi:10.1137/0111030](https://doi.org/10.1137/0111030). +[doi:10.1090/S0025-5718-1980-0572855-7](https://doi.org/10.1090/S0025-5718-1980-0572855-7). Its +initial inverse Hessian is the target's sparse normal matrix - the X-ray target's curvature per atom +on the diagonal and the restraints' Gauss-Newton matrix in full - as REFMAC refines on its normal +matrix (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients: M. R. Hestenes and +E. Stiefel, "Methods of conjugate gradients for solving linear systems" (1952), J. Res. Natl. Bur. +Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). **Geometry restraints** — those of the CCP4 Monomer Library, a subset of which rugnux carries (see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)). A. A. Vagin, R. A. Steiner, A. A. Lebedev, diff --git a/docs/CPU_DATA_ANALYSIS.md b/docs/CPU_DATA_ANALYSIS.md index bfc7217c1..503030a2f 100644 --- a/docs/CPU_DATA_ANALYSIS.md +++ b/docs/CPU_DATA_ANALYSIS.md @@ -66,12 +66,13 @@ The methods draw on, and in places reimplement, solutions from: - H. F. Walker & P. Ni, "Anderson acceleration for fixed-point iterations", *SIAM J. Numer. Anal.* **49** (2011), 1715-1735, after D. G. Anderson, "Iterative procedures for nonlinear integral equations", *J. ACM* **12** (1965), 547-560 (the acceleration of the per-frame scaling loops, §10.6). - R. A. Fisher, "Frequency distribution of the values of the correlation coefficient in samples from an indefinitely large population", *Biometrika* **10** (1915), 507-521 (the z-transformation on which the correction surfaces' held-out half-set CC1/2 is compared). - R. C. Agarwal, "A new least-squares refinement technique based on the fast Fourier transform algorithm", *Acta Cryst.* **A34** (1978), 791-809 (the gradient of the refinement target, §15). -- N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and the B restraints, §15). +- N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target, the B restraints and the minimiser's sparse normal matrix, §15). - V. Y. Lunin & T. P. Skovoroda, "R-free likelihood-based estimates of errors for phases calculated from atomic models", *Acta Cryst.* **A51** (1995), 880-887 (the likelihood parameters from the free set, §15). - A. A. Vagin et al., "REFMAC5 dictionary: organization of prior chemical knowledge and guidelines for its use", *Acta Cryst.* **D60** (2004), 2184-2195 (the restraint dictionary, §15). - P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (the weight chosen among refinements by R-free, §15). - D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782 (the two-loop recursion of the minimiser, §15.6). +- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6). - G. N. Murshudov, P. Skubák, A. A. Lebedev et al., "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (the likelihood's D from the working set and its variance from the free, and the Kullback-Leibler B restraints, §15.3, §15.5). - P. Roversi, E. Blanc, S. Johnson & S. M. Lea, "Tetartohedral twinning could happen to you too", *Acta Cryst.* **D68** (2012), 418-424 (twin refinement with more than two domains, and the free set closed under the twin domains, §15.12). - K. A. Sultana, I. Alexeev, I. Kursula, P. Mäntsälä, J. Niemi & G. Schneider, "Structure determination by multiwavelength anomalous diffraction of aclacinomycin oxidoreductase: indications of multidomain pseudomerohedral twinning", *Acta Cryst.* **D63** (2007), 149-159 (a six-domain twin of a monoclinic crystal on a hexagonal metric, §15.12). @@ -120,7 +121,6 @@ The methods draw on, and in places reimplement, solutions from: - V. Luzzati, "Traitement statistique des erreurs dans la détermination des structures cristallines", *Acta Cryst.* **5** (1952), 802-810 (the fall-off of $\sigma_A$ with resolution for a model's coordinate error, Luzzati's $D$, in the likelihood of a placement, §14.10). - D. Liebschner, P. V. Afonine, A. G. Urzhumtsev & P. D. Adams, "Implementation of the riding hydrogen model in CCTBX to support the next generation of X-ray and neutron joint refinement in Phenix", *Methods Enzymol.* **634** (2020), 177-199 (the riding hydrogens, placed by GEMMI's code, §15.2). - D. L. Theobald, "Rapid calculation of RMSDs using a quaternion-based characteristic polynomial", *Acta Cryst.* **A61** (2005), 478-480, and P. Liu, D. K. Agrafiotis & D. L. Theobald, *J. Comput. Chem.* **31** (2010), 1561-1563 (the QCP superposition behind the NCS rms criterion, via GEMMI, §15.5). -- S. S. Oren & D. G. Luenberger, "Self-scaling variable metric (SSVM) algorithms. Part I", *Management Sci.* **20** (1974), 845-862, and D. F. Shanno & K.-H. Phua, "Matrix conditioning and nonlinear optimization", *Math. Program.* **14** (1978), 149-160 (the rescaling of the L-BFGS initial inverse Hessian by the latest pair, §15.6). (list is not exhaustive; the full citations, with DOIs, are in [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md)) diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 0a03eb028..293b59be8 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -286,26 +286,41 @@ Whether the restraints are used at all, and how tightly ($\sigma$), the weight t ### 15.6 The minimiser -Positions and B factors are minimised in turn, each by an L-BFGS of rugnux's own (`RefinementMinimiser`), -with the two-loop recursion of Nocedal and of Liu and Nocedal, a memory of 20 pairs and a backtracking line -search from the full step to sufficient decrease, its first trial moving no position by more than 0.3 Å and -no B by more than 10 Ų. What sets it apart from a textbook L-BFGS: +Positions, B factors, anisotropic U and TLS parameters are minimised in turn, each by an L-BFGS of rugnux's own +(`RefinementMinimiser`), with the two-loop recursion of Nocedal and of Liu and Nocedal, a memory of 20 pairs and a +backtracking line search from the full step to sufficient decrease, its first trial moving no position by more +than 0.3 Å and no B by more than 10 Ų. What sets it apart from a textbook L-BFGS is its **initial inverse +Hessian: the target's Gauss-Newton normal matrix**, as REFMAC refines on its sparse normal matrix (Murshudov, +Vagin & Dodson 1997). The matrix holds what is known of the curvature in advance; the L-BFGS pairs add what it +leaves out. -- **Its memory is kept from one macro-cycle to the next.** A macro-cycle is a few dozen iterations on a target - whose scale and likelihood parameters were refitted in between; starting each from what the earlier ones - learnt of the curvature, instead of from a steepest-descent step, is what lets two macro-cycles of a - weight trial get as far as many restarted ones. Each weight trial starts with an empty memory. -- **The B steps are scaled by their curvature.** The initial inverse Hessian of the B factors is a - diagonal: the X-ray target's curvature in each atom's B - its shape from the likelihood's curvature in - $|F_\mathrm{model}|$ and the atom's own scattering, summed in resolution shells, calibrated once per trial - against central differences of the gradient along a random $\pm1$ direction - plus the B restraints' - curvature, each element at least a thousandth of the median. The positions take none: their restraints' - curvature is that of stiff bonds and angles, which move atoms together, and as a diagonal it slows the soft - (torsion and X-ray) directions rather than scaling them. +- **The restraints' part is their Gauss-Newton matrix $J^\mathsf{T}J$, in full.** Every restraint is the square of + a residual (or close to it: the NCS distances' robust loss), so its gradient is $2r\,\nabla r$ and + $J = \nabla E / \sqrt{2E}$ is its row of the Jacobian, $\sqrt2\,\nabla r$; a plane contributes one row per + atom. $J^\mathsf{T}J$ is positive semi-definite where the exact Hessian is not (torsions, repulsions), and near + the ideal geometry it is the exact Hessian. Its products $J^\mathsf{T}(Jv)$ are two gathers - over the rows, + then over each atom's entries - on the slot's threads, or on its GPU. +- **The X-ray part is per atom, on the diagonal.** The X-ray target's curvature in an atom's $x$, $y$ and $z$ is + the likelihood's curvature in $|F_\mathrm{model}|$ summed in resolution shells against the atom's own part of + it, $(2\pi)^2 s^2/3\,f(s)^2\,e^{-Bs^2/2}$ (the reflections taken as spread evenly over directions); in its B, + $(s^2/4)^2 f(s)^2 e^{-Bs^2/2}$, a third of it to each element of a U; and a TLS body's dense $20\times20$ block + is that curvature carried through the derivatives of its atoms' U in the body's parameters. Each is calibrated + once per trial against central differences of the gradient along a random $\pm1$ direction. The coupling of + neighbouring atoms through the X-ray target is not in it; the pairs learn it. +- **The anisotropic U restraints** are quadratic in U, so their Hessian is exact by one difference of their + gradient. +- **The matrix is solved, not inverted.** Each L-BFGS direction solves the normal equations by conjugate + gradients to a tenth of the residual (at most 50 iterations), preconditioned by each atom's $3\times3$ block for + the positions and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser stands on. +- **Its memory is kept from one macro-cycle to the next.** Each weight trial starts with an empty memory. -The diagonal is rescaled at each iteration by the latest pair (Oren-Shanno scaling in that metric). A solve -stops after its iteration budget (30 for the positions and 20 for the B factors per macro-cycle) or where an -iteration lowers the target by less than $10^{-7}$ of it. +A solve stops after its iteration budget (10 for the positions and 7 for the B factors, U or TLS per +macro-cycle) or where an iteration lowers the target by less than $10^{-7}$ of it. Against the plain L-BFGS this +replaced (a diagonal initial inverse Hessian for the B factors, none for the positions, 30 and 20 iterations), a +trial takes about a third of the evaluations. With it the population (§15.7) no longer gains, and the protocol +is the same on the CPU and the GPU - two rounds, no population - at a third of the evaluations the GPU's protocol +took before and R-free 0.0005 lower in the mean over 19 refinements (-LL free lower on 11 of them), and at fewer +evaluations than the CPU's former one-round protocol. ### 15.7 The weights, decided by trial @@ -337,8 +352,10 @@ are: stays within 0.001 of the best so far, and the trajectory ends at its best step. On a deposited model, whose free set the depositor's refinement has already fitted, R-work falls while R-free rises, and the trajectory ends where it began. -3. **The population**, from where the trajectory ended: twelve members (at most as many as the trial slots, so - that they run in one wave; none below two) refined side by side, each a trial at the +3. **The population** (off by default, `RefinementSettings::population`: with the minimiser of §15.6 its members + took nothing off R-free over 19 refinements that the second round did not), from where the trajectory ended: + up to twelve members (at most as many as the trial slots, so that they run in one wave; none below two) refined + side by side, each a trial at the trajectory's weights but on its own path - the positions as they are or kicked at random by up to 0.2-0.3 Å in each coordinate, on the data as they are or **blurred** along a schedule, 60 then 20 Ų (the landscape of 4 Å, then of 2.2 Å), 100 then 30, or 30 then 10. The blur raises the likelihood's variance of each reflection @@ -741,7 +758,7 @@ What the refinement does not do yet - each a known gap, not a design choice: - L. Piela, J. Kostrowicki & H. A. Scheraga, "On the multiple-minima problem in the conformational analysis of molecules: deformation of the potential energy hypersurface by the diffusion equation method", *J. Phys. Chem.* **93** (1989), 3339-3346, and A. Jack & M. Levitt, "Refinement of large structures by simultaneous minimization of energy and R factor", *Acta Cryst.* **A34** (1978), 931-935 (the members refined on blurred data, §15.7). - J. Praznikar, P. V. Afonine, G. Guncar, P. D. Adams & D. Turk, "Averaged kick maps: less noise, more signal... and probably less bias", *Acta Cryst.* **D65** (2009), 921-931, and J. C. Nwachukwu, M. R. Southern, J. R. Kiefer, P. V. Afonine, P. D. Adams, T. C. Terwilliger & K. W. Nettles, "Improved crystallographic structures using extensive combinatorial refinement", *Structure* **21** (2013), 1923-1930 (the kicked members, their averaged map and the crossover residue by residue, §15.7). - J. Desmet, M. De Maeyer, B. Hazes & I. Lasters, "The dead-end elimination theorem and its use in protein side-chain positioning", *Nature* **356** (1992), 539-542; J. Besag, "On the statistical analysis of dirty pictures", *J. R. Statist. Soc.* **B48** (1986), 259-279; R. P. Joosten, K. Joosten, S. X. Cohen, G. Vriend & A. Perrakis, "Automatic rebuilding and optimization of crystallographic structures in the Protein Data Bank", *Bioinformatics* **27** (2011), 3392-3398 (the side-chain field: states, the conditional-modes solver and its order, §15.5). -- N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and its figure of merit, §15.2, §15.8). +- N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and its figure of merit, §15.2, §15.8; the minimiser's sparse normal matrix, §15.6). - G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5; the twinned refinement §15.12 simplifies, its twin laws pruned by fraction and its fractions refined by least squares). - R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with errors", *Acta Cryst.* **A42** (1986), 140-149 (the m, D weighting of the map coefficients, §15.8). - T. O. Yeates, "Detecting and overcoming crystal twinning", *Methods Enzymol.* **276** (1997), 344-358 (the twinned intensity, §15.12). @@ -762,6 +779,7 @@ What the refinement does not do yet - each a known gap, not a design choice: - Z. Dauter & M. Dauter, "Entering a new phase: using solvent halide ions in protein structure determination", *Structure* **9** (2001), R21-R26 (halides at sites with few donors, §15.10). - N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6). +- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6). - A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7). - P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7). - A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free). diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index c64486ade..8b5c834db 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -100,12 +99,6 @@ constexpr double MAX_STEP_XYZ = 0.3, MAX_STEP_B = 10.0; // The loosest NCS restraint a trajectory step tries (A); looser than that, the copies are refined independently. constexpr double NCS_SIGMA_MAX = 0.4; -// PROTOTYPE instrumentation (lane k): MINSTAT/TRIALSTAT/... lines on stderr where RUGNUX_REFINE_STATS is set. -bool RefineStats() { - static const bool on = getenv("RUGNUX_REFINE_STATS") != nullptr; - return on; -} - // The figure of merit of the likelihood, , at X = 2 Fo D Fc / var: I1(X)/I0(X) for an acentric // reflection, tanh(X / 2) for a centric one. // Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 @@ -619,8 +612,7 @@ struct Context { std::vector XrayBCurvatureShape() const { return XrayCurvatureShape(rows, false); } // The same for each of an atom's x, y and z: its own part of the likelihood's curvature in |F_model| is then // (2 pi)^2 s^2 / 3 f(s)^2 exp(-B s^2 / 2), the reflections taken as spread evenly over directions - the X-ray - // block of the normal matrix as a multiple of I per atom (Murshudov, Vagin & Dodson 1997 keep the X-ray blocks - // of atoms on the diagonal). `r`: the row model the engine has (blurred or not). + // block of the normal matrix as a multiple of I per atom. `r`: the row model the engine has (blurred or not). std::vector XrayXyzCurvatureShape(const XrayRowModel &r) const { return XrayCurvatureShape(r, true); } std::vector XrayCurvatureShape(const XrayRowModel &rows, bool xyz) const { constexpr int SHELLS = 32; @@ -665,144 +657,6 @@ struct Context { return shape; } - // The X-ray normal matrix between the positions of neighbouring atoms of `params` (riders left out), in the units of - // XrayXyzCurvatureShape(): for atoms k and l a vector r apart, the sum over the shells of - // w f_k f_l exp(-(B_k + B_l) s^2 / 4) (2 pi s)^2 [(j1(x) / x) I - j2(x) r r^T / r^2], x = 2 pi s r - the overlap of - // their density derivatives, the reflections spread evenly over directions; at r = 0 the shape's own (2 pi)^2 s^2 / 3. - // Out to where it has fallen below a hundredth of its value at 0 for two carbons at the median B, and stays there. - // Per parameter index its neighbours and blocks (xx yy zz xy xz yz), both ways round. - struct XrayCoupling { - std::vector start, other; - std::vector> block; - }; - XrayCoupling XrayXyzCoupling(const XrayRowModel &rows, const std::vector ¶ms) const { - constexpr int SHELLS = 32; - double s2_max = 0; - for (double s2 : refl.inv_d2) - s2_max = std::max(s2_max, s2); - std::vector w(SHELLS, 0), s2_sum(SHELLS, 0); - std::vector n(SHELLS, 0); - for (size_t m = 0; m < refl.Size(); m++) { - if (refl.free[m]) - continue; - const int k = std::min(SHELLS - 1, static_cast(refl.inv_d2[m] / s2_max * SHELLS)); - const double kd = rows.k_total[m] * rows.d[m]; - w[k] += 2.0 * kd * kd / rows.var[m]; - s2_sum[k] += refl.inv_d2[m]; - n[k]++; - } - std::vector s(SHELLS, 0); - for (int k = 0; k < SHELLS; k++) - s[k] = n[k] > 0 ? std::sqrt(s2_sum[k] / n[k]) : 0.0; - std::map, std::vector> fs; // per element and charge, f per shell - auto form = [&](const gemmi::Atom &a) -> const std::vector & { - std::vector &f = fs[{static_cast(a.element.elem), a.charge}]; - if (f.empty()) { - const auto &coef = IonFormFactor(a.element.elem, a.charge); - f.resize(SHELLS, 0); - for (int k = 0; k < SHELLS; k++) - f[k] = coef.calculate_sf(s[k] * s[k] / 4.0); - } - return f; - }; - // a = j1(x) / x and b = j2(x) per shell at distance r, for f_k f_l exp(-(B_k + B_l) s^2 / 4) given per shell - auto kernel = [&](const std::vector &ff, double r, double &a_sum, double &b_sum) { - a_sum = b_sum = 0; - for (int k = 0; k < SHELLS; k++) { - if (n[k] == 0) - continue; - const double x = 2 * M_PI * s[k] * r, c = w[k] * ff[k] * 4 * M_PI * M_PI * s[k] * s[k]; - double a, b; - if (x < 1e-3) { - a = 1.0 / 3.0 - x * x / 30.0; - b = x * x / 15.0; - } else { - const double sx = std::sin(x), cx = std::cos(x); - a = (sx / (x * x) - cx / x) / x; - b = (3.0 / (x * x) - 1.0) * sx / x - 3.0 * cx / (x * x); - } - a_sum += c * a; - b_sum += c * b; - } - }; - // the cutoff, for two carbons at the median B - std::vector bs; - for (int i : params) - bs.push_back(atoms[i]->b_iso); - XrayCoupling out; - out.start.assign(params.size() + 1, 0); - if (bs.empty()) - return out; - std::nth_element(bs.begin(), bs.begin() + bs.size() / 2, bs.end()); - const double b_med = bs[bs.size() / 2]; - gemmi::Atom carbon; - carbon.element = gemmi::El::C; - std::vector ff(SHELLS); - const std::vector &fc = form(carbon); - for (int k = 0; k < SHELLS; k++) - ff[k] = fc[k] * fc[k] * std::exp(-b_med * s[k] * s[k] / 2.0); - double a0, b0; - kernel(ff, 0.0, a0, b0); - double cutoff = 1.0; - for (double r = 1.0; r <= 8.0; r += 0.1) { - double a, b; - kernel(ff, r, a, b); - if (std::max(std::fabs(a), std::fabs(a - b)) > 0.01 * a0) - cutoff = r + 0.1; - } - // the neighbours by a grid of cells one cutoff wide - std::map, std::vector> cells; - auto cell_of = [&](const gemmi::Position &p) { - return std::array{static_cast(std::floor(p.x / cutoff)), static_cast(std::floor(p.y / cutoff)), - static_cast(std::floor(p.z / cutoff))}; - }; - auto is_param = [&](int i) { return rider_parent.empty() || rider_parent[i] < 0; }; - for (size_t k = 0; k < params.size(); k++) - if (is_param(params[k])) - cells[cell_of(atoms[params[k]]->pos)].push_back(static_cast(k)); - std::vector>>> rows_of(params.size()); - for (size_t k = 0; k < params.size(); k++) { - const gemmi::Atom &ak = *atoms[params[k]]; - if (!is_param(params[k])) - continue; - const std::array c = cell_of(ak.pos); - const std::vector &fk = form(ak); - for (int dx = -1; dx <= 1; dx++) - for (int dy = -1; dy <= 1; dy++) - for (int dz = -1; dz <= 1; dz++) { - const auto it = cells.find({c[0] + dx, c[1] + dy, c[2] + dz}); - if (it == cells.end()) - continue; - for (int l : it->second) { - if (l == static_cast(k)) - continue; - const gemmi::Atom &al = *atoms[params[l]]; - const gemmi::Position d = al.pos - ak.pos; - const double r = d.length(); - if (r >= cutoff || r <= 0) - continue; - const std::vector &fl = form(al); - for (int q = 0; q < SHELLS; q++) - ff[q] = fk[q] * fl[q] * ak.occ * al.occ * std::exp(-(ak.b_iso + al.b_iso) * s[q] * s[q] / 4.0); - double a, b; - kernel(ff, r, a, b); - const double u[3] = {d.x / r, d.y / r, d.z / r}; - rows_of[k].push_back({l, {a - b * u[0] * u[0], a - b * u[1] * u[1], a - b * u[2] * u[2], - -b * u[0] * u[1], -b * u[0] * u[2], -b * u[1] * u[2]}}); - } - } - std::sort(rows_of[k].begin(), rows_of[k].end(), [](const auto &x, const auto &y) { return x.first < y.first; }); - } - for (size_t k = 0; k < params.size(); k++) { - out.start[k + 1] = out.start[k] + static_cast(rows_of[k].size()); - for (const auto &[l, blk] : rows_of[k]) { - out.other.push_back(l); - out.block.push_back(blk); - } - } - return out; - } - // The factor that turns XrayBCurvatureShape() into the X-ray target's curvature: its second derivative along a // random +-1 direction of the B factors of `params`, by central differences of the gradient, over what the shape // predicts for it. @@ -953,7 +807,6 @@ public: gradient != nullptr ? &gr : nullptr, restraints); } *cost = weight_ * xray + restraints; - last_xray_ = xray; if (gradient == nullptr) return std::isfinite(*cost); ctx_.FoldRiders(gx); @@ -981,9 +834,6 @@ public: } Mode GetMode() const { return mode_; } - // The X-ray target and its gradient per atom (riders folded) at the last Evaluate(). - double LastXray() const { return last_xray_; } - const std::vector> &LastXrayGradient() const { return gx_; } double Weight() const { return weight_; } const std::vector &Atoms() const { return param_atoms_; } @@ -1002,66 +852,6 @@ private: // the gradients per atom, kept from evaluation to evaluation (7 MB each on a large model: made anew, their // pages fault in every time) mutable std::vector> gx_, gr_; - mutable double last_xray_ = 0; -}; - -// PROTOTYPE (RUGNUX_NEWTON_JOINT=1): the positions and the B factors at once, against X + R_geometry / w + R_B / adp_w - -// stationary where the positions' target (w X + R_geometry) and the B factors' (adp_w X + R_B) both are, so that one -// evaluation moves both. The parameters: x y z of every atom, then its B in units of B_UNIT (one step bound for all). -class XyzBTarget { -public: - static constexpr double B_UNIT = MAX_STEP_B / MAX_STEP_XYZ; - XyzBTarget(Context &ctx, const std::vector ¶m_atoms, double w, double adp_w, const RefinementTopology &topo) - : ctx_(ctx), xyz_(ctx, AtomTarget::Mode::Xyz, param_atoms, w, 0, topo), - b_(ctx, AtomTarget::Mode::B, param_atoms, adp_w, 1, topo), n_(param_atoms.size()), w_(w), adp_w_(adp_w) {} - - std::vector Get() const { - std::vector p = xyz_.Get(); - for (double b : b_.Get()) - p.push_back(b / B_UNIT); - return p; - } - void Set(const double *p) const { - std::vector b(p + 3 * n_, p + 4 * n_); - for (double &x : b) - x *= B_UNIT; - xyz_.Set(p); - b_.Set(b.data()); - } - double MaxStep() const { return MAX_STEP_XYZ; } - - bool Evaluate(const double *p, double *cost, double *gradient) const { - // each part's own evaluation at the same point gives w X + R_geometry and adp_w X + R_B, and their gradients - std::vector g_xyz(3 * n_), g_b(n_), b(p + 3 * n_, p + 4 * n_); - for (double &x : b) - x *= B_UNIT; - b_.Set(b.data()); - double f_xyz = 0; - if (!xyz_.Evaluate(p, &f_xyz, g_xyz.data())) - return false; - // the B part at the same model: its X-ray target is the same one, only its restraints are added - std::vector> gr; - const double r_b = ctx_.engine->EvaluateRestraints(1, ctx_.XyzB(), &gr); - const double xray = xyz_.LastXray(); - *cost = xray + (f_xyz - w_ * xray) / w_ + r_b / adp_w_; - const std::vector> &gx = xyz_.LastXrayGradient(); - for (size_t k = 0; k < 3 * n_; k++) - gradient[k] = g_xyz[k] / w_; - for (size_t k = 0; k < n_; k++) { - const int i = xyz_.Atoms()[k]; - const double g = gx[i][3] + gr[i][3] / adp_w_; - const double bk = b[k]; - const bool held = (bk <= ctx_.BLow(i) && g > 0) || (bk >= ctx_.BHigh(i) && g < 0); - gradient[3 * n_ + k] = held ? 0.0 : g * B_UNIT; - } - return std::isfinite(*cost); - } - -private: - Context &ctx_; - AtomTarget xyz_, b_; - size_t n_; - double w_, adp_w_; }; // The X-ray target's derivatives in U of every atom, with what the target says of a riding hydrogen's B (its parent's @@ -1203,114 +993,6 @@ private: std::vector radius_; }; -// The minimiser's diagonal for the B factors of `param_atoms` (RefinementMinimiser::SetDiagonal): weight times the -// X-ray target's curvature in B plus the B restraints', each element at least a thousandth of the median. The -// positions take none: their restraints' curvature is that of stiff bonds and angles, which move atoms together, so -// as a diagonal it slows rather than scales the minimiser. -std::vector BDiagonal(const std::vector &xray, const std::vector &restraints, - const std::vector ¶m_atoms, double weight) { - std::vector h; - for (int i : param_atoms) - h.push_back(weight * xray[i] + restraints[i]); - if (h.empty()) // no atom the dictionary describes - return h; - std::vector sorted = h; - std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); - const double floor = std::max(1e-3 * sorted[sorted.size() / 2], 1e-12); - for (double &v : h) - v = std::max(v, floor); - return h; -} - -// `minimiser` carries its memory from the call before; `diagonal` is the curvature it scales the steps by (none: 1). -template -void Minimise(Context &ctx, const Target &target, int iterations, RefinementMinimiser &minimiser, - std::vector diagonal = {}, const char *tag = "?") { - std::vector p = target.Get(); - if (p.empty()) - return; - minimiser.SetDiagonal(diagonal.empty() ? std::vector(p.size(), 1.0) : std::move(diagonal)); - const RefinementMinimiser::Outcome o = minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { - return target.Evaluate(x.data(), &f, g.data()); - }, - p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); - target.Set(p.data()); // where the minimiser ended, not where its last line search probed - if (RefineStats()) fprintf(stderr, "MINSTAT lbfgs.%s it=%d ev=%d lim=%d n=%zu f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, - iterations, p.size(), o.f_start, o.f_end); -} - -// PROTOTYPE (RUGNUX_NEWTON=1): the minimisation by damped Newton steps (RefinementNewton) on the Gauss-Newton normal -// matrix, as REFMAC refines on its sparse normal matrix: the X-ray target's curvature per parameter on the diagonal (no -// coupling between atoms), the restraints' J^T J in full. -// Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 -struct NewtonSettings { - int xyz_iterations = 30, b_iterations = 20; // the most per macro-cycle, as the L-BFGS's - double relative_stop = 0.0; - double step_scale = 1.0; // the step bound, times the L-BFGS's -}; -const NewtonSettings &Newton() { - static const NewtonSettings s = [] { - NewtonSettings n; - if (const char *e = getenv("RUGNUX_NEWTON_XYZ")) - n.xyz_iterations = std::atoi(e); - if (const char *e = getenv("RUGNUX_NEWTON_B")) - n.b_iterations = std::atoi(e); - if (const char *e = getenv("RUGNUX_NEWTON_RELSTOP")) - n.relative_stop = std::atof(e); - if (const char *e = getenv("RUGNUX_NEWTON_STEPSCALE")) - n.step_scale = std::atof(e); - return n; - }(); - return s; -} - -bool NewtonQuasi(); -// `target` by `newton` from where it is, with H v = product and D = diagonal (each element > 0). -template -void MinimiseNewton(Context &ctx, const Target &target, RefinementNewton &newton, const RefinementNewton::Product &product, - const std::vector &diagonal, int iterations, const char *tag, - const RefinementNewton::Preconditioner &precondition = nullptr) { - std::vector p = target.Get(); - if (p.empty()) - return; - if (NewtonQuasi() && newton.lbfgs != nullptr) { - // RUGNUX_NEWTON=2: the L-BFGS, its initial inverse Hessian the Gauss-Newton matrix solved by CG - RefinementMinimiser &m = *newton.lbfgs; - if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) - newton.cg_tolerance = std::atof(e); - const int cg_before = newton.cg_iterations; - m.SetPreconditioner([&](const std::vector &x, std::vector &q) { - std::vector z; - newton.SolveUndamped(product, diagonal, precondition, x, q, z); - q.swap(z); - }); - const RefinementMinimiser::Outcome o = m.Minimise( - [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, - p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); - m.SetPreconditioner(nullptr); - target.Set(p.data()); - if (RefineStats()) - fprintf(stderr, "MINSTAT qn.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, - iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); - return; - } - newton.relative_stop = Newton().relative_stop; - newton.trace = getenv("RUGNUX_NEWTON_TRACE") != nullptr; - if (const char *e = getenv("RUGNUX_NEWTON_CGMAX")) - newton.cg_max = std::atoi(e); - if (const char *e = getenv("RUGNUX_NEWTON_CGTOL")) - newton.cg_tolerance = std::atof(e); - const int cg_before = newton.cg_iterations; - const RefinementMinimiser::Outcome o = newton.Minimise( - [&](const std::vector &x, double &f, std::vector &g) { return target.Evaluate(x.data(), &f, g.data()); }, - product, diagonal, p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep() * Newton().step_scale, - [&]() { return ctx.Cancelled(); }, precondition); - target.Set(p.data()); - if (RefineStats()) - fprintf(stderr, "MINSTAT newton.%s it=%d ev=%d lim=%d cg=%d f0=%.6g f1=%.6g\n", tag, o.iterations, o.evaluations, - iterations, newton.cg_iterations - cg_before, o.f_start, o.f_end); -} - // Each element at least a thousandth of the median (a parameter the target does not see still takes a finite step). std::vector FlooredDiagonal(std::vector d) { if (d.empty()) @@ -1323,6 +1005,37 @@ std::vector FlooredDiagonal(std::vector d) { return d; } +// The minimisers' initial inverse Hessian (RefinementMinimiser::SetInverseHessian) is the Gauss-Newton normal matrix of +// the target, as REFMAC refines on its sparse normal matrix: the X-ray target's curvature per atom on the diagonal and +// the restraints' J^T J in full, solved for each direction by conjugate gradients to a tenth of the residual. +// Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 +constexpr int NORMAL_CG_ITERATIONS = 50; +constexpr double NORMAL_CG_TOLERANCE = 0.1; + +// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: +// hv = H v at x (product), z = M^-1 r for a preconditioner M of it (precondition). +template +void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations, + const std::function &, const std::vector &, + std::vector &)> &product, + const std::function &, std::vector &)> &precondition) { + std::vector p = target.Get(); + if (p.empty()) + return; + minimiser.SetInverseHessian([&](const std::vector &x, std::vector &q) { + std::vector z; + ConjugateGradients([&](const std::vector &v, std::vector &hv) { product(x, v, hv); }, precondition, + q, z, NORMAL_CG_ITERATIONS, NORMAL_CG_TOLERANCE); + q.swap(z); + }); + minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { + return target.Evaluate(x.data(), &f, g.data()); + }, + p, iterations, MINIMISER_F_TOLERANCE, target.MaxStep(), [&]() { return ctx.Cancelled(); }); + minimiser.SetInverseHessian(nullptr); + target.Set(p.data()); // where the minimiser ended, not where its last line search probed +} + // The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the // point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on // a GPU, on its card (RestraintNormalMatrixGPU). @@ -1340,7 +1053,7 @@ public: } void At(const RestraintSet &set, const std::vector> &xyzb) { - const RefinementStage stage("newton.jacobian"); + const RefinementStage stage("normal.jacobian"); #ifdef JFJOCH_USE_CUDA if (gpu_) return gpu_->At(set, xyzb); @@ -1371,7 +1084,7 @@ public: // hv = J^T J v: J v per row, then per parameter atom the sum over its entries void Multiply(const std::vector &v, std::vector &hv) { - const RefinementStage stage("newton.product"); + const RefinementStage stage("normal.product"); #ifdef JFJOCH_USE_CUDA if (gpu_) return gpu_->Multiply(v, hv); @@ -1444,27 +1157,6 @@ private: #endif }; -// hv[offset + 3k ..] += scale sum over the neighbours l of k of the X-ray block (k, l) times v[offset + 3l ..]. -void AddXrayCoupling(const Context::XrayCoupling &c, double scale, const std::vector &v, size_t offset, - std::vector &hv, size_t nthreads) { - const int n = static_cast(c.start.size()) - 1; - ParallelChunks(n, ThreadsForWork(n, nthreads, 4096), [&](int lo, int hi) { - for (int k = lo; k < hi; k++) { - double s0 = 0, s1 = 0, s2 = 0; - for (int q = c.start[k]; q < c.start[k + 1]; q++) { - const std::array &m = c.block[q]; - const double *x = &v[offset + 3 * c.other[q]]; - s0 += m[0] * x[0] + m[3] * x[1] + m[4] * x[2]; - s1 += m[3] * x[0] + m[1] * x[1] + m[5] * x[2]; - s2 += m[4] * x[0] + m[5] * x[1] + m[2] * x[2]; - } - hv[offset + 3 * k] += scale * s0; - hv[offset + 3 * k + 1] += scale * s1; - hv[offset + 3 * k + 2] += scale * s2; - } - }); -} - // z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. std::array Solve3(const std::array &m, double r0, double r1, double r2) { const double a = m[0], b = m[1], c = m[2], d = m[3], e = m[4], f = m[5]; @@ -1475,43 +1167,40 @@ std::array Solve3(const std::array &m, double r0, double r (c02 * r0 + c12 * r1 + c22 * r2) / det}; } -// The positions' preconditioner: per atom its 3 x 3 block of H (`blocks`, the restraints' over `scale`, plus the X-ray -// curvature `h`) and lambda times the damping's diagonal `diagonal`; parameters 3k..3k+2 of atom k from `offset` on. -void PreconditionBlocks(const std::vector> &blocks, double scale, const std::vector &h, - const std::vector &diagonal, size_t offset, const std::vector &r, double lambda, - std::vector &z) { +// The positions' preconditioner: per atom its 3 x 3 block of H, the restraints' (`blocks`) plus the X-ray curvature `h`. +void PreconditionBlocks(const std::vector> &blocks, const std::vector &h, + const std::vector &r, std::vector &z) { for (size_t k = 0; k < blocks.size(); k++) { - const size_t o = offset + 3 * k; - std::array m; - for (int c = 0; c < 6; c++) - m[c] = blocks[k][c] / scale; + std::array m = blocks[k]; for (int c = 0; c < 3; c++) - m[c] += h[o + c] + lambda * diagonal[o + c]; - const std::array s = Solve3(m, r[o], r[o + 1], r[o + 2]); - z[o] = s[0]; - z[o + 1] = s[1]; - z[o + 2] = s[2]; + m[c] += h[3 * k + c]; + const std::array s = Solve3(m, r[3 * k], r[3 * k + 1], r[3 * k + 2]); + z[3 * k] = s[0]; + z[3 * k + 1] = s[1]; + z[3 * k + 2] = s[2]; } } -// The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`. -void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewton &newton, const std::vector &xray, - const RestraintSet &set, int iterations, const Context::XrayCoupling *coupling = nullptr, - double coupling_scale = 0) { +// The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`, +// the Jacobian at the point of the product. Preconditioned per atom: by its 3 x 3 block of H at the start for the +// positions, by the diagonal for the B factors. +void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &minimiser, const std::vector &xray, + const RestraintSet &set, int iterations) { const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; const int per = xyz ? 3 : 1; - std::vector p = target.Get(); - const size_t n = p.size(); + std::vector x_at = target.Get(); + const size_t n = x_at.size(); std::vector h(n); for (size_t k = 0; k < n; k++) h[k] = xray[target.Atoms()[k / per]]; RestraintNormalMatrix normal(ctx, target.Atoms(), xyz); - std::vector x_at = p; - target.Set(p.data()); + target.Set(x_at.data()); normal.At(set, ctx.XyzB()); std::vector diagonal = normal.Diagonal(); for (size_t k = 0; k < n; k++) diagonal[k] += h[k]; + diagonal = FlooredDiagonal(diagonal); + const std::vector> blocks = xyz ? normal.Blocks() : std::vector>(); auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { if (x != x_at) { x_at = x; @@ -1521,86 +1210,22 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto normal.Multiply(v, hv); for (size_t k = 0; k < n; k++) hv[k] += h[k] * v[k]; - if (coupling != nullptr) - AddXrayCoupling(*coupling, coupling_scale, v, 0, hv, ctx.nthreads); }; - diagonal = FlooredDiagonal(diagonal); - if (!xyz) { - MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "b"); - return; - } - // each atom's 3 x 3 block of H at the start, fixed for the call - const std::vector> blocks = normal.Blocks(); - MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "xyz", - [&](const std::vector &r, double lambda, std::vector &z) { - PreconditionBlocks(blocks, 1.0, h, diagonal, 0, r, lambda, z); - }); -} - -// The positions and the B factors at once (XyzBTarget): H the two blocks of MinimiseAtomsNewton(), each over its weight -// (`xray_xyz`, `xray_b`: per atom, the X-ray curvature alone), no coupling between them. -void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vector &atoms, RefinementNewton &newton, - const std::vector &xray_xyz, const std::vector &xray_b, const RestraintSet &set_xyz, - const RestraintSet &set_b, double w, double adp_w, int iterations, - const Context::XrayCoupling *coupling = nullptr, double coupling_scale = 0) { - const size_t n = atoms.size(); - const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT; - RestraintNormalMatrix normal_xyz(ctx, atoms, true), normal_b(ctx, atoms, false); - std::vector x_at = target.Get(); - auto at = [&](const std::vector &x) { - target.Set(x.data()); - const std::vector> xyzb = ctx.XyzB(); - normal_xyz.At(set_xyz, xyzb); - normal_b.At(set_b, xyzb); + auto precondition = [&](const std::vector &r, std::vector &z) { + z.resize(r.size()); + if (xyz) + PreconditionBlocks(blocks, h, r, z); + else + for (size_t k = 0; k < n; k++) + z[k] = r[k] / diagonal[k]; }; - at(x_at); - std::vector diagonal(4 * n); - { - const std::vector dx = normal_xyz.Diagonal(), db = normal_b.Diagonal(); - for (size_t k = 0; k < n; k++) { - for (int c = 0; c < 3; c++) - diagonal[3 * k + c] = xray_xyz[atoms[k]] + dx[3 * k + c] / w; - diagonal[3 * n + k] = u2 * (xray_b[atoms[k]] + db[k] / adp_w); - } - } - std::vector vx(3 * n), vb(n), hx, hb; - auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { - if (x != x_at) { - x_at = x; - at(x); - } - std::copy(v.begin(), v.begin() + 3 * n, vx.begin()); - std::copy(v.begin() + 3 * n, v.end(), vb.begin()); - normal_xyz.Multiply(vx, hx); - normal_b.Multiply(vb, hb); - hv.resize(4 * n); - for (size_t k = 0; k < n; k++) { - for (int c = 0; c < 3; c++) - hv[3 * k + c] = hx[3 * k + c] / w + xray_xyz[atoms[k]] * v[3 * k + c]; - hv[3 * n + k] = u2 * (hb[k] / adp_w + xray_b[atoms[k]] * v[3 * n + k]); - } - if (coupling != nullptr) - AddXrayCoupling(*coupling, coupling_scale, v, 0, hv, ctx.nthreads); - }; - diagonal = FlooredDiagonal(diagonal); - // the positions' 3 x 3 blocks of H at the start, fixed for the call; the B factors' diagonal - const std::vector> blocks = normal_xyz.Blocks(); - std::vector h(3 * n); - for (size_t k = 0; k < n; k++) - for (int c = 0; c < 3; c++) - h[3 * k + c] = xray_xyz[atoms[k]]; - MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "xyzb", - [&](const std::vector &r, double lambda, std::vector &z) { - PreconditionBlocks(blocks, w, h, diagonal, 0, r, lambda, z); - for (size_t k = 3 * n; k < 4 * n; k++) - z[k] = r[k] / ((1 + lambda) * diagonal[k]); - }); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); } // Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their -// gradient (they are quadratic in U); D the diagonal the L-BFGS takes. -void MinimiseUNewton(Context &ctx, const AdpTarget &target, RefinementNewton &newton, const std::vector &xray, - const std::vector &diagonal, int iterations) { +// gradient (they are quadratic in U); preconditioned by `diagonal`. +void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, const std::vector &xray, + const std::vector &diagonal, int iterations) { const size_t n = xray.size(); std::vector x_at, g_at(n), xp(n), gp(n); auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { @@ -1623,13 +1248,19 @@ void MinimiseUNewton(Context &ctx, const AdpTarget &target, RefinementNewton &ne for (size_t k = 0; k < n; k++) hv[k] += xray[k] * v[k]; }; - MinimiseNewton(ctx, target, newton, product, diagonal, iterations, "u"); + auto precondition = [&](const std::vector &r, std::vector &z) { + z.resize(n); + for (size_t k = 0; k < n; k++) + z[k] = r[k] / diagonal[k]; + }; + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); } // The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's -// parameters (linear in them), C the X-ray curvature of its U per element (`xray_b`: of its B, a third of it to each). -void MinimiseTlsNewton(Context &ctx, const AdpTarget &target, RefinementNewton &newton, const std::vector &xray_b, - int iterations) { +// parameters (linear in them), C the X-ray curvature of its U per element (`xray_b`: of its B, a third of it to each); +// one dense block per body, preconditioned by its diagonal. +void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, const std::vector &xray_b, + int iterations) { constexpr int P = TlsBody::PARAMETERS; const size_t nb = ctx.tls.size(); std::vector> blocks(nb); @@ -1654,6 +1285,7 @@ void MinimiseTlsNewton(Context &ctx, const AdpTarget &target, RefinementNewton & for (size_t b = 0; b < nb; b++) for (int k = 0; k < P; k++) diagonal[b * P + k] = blocks[b][k * P + k]; + diagonal = FlooredDiagonal(diagonal); auto product = [&](const std::vector &, const std::vector &v, std::vector &hv) { hv.assign(v.size(), 0.0); for (size_t b = 0; b < nb; b++) @@ -1661,7 +1293,12 @@ void MinimiseTlsNewton(Context &ctx, const AdpTarget &target, RefinementNewton & for (int l = 0; l < P; l++) hv[b * P + k] += blocks[b][k * P + l] * v[b * P + l]; }; - MinimiseNewton(ctx, target, newton, product, FlooredDiagonal(diagonal), iterations, "tls"); + auto precondition = [&](const std::vector &r, std::vector &z) { + z.resize(r.size()); + for (size_t k = 0; k < r.size(); k++) + z[k] = r[k] / diagonal[k]; + }; + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis @@ -1840,18 +1477,6 @@ double BalancedWeight(Context &ctx, const std::vector &refinable) { std::vector> gx, gr; ctx.engine->Evaluate(ctx.Model(), &gx); std::vector> xyzb = ctx.XyzB(); - { // MEASUREMENT (lane k): the ratio of the gradients' norms at the model as given, unshaken - std::vector> g0; - ctx.engine->EvaluateRestraints(0, xyzb, &g0); - double a = 0, b = 0; - for (int i : refinable) - for (int c = 0; c < 3; c++) { - a += gx[i][c] * gx[i][c]; - b += g0[i][c] * g0[i][c]; - } - if (RefineStats()) fprintf(stderr, "WEIGHTSTAT unshaken_ratio=%.4g |gx|=%.4g |gr0|=%.4g\n", a > 0 ? std::sqrt(b / a) : 0.0, - std::sqrt(a), std::sqrt(b)); - } std::mt19937 rng(1); std::normal_distribution shake(0.0, 0.1); for (std::array &a : xyzb) @@ -2496,40 +2121,13 @@ double XrayGradientNorm(Context &ctx, const std::vector &refinable) { return n; } -// PROTOTYPE switch: RUGNUX_NEWTON=1 the positions, the B factors, the U and the TLS bodies by damped Newton steps. -bool NewtonOn() { - static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) != "0"; - return on; -} -// RUGNUX_NEWTON_PARTS (default "xyz,b,tls,u"): which of the minimisations take the Newton steps. -bool NewtonOn(const char *part) { - static const std::string parts = getenv("RUGNUX_NEWTON_PARTS") != nullptr ? getenv("RUGNUX_NEWTON_PARTS") : "xyz,b,tls,u"; - return NewtonOn() && parts.find(part) != std::string::npos; -} -// RUGNUX_NEWTON=2: the L-BFGS on top of the Gauss-Newton matrix (MinimiseNewton). -bool NewtonQuasi() { - static const bool on = getenv("RUGNUX_NEWTON") != nullptr && std::string(getenv("RUGNUX_NEWTON")) == "2"; - return on; -} -// RUGNUX_NEWTON_COUPLING=1: the X-ray normal matrix between neighbouring atoms' positions in H (XrayXyzCoupling). -bool NewtonCoupling() { - static const bool on = getenv("RUGNUX_NEWTON_COUPLING") != nullptr; - return on; -} -// RUGNUX_NEWTON_JOINT=1: where the B factors are refined isotropically, they and the positions at once (XyzBTarget). -bool NewtonJoint() { - static const bool on = getenv("RUGNUX_NEWTON_JOINT") != nullptr; - return on; -} // The X-ray curvature per atom in each of x, y, z at weight w, for the row model `rows` the engine has; `scale` // calibrated at the first call of the trial (negative: not yet). -std::vector NewtonXyzCurvature(Context &ctx, const std::vector &refinable, const XrayRowModel &rows, - double w, double &scale) { +std::vector XyzCurvature(Context &ctx, const std::vector &refinable, const XrayRowModel &rows, double w, + double &scale) { std::vector shape = ctx.XrayXyzCurvatureShape(rows); - if (scale < 0) { + if (scale < 0) scale = ctx.CalibrateXyzCurvature(refinable, shape); - if (RefineStats()) fprintf(stderr, "NEWTONCAL xyz_scale=%.4g\n", scale); - } for (double &x : shape) x *= w * scale; return shape; @@ -2584,13 +2182,9 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s b(ctx, AtomTarget::Mode::B, refinable, adp_w, 1, topo); const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); - RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); - RefinementNewton xyz_newton, b_newton, tls_newton, u_newton; - xyz_newton.lbfgs = &xyz_minimiser; - b_newton.lbfgs = &b_minimiser; - tls_newton.lbfgs = &tls_minimiser; - u_newton.lbfgs = &b_minimiser; - double xray_scale = 0, xyz_scale = -1, tls_scale = -1; + RefinementMinimiser xyz_minimiser, b_minimiser, tls_minimiser; + // the X-ray curvature's calibrations (CalibrateXyzCurvature, CalibrateBCurvature), once per trial + double xyz_scale = -1, b_scale = -1; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { PlaceRiders(ctx, topo); if (cycle == 0 && fit != nullptr) @@ -2603,15 +2197,11 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s } if (adp_w > 0) ctx.MoveOverallB(); - bool joint_done = false; // the B factors refined with the positions (MinimiseJointNewton) if (!hold_positions) { - RestraintSet xyz_set; + const RestraintSet set = RestraintsNow(ctx, topo, ncs, ncs_sigma); { - const RestraintSet set = RestraintsNow(ctx, topo, ncs, ncs_sigma); const RefinementStage s1("set_restraints"); ctx.engine->SetRestraints(0, set); - if (NewtonOn()) - xyz_set = set; } const RefinementStage s2("minimise.xyz"); const double b_h = cycle < static_cast(tw.blur.size()) ? tw.blur[cycle] : 0.0; @@ -2626,62 +2216,33 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s ctx.engine->SetRowModel(blurred); const double nh = XrayGradientNorm(ctx, refinable); const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); - if (NewtonOn("xyz")) { - const std::vector h = NewtonXyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale); - const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(blurred, refinable) - : Context::XrayCoupling(); - MinimiseAtomsNewton(ctx, xyz_h, xyz_newton, h, xyz_set, Newton().xyz_iterations, - NewtonCoupling() ? &c : nullptr, xyz_h.Weight() * xyz_scale); - } - else - Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser, {}, "xyz_blur"); + MinimiseAtoms(ctx, xyz_h, xyz_minimiser, XyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), + set, settings.xyz_iterations); ctx.engine->SetRowModel(ctx.rows); - } else if (NewtonOn() && NewtonJoint() && kind == AdpHypothesis::Kind::Isotropic && adp_w > 0) { - const RestraintSet b_set = BRestraintsNow(ctx, topo); - ctx.engine->SetRestraints(1, b_set); - std::vector xray_b = ctx.XrayBCurvatureShape(); - if (cycle == 0) - xray_scale = ctx.CalibrateBCurvature(refinable, xray_b); - for (double &x : xray_b) - x *= xray_scale; - const XyzBTarget xyz_b(ctx, refinable, w, adp_w, topo); - const std::vector h = NewtonXyzCurvature(ctx, refinable, ctx.rows, 1.0, xyz_scale); - const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(ctx.rows, refinable) - : Context::XrayCoupling(); - MinimiseJointNewton(ctx, xyz_b, refinable, xyz_newton, h, xray_b, xyz_set, b_set, w, adp_w, - Newton().xyz_iterations, NewtonCoupling() ? &c : nullptr, xyz_scale); - joint_done = true; - } else if (NewtonOn("xyz")) { - const std::vector h = NewtonXyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale); - const Context::XrayCoupling c = NewtonCoupling() ? ctx.XrayXyzCoupling(ctx.rows, refinable) - : Context::XrayCoupling(); - MinimiseAtomsNewton(ctx, xyz, xyz_newton, h, xyz_set, Newton().xyz_iterations, - NewtonCoupling() ? &c : nullptr, w * xyz_scale); - } - else - Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser, {}, "xyz"); + } else + MinimiseAtoms(ctx, xyz, xyz_minimiser, XyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), set, + settings.xyz_iterations); } + // the X-ray curvature in B for the atoms as they are, which TLS, U and B take a share of + auto b_curvature = [&]() { + std::vector c = ctx.XrayBCurvatureShape(); + if (b_scale < 0) + b_scale = ctx.CalibrateBCurvature(refinable, c); + for (double &x : c) + x *= b_scale; + return c; + }; if (kind == AdpHypothesis::Kind::Tls) { ctx.ApplyTls(); // the bodies' U at the atoms as they are const RefinementStage s4("minimise.tls"); - if (NewtonOn("tls")) { - std::vector xray = ctx.XrayBCurvatureShape(); - if (tls_scale < 0) - tls_scale = ctx.CalibrateBCurvature(refinable, xray); - for (double &x : xray) - x *= tls_scale; - MinimiseTlsNewton(ctx, tls_target, tls_newton, xray, Newton().b_iterations); - } else - Minimise(ctx, tls_target, settings.b_iterations, tls_minimiser, {}, "tls"); + MinimiseTls(ctx, tls_target, tls_minimiser, b_curvature(), settings.b_iterations); } if (adp_w > 0 && kind == AdpHypothesis::Kind::Anisotropic) { - // each U's six elements take a third of the curvature of its B - std::vector xray = ctx.XrayBCurvatureShape(); - if (cycle == 0) - xray_scale = ctx.CalibrateBCurvature(refinable, xray); - // and the restraints' curvature in 8 pi^2 U: per pair 2 / sigma^2 of the similarity and about a third of - // the rigid bond's, per isotropic atom 2 / sigma^2 - std::vector restraint(xray.size(), 0.0); + const std::vector xray_b = b_curvature(); + // each U's six elements take a third of the curvature of its B; and the restraints' curvature in 8 pi^2 U: + // per pair 2 / sigma^2 of the similarity and about a third of the rigid bond's, per isotropic atom + // 2 / sigma^2 - the preconditioner's diagonal + std::vector restraint(xray_b.size(), 0.0); const double u2 = gemmi::u_to_b() * gemmi::u_to_b(); const double per_pair = (2.0 / (URestraints::SIGMA_SIMILAR * URestraints::SIGMA_SIMILAR) + 2.0 / (3.0 * URestraints::SIGMA_RIGID * URestraints::SIGMA_RIGID)) / u2; @@ -2691,35 +2252,21 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s } for (int i : u_restraints.isotropic) restraint[i] += 2.0 / (URestraints::SIGMA_ISOTROPIC * URestraints::SIGMA_ISOTROPIC) / u2; - for (double &x : xray) - x *= xray_scale / 3.0; - std::vector diagonal; - for (double h : BDiagonal(xray, restraint, u_atoms, adp_w)) - diagonal.insert(diagonal.end(), 6, h); + std::vector h, diagonal; + for (int i : u_atoms) { + h.insert(h.end(), 6, adp_w * xray_b[i] / 3.0); + diagonal.insert(diagonal.end(), 6, adp_w * xray_b[i] / 3.0 + restraint[i]); + } const RefinementStage s3("minimise.u"); - if (NewtonOn("u")) { - std::vector h; - for (int i : u_atoms) - h.insert(h.end(), 6, adp_w * xray[i]); - MinimiseUNewton(ctx, u_target, u_newton, h, diagonal, Newton().b_iterations); - } else - Minimise(ctx, u_target, settings.b_iterations, b_minimiser, diagonal, "u"); - } else if (adp_w > 0 && !joint_done) { + MinimiseU(ctx, u_target, b_minimiser, h, FlooredDiagonal(diagonal), settings.b_iterations); + } else if (adp_w > 0) { const RestraintSet b_set = BRestraintsNow(ctx, topo); ctx.engine->SetRestraints(1, b_set); - std::vector xray = ctx.XrayBCurvatureShape(); - if (cycle == 0) - xray_scale = ctx.CalibrateBCurvature(refinable, xray); - for (double &x : xray) - x *= xray_scale; + std::vector xray_b = b_curvature(); + for (double &x : xray_b) + x *= adp_w; const RefinementStage s3("minimise.b"); - if (NewtonOn("b")) { - for (double &x : xray) - x *= adp_w; - MinimiseAtomsNewton(ctx, b, b_newton, xray, b_set, Newton().b_iterations); - } else - Minimise(ctx, b, settings.b_iterations, b_minimiser, - BDiagonal(xray, BRestraintCurvature(b_set, ctx.XyzB()), refinable, adp_w), "b"); + MinimiseAtoms(ctx, b, b_minimiser, xray_b, b_set, settings.b_iterations); } } PlaceRiders(ctx, topo); @@ -2764,8 +2311,6 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s out.fit = ctx.Fitted(); out.ran = true; out.evaluations = ctx.evaluations - evaluations_before; - if (RefineStats()) fprintf(stderr, "TRIALSTAT kind=%d adp_w=%.3g w=%.3g cycles=%d ev=%d rfree=%.4f\n", static_cast(kind), adp_w, w, cycles, - out.evaluations, t.r_free); return out; } @@ -2831,11 +2376,6 @@ std::vector RunTrials(const std::vector &trial_slots, for (const std::exception_ptr &e : errors) if (e) std::rethrow_exception(e); - int ev = 0; - for (const TrialOutcome &o : out) - ev += o.evaluations; - if (RefineStats()) fprintf(stderr, "TRIALSSET t=%.3f n=%zu cycles=%d ev=%d\n", - std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(), out.size(), cycles, ev); return out; } @@ -3862,10 +3402,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector(std::max(settings.population, 0)) - : std::min(settings.population, trial_slots.size()); + const size_t population = std::min(std::max(settings.population, 0), trial_slots.size()); if (population > 1 && state.ran && !ctx.Cancelled()) { const RefinementStage pop_stage("population"); const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma; @@ -5027,30 +4564,11 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector &data = moved > 0 ? closed : merged; - // RefinementSettings::lean_on_cpu #ifdef JFJOCH_USE_CUDA const bool on_cpu = !settings.gpu || get_gpu_count() == 0; #else const bool on_cpu = true; #endif - if (settings.lean_on_cpu && on_cpu) { - settings.population = 0; - settings.rounds = 1; - logger.Info("Refinement on the CPU: no population and one round (half of the evaluations)"); - } - // PROTOTYPE (lane k): RUGNUX_REFINE_PROTOCOL=,[,] on the CPU and the GPU alike - if (const char *e = getenv("RUGNUX_REFINE_PROTOCOL")) { - int v[5] = {settings.population, settings.rounds, settings.macro_cycles, settings.xyz_iterations, - settings.b_iterations}; - sscanf(e, "%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4]); - settings.population = v[0]; - settings.rounds = v[1]; - settings.macro_cycles = v[2]; - settings.xyz_iterations = v[3]; - settings.b_iterations = v[4]; - logger.Info("Refinement EXPERIMENT protocol: population {}, {} rounds, {} macro-cycles", settings.population, - settings.rounds, settings.macro_cycles); - } // Each arm, untwinned and twinned, is a whole refinement - the solvent built and the displacement model chosen // against its own target - and the decision is taken on where they ended. A model only scored is scored both // ways and decided by the same rule, so the R-free it reports is not an untwinned one by default. diff --git a/image_analysis/structure_refinement/Refinement.h b/image_analysis/structure_refinement/Refinement.h index 484001fd5..65f3f158e 100644 --- a/image_analysis/structure_refinement/Refinement.h +++ b/image_analysis/structure_refinement/Refinement.h @@ -48,8 +48,10 @@ struct RefinementSettings { bool score_only = false; // only score the model as given (RefinementResult::score_start), no stage bool group_b = true; // the placement stage's B shift per residue int macro_cycles = 2; // per weight trial, from the model the trial starts from - int xyz_iterations = 30; // L-BFGS iterations per macro-cycle - int b_iterations = 20; + // L-BFGS iterations per macro-cycle (on the Gauss-Newton normal matrix: RefinementMinimiser), of the positions and of + // the B factors, U and TLS + int xyz_iterations = 10; + int b_iterations = 7; // The first trials, from the placed model: the positions' X-ray weight, times the gradient-balanced one // (BalancedWeight), each with the B factors' X-ray weight their own balanced one times each of adp_ladder (0: B // held). @@ -84,8 +86,9 @@ struct RefinementSettings { // The population (RefineOnce, 2a.): this many members at most, refined from where the weight trajectory ended in // one wave, each on the data blurred along a schedule of its own and its positions kicked or not, then crossed // residue by residue on their averaged map and refined once more; the child kept where the free set prefers it. - // Capped at the trial slots; fewer than two: no population. - int population = 12; + // Capped at the trial slots; fewer than two: no population (the default: with the positions minimised on the + // Gauss-Newton matrix, the members took nothing off R-free over 19 sets that the two rounds did not). + int population = 0; // A refinement against the data as twinned by each twin law the lattice admits (RefinementTwinLaws), beside the // untwinned one, and the model of whichever the evidence favours (RefineModel). bool twin = true; @@ -97,11 +100,6 @@ struct RefinementSettings { // model of this crystal scored on a free set drawn here). bool as_given_competes = true; bool gpu = true; - // Where the refinement runs on the CPU (no GPU wanted, or none to run on), where an evaluation costs some thirty - // times a GPU's: no population and one round, for about half of the evaluations, at an R-free 0.001 higher in the - // median of 19 sets (at most 0.015, a placed prediction the population helps). The twinned data are still refined - // against: without that arm a crystal 30% twinned is refined as untwinned. - bool lean_on_cpu = true; size_t nthreads = 1; size_t gpu_resident_bytes = 0; // held on the card by what runs beside // The most weight trials run side by side, 0 for as many as the ladders and the cards take; the result does not diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index eeb4e85be..6353fff8d 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -4,41 +4,34 @@ #pragma once #include -#include #include #include #include #include -#include "../../common/ParallelFor.h" - -// Limited-memory BFGS whose memory outlives one call, so that a refinement's macro-cycles - each a few dozen -// iterations on a target whose scale and likelihood parameters were refitted in between - go on from what the -// earlier ones learnt of the curvature instead of starting each from a steepest-descent step. The initial inverse -// Hessian is a diagonal the caller gives (SetDiagonal: an estimate of the Hessian's diagonal, so that a parameter -// with ten times the curvature of another takes a tenth of its step), scaled each iteration by the latest pair -// (Oren-Shanno in that metric). The line search backtracks from the full step to sufficient decrease. +// Limited-memory BFGS whose initial inverse Hessian is the caller's (SetInverseHessian: H0 q, here the Gauss-Newton +// normal matrix solved by ConjugateGradients), and whose memory outlives one call, so that a refinement's +// macro-cycles - each a few iterations on a target whose scale and likelihood parameters were refitted in between - +// go on from what the earlier ones learnt. The normal matrix holds what is known of the curvature in advance (the +// restraints exactly, the X-ray target per atom); the pairs add what it leaves out (the X-ray target's coupling of +// neighbouring atoms). The line search backtracks from the full step to sufficient decrease. // Following Nocedal (1980) Math. Comp. 35, 773-782 and Liu & Nocedal (1989) Math. Program. 45, 503-528 class RefinementMinimiser { public: // f(x, g): the target at x, and its gradient in g (sized as x); false where it cannot be evaluated there. using Target = std::function &x, double &f, std::vector &g)>; + // q = H0 q at x. + using InverseHessian = std::function &x, std::vector &q)>; - // `nthreads`: the threads the sweeps over the parameters run on (the dot products stay serial chains). - explicit RefinementMinimiser(size_t memory = 20, size_t nthreads = 1) : memory_(memory), nthreads_(nthreads) {} + explicit RefinementMinimiser(size_t memory = 20) : memory_(memory) {} - // Forget the pairs (the target changed beyond what the memory still describes, e.g. another weight). + // Forget the pairs (the target changed beyond what the memory still describes). void Reset() { s_.clear(); y_.clear(); sy_.clear(); } - // The estimated diagonal of the Hessian; every element > 0. - void SetDiagonal(std::vector h) { h_ = std::move(h); } - // The initial inverse Hessian as an operator instead of the diagonal: q = H0 q at x (null: the diagonal). It is - // taken as it is, without the Oren-Shanno factor. - using Preconditioner = std::function &x, std::vector &q)>; - void SetPreconditioner(Preconditioner p) { precondition_ = std::move(p); } + void SetInverseHessian(InverseHessian h0) { h0_ = std::move(h0); } struct Outcome { int iterations = 0; @@ -52,8 +45,6 @@ public: Outcome Minimise(const Target &target, std::vector &x, int iterations, double f_tolerance, double max_step, const std::function &stop = nullptr) { const size_t n = x.size(); - if (h_.size() != n) - h_.assign(n, 1.0); Outcome out; double f = 0; std::vector g(n), d(n), x_new(n), g_new(n); @@ -66,7 +57,7 @@ public: break; Direction(x, g, d); double gd = Dot(g, d); - if (!(gd < 0)) { // not a descent direction: the memory no longer fits, start again from the diagonal + if (!(gd < 0)) { // not a descent direction: the memory no longer fits, start again from H0 Reset(); Direction(x, g, d); gd = Dot(g, d); @@ -80,7 +71,8 @@ public: double f_new = 0; bool accepted = false; for (int ls = 0; ls < 10; ls++) { - Sweep(n, [&](size_t k) { x_new[k] = x[k] + alpha * d[k]; }); + for (size_t k = 0; k < n; k++) + x_new[k] = x[k] + alpha * d[k]; const bool ok = target(x_new, f_new, g_new); out.evaluations++; if (ok && f_new <= f + 1e-4 * alpha * gd) { @@ -93,29 +85,21 @@ public: } if (!accepted) break; - // the pair's vectors are the ones the memory last let go, where it did: made anew, their pages fault in - std::vector s = std::move(spare_s_), y = std::move(spare_y_); - s.resize(n); - y.resize(n); - Sweep(n, [&](size_t k) { + std::vector s(n), y(n); + for (size_t k = 0; k < n; k++) { s[k] = x_new[k] - x[k]; y[k] = g_new[k] - g[k]; - }); + } const double sy = Dot(s, y); if (sy > 1e-10 * std::sqrt(Dot(s, s) * Dot(y, y))) { s_.push_back(std::move(s)); y_.push_back(std::move(y)); sy_.push_back(sy); if (s_.size() > memory_) { - spare_s_ = std::move(s_.front()); - spare_y_ = std::move(y_.front()); s_.pop_front(); y_.pop_front(); sy_.pop_front(); } - } else { - spare_s_ = std::move(s); - spare_y_ = std::move(y); } const double decrease = f - f_new; x.swap(x_new); @@ -137,82 +121,8 @@ private: return s; } - // fn(k) for every k below n on the threads, each element on its own: the serial loop's bits. - template - void Sweep(size_t n, Fn fn) const { - ParallelChunks(static_cast(n), ThreadsForWork(n, nthreads_), [&](int lo, int hi) { - for (int k = lo; k < hi; k++) - fn(static_cast(k)); - }); - } - - // d = -H g by the two-loop recursion, H0 = gamma diag(1/h). Every dot product is a serial chain of n - // additions, so each sweep over the parameters takes the next pair's dot product while it applies the - // previous pair's update to q - the same operations on each element in the same order as a sweep for - // each would do, with the update's memory traffic hidden under the chain. + // d = -H g by the two-loop recursion, H0 the caller's. void Direction(const std::vector &x, const std::vector &g, std::vector &d) const { - if (precondition_) { - DirectionPreconditioned(x, g, d); - return; - } - const size_t n = g.size(), m = s_.size(); - std::vector &q = q_; // kept from call to call: made anew, its pages fault in every time - q.assign(g.begin(), g.end()); - std::vector a(m), rho(m); - for (size_t i = 0; i < m; i++) - rho[i] = 1.0 / sy_[i]; - double dot = m > 0 ? Dot(s_[m - 1], q) : 0.0, yhy = 0.0; - for (size_t i = m; i-- > 0;) { - a[i] = rho[i] * dot; - const double ai = a[i]; - const double *y = y_[i].data(); - dot = 0.0; - if (i > 0) { - const double *s = s_[i - 1].data(); - for (size_t k = 0; k < n; k++) { - q[k] -= ai * y[k]; - dot += s[k] * q[k]; - } - } else { - const double *yl = y_[m - 1].data(); - for (size_t k = 0; k < n; k++) { - q[k] -= ai * y[k]; - yhy += yl[k] * yl[k] / h_[k]; - } - } - } - const double gamma = m > 0 ? sy_[m - 1] / yhy : 1.0; - if (m > 0) { - const double *y = y_[0].data(); - for (size_t k = 0; k < n; k++) { - q[k] *= gamma / h_[k]; - dot += y[k] * q[k]; - } - } else - for (size_t k = 0; k < n; k++) - q[k] *= gamma / h_[k]; - for (size_t i = 0; i < m; i++) { - // b on its own and (a[i] - b) in the loop, as before: written as a[i] - rho[i] * dot the compiler - // contracts it into a fused multiply-add, and the bits change - const double b = rho[i] * dot; - const double *s = s_[i].data(); - dot = 0.0; - if (i + 1 < m) { - const double *y = y_[i + 1].data(); - for (size_t k = 0; k < n; k++) { - q[k] += s[k] * (a[i] - b); - dot += y[k] * q[k]; - } - } else - for (size_t k = 0; k < n; k++) - q[k] += s[k] * (a[i] - b); - } - for (size_t k = 0; k < n; k++) - d[k] = -q[k]; - } - - // The two-loop recursion with H0 the preconditioner. - void DirectionPreconditioned(const std::vector &x, const std::vector &g, std::vector &d) const { const size_t m = s_.size(), n = g.size(); std::vector q = g, a(m); for (size_t i = m; i-- > 0;) { @@ -220,7 +130,7 @@ private: for (size_t k = 0; k < n; k++) q[k] -= a[i] * y_[i][k]; } - precondition_(x, q); + h0_(x, q); for (size_t i = 0; i < m; i++) { const double b = Dot(y_[i], q) / sy_[i]; for (size_t k = 0; k < n; k++) @@ -231,159 +141,50 @@ private: d[k] = -q[k]; } - Preconditioner precondition_; - size_t memory_, nthreads_; + size_t memory_; + InverseHessian h0_; std::deque> s_, y_; std::deque sy_; // s.y of each pair, as taken when it was kept - std::vector spare_s_, spare_y_; // the pair the memory last let go, for the next one to reuse - mutable std::vector q_; // Direction()'s working vector - std::vector h_; }; -// Damped Newton steps (Levenberg-Marquardt) on a positive semi-definite approximation of the Hessian: each iteration -// solves (H + lambda D) d = -g by conjugate gradients preconditioned with (1 + lambda) D - H v the caller's product at -// the point of the last accepted evaluation, D H's diagonal - and takes d where it lowers f. lambda falls after a -// step the quadratic model predicted well and rises after one it did not; a step that does not lower f is solved again -// with the larger lambda, without a new gradient. An iteration is one evaluation of the target where its step is -// taken. A solve stops where an iteration lowers f by less than `relative_stop` of what the call has lowered it by. -// Following Levenberg (1944) Q. Appl. Math. 2, 164-168 and Marquardt (1963) J. SIAM 11, 431-441 -class RefinementNewton { -public: - using Target = RefinementMinimiser::Target; - // hv = H v at x, the point of the last accepted evaluation. - using Product = std::function &x, const std::vector &v, std::vector &hv)>; - using Outcome = RefinementMinimiser::Outcome; - - int cg_iterations = 0; // summed over the calls, for the log - RefinementMinimiser *lbfgs = nullptr; // PROTOTYPE (RUGNUX_NEWTON=2): the L-BFGS the GN matrix preconditions - int cg_max = 50; - double cg_tolerance = 0.1; - double relative_stop = 0.0; - bool trace = false; - - // z = M^-1 r for M an approximation of H + lambda D that is cheap to solve (null: (1 + lambda) D). - using Preconditioner = std::function &r, double lambda, std::vector &z)>; - - Outcome Minimise(const Target &target, const Product &product, const std::vector &diagonal, - std::vector &x, int iterations, double f_tolerance, double max_step, - const std::function &stop = nullptr, const Preconditioner &precondition = nullptr) { - const size_t n = x.size(); - Outcome out; - double f = 0; - std::vector g(n), d(n), x_new(n), g_new(n), hd(n); - if (!target(x, f, g)) - return out; - out.evaluations++; - out.f_start = out.f_end = f; - for (int it = 0; it < iterations; it++) { - if (stop && stop()) - break; - bool accepted = false; - double f_new = 0; - for (int attempt = 0; attempt < 6 && !accepted; attempt++) { - Solve(product, diagonal, precondition, x, g, d); - double d_max = 0; - for (double v : d) - d_max = std::max(d_max, std::fabs(v)); - if (d_max > max_step) - for (double &v : d) - v *= max_step / d_max; - product(x, d, hd); - const double gd = Dot(g, d), predicted = -(gd + 0.5 * Dot(d, hd)); - if (!(gd < 0)) - break; - for (size_t k = 0; k < n; k++) - x_new[k] = x[k] + d[k]; - const bool ok = target(x_new, f_new, g_new); - out.evaluations++; - const double rho = ok ? (f - f_new) / predicted : -1.0; - if (trace) - fprintf(stderr, "NTRACE it=%d att=%d dmax=%.3g lambda=%.3g pred=%.4g act=%.4g rho=%.3f\n", it, attempt, - d_max, lambda_, predicted, f - f_new, rho); - if (ok && f_new <= f + 1e-4 * gd) { - accepted = true; - if (rho > 0.75) - lambda_ = std::max(lambda_ / 3.0, 1e-4); - else if (rho < 0.25) - lambda_ *= 2.0; - } else - lambda_ = std::max(4.0 * lambda_, 0.1); - } - if (!accepted) - break; - const double decrease = f - f_new; - x.swap(x_new); - g.swap(g_new); - f = f_new; - out.iterations++; - out.f_end = f; - if (decrease <= f_tolerance * std::max(std::fabs(f), 1.0) || decrease < relative_stop * (out.f_start - f)) - break; - } - return out; - } - - // z = H^-1 q (lambda 0), by the same conjugate gradients: H0 for the L-BFGS (RefinementMinimiser::SetPreconditioner). - void SolveUndamped(const Product &product, const std::vector &diagonal, const Preconditioner &precondition, - const std::vector &x, const std::vector &q, std::vector &z) { - const double l = lambda_; - lambda_ = 0; - std::vector minus(q.size()); - for (size_t k = 0; k < q.size(); k++) - minus[k] = -q[k]; - Solve(product, diagonal, precondition, x, minus, z); - lambda_ = l; - } - -private: - static double Dot(const std::vector &a, const std::vector &b) { +// z with H z = b to `tolerance` of |b| (at most `max_iterations`), by conjugate gradients preconditioned with M: +// hv = H v (product), z = M^-1 r (precondition), both symmetric positive definite. Returns the iterations taken. +// Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436 +inline int ConjugateGradients(const std::function &v, std::vector &hv)> &product, + const std::function &r, std::vector &z)> &precondition, + const std::vector &b, std::vector &x, int max_iterations, double tolerance) { + auto dot = [](const std::vector &u, const std::vector &v) { double s = 0; - for (size_t k = 0; k < a.size(); k++) - s += a[k] * b[k]; + for (size_t k = 0; k < u.size(); k++) + s += u[k] * v[k]; return s; - } - - // d with (H + lambda D) d = -g to `cg_tolerance` of |g|, by preconditioned conjugate gradients. - void Solve(const Product &product, const std::vector &diagonal, const Preconditioner &precondition, - const std::vector &x, const std::vector &g, std::vector &d) { - const size_t n = g.size(); - d.assign(n, 0.0); - std::vector r(n), z(n), p(n), hp(n); - auto apply = [&]() { - if (precondition) - precondition(r, lambda_, z); - else - for (size_t j = 0; j < n; j++) - z[j] = r[j] / ((1 + lambda_) * diagonal[j]); - }; - for (size_t k = 0; k < n; k++) - r[k] = -g[k]; - apply(); - p = z; - double rz = Dot(r, z); - const double r0 = std::sqrt(Dot(r, r)); - for (int k = 0; k < cg_max; k++) { - cg_iterations++; - product(x, p, hp); - for (size_t j = 0; j < n; j++) - hp[j] += lambda_ * diagonal[j] * p[j]; - const double php = Dot(p, hp); - if (!(php > 0)) - break; - const double a = rz / php; - for (size_t j = 0; j < n; j++) { - d[j] += a * p[j]; - r[j] -= a * hp[j]; - } - if (std::sqrt(Dot(r, r)) < cg_tolerance * r0) - break; - apply(); - const double rz_new = Dot(r, z); - for (size_t j = 0; j < n; j++) - p[j] = z[j] + rz_new / rz * p[j]; - rz = rz_new; + }; + const size_t n = b.size(); + x.assign(n, 0.0); + std::vector r = b, z(n), p, hp(n); + precondition(r, z); + p = z; + double rz = dot(r, z); + const double r0 = std::sqrt(dot(r, r)); + int k = 0; + while (k < max_iterations) { + k++; + product(p, hp); + const double php = dot(p, hp); + if (!(php > 0)) + break; + const double a = rz / php; + for (size_t j = 0; j < n; j++) { + x[j] += a * p[j]; + r[j] -= a * hp[j]; } + if (std::sqrt(dot(r, r)) < tolerance * r0) + break; + precondition(r, z); + const double rz_new = dot(r, z); + for (size_t j = 0; j < n; j++) + p[j] = z[j] + rz_new / rz * p[j]; + rz = rz_new; } - - double lambda_ = 0.01; // kept from call to call, as the L-BFGS memory is -}; + return k; +} diff --git a/image_analysis/structure_refinement/RefinementRestraints.cpp b/image_analysis/structure_refinement/RefinementRestraints.cpp index 7d0813183..73cc05952 100644 --- a/image_analysis/structure_refinement/RefinementRestraints.cpp +++ b/image_analysis/structure_refinement/RefinementRestraints.cpp @@ -551,18 +551,6 @@ RestraintJacobian RestraintGaussNewton(const RestraintSet &set, const std::vecto return jac; } -std::vector BRestraintCurvature(const RestraintSet &set, const std::vector> &xyzb) { - std::vector h(xyzb.size(), 0.0); - for (const RestraintRecord &r : set.records) - if (r.kind == static_cast(RestraintKind::BSimilarity)) { - // d2E/dB_i^2 of BSimilarityTerm = 2 B_j / (kappa^2 B_i^3) - const double bi = xyzb[r.atom[0]][3], bj = xyzb[r.atom[1]][3], k2 = double(r.sigma) * r.sigma; - h[r.atom[0]] += 2.0 * bj / (k2 * bi * bi * bi); - h[r.atom[1]] += 2.0 * bi / (k2 * bj * bj * bj); - } - return h; -} - gemmi::Position SpecialPosition::OnSite(const gemmi::Position &p) const { gemmi::Vec3 sum = p; for (const gemmi::Transform &t : ops) diff --git a/image_analysis/structure_refinement/RefinementRestraints.h b/image_analysis/structure_refinement/RefinementRestraints.h index 23e8ff026..6fce88e58 100644 --- a/image_analysis/structure_refinement/RefinementRestraints.h +++ b/image_analysis/structure_refinement/RefinementRestraints.h @@ -49,7 +49,7 @@ double RestraintValue(const RestraintSet &set, const RestraintRecord &r, const s double EvaluateRestraints(const RestraintSet &set, const std::vector> &xyzb, std::vector> *grad); -// PROTOTYPE: the Gauss-Newton approximation of the set's Hessian, sum over rows of J J^T: per row (a restraint's +// The Gauss-Newton approximation of the set's Hessian, sum over rows of J J^T: per row (a restraint's // residual) its derivatives in (x, y, z, B) of the atoms atom[start[r] ... start[r + 1]). struct RestraintJacobian { std::vector start; @@ -58,9 +58,6 @@ struct RestraintJacobian { }; RestraintJacobian RestraintGaussNewton(const RestraintSet &set, const std::vector> &xyzb); -// The diagonal of the Hessian of the set's B similarity restraints with respect to each atom's B, at `xyzb`. -std::vector BRestraintCurvature(const RestraintSet &set, const std::vector> &xyzb); - // An atom on a crystallographic special position: one that some symmetry operations of the group, each with a // lattice translation, take onto itself - the site's symmetry. Its position moves only within the subspace those // operations leave as it is (a 2-fold axis: along the axis; a 4-fold's: along it; a centre of inversion: not at all), diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index e34653877..b681cd69b 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -788,6 +788,55 @@ TEST_CASE("RefinementRestraints_deposited_ligands", "[Refinement]") { } } +TEST_CASE("RefinementRestraints_gauss_newton", "[Refinement]") { + // near the ideal geometry the restraints' Hessian is their Gauss-Newton J^T J (the residuals' own curvature times + // residuals that are small): v.J^T J v against central differences of the gradient along v, for random v + const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); + gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); + st.models[0] = shaken(st.models[0], 0.005, 0.0, 2); + const RefinementTopology topo = BuildRefinementTopology(st); + RestraintSet set = topo.geometry; + set.Finalize(atoms_of(st.models[0]).size()); + std::vector> xyzb; + for (gemmi::Atom *a : atoms_of(st.models[0])) + xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); + const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); + REQUIRE(jac.start.size() > 10); + std::mt19937 rng(4); + std::normal_distribution normal(0.0, 1.0); + for (int trial = 0; trial < 5; trial++) { + std::vector> v(xyzb.size(), {0, 0, 0, 0}); + for (auto &e : v) + for (int c = 0; c < 3; c++) + e[c] = normal(rng); + double gn = 0; + for (size_t r = 0; r + 1 < jac.start.size(); r++) { + double jv = 0; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + for (int c = 0; c < 3; c++) + jv += jac.j[e][c] * v[jac.atom[e]][c]; + gn += jv * jv; + } + const double h = 1e-5; + std::array>, 2> g; + for (int side = 0; side < 2; side++) { + auto x = xyzb; + for (size_t i = 0; i < x.size(); i++) + for (int c = 0; c < 3; c++) + x[i][c] += (side == 0 ? h : -h) * v[i][c]; + g[side].assign(x.size(), {0, 0, 0, 0}); + EvaluateRestraints(set, x, &g[side]); + } + double exact = 0; + for (size_t i = 0; i < xyzb.size(); i++) + for (int c = 0; c < 3; c++) + exact += v[i][c] * (g[0][i][c] - g[1][i][c]) / (2 * h); + CAPTURE(trial, gn, exact); + CHECK(gn > 0); + CHECK(gn == Catch::Approx(exact).epsilon(0.1)); + } +} + TEST_CASE("RefinementRestraints_gradient", "[Refinement]") { const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); @@ -1161,18 +1210,21 @@ TEST_CASE("RefinementMinimiser_quadratic", "[Refinement]") { return d; }; - // the exact diagonal: one step + // the exact inverse Hessian: one step RefinementMinimiser exact; - std::vector h(n); - for (size_t k = 0; k < n; k++) - h[k] = 2 * a[k]; - exact.SetDiagonal(h); + exact.SetInverseHessian([&](const std::vector &, std::vector &q) { + for (size_t k = 0; k < n; k++) + q[k] /= 2 * a[k]; + }); std::vector x(n, 0.0); exact.Minimise(target, x, 1, 0.0, 10.0); CHECK(distance(x) < 1e-9); - // without it, the memory carried from one call to the next does what one long call does + // with the identity, the memory carried from one call to the next does what one long call does RefinementMinimiser split, whole; + const RefinementMinimiser::InverseHessian identity = [](const std::vector &, std::vector &) {}; + split.SetInverseHessian(identity); + whole.SetInverseHessian(identity); std::vector xs(n, 0.0), xw(n, 0.0); for (int call = 0; call < 6; call++) split.Minimise(target, xs, 10, 0.0, 10.0); @@ -1182,6 +1234,44 @@ TEST_CASE("RefinementMinimiser_quadratic", "[Refinement]") { CHECK(xs[k] == Catch::Approx(xw[k]).margin(1e-12)); } +TEST_CASE("RefinementMinimiser_conjugate_gradients", "[Refinement]") { + // a tridiagonal positive definite system, solved to the tolerance with and without a (Jacobi) preconditioner + const size_t n = 200; + auto product = [&](const std::vector &v, std::vector &hv) { + hv.assign(n, 0.0); + for (size_t k = 0; k < n; k++) { + hv[k] = (2.0 + 0.01 * k) * v[k]; + if (k > 0) + hv[k] -= v[k - 1]; + if (k + 1 < n) + hv[k] -= v[k + 1]; + } + }; + std::vector b(n); + for (size_t k = 0; k < n; k++) + b[k] = std::sin(0.1 * k); + auto residual = [&](const std::vector &x) { + std::vector hx; + product(x, hx); + double r = 0, b2 = 0; + for (size_t k = 0; k < n; k++) { + r += (hx[k] - b[k]) * (hx[k] - b[k]); + b2 += b[k] * b[k]; + } + return std::sqrt(r / b2); + }; + std::vector x; + ConjugateGradients(product, [](const std::vector &r, std::vector &z) { z = r; }, b, x, 500, 1e-8); + CHECK(residual(x) < 1e-8); + const int jacobi = ConjugateGradients(product, [&](const std::vector &r, std::vector &z) { + z.resize(n); + for (size_t k = 0; k < n; k++) + z[k] = r[k] / (2.0 + 0.01 * k); + }, b, x, 500, 0.1); + CHECK(residual(x) < 0.1); + CHECK(jacobi < 50); +} + TEST_CASE("Refinement_restrained_recovers_model", "[Refinement]") { const gemmi::UnitCell cell(36, 40, 44, 90, 90, 90); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 21 21 21"); From 00e9e0fcc796308aa1cb5b2688c6a65a5f32a418 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 15:03:35 +0100 Subject: [PATCH 06/12] Refinement minimiser: the sweeps over the parameters on the slot's threads, the normal matrix taken once per direction The L-BFGS's and the conjugate gradients' vector sweeps run on the threads, their dot products folded in fixed blocks so that the bits do not depend on the thread count; the Jacobian is taken where the minimiser stands once per direction rather than checked at every product. On a ribosome (4ybb, refine arm, GPU, quiet) 491 s against 455 s for the former minimiser and protocol (R-free 0.2442 against 0.2472, 1820 against 4790 evaluations): the host side of the normal matrix is what a model this large still pays for. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 59 ++++++++------- .../RefinementMinimiser.h | 73 +++++++++++-------- 2 files changed, 73 insertions(+), 59 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 966436dba..7637610a8 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -1012,20 +1011,21 @@ std::vector FlooredDiagonal(std::vector d) { constexpr int NORMAL_CG_ITERATIONS = 50; constexpr double NORMAL_CG_TOLERANCE = 0.1; -// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: -// hv = H v at x (product), z = M^-1 r for a preconditioner M of it (precondition). +// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x) +// takes the matrix at x (where the minimiser stands), then hv = H v (product) and z = M^-1 r for a preconditioner M of +// it (precondition). template void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations, - const std::function &, const std::vector &, - std::vector &)> &product, + const std::function &)> &at, + const std::function &, std::vector &)> &product, const std::function &, std::vector &)> &precondition) { std::vector p = target.Get(); if (p.empty()) return; minimiser.SetInverseHessian([&](const std::vector &x, std::vector &q) { + at(x); std::vector z; - ConjugateGradients([&](const std::vector &v, std::vector &hv) { product(x, v, hv); }, precondition, - q, z, NORMAL_CG_ITERATIONS, NORMAL_CG_TOLERANCE); + ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, NORMAL_CG_TOLERANCE, ctx.nthreads); q.swap(z); }); minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { @@ -1169,8 +1169,8 @@ std::array Solve3(const std::array &m, double r0, double r // The positions' preconditioner: per atom its 3 x 3 block of H, the restraints' (`blocks`) plus the X-ray curvature `h`. void PreconditionBlocks(const std::vector> &blocks, const std::vector &h, - const std::vector &r, std::vector &z) { - for (size_t k = 0; k < blocks.size(); k++) { + const std::vector &r, std::vector &z, size_t nthreads) { + refine_vec::Each(blocks.size(), nthreads, [&](size_t k) { std::array m = blocks[k]; for (int c = 0; c < 3; c++) m[c] += h[3 * k + c]; @@ -1178,7 +1178,7 @@ void PreconditionBlocks(const std::vector> &blocks, const z[3 * k] = s[0]; z[3 * k + 1] = s[1]; z[3 * k + 2] = s[2]; - } + }); } // The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`, @@ -1201,25 +1201,25 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & diagonal[k] += h[k]; diagonal = FlooredDiagonal(diagonal); const std::vector> blocks = xyz ? normal.Blocks() : std::vector>(); - auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + auto at = [&](const std::vector &x) { if (x != x_at) { x_at = x; target.Set(x.data()); normal.At(set, ctx.XyzB()); } + }; + auto product = [&](const std::vector &v, std::vector &hv) { normal.Multiply(v, hv); - for (size_t k = 0; k < n; k++) - hv[k] += h[k] * v[k]; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] += h[k] * v[k]; }); }; auto precondition = [&](const std::vector &r, std::vector &z) { z.resize(r.size()); if (xyz) - PreconditionBlocks(blocks, h, r, z); + PreconditionBlocks(blocks, h, r, z, ctx.nthreads); else - for (size_t k = 0; k < n; k++) - z[k] = r[k] / diagonal[k]; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; }); }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); } // Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their @@ -1228,32 +1228,30 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim const std::vector &diagonal, int iterations) { const size_t n = xray.size(); std::vector x_at, g_at(n), xp(n), gp(n); - auto product = [&](const std::vector &x, const std::vector &v, std::vector &hv) { + auto at = [&](const std::vector &x) { if (x != x_at) { x_at = x; target.RestraintGradient(x.data(), g_at.data()); } + }; + auto product = [&](const std::vector &v, std::vector &hv) { double v_max = 0; for (double e : v) v_max = std::max(v_max, std::fabs(e)); hv.assign(n, 0.0); if (v_max > 0) { const double t = 1.0 / v_max; - for (size_t k = 0; k < n; k++) - xp[k] = x[k] + t * v[k]; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { xp[k] = x_at[k] + t * v[k]; }); target.RestraintGradient(xp.data(), gp.data()); - for (size_t k = 0; k < n; k++) - hv[k] = (gp[k] - g_at[k]) / t; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] = (gp[k] - g_at[k]) / t; }); } - for (size_t k = 0; k < n; k++) - hv[k] += xray[k] * v[k]; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] += xray[k] * v[k]; }); }; auto precondition = [&](const std::vector &r, std::vector &z) { z.resize(n); - for (size_t k = 0; k < n; k++) - z[k] = r[k] / diagonal[k]; + refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; }); }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); } // The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's @@ -1286,7 +1284,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (int k = 0; k < P; k++) diagonal[b * P + k] = blocks[b][k * P + k]; diagonal = FlooredDiagonal(diagonal); - auto product = [&](const std::vector &, const std::vector &v, std::vector &hv) { + auto product = [&](const std::vector &v, std::vector &hv) { hv.assign(v.size(), 0.0); for (size_t b = 0; b < nb; b++) for (int k = 0; k < P; k++) @@ -1298,7 +1296,8 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (size_t k = 0; k < r.size(); k++) z[k] = r[k] / diagonal[k]; }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, [](const std::vector &) {}, product, + precondition); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis @@ -2182,7 +2181,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s b(ctx, AtomTarget::Mode::B, refinable, adp_w, 1, topo); const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); - RefinementMinimiser xyz_minimiser, b_minimiser, tls_minimiser; + RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); // the X-ray curvature's calibrations (CalibrateXyzCurvature, CalibrateBCurvature), once per trial double xyz_scale = -1, b_scale = -1; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 6353fff8d..b79bd1c36 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -9,6 +9,34 @@ #include #include +#include "../../common/ParallelFor.h" + +// Sweeps over the parameters on `nthreads` threads: each element on its own, and dot products folded in fixed +// blocks (ParallelBlocks), so that the bits do not depend on the number of threads. +namespace refine_vec { +template +void Each(size_t n, size_t nthreads, Fn fn) { + ParallelChunks(static_cast(n), ThreadsForWork(n, nthreads, 16384), [&](int lo, int hi) { + for (int k = lo; k < hi; k++) + fn(static_cast(k)); + }); +} +inline double Dot(const std::vector &a, const std::vector &b, size_t nthreads) { + const int n = static_cast(a.size()); + std::vector part(ReductionBlocks(n), 0.0); + ParallelBlocks(n, nthreads, [&](int block, int lo, int hi) { + double s = 0; + for (int k = lo; k < hi; k++) + s += a[k] * b[k]; + part[block] = s; + }); + double s = 0; + for (double v : part) + s += v; + return s; +} +} // namespace refine_vec + // Limited-memory BFGS whose initial inverse Hessian is the caller's (SetInverseHessian: H0 q, here the Gauss-Newton // normal matrix solved by ConjugateGradients), and whose memory outlives one call, so that a refinement's // macro-cycles - each a few iterations on a target whose scale and likelihood parameters were refitted in between - @@ -23,7 +51,8 @@ public: // q = H0 q at x. using InverseHessian = std::function &x, std::vector &q)>; - explicit RefinementMinimiser(size_t memory = 20) : memory_(memory) {} + // `nthreads`: the threads the sweeps over the parameters run on. + explicit RefinementMinimiser(size_t memory = 20, size_t nthreads = 1) : memory_(memory), nthreads_(nthreads) {} // Forget the pairs (the target changed beyond what the memory still describes). void Reset() { @@ -71,8 +100,7 @@ public: double f_new = 0; bool accepted = false; for (int ls = 0; ls < 10; ls++) { - for (size_t k = 0; k < n; k++) - x_new[k] = x[k] + alpha * d[k]; + refine_vec::Each(n, nthreads_, [&](size_t k) { x_new[k] = x[k] + alpha * d[k]; }); const bool ok = target(x_new, f_new, g_new); out.evaluations++; if (ok && f_new <= f + 1e-4 * alpha * gd) { @@ -86,10 +114,10 @@ public: if (!accepted) break; std::vector s(n), y(n); - for (size_t k = 0; k < n; k++) { + refine_vec::Each(n, nthreads_, [&](size_t k) { s[k] = x_new[k] - x[k]; y[k] = g_new[k] - g[k]; - } + }); const double sy = Dot(s, y); if (sy > 1e-10 * std::sqrt(Dot(s, s) * Dot(y, y))) { s_.push_back(std::move(s)); @@ -114,12 +142,7 @@ public: } private: - static double Dot(const std::vector &a, const std::vector &b) { - double s = 0; - for (size_t k = 0; k < a.size(); k++) - s += a[k] * b[k]; - return s; - } + double Dot(const std::vector &a, const std::vector &b) const { return refine_vec::Dot(a, b, nthreads_); } // d = -H g by the two-loop recursion, H0 the caller's. void Direction(const std::vector &x, const std::vector &g, std::vector &d) const { @@ -127,21 +150,18 @@ private: std::vector q = g, a(m); for (size_t i = m; i-- > 0;) { a[i] = Dot(s_[i], q) / sy_[i]; - for (size_t k = 0; k < n; k++) - q[k] -= a[i] * y_[i][k]; + refine_vec::Each(n, nthreads_, [&](size_t k) { q[k] -= a[i] * y_[i][k]; }); } h0_(x, q); for (size_t i = 0; i < m; i++) { const double b = Dot(y_[i], q) / sy_[i]; - for (size_t k = 0; k < n; k++) - q[k] += s_[i][k] * (a[i] - b); + refine_vec::Each(n, nthreads_, [&](size_t k) { q[k] += s_[i][k] * (a[i] - b); }); } d.resize(n); - for (size_t k = 0; k < n; k++) - d[k] = -q[k]; + refine_vec::Each(n, nthreads_, [&](size_t k) { d[k] = -q[k]; }); } - size_t memory_; + size_t memory_, nthreads_; InverseHessian h0_; std::deque> s_, y_; std::deque sy_; // s.y of each pair, as taken when it was kept @@ -152,13 +172,9 @@ private: // Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436 inline int ConjugateGradients(const std::function &v, std::vector &hv)> &product, const std::function &r, std::vector &z)> &precondition, - const std::vector &b, std::vector &x, int max_iterations, double tolerance) { - auto dot = [](const std::vector &u, const std::vector &v) { - double s = 0; - for (size_t k = 0; k < u.size(); k++) - s += u[k] * v[k]; - return s; - }; + const std::vector &b, std::vector &x, int max_iterations, double tolerance, + size_t nthreads = 1) { + auto dot = [&](const std::vector &u, const std::vector &v) { return refine_vec::Dot(u, v, nthreads); }; const size_t n = b.size(); x.assign(n, 0.0); std::vector r = b, z(n), p, hp(n); @@ -174,16 +190,15 @@ inline int ConjugateGradients(const std::function if (!(php > 0)) break; const double a = rz / php; - for (size_t j = 0; j < n; j++) { + refine_vec::Each(n, nthreads, [&](size_t j) { x[j] += a * p[j]; r[j] -= a * hp[j]; - } + }); if (std::sqrt(dot(r, r)) < tolerance * r0) break; precondition(r, z); const double rz_new = dot(r, z); - for (size_t j = 0; j < n; j++) - p[j] = z[j] + rz_new / rz * p[j]; + refine_vec::Each(n, nthreads, [&](size_t j) { p[j] = z[j] + rz_new / rz * p[j]; }); rz = rz_new; } return k; From b67f48848fbcc4fc4581df29c9c81ab5f61de6dc Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 15:24:40 +0100 Subject: [PATCH 07/12] Refinement minimiser: sweeps threaded only from 65536 parameters a thread Below that waking the workers costs more than the pass (the dot products' fixed blocks with them). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- image_analysis/structure_refinement/RefinementMinimiser.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index b79bd1c36..6bedb58fc 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -14,22 +14,24 @@ // Sweeps over the parameters on `nthreads` threads: each element on its own, and dot products folded in fixed // blocks (ParallelBlocks), so that the bits do not depend on the number of threads. namespace refine_vec { +// a pass over fewer than this many parameters per thread costs less than waking the threads +constexpr size_t SWEEP_PER_THREAD = 65536; template void Each(size_t n, size_t nthreads, Fn fn) { - ParallelChunks(static_cast(n), ThreadsForWork(n, nthreads, 16384), [&](int lo, int hi) { + ParallelChunks(static_cast(n), ThreadsForWork(n, nthreads, SWEEP_PER_THREAD), [&](int lo, int hi) { for (int k = lo; k < hi; k++) fn(static_cast(k)); }); } inline double Dot(const std::vector &a, const std::vector &b, size_t nthreads) { const int n = static_cast(a.size()); - std::vector part(ReductionBlocks(n), 0.0); + std::vector part(ReductionBlocks(n, SWEEP_PER_THREAD), 0.0); ParallelBlocks(n, nthreads, [&](int block, int lo, int hi) { double s = 0; for (int k = lo; k < hi; k++) s += a[k] * b[k]; part[block] = s; - }); + }, SWEEP_PER_THREAD); double s = 0; for (double v : part) s += v; From 92f6bcdc750bf583fa7302ad18a6d9c429171a86 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 18:13:08 +0100 Subject: [PATCH 08/12] Refinement minimiser: a per-atom trust floor on the normal matrix, a per-residue preconditioner, a fixed CG count Traced: with the X-ray curvature per atom as the only curvature of a soft mode (a high-B side chain turning about a single bond, which the restraints leave free), the normal matrix's inverse made steps of tens to hundreds of A there, and the global step bound then scaled every other atom's step down to a few percent; a rounding difference of 1e-7 in the gradient moved a trial's R-free by up to 0.004. - Each atom's diagonal is raised, once per call, to at least |g_i| over the step bound, so that such an atom's step is held by its own curvature: the first trial's iterations take the full step, and a 1e-7 perturbation stays at 1e-7. - The CG is preconditioned per residue (the residue's own J^T J and X-ray diagonal, by Cholesky), and runs a fixed 20 iterations instead of to a tolerance, whose iteration count a rounding difference can flip. - The per-atom 3 x 3 blocks (CPU and GPU) are gone with the preconditioner they served. GPU, 19 sets, against the rc188-merge protocol and minimiser: R-free -0.0014 mean, -0.0008 median, worse by > 0.002 on 2 and better on 7 (the experiment's build); 158k -> 47k evaluations. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 170 +++++++++++------- .../RefinementNewtonGPU.cu | 36 ---- .../RefinementNewtonGPU.h | 3 +- tests/RefinementTest.cpp | 30 +--- 4 files changed, 109 insertions(+), 130 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 7637610a8..d05c9c53d 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -1006,26 +1007,26 @@ std::vector FlooredDiagonal(std::vector d) { // The minimisers' initial inverse Hessian (RefinementMinimiser::SetInverseHessian) is the Gauss-Newton normal matrix of // the target, as REFMAC refines on its sparse normal matrix: the X-ray target's curvature per atom on the diagonal and -// the restraints' J^T J in full, solved for each direction by conjugate gradients to a tenth of the residual. +// the restraints' J^T J in full, solved for each direction by a fixed number of conjugate-gradient iterations - not to a +// tolerance, whose iteration count, and with it the direction, a rounding difference can flip. // Following Murshudov, Vagin & Dodson (1997) Acta Cryst. D53, 240-255 -constexpr int NORMAL_CG_ITERATIONS = 50; -constexpr double NORMAL_CG_TOLERANCE = 0.1; +constexpr int NORMAL_CG_ITERATIONS = 20; -// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x) -// takes the matrix at x (where the minimiser stands), then hv = H v (product) and z = M^-1 r for a preconditioner M of -// it (precondition). +// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q) +// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then hv = H v +// (product) and z = M^-1 r for a preconditioner M of it (precondition). template void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations, - const std::function &)> &at, + const std::function &, const std::vector &)> &at, const std::function &, std::vector &)> &product, const std::function &, std::vector &)> &precondition) { std::vector p = target.Get(); if (p.empty()) return; minimiser.SetInverseHessian([&](const std::vector &x, std::vector &q) { - at(x); + at(x, q); std::vector z; - ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, NORMAL_CG_TOLERANCE, ctx.nthreads); + ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads); q.swap(z); }); minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { @@ -1112,26 +1113,6 @@ public: }); } - // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. - std::vector> Blocks() const { -#ifdef JFJOCH_USE_CUDA - if (gpu_) - return gpu_->Blocks(); -#endif - std::vector> b(n_params_, {0, 0, 0, 0, 0, 0}); - for (size_t k = 0; k < n_params_; k++) - for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) { - const std::array &j = jac_.j[entry_[q]]; - b[k][0] += j[0] * j[0]; - b[k][1] += j[1] * j[1]; - b[k][2] += j[2] * j[2]; - b[k][3] += j[0] * j[1]; - b[k][4] += j[0] * j[2]; - b[k][5] += j[1] * j[2]; - } - return b; - } - std::vector Diagonal() const { #ifdef JFJOCH_USE_CUDA if (gpu_) @@ -1157,33 +1138,76 @@ private: #endif }; -// z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. -std::array Solve3(const std::array &m, double r0, double r1, double r2) { - const double a = m[0], b = m[1], c = m[2], d = m[3], e = m[4], f = m[5]; - const double c00 = b * c - f * f, c01 = e * f - d * c, c02 = d * f - b * e; - const double c11 = a * c - e * e, c12 = d * e - a * f, c22 = a * b - d * d; - const double det = a * c00 + d * c01 + e * c02; - return {(c00 * r0 + c01 * r1 + c02 * r2) / det, (c01 * r0 + c11 * r1 + c12 * r2) / det, - (c02 * r0 + c12 * r1 + c22 * r2) / det}; -} +// The positions' preconditioner: per residue the dense block of H over its atoms' x y z - the restraints within the +// residue (J^T J of the rows whose atoms it holds, at the point given) and the X-ray diagonal `h` - by its Cholesky +// factor. What couples residues (peptide bonds, NCS, contacts) is left to the conjugate gradients; the stiff bonds +// and angles within a residue, which a diagonal or a block per atom leaves to them too, are solved here. +class ResidueBlocks { +public: + ResidueBlocks(Context &ctx, const std::vector ¶m_atoms, const RestraintSet &set, const std::vector &h) { + std::vector residue_of; // per atom, in model order + for (const gemmi::Chain &ch : ctx.Model().chains) + for (const gemmi::Residue &r : ch.residues) { + residue_of.insert(residue_of.end(), r.atoms.size(), static_cast(params_.size())); + params_.emplace_back(); + } + std::vector local(param_atoms.size()), group(param_atoms.size()), param_of(ctx.atoms.size(), -1); + for (size_t k = 0; k < param_atoms.size(); k++) { + group[k] = residue_of[param_atoms[k]]; + local[k] = static_cast(params_[group[k]].size()); + params_[group[k]].push_back(static_cast(k)); + param_of[param_atoms[k]] = static_cast(k); + } + std::vector block(params_.size()); + for (size_t g = 0; g < params_.size(); g++) { + const int m = 3 * static_cast(params_[g].size()); + block[g] = Eigen::MatrixXd::Zero(m, m); + for (size_t l = 0; l < params_[g].size(); l++) + for (int c = 0; c < 3; c++) + block[g](3 * l + c, 3 * l + c) = h[3 * params_[g][l] + c]; + } + const RestraintJacobian jac = RestraintGaussNewton(set, ctx.XyzB()); + for (size_t row = 0; row + 1 < jac.start.size(); row++) + for (int e1 = jac.start[row]; e1 < jac.start[row + 1]; e1++) + for (int e2 = jac.start[row]; e2 < jac.start[row + 1]; e2++) { + const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]]; + if (k1 < 0 || k2 < 0 || group[k1] != group[k2]) + continue; + Eigen::MatrixXd &b = block[group[k1]]; + for (int a = 0; a < 3; a++) + for (int c = 0; c < 3; c++) + b(3 * local[k1] + a, 3 * local[k2] + c) += jac.j[e1][a] * jac.j[e2][c]; + } + llt_.resize(params_.size()); + ParallelFor(static_cast(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); }); + } -// The positions' preconditioner: per atom its 3 x 3 block of H, the restraints' (`blocks`) plus the X-ray curvature `h`. -void PreconditionBlocks(const std::vector> &blocks, const std::vector &h, - const std::vector &r, std::vector &z, size_t nthreads) { - refine_vec::Each(blocks.size(), nthreads, [&](size_t k) { - std::array m = blocks[k]; - for (int c = 0; c < 3; c++) - m[c] += h[3 * k + c]; - const std::array s = Solve3(m, r[3 * k], r[3 * k + 1], r[3 * k + 2]); - z[3 * k] = s[0]; - z[3 * k + 1] = s[1]; - z[3 * k + 2] = s[2]; - }); -} + void Apply(const std::vector &r, std::vector &z, size_t nthreads) const { + z.resize(r.size()); + ParallelFor(static_cast(params_.size()), nthreads, [&](int g) { + const std::vector &p = params_[g]; + Eigen::VectorXd rg(3 * p.size()); + for (size_t l = 0; l < p.size(); l++) + for (int c = 0; c < 3; c++) + rg(3 * l + c) = r[3 * p[l] + c]; + const Eigen::VectorXd zg = llt_[g].solve(rg); + for (size_t l = 0; l < p.size(); l++) + for (int c = 0; c < 3; c++) + z[3 * p[l] + c] = zg(3 * l + c); + }); + } + +private: + std::vector> params_; // per residue its parameter atoms (their index among the parameters) + std::vector> llt_; +}; // The positions or the B factors: H = diag(xray) (per atom, already times the weight) + the restraints' J^T J of `set`, -// the Jacobian at the point of the product. Preconditioned per atom: by its 3 x 3 block of H at the start for the -// positions, by the diagonal for the B factors. +// the Jacobian at the point of each direction. The diagonal is raised per atom, once per call, to at least |g_i| over the +// step bound (g_i the atom's part of the first direction's gradient): where nothing else curves the target (a high-B +// side chain turning about a single bond, which the restraints leave free and the X-ray target hardly sees) the step +// is then held to the bound by the atom's own curvature, instead of one such atom scaling down the whole step. +// Preconditioned per residue for the positions (ResidueBlocks), by the diagonal for the B factors. void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &minimiser, const std::vector &xray, const RestraintSet &set, int iterations) { const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; @@ -1196,12 +1220,25 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & RestraintNormalMatrix normal(ctx, target.Atoms(), xyz); target.Set(x_at.data()); normal.At(set, ctx.XyzB()); - std::vector diagonal = normal.Diagonal(); - for (size_t k = 0; k < n; k++) - diagonal[k] += h[k]; - diagonal = FlooredDiagonal(diagonal); - const std::vector> blocks = xyz ? normal.Blocks() : std::vector>(); - auto at = [&](const std::vector &x) { + const std::vector restraint_diagonal = normal.Diagonal(); + std::vector diagonal; + std::unique_ptr residues; + auto at = [&](const std::vector &x, const std::vector &q) { + if (diagonal.empty()) { // the first direction: the floor, then the preconditioner with it + for (size_t a = 0; a < n / per; a++) { + double g2 = 0; + for (int c = 0; c < per; c++) + g2 += q[per * a + c] * q[per * a + c]; + for (int c = 0; c < per; c++) + h[per * a + c] = std::max(h[per * a + c], std::sqrt(g2) / target.MaxStep()); + } + diagonal.resize(n); + for (size_t k = 0; k < n; k++) + diagonal[k] = restraint_diagonal[k] + h[k]; + diagonal = FlooredDiagonal(diagonal); + if (xyz) + residues = std::make_unique(ctx, target.Atoms(), set, h); + } if (x != x_at) { x_at = x; target.Set(x.data()); @@ -1213,11 +1250,12 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & refine_vec::Each(n, ctx.nthreads, [&](size_t k) { hv[k] += h[k] * v[k]; }); }; auto precondition = [&](const std::vector &r, std::vector &z) { - z.resize(r.size()); - if (xyz) - PreconditionBlocks(blocks, h, r, z, ctx.nthreads); - else + if (residues) + residues->Apply(r, z, ctx.nthreads); + else { + z.resize(r.size()); refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; }); + } }; MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); } @@ -1228,7 +1266,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim const std::vector &diagonal, int iterations) { const size_t n = xray.size(); std::vector x_at, g_at(n), xp(n), gp(n); - auto at = [&](const std::vector &x) { + auto at = [&](const std::vector &x, const std::vector &) { if (x != x_at) { x_at = x; target.RestraintGradient(x.data(), g_at.data()); @@ -1296,8 +1334,8 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (size_t k = 0; k < r.size(); k++) z[k] = r[k] / diagonal[k]; }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, [](const std::vector &) {}, product, - precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, + [](const std::vector &, const std::vector &) {}, product, precondition); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index a3c61554e..a11281e46 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -197,24 +197,6 @@ __global__ void DiagonalKernel(const int *entry_start, const int *entry, const f } } -__global__ void BlocksKernel(const int *entry_start, const int *entry, const float4 *j, float *b, int n) { - const int k = blockIdx.x * blockDim.x + threadIdx.x; - if (k >= n) - return; - float s[6] = {0, 0, 0, 0, 0, 0}; - for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const float4 x = j[entry[q]]; - s[0] += x.x * x.x; - s[1] += x.y * x.y; - s[2] += x.z * x.z; - s[3] += x.x * x.y; - s[4] += x.x * x.z; - s[5] += x.y * x.z; - } - for (int c = 0; c < 6; c++) - b[6 * k + c] = s[c]; -} - template CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { CudaDevicePtr d(std::max(host.size(), 1)); @@ -335,21 +317,3 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { return out; } -std::vector> RestraintNormalMatrixGPU::Blocks() const { - DeviceGuard guard(dev_->device); - Device &d = *dev_; - const int n = static_cast(n_params_); - std::vector> out(n_params_); - if (n == 0) - return out; - CudaDevicePtr b(6 * n_params_); - std::vector host(6 * n_params_); - BlocksKernel<<>>(d.entry_start, d.entry, d.j, b, n); - cuda_err(cudaGetLastError()); - cuda_err(cudaMemcpyAsync(host.data(), b.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); - cuda_err(cudaStreamSynchronize(d.stream)); - for (size_t k = 0; k < n_params_; k++) - for (int c = 0; c < 6; c++) - out[k][c] = host[6 * k + c]; - return out; -} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h index 3136ed029..317cfa8f9 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.h +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -31,8 +31,7 @@ public: void At(const RestraintSet &set, const std::vector> &xyzb); // hv = J^T J v void Multiply(const std::vector &v, std::vector &hv); - // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. - std::vector> Blocks() const; + // The diagonal of J^T J. std::vector Diagonal() const; private: diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index 4597972f6..c53a232e3 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -1071,9 +1071,8 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { std::vector v(n); for (size_t i = 0; i < n; i++) v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen); - // J^T J v, the diagonal and the 3 x 3 blocks on the CPU, each entry in turn + // J^T J v and the diagonal on the CPU, each entry in turn std::vector hv(n, 0.0), diagonal(n, 0.0); - std::vector> blocks(params.size(), {0, 0, 0, 0, 0, 0}); for (size_t r = 0; r + 1 < jac.start.size(); r++) { double t = 0; for (int e = jac.start[r]; e < jac.start[r + 1]; e++) @@ -1090,12 +1089,6 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { const std::array &j = jac.j[e]; for (int c = 0; c < per; c++) diagonal[per * k + c] += j[c0 + c] * j[c0 + c]; - blocks[k][0] += j[0] * j[0]; - blocks[k][1] += j[1] * j[1]; - blocks[k][2] += j[2] * j[2]; - blocks[k][3] += j[0] * j[1]; - blocks[k][4] += j[0] * j[2]; - blocks[k][5] += j[1] * j[2]; } RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); gpu.At(set, xyzb); @@ -1105,32 +1098,17 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { REQUIRE(hv_gpu.size() == n); REQUIRE(diagonal_gpu.size() == n); // In float, from float positions: within a few float epsilons of the double product in norm (in refinement - // its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself, each block - // relative to its own size. - double num = 0, den = 0, worst_diagonal = 0, worst_block = 0; + // its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself. + double num = 0, den = 0, worst_diagonal = 0; for (size_t i = 0; i < n; i++) { num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]); den += hv[i] * hv[i]; if (diagonal[i] > 0) worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]); } - if (xyz) { - const std::vector> blocks_gpu = gpu.Blocks(); - REQUIRE(blocks_gpu.size() == params.size()); - for (size_t k = 0; k < params.size(); k++) { - double d2 = 0, b2 = 0; - for (int c = 0; c < 6; c++) { - d2 += (blocks_gpu[k][c] - blocks[k][c]) * (blocks_gpu[k][c] - blocks[k][c]); - b2 += blocks[k][c] * blocks[k][c]; - } - if (b2 > 0) - worst_block = std::max(worst_block, std::sqrt(d2 / b2)); - } - } - CAPTURE(std::sqrt(num / den), worst_diagonal, worst_block); + CAPTURE(std::sqrt(num / den), worst_diagonal); CHECK(std::sqrt(num / den) < 1e-5); CHECK(worst_diagonal < 1e-5); - CHECK(worst_block < 1e-5); } } From b3d26b2c87671f77b839c625250ebda292092165 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 18:13:54 +0100 Subject: [PATCH 09/12] Docs: the minimiser's trust floor per atom and per-residue preconditioner; Levenberg/Marquardt credited for the floor Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 7 ++++++- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 21 ++++++++++++++----- .../structure_refinement/Refinement.cpp | 2 ++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 0c86fbbd6..8359d48d4 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -603,7 +603,12 @@ initial inverse Hessian is the target's sparse normal matrix - the X-ray target' on the diagonal and the restraints' Gauss-Newton matrix in full - as REFMAC refines on its normal matrix (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients: M. R. Hestenes and E. Stiefel, "Methods of conjugate gradients for solving linear systems" (1952), J. Res. Natl. Bur. -Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). +Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). Each atom's +diagonal is raised to hold its step within the step bound, a trust region per atom after the damping +of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" (1944), +Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and D. W. +Marquardt, "An algorithm for least-squares estimation of nonlinear parameters" (1963), J. Soc. Ind. +Appl. Math. 11, 431-441 [doi:10.1137/0111030](https://doi.org/10.1137/0111030). **Geometry restraints** — those of the CCP4 Monomer Library, a subset of which rugnux carries (see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)). A. A. Vagin, R. A. Steiner, A. A. Lebedev, diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 293b59be8..13879eb05 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -309,9 +309,19 @@ leaves out. neighbouring atoms through the X-ray target is not in it; the pairs learn it. - **The anisotropic U restraints** are quadratic in U, so their Hessian is exact by one difference of their gradient. -- **The matrix is solved, not inverted.** Each L-BFGS direction solves the normal equations by conjugate - gradients to a tenth of the residual (at most 50 iterations), preconditioned by each atom's $3\times3$ block for - the positions and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser stands on. +- **A trust floor per atom.** Where nothing but the X-ray target curves a mode - a high-B side chain turning about + a single bond, which the restraints leave free and the X-ray target hardly sees - the matrix's inverse would + make a step of tens of ångströms there, and the step bound would then shrink every other atom's step with it. + So each atom's diagonal is raised, once per macro-cycle, to at least $|g_i|/\Delta$ ($g_i$ its gradient, + $\Delta$ the step bound): its step is held by its own curvature, a trust region per atom in the manner of + Levenberg and Marquardt's damping. Without it a rounding difference of $10^{-7}$ in the gradient moved a weight + trial's R-free by up to 0.004; with it the perturbation stays where it was. +- **The matrix is solved, not inverted.** Each L-BFGS direction solves the normal equations by a fixed 20 + conjugate-gradient iterations - not to a tolerance, whose iteration count, and with it the direction, a rounding + difference can flip - preconditioned per residue for the positions (the residue's own $J^\mathsf{T}J$ and X-ray + diagonal, by Cholesky: the stiff bonds and angles inside a residue are solved there, what couples residues is + left to the iterations) and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser + stands on. - **Its memory is kept from one macro-cycle to the next.** Each weight trial starts with an empty memory. A solve stops after its iteration budget (10 for the positions and 7 for the B factors, U or TLS per @@ -319,8 +329,8 @@ macro-cycle) or where an iteration lowers the target by less than $10^{-7}$ of i replaced (a diagonal initial inverse Hessian for the B factors, none for the positions, 30 and 20 iterations), a trial takes about a third of the evaluations. With it the population (§15.7) no longer gains, and the protocol is the same on the CPU and the GPU - two rounds, no population - at a third of the evaluations the GPU's protocol -took before and R-free 0.0005 lower in the mean over 19 refinements (-LL free lower on 11 of them), and at fewer -evaluations than the CPU's former one-round protocol. +took before and R-free 0.0014 lower in the mean over 19 refinements, and at fewer evaluations than the CPU's +former one-round protocol. ### 15.7 The weights, decided by trial @@ -780,6 +790,7 @@ What the refinement does not do yet - each a known gap, not a design choice: - N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6). - M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6). +- K. Levenberg, "A method for the solution of certain non-linear problems in least squares", *Q. Appl. Math.* **2** (1944), 164-168, and D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters", *J. Soc. Ind. Appl. Math.* **11** (1963), 431-441 (the minimiser's trust floor per atom, §15.6). - A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7). - P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7). - A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free). diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index d05c9c53d..d360c8f59 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1208,6 +1208,8 @@ private: // side chain turning about a single bond, which the restraints leave free and the X-ray target hardly sees) the step // is then held to the bound by the atom's own curvature, instead of one such atom scaling down the whole step. // Preconditioned per residue for the positions (ResidueBlocks), by the diagonal for the B factors. +// The floor a trust region per atom, after Levenberg (1944) Q. Appl. Math. 2, 164-168 and Marquardt (1963) J. SIAM +// 11, 431-441 void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &minimiser, const std::vector &xray, const RestraintSet &set, int iterations) { const bool xyz = target.GetMode() == AtomTarget::Mode::Xyz; From dc88610e0dfd2e9d89dacb2ecca3cd87b14c3e57 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 19:18:12 +0100 Subject: [PATCH 10/12] Refinement on the CPU: the arms take half of the threads each only where the lattice admits a twin law Without one the twinned arm ends at once and its half sat idle (found by lane t: about 6 of 12 threads busy through the whole refinement). Identical R-free and evaluations; at 12 threads thau 125 -> 92 s, 9zlo 143 -> 90 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- image_analysis/structure_refinement/Refinement.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index d360c8f59..609a0fcd5 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -4645,7 +4645,10 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector 1; + // on the CPU each arm on half of the threads - where the lattice admits a twin law, without which the twinned + // arm ends at once and its half would sit idle + const bool halves = on_cpu && settings.population <= 1 && settings.nthreads > 1 && + !RefinementTwinLaws(cell, sg).empty(); RefinementSettings untwinned_settings = settings, twin_settings = settings; if (halves) { twin_settings.nthreads = settings.nthreads / 2; From 931896963e9dc889686f5c67a764f0e5b8c9cb9f Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 19:54:46 +0100 Subject: [PATCH 11/12] Refinement minimiser: the trust floor for the U and the TLS parameters too A TLS trial's first macro-cycles were where a 1e-7 perturbation of the X-ray gradient still grew (cytc end to end: R-free 0.3410 against 0.3460); with the floor on the U and the TLS parameters as on the positions and B, 0.3411 against 0.3421. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 609a0fcd5..ae6ecbac0 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1263,12 +1263,27 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & } // Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their -// gradient (they are quadratic in U); preconditioned by `diagonal`. -void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, const std::vector &xray, - const std::vector &diagonal, int iterations) { +// gradient (they are quadratic in U); the diagonal raised to the trust floor as MinimiseAtoms() raises it; +// preconditioned by `diagonal`. +void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, std::vector xray, + std::vector diagonal, int iterations) { const size_t n = xray.size(); std::vector x_at, g_at(n), xp(n), gp(n); - auto at = [&](const std::vector &x, const std::vector &) { + bool first = true; + auto at = [&](const std::vector &x, const std::vector &q) { + if (first) { // the trust floor per atom, as MinimiseAtoms() takes it + first = false; + for (size_t a = 0; a < n / 6; a++) { + double g2 = 0; + for (int c = 0; c < 6; c++) + g2 += q[6 * a + c] * q[6 * a + c]; + const double floor = std::sqrt(g2) / target.MaxStep(); + for (int c = 0; c < 6; c++) { + diagonal[6 * a + c] += std::max(floor - xray[6 * a + c], 0.0); + xray[6 * a + c] = std::max(xray[6 * a + c], floor); + } + } + } if (x != x_at) { x_at = x; target.RestraintGradient(x.data(), g_at.data()); @@ -1296,7 +1311,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim // The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's // parameters (linear in them), C the X-ray curvature of its U per element (`xray_b`: of its B, a third of it to each); -// one dense block per body, preconditioned by its diagonal. +// one dense block per body, its diagonal raised to the trust floor, preconditioned by its diagonal. void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &minimiser, const std::vector &xray_b, int iterations) { constexpr int P = TlsBody::PARAMETERS; @@ -1324,6 +1339,19 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (int k = 0; k < P; k++) diagonal[b * P + k] = blocks[b][k * P + k]; diagonal = FlooredDiagonal(diagonal); + // the trust floor per parameter, as MinimiseAtoms() takes it per atom + bool first = true; + auto at = [&](const std::vector &, const std::vector &q) { + if (!first) + return; + first = false; + for (size_t b = 0; b < nb; b++) + for (int k = 0; k < P; k++) { + const double floor = std::fabs(q[b * P + k]) / target.MaxStep(); + diagonal[b * P + k] += std::max(floor - blocks[b][k * P + k], 0.0); + blocks[b][k * P + k] = std::max(blocks[b][k * P + k], floor); + } + }; auto product = [&](const std::vector &v, std::vector &hv) { hv.assign(v.size(), 0.0); for (size_t b = 0; b < nb; b++) @@ -1336,8 +1364,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (size_t k = 0; k < r.size(); k++) z[k] = r[k] / diagonal[k]; }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, - [](const std::vector &, const std::vector &) {}, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis From 13c819e43d421f3fbe983b50b3f97bd694dc2381 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 20:21:31 +0100 Subject: [PATCH 12/12] Docs: the trust floor for U and TLS Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 13879eb05..225816d63 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -314,7 +314,7 @@ leaves out. make a step of tens of ångströms there, and the step bound would then shrink every other atom's step with it. So each atom's diagonal is raised, once per macro-cycle, to at least $|g_i|/\Delta$ ($g_i$ its gradient, $\Delta$ the step bound): its step is held by its own curvature, a trust region per atom in the manner of - Levenberg and Marquardt's damping. Without it a rounding difference of $10^{-7}$ in the gradient moved a weight + Levenberg and Marquardt's damping. A U (per atom) and a TLS body's parameters (each) take the same floor. Without it a rounding difference of $10^{-7}$ in the gradient moved a weight trial's R-free by up to 0.004; with it the perturbation stays where it was. - **The matrix is solved, not inverted.** Each L-BFGS direction solves the normal equations by a fixed 20 conjugate-gradient iterations - not to a tolerance, whose iteration count, and with it the direction, a rounding