diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index e92060510..ac0ffc781 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include @@ -808,8 +810,10 @@ public: double xray = 0, restraints = 0; { const RefinementStage s1("eval.engine"); + ctx_.engine->SetXyzGradient(mode_ == Mode::Xyz); // B alone reads only the B derivatives xray = ctx_.engine->EvaluateWithRestraints(ctx_.Model(), restraint_set_, gradient != nullptr ? &gx : nullptr, gradient != nullptr ? &gr : nullptr, restraints); + ctx_.engine->SetXyzGradient(true); } *cost = weight_ * xray + restraints; if (gradient == nullptr) @@ -1905,6 +1909,11 @@ constexpr size_t TWIN_NULL_SHIFT = 2; // evidence (AdoptTwinLaw). constexpr double TWIN_NULL_Z = 3.0; constexpr double TWIN_NULL_Z_DATA = 2.0; +// A twinned refinement stops after its first ladder where the law is ahead of its null by less than this many +// standard errors there (RefineOnce, 1a.): 1.5 under the lowest bar a law is taken at (TWIN_NULL_Z_DATA). From the +// ladder's end to the refinement's, the law's lead rose by 1.35 at most (on 19 refinements with a law, 3 twinned: those +// 8.6-17.7 at the ladder's end, the others under 2.4). +constexpr double TWIN_STOP_Z = 0.5; // `first`: the first law whose mates are replaced - the laws before it keep theirs (the null of the domains added to // them). With several laws a twin group holds up to Laws() + 1 reflections, kept next to each other, and law first + k // is paired (Laws() + 1) (k + 1) apart: past the reflection's own group, and no two laws drawing the same partner. @@ -2249,18 +2258,51 @@ double XrayGradientNorm(Context &ctx, const std::vector &refinable) { } // The X-ray curvature per atom in each of x, y, z at weight w, for the row model `rows` the engine has; `scale` -// calibrated at the first call of the trial (negative: not yet). -std::vector XyzCurvature(Context &ctx, const std::vector &refinable, const XrayRowModel &rows, double w, - double &scale) { +// calibrated by `calibrate` (from the shape) at the first call of the trial (negative: not yet). +std::vector XyzCurvature(Context &ctx, const XrayRowModel &rows, double w, double &scale, + const std::function &)> &calibrate) { std::vector shape = ctx.XrayXyzCurvatureShape(rows); if (scale < 0) - scale = ctx.CalibrateXyzCurvature(refinable, shape); + scale = calibrate(shape); for (double &x : shape) x *= w * scale; return shape; } -TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const std::vector &pos0, +// The calibrations of the X-ray curvature in x, y, z (CalibrateXyzCurvature) of the trials RunTrials() runs together: +// trials that start from the same model with the same B and displacement model calibrate the same, so the first to +// need one makes it and the others wait for it. +class XyzCalibrations { +public: + // whether the B factors are refined (MoveOverallB), the displacement hypothesis, the kick, its seed and the first + // macro-cycle's blur + using Key = std::tuple; + double Get(const Key &key, const std::function &calibrate) { + std::unique_lock lock(m_); + if (auto it = made_.find(key); it != made_.end()) { + const std::shared_future f = it->second; + lock.unlock(); + return f.get(); + } + std::promise p; + made_.emplace(key, p.get_future().share()); + lock.unlock(); + try { + const double scale = calibrate(); + p.set_value(scale); + return scale; + } catch (...) { + p.set_exception(std::current_exception()); // to those waiting for it + throw; + } + } + +private: + std::mutex m_; + std::map> made_; +}; + +TrialOutcome RunTrial(TrialSlot slot, XyzCalibrations &calibrations, const std::vector &refinable, const std::vector &pos0, const std::vector &b0, const std::vector &aniso0, const Context::Fit *fit, const TrialWeights &tw, int cycles, const RefinementSettings &settings, const NcsCopies *ncs, const AdpHypothesis *hyp = nullptr, bool hold_positions = false) { @@ -2294,6 +2336,8 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s PlaceRiders(ctx, topo); ctx.tls = hyp->bodies; ctx.tls_b = hyp->tls_b; + // the bodies' U bounded by the start's floor, not by the one the slot's last trial left + ctx.b_floor = fit != nullptr ? fit->b_floor : B_FLOOR; ctx.ApplyTls(); } else if (kind == AdpHypothesis::Kind::Anisotropic) { if (ctx.aniso.size() != ctx.atoms.size()) @@ -2314,8 +2358,15 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s const AdpTarget tls_target(ctx, AdpTarget::Mode::Tls, refinable, 1.0, nullptr); const AdpTarget u_target(ctx, AdpTarget::Mode::U, u_atoms, adp_w, &u_restraints); RefinementMinimiser xyz_minimiser(20, ctx.nthreads), b_minimiser(20, ctx.nthreads), tls_minimiser(20, ctx.nthreads); - // the X-ray curvature's calibrations (CalibrateXyzCurvature, CalibrateBCurvature), once per trial + // the X-ray curvature's calibrations (CalibrateXyzCurvature, CalibrateBCurvature), once per trial - that in x, y, + // z shared with the trials that start alike double xyz_scale = -1, b_scale = -1; + auto calibrate_xyz = [&](const std::vector &shape) { + const XyzCalibrations::Key key{adp_w > 0, hyp, tw.kick, tw.seed, tw.blur.empty() ? 0.0 : tw.blur[0]}; + const double scale = calibrations.Get(key, [&]() { return ctx.CalibrateXyzCurvature(refinable, shape); }); + ctx.MoveRiders(); // the riders where the calibration leaves them, wherever it ran + return scale; + }; for (int cycle = 0; cycle < cycles && !ctx.Cancelled(); cycle++) { PlaceRiders(ctx, topo); if (cycle == 0 && fit != nullptr) @@ -2347,11 +2398,11 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s ctx.engine->SetRowModel(blurred); const double nh = XrayGradientNorm(ctx, refinable); const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); - MinimiseAtoms(ctx, xyz_h, xyz_minimiser, XyzCurvature(ctx, refinable, blurred, xyz_h.Weight(), xyz_scale), + MinimiseAtoms(ctx, xyz_h, xyz_minimiser, XyzCurvature(ctx, blurred, xyz_h.Weight(), xyz_scale, calibrate_xyz), set, settings.xyz_iterations); ctx.engine->SetRowModel(ctx.rows); } else - MinimiseAtoms(ctx, xyz, xyz_minimiser, XyzCurvature(ctx, refinable, ctx.rows, w, xyz_scale), set, + MinimiseAtoms(ctx, xyz, xyz_minimiser, XyzCurvature(ctx, ctx.rows, w, xyz_scale, calibrate_xyz), set, settings.xyz_iterations); } // the X-ray curvature in B for the atoms as they are, which TLS, U and B take a share of @@ -2462,9 +2513,11 @@ std::vector RunTrials(const std::vector &trial_slots, // every thread is dealt out, and a slot that has no trial left gives its threads to those still running. const size_t running = std::min(trial_slots.size(), trials.size()); const bool on_cpu = !trial_slots[0].ctx->engine->OnGPU(); + const size_t nthreads = + settings.nthreads + (settings.spare_arm_threads != nullptr ? settings.spare_arm_threads->load() : 0); std::atomic spare_threads{0}; for (size_t s = 0; s < running; s++) { - const size_t share = settings.nthreads / running + (on_cpu && s < settings.nthreads % running ? 1 : 0); + const size_t share = nthreads / running + (on_cpu && s < nthreads % running ? 1 : 0); trial_slots[s].ctx->SetThreads(std::max(1, share)); if (on_cpu) trial_slots[s].ctx->spare_threads = &spare_threads; @@ -2480,6 +2533,7 @@ std::vector RunTrials(const std::vector &trial_slots, std::iota(order.begin(), order.end(), 0); std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) { return length(a) > length(b); }); std::atomic next{trial_slots.size()}; + XyzCalibrations calibrations; auto run_slot = [&](size_t s) { try { #ifdef JFJOCH_USE_CUDA @@ -2487,8 +2541,8 @@ std::vector RunTrials(const std::vector &trial_slots, #endif for (size_t j = s; j < trials.size(); j = next++) if (const size_t i = order[j]; !trial_slots[s].ctx->Cancelled()) - out[i] = RunTrial(trial_slots[s], refinable, pos0, b0, aniso0, fit, trials[i], cycles, settings, ncs, - hyps.empty() ? nullptr : hyps[i], hold_positions); + out[i] = RunTrial(trial_slots[s], calibrations, refinable, pos0, b0, aniso0, fit, trials[i], cycles, + settings, ncs, hyps.empty() ? nullptr : hyps[i], hold_positions); } catch (...) { errors[s] = std::current_exception(); } @@ -3357,13 +3411,15 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector cards; for (const TrialSlot &s : trial_slots) cards.insert(s.ctx->device); const bool one_card = cards.size() == 1 && *cards.begin() >= 0; - auto ahead = [&](size_t trials) { return !one_card && trial_slots.size() >= trials; }; + const bool on_gpu = ctx.engine->OnGPU(); + auto ahead = [&](size_t trials) { return on_gpu && !one_card && trial_slots.size() >= trials; }; int ahead_evaluations = 0; auto drop_ahead = [&](TrialOutcome &o) { ahead_evaluations += o.evaluations; @@ -3473,6 +3529,22 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector= 0 ? std::move(outcomes[k]) : TrialOutcome(); }; TrialOutcome state = ladder_choice(pos0, b0, aniso0, nullptr); + // 1a. A twinned refinement (DecideTwin) stops where the ladder leaves its law far from being taken, hardly + // ahead of its null (TwinNull, TWIN_STOP_Z) - its threads go to the untwinned refinement beside it, and + // the law is decided where it stopped. + bool stopped = false; + if (settings.twin_arm_stops && state.ran && !ctx.Cancelled()) { + ctx.SetPositions(state.pos); + ctx.SetB(state.b); + ctx.SetAniso(state.aniso); + ctx.Restore(state.fit); + ctx.FModel(); + RefinementResult null; + TwinNull(ctx, null, null_first); + stopped = null.twin_null_z < TWIN_STOP_Z; + logger.Info("Refinement, twin law: after the ladder fraction {:.3f}, ahead of its null by {:.1f} standard " + "errors{}", FractionSum(ctx.rows), null.twin_null_z, stopped ? " - not taken further" : ""); + } // 2. The trajectory: steps of a few macro-cycles from where the last one ended, each trying the weights as they // are and, in turn, the positions' X-ray weight halved and doubled (halved only in the first step, from the @@ -3538,7 +3610,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector(std::max(settings.population, 0), trial_slots.size()); - if (population > 1 && state.ran && !ctx.Cancelled()) { + if (population > 1 && !stopped && state.ran && !ctx.Cancelled()) { const RefinementStage pop_stage("population"); const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma; // the members, the most different first: the positions as they are and kicked, on the data as they are @@ -3633,7 +3706,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector trials; for (double f : settings.final_ladder) @@ -4253,7 +4326,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector own_tls = state.tls; @@ -4427,7 +4500,8 @@ double FinalRWork(const RefinementResult &r) { // standard errors, each added domain at a fraction of TWIN_DOMAIN_MIN at least. void DecideTwin(const gemmi::Structure &placed, const std::vector &merged, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double wavelength_A, Logger &logger, - const RefinementSettings &settings, RefinementResult &result, const std::shared_future &untwinned) { + const RefinementSettings &settings, RefinementResult &result, const std::shared_future &untwinned, + const std::function &arm_done) { const RefinementStage stage("twin"); const std::vector laws = RefinementTwinLaws(cell, sg); const std::vector domains = RefinementTwinDomains(cell, sg); @@ -4489,7 +4563,10 @@ void DecideTwin(const gemmi::Structure &placed, const std::vector 1 && !RefinementTwinLaws(cell, sg).empty(); RefinementSettings untwinned_settings = settings, twin_settings = settings; + // and each arm, done, gives its threads to the other + std::atomic spare_for_untwinned{0}, spare_for_twin{0}; if (halves) { twin_settings.nthreads = settings.nthreads / 2; untwinned_settings.nthreads = settings.nthreads - twin_settings.nthreads; + untwinned_settings.spare_arm_threads = &spare_for_untwinned; + twin_settings.spare_arm_threads = &spare_for_twin; } + auto twin_done = [&]() { + if (halves) + spare_for_untwinned = twin_settings.nthreads; + }; Logger twin_logger = Logger::Buffered(); CardHeld held; std::future held_bytes = held.bytes.get_future(); @@ -4795,10 +4880,12 @@ RefinementResult RefineModel(const gemmi::Structure &placed, const std::vector *cancel = nullptr; + // Where two refinements run side by side on the CPU (RefineModel's arms): the threads the other one has finished + // with, which this one's weight trials take up from then on (the result does not depend on it). Null: none. + const std::atomic *spare_arm_threads = nullptr; + // A twinned refinement beside the untwinned one (DecideTwin): it stops after its first ladder where that leaves + // the law far from being taken. + bool twin_arm_stops = false; // The user's restraint dictionary, over the embedded one (BuildRefinementTopology) RestraintDictionary dictionary; }; diff --git a/image_analysis/structure_refinement/RefinementXray.cpp b/image_analysis/structure_refinement/RefinementXray.cpp index 956d7c9ae..8bf4daa4d 100644 --- a/image_analysis/structure_refinement/RefinementXray.cpp +++ b/image_analysis/structure_refinement/RefinementXray.cpp @@ -143,6 +143,7 @@ public: void SetRowModel(const XrayRowModel &rows) override { rows_ = rows; } bool OnGPU() const override { return false; } void SetThreads(size_t nthreads) override { nthreads_ = nthreads; } + void SetXyzGradient(bool xyz) override { xyz_gradient_ = xyz; } void Fcalc(const gemmi::Model &model, std::vector> &f) override { std::vector> fd; @@ -237,7 +238,7 @@ public: map_.data[i] *= norm; }); const bool aniso = AnyAniso(); - AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr, true); + AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr, true, xyz_gradient_); if (!aniso) grad_u_.clear(); return sum; @@ -265,6 +266,7 @@ private: double d_min_; const RefinementReflections &refl_; size_t nthreads_; + bool xyz_gradient_ = true; // SetXyzGradient() gemmi::Grid grid_; SymmetryComposition composition_; std::vector orbit_leaders_; @@ -1016,7 +1018,7 @@ bool AtomGradientsFromMapMetal(const gemmi::Grid &map, const gemmi::Densi void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads, std::vector> &grad, std::vector> *grad_u, - [[maybe_unused]] bool gpu) { + [[maybe_unused]] bool gpu, bool xyz) { const gemmi::DensityCalculator dc = Calculator(map.unit_cell, *map.spacegroup, 1.0, blur); std::vector atoms; for (const gemmi::Chain &ch : model.chains) @@ -1071,20 +1073,30 @@ void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &mod const float a = precal.a[q], bq = precal.b[q]; for (size_t k = 0; k < np; k++) p.exponent[k] = std::max(bq * p.r2[k], -88.f); - for (size_t k = 0; k < np; k++) { - const double e = a * gemmi::unsafe_expapprox(p.exponent[k]); - sum_dr2[k] += e * bq; - sum_db[k] += e * bq * (1.5 + bq * p.r2[k]); - } + if (xyz) + for (size_t k = 0; k < np; k++) { + const double e = a * gemmi::unsafe_expapprox(p.exponent[k]); + sum_dr2[k] += e * bq; + sum_db[k] += e * bq * (1.5 + bq * p.r2[k]); + } + else + for (size_t k = 0; k < np; k++) { + const double e = a * gemmi::unsafe_expapprox(p.exponent[k]); + sum_db[k] += e * bq * (1.5 + bq * p.r2[k]); + } } double gx = 0, gy = 0, gz = 0, gb = 0; - for (size_t k = 0; k < np; k++) { - const double v = p.value[k]; - gx += v * sum_dr2[k] * p.x[k]; - gy += v * sum_dr2[k] * p.y[k]; - gz += v * sum_dr2[k] * p.z[k]; - gb += v * sum_db[k]; - } + if (xyz) + for (size_t k = 0; k < np; k++) { + const double v = p.value[k]; + gx += v * sum_dr2[k] * p.x[k]; + gy += v * sum_dr2[k] * p.y[k]; + gz += v * sum_dr2[k] * p.z[k]; + gb += v * sum_db[k]; + } + else + for (size_t k = 0; k < np; k++) + gb += p.value[k] * sum_db[k]; // rho = occ sum_q a_q exp(b_q r^2): d rho / d(centre) = occ 2 (sum a_q b_q e_q) (centre - point), and // with b_q = -4 pi^2 / (B_q + B + blur), d rho / dB = occ sum a_q e_q b_q (1.5 + b_q r^2) / (4 pi^2). grad[i] = {2.0 * atom.occ * gx, 2.0 * atom.occ * gy, 2.0 * atom.occ * gz, atom.occ * gb / FOUR_PI2}; diff --git a/image_analysis/structure_refinement/RefinementXray.h b/image_analysis/structure_refinement/RefinementXray.h index 3294fcc19..991dc7c8f 100644 --- a/image_analysis/structure_refinement/RefinementXray.h +++ b/image_analysis/structure_refinement/RefinementXray.h @@ -208,6 +208,9 @@ public: virtual bool OnGPU() const = 0; // The threads the engine's evaluations are spread over from now on (the CPU's; a GPU engine keeps its own). virtual void SetThreads(size_t) {} + // false: the evaluations from now on may leave the x, y, z derivatives out (where only B is refined); the B + // derivatives are the same. The CPU engine's; a GPU engine computes them all. + virtual void SetXyzGradient(bool) {} // Which atoms are anisotropic and their shapes, in model order (AtomAniso), read at every Fcalc() and // Evaluate() until it is set again; null or empty: every atom isotropic (the model's own aniso is not read). @@ -290,7 +293,8 @@ std::unique_ptr MakeRefinementEngineCPU(const gemmi::UnitCell // PutModelDensityOnGrid() grids it: its B derivative is that along U + dB / (8 pi^2) I, and with `grad_u` (resized // to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23. With `gpu`, on the Apple GPU // where there is one (AtomGradientMetal.h, macOS): the same points summed in single precision, the CPU's gradient to -// a few parts in 1e5 rather than bit for bit. +// a few parts in 1e5 rather than bit for bit. `xyz` false: an isotropic atom's x, y, z derivatives left at zero, its B +// derivative the same. void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads, std::vector> &grad, - std::vector> *grad_u = nullptr, bool gpu = false); + std::vector> *grad_u = nullptr, bool gpu = false, bool xyz = true);