From 0f707caab951f08de144738798fe7de78a3d9c3f Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 9 Oct 2026 13:08:09 +0200 Subject: [PATCH] ModelValidation: the null's structure factors on the GPU too The null - nine random placements of the model, and the real model's side of the comparison - is scored to a coarser limit (3.5 A, or where the free set allows), and those structure factors still ran on the CPU, one model at a time on each replicate thread. Where the d_min structure factors are on the GPU, a second ModelStructureFactorsGPU engine at the null's limit is now made, on the same card, before any replicate starts; every replicate and the real side go through it, so all of them are computed the same way. It is decided like the first engine, from the card's total memory: the two together must stay within half of it, else the null stays on the CPU (logged). The replicates' threads may sit on other cards (modelpar's 8d6e9d3a4); each call works on the engine's own device. Where the null is scored to d_min it shares the d_min engine, as before. Measured (validation time, before -> after; CPU path for reference): P3_2 1.55 A: 11.2 -> 9.0 s (CPU 17.7) P2_1 oblique: 5.1 -> 4.5 s (CPU 7.9) C2 1.11 A: 12.9 -> 13.2 s (CPU 23.3) - its null time is in the scale fits, which stay on the CPU p.mtz md5-identical in all four variants per set (before, after, CPU, -N 8); maps and map MTZ bit-identical after vs -N 8; the null's sigma moves only in the second decimal (+113.39 -> +113.38, +86.13 -> +86.14). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi --- .../structure_refinement/ModelValidation.cpp | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index 95a4550a3..f557b05e3 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -357,16 +357,17 @@ double frame_probe_r(const gemmi::Structure &st, const gemmi::SpaceGroup *sg, // once, before any of them is computed, and never revisited: nothing is moved to the CPU part way through. // Decided on what the GPU path needs against the card's TOTAL memory and not against what happens to be // free at the moment, so the same input on the same machine always takes the same path, whatever else is -// running beside it. The structure factors may take half the card; the rigid body's engines take at most -// a quarter beside them (RigidBodyGPUPool::Create), and the last quarter is left to everything else. +// running beside it. The structure factors may take half the card - `reserved` is what an engine of the +// same validation already holds there; the rigid body's engines take at most a quarter beside them +// (RigidBodyGPUPool::Create), and the last quarter is left to everything else. std::unique_ptr StructureFactorsGPU(int device, const gemmi::Model &model, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, - double d_min, Logger &logger) { + double d_min, size_t reserved, Logger &logger) { auto sf = std::make_unique(device, model, cell, sg, d_min); const std::array n = sf->GridSize(); if (!sf->Supported()) { - logger.Info("Model validation: structure factors and maps on the CPU - the cell is too small for the " - "GPU's gridding"); + logger.Info("Model validation: structure factors to {:.2f} A on the CPU - the cell is too small for the " + "GPU's gridding", d_min); return nullptr; } size_t total = 0; @@ -375,10 +376,10 @@ std::unique_ptr StructureFactorsGPU(int device, const } catch (const JFJochException &e) { throw RigidBodyGPUFailure(e.what()); } - if (sf->DeviceBytes() > total / 2) { - logger.Info("Model validation: structure factors and maps on the CPU - on a {}x{}x{} grid they need " - "{:.2f} GB of GPU memory, over half of the card's {:.2f} GB", n[0], n[1], n[2], - sf->DeviceBytes() / 1e9, total / 1e9); + if (reserved + sf->DeviceBytes() > total / 2) { + logger.Info("Model validation: structure factors to {:.2f} A on the CPU - on a {}x{}x{} grid they need " + "{:.2f} GB of GPU memory beside {:.2f} GB already reserved, over half of the card's {:.2f} GB", + d_min, n[0], n[1], n[2], sf->DeviceBytes() / 1e9, reserved / 1e9, total / 1e9); return nullptr; } try { @@ -386,8 +387,8 @@ std::unique_ptr StructureFactorsGPU(int device, const } catch (const JFJochException &e) { throw RigidBodyGPUFailure(e.what()); } - logger.Info("Model validation: structure factors and maps on the GPU - a {}x{}x{} grid, {:.2f} GB of the " - "card's {:.2f} GB", n[0], n[1], n[2], sf->DeviceBytes() / 1e9, total / 1e9); + logger.Info("Model validation: structure factors to {:.2f} A on the GPU - a {}x{}x{} grid, {:.2f} GB of the " + "card's {:.2f} GB", d_min, n[0], n[1], n[2], sf->DeviceBytes() / 1e9, total / 1e9); return sf; } #endif @@ -586,15 +587,16 @@ ModelValidationResult Validate(const std::vector &merged, gemmi::count_atom_sites(st.models[0]), ucell.a, ucell.b, ucell.c, sg->hm, d_min); // The structure factors to d_min and the maps on the GPU, where `rigid_body_gpu` allows it and the - // card can take them; decided here, before any is computed. Anything to a coarser limit - the null's, - // the indexing probe's - stays on the CPU, where it is cheap, so that every replicate of the null and - // the real model's side of it are computed the same way. - ModelStructureFactorsGPU *sf_gpu = nullptr; + // card can take them; decided here, before any is computed. The null's, to its coarser limit, get an + // engine of their own further down, decided the same way before any replicate starts - for all of + // them and the real model's side of the null together, so that every one is computed the same way. + ModelStructureFactorsGPU *sf_gpu = nullptr, *sf_null = nullptr; #ifdef JFJOCH_USE_CUDA - std::unique_ptr sf_engine; + std::unique_ptr sf_engine, sf_null_engine; // On the current device, as the rigid body's engines take it: one card for the whole validation. if (rigid_body_gpu && get_gpu_count() > 0) - sf_engine = StructureFactorsGPU(RigidBodyGPUEngine::CurrentDevice(), st.models[0], ucell, *sg, d_min, logger); + sf_engine = StructureFactorsGPU(RigidBodyGPUEngine::CurrentDevice(), st.models[0], ucell, *sg, d_min, 0, + logger); sf_gpu = sf_engine.get(); #endif @@ -603,9 +605,12 @@ ModelValidationResult Validate(const std::vector &merged, // takes the state to work on so that a null replicate can run it on its own copy. --- auto compute_model_factors = [&](ModelState &ms, double to_d) { #ifdef JFJOCH_USE_CUDA - if (sf_gpu != nullptr && to_d == sf_gpu->DMin()) { + ModelStructureFactorsGPU *gpu = sf_gpu != nullptr && to_d == sf_gpu->DMin() ? sf_gpu + : sf_null != nullptr && to_d == sf_null->DMin() ? sf_null + : nullptr; + if (gpu != nullptr) { try { - sf_gpu->Compute(ms.st.models[0], ms.fcalc, ms.fmask); + gpu->Compute(ms.st.models[0], ms.fcalc, ms.fmask); } catch (const JFJochException &e) { throw RigidBodyGPUFailure(e.what()); } @@ -960,6 +965,14 @@ ModelValidationResult Validate(const std::vector &merged, for (const MergedReflection &r : obs) if (r.d >= null_d_min) obs_null.push_back(r); +#ifdef JFJOCH_USE_CUDA + // On the card the d_min engine is on; the replicates' threads may sit on other cards, and each + // call works on the engine's own device. Where the null is scored to d_min it shares that engine. + if (sf_gpu != nullptr && null_d_min != d_min) + sf_null_engine = StructureFactorsGPU(sf_gpu->Device(), st.models[0], ucell, *sg, null_d_min, + sf_gpu->DeviceBytes(), logger); + sf_null = sf_null_engine.get(); +#endif } const gemmi::Structure null_model = st; // the model as read, which the replicates are made from // Each replicate works on its own copy of the model as read and of its structure factors, and the