diff --git a/broker/JFJochServices.cpp b/broker/JFJochServices.cpp index 31dfaee5f..b38d4e5dc 100644 --- a/broker/JFJochServices.cpp +++ b/broker/JFJochServices.cpp @@ -4,9 +4,12 @@ #include "JFJochServices.h" #include "../common/JFJochException.h" +#include "../image_analysis/structure_refinement/ModelValidation.h" #include "../detector_control/SLSDetectorWrapper.h" #include "../detector_control/DectrisDetectorWrapper.h" +#include + JFJochServices::JFJochServices(Logger &in_logger) : logger(in_logger) {} void JFJochServices::Preflight(const DiffractionExperiment& experiment) { @@ -392,6 +395,8 @@ void JFJochServices::SetRugnuxModel(std::shared_ptr model, in s.model = std::move(model); s.model_size_bytes = s.model ? size_bytes : 0; SetRugnuxLive(s); + // Before the collection it is for starts, so the run's validation is not the process's first. + WarmUpModelValidation(s.model, std::thread::hardware_concurrency(), logger); } LiveCollection *JFJochServices::GetLiveCollection() { diff --git a/image_analysis/structure_refinement/ModelValidation.cpp b/image_analysis/structure_refinement/ModelValidation.cpp index 25c96b7fb..0ec007e9d 100644 --- a/image_analysis/structure_refinement/ModelValidation.cpp +++ b/image_analysis/structure_refinement/ModelValidation.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -1948,6 +1950,19 @@ std::vector OtherCardsTotalMemory(int device) { return out; } +namespace { + +// The warm-up last started (WarmUpModelValidation); invalid before the first. +std::mutex warm_up_m; +std::shared_future warm_up; + +std::shared_future LastWarmUp() { + std::lock_guard lock(warm_up_m); + return warm_up; +} + +} // namespace + ModelValidationResult ValidateAgainstModel(const std::vector &merged, const UnitCell &cell, const RugnuxModel &model, @@ -1960,6 +1975,8 @@ ModelValidationResult ValidateAgainstModel(const std::vector & const ModelValidationSchedule &schedule, const gemmi::Op *twin_law, const gemmi::Structure *placed) { + if (const auto w = LastWarmUp(); w.valid()) + w.wait(); #ifdef JFJOCH_USE_CUDA // A CUDA failure ends the validation, not the run, and nothing is moved to the CPU part way through: // the validation reports why it did not finish, and a validation that did not finish decides nothing, @@ -1983,6 +2000,47 @@ ModelValidationResult ValidateAgainstModel(const std::vector & #endif } +void WarmUpModelValidation(std::shared_ptr model, size_t nthreads, Logger logger) { +#ifdef JFJOCH_USE_CUDA + if (!model || get_gpu_count() == 0) + return; + std::lock_guard lock(warm_up_m); + std::shared_future previous = warm_up; + warm_up = std::async(std::launch::async, [model, nthreads, logger, previous]() mutable { + if (previous.valid()) + previous.wait(); + const auto start = std::chrono::steady_clock::now(); + // Held for as long as it runs: work beside a run waits for it where the card is too small for both. + GPUWorkBeside beside; + Logger quiet = Logger::Buffered(); + const double d_min = model->structure.resolution > 0.0 ? model->structure.resolution : 2.0; + std::vector fake = ModelReferenceIntensities(*model, std::nullopt, nullptr, d_min, quiet); + for (size_t i = 0; i < fake.size(); i++) { + fake[i].F = std::sqrt(fake[i].I); + fake[i].rfree_flag = i % 20 == 0; + } + const gemmi::UnitCell &c = model->structure.cell; + const UnitCell cell{static_cast(c.a), static_cast(c.b), static_cast(c.c), + static_cast(c.alpha), static_cast(c.beta), static_cast(c.gamma)}; + std::string failure; + try { + failure = Validate(fake, cell, *model, quiet, nullptr, true, nthreads, 0.0, {}, {}, nullptr, true, + nullptr).failure_reason; + } catch (const std::exception &e) { + cuda_clear_error(); + failure = e.what(); + } + logger.Info("Model validation warm-up on the GPU: {:.2f} s{}", + std::chrono::duration(std::chrono::steady_clock::now() - start).count(), + failure.empty() ? "" : " (did not finish: " + failure + ")"); + }).share(); +#else + (void) model; + (void) nthreads; + (void) logger; +#endif +} + namespace { // The reindexing operator as it reads on Miller indices ("k,h,-l" rather than "y,x,-z"). diff --git a/image_analysis/structure_refinement/ModelValidation.h b/image_analysis/structure_refinement/ModelValidation.h index d059d31d5..518114823 100644 --- a/image_analysis/structure_refinement/ModelValidation.h +++ b/image_analysis/structure_refinement/ModelValidation.h @@ -413,3 +413,12 @@ std::vector ModelReferenceIntensities(const RugnuxModel &model const gemmi::SpaceGroup *space_group, double d_min, Logger &logger); + +// Run one validation of `model` against its own calculated intensities (to the model's stated +// resolution, 2 A where it states none) on a background thread, and throw it away. A process's first +// validation pays for what the GPU loads on first use - kernels, cuFFT plans and their sizes, on every +// card the null's replicates are spread over - and this pays it ahead, where nobody waits: the broker +// calls it when a model is uploaded, before the collection starts. ValidateAgainstModel waits for a +// warm-up still running, so a real validation never shares the cards with one and computes exactly +// what it would have without it. Its time goes to `logger` when it ends. No-op without a GPU. +void WarmUpModelValidation(std::shared_ptr model, size_t nthreads, Logger logger); diff --git a/tests/ModelValidationTest.cpp b/tests/ModelValidationTest.cpp index 1697896d9..78ed55c64 100644 --- a/tests/ModelValidationTest.cpp +++ b/tests/ModelValidationTest.cpp @@ -864,6 +864,41 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]") std::filesystem::remove(path); } +// A warm-up (WarmUpModelValidation) only loads what the GPU loads on first use: the validation after it, +// which waits for it, gives what it gives without one. +TEST_CASE("ModelValidation_WarmUpChangesNothing", "[ModelValidation]") { + Logger logger("ModelValidation_WarmUpChangesNothing"); + + const auto path = WriteTemp("warm_up_test.pdb", ClusterPdb().c_str()); + auto obs = ModelReferenceIntensities(ReadModel(path), {}, {}, 2.5, logger); + REQUIRE(obs.size() > 1000); + std::mt19937 rng(20261019); + std::normal_distribution normal(0.0, 1.0); + for (size_t i = 0; i < obs.size(); i++) { + obs[i].I = static_cast(obs[i].I * (1.0 + 0.1 * normal(rng))); + obs[i].F = std::sqrt(std::max(0.0f, obs[i].I)); + obs[i].rfree_flag = (i % 20) == 0; + } + const UnitCell cell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90}; + const auto model = std::make_shared(ReadModel(path)); + const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 21 21 21"); + + const auto cold = ValidateAgainstModel(obs, cell, *model, logger, sg, false, 4); + WarmUpModelValidation(model, 4, logger); + const auto warm = ValidateAgainstModel(obs, cell, *model, logger, sg, false, 4); + + REQUIRE(cold.ok); + REQUIRE(warm.ok); + CHECK(warm.r_work == cold.r_work); + CHECK(warm.r_free == cold.r_free); + CHECK(warm.rigid_body_angle_deg == cold.rigid_body_angle_deg); + CHECK(warm.null_r_work_mean == cold.null_r_work_mean); + CHECK(warm.null_r_work_sd == cold.null_r_work_sd); + CHECK(warm.r_work_sigma == cold.r_work_sigma); + + std::filesystem::remove(path); +} + // The model path's structure factors come from FFTW; they must be gemmi's own transform to float // precision, in the same layout, so prepare_asu_data() reads the same reflections from either.