Model validation: warm up on the GPU when the broker gets a model

WarmUpModelValidation runs one throwaway validation of the uploaded model against its own
calculated intensities on a background thread (holding a GPUWorkBeside), so kernels, cuFFT
kernels and contexts on every card the null's replicates use are loaded before the collection's
validation. ValidateAgainstModel waits for a warm-up still running, so a real validation never
shares the cards with it and computes what it computes without one (test
ModelValidation_WarmUpChangesNothing; CLI run with and without warm-up: identical p.mtz,
p_maps.mtz, report and validation lines). The broker logs the warm-up's time at upload, which
measures the first-use cost on the machine directly (local RTX 5080: 0.88 s first, 0.78 s second).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

image_analysis/structure_refinement/ModelValidation.cpp

@@ -10,6 +10,8 @@
 #include <cstdio>
 #include <array>
 #include <atomic>
+#include <chrono>
+#include <mutex>
 #include <functional>
 #include <memory>
 #include <future>
@@ -1948,6 +1950,19 @@ std::vector<size_t> OtherCardsTotalMemory(int device) {
     return out;
 }
 
+namespace {
+
+// The warm-up last started (WarmUpModelValidation); invalid before the first.
+std::mutex warm_up_m;
+std::shared_future<void> warm_up;
+
+std::shared_future<void> LastWarmUp() {
+    std::lock_guard lock(warm_up_m);
+    return warm_up;
+}
+
+} // namespace
+
 ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &merged,
                                            const UnitCell &cell,
                                            const RugnuxModel &model,
@@ -1960,6 +1975,8 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
                                            const ModelValidationSchedule &schedule,
                                            const gemmi::Op *twin_law,
                                            const gemmi::Structure *placed) {
+    if (const auto w = LastWarmUp(); w.valid())
+        w.wait();
 #ifdef JFJOCH_USE_CUDA
     // A CUDA failure ends the validation, not the run, and nothing is moved to the CPU part way through:
     // the validation reports why it did not finish, and a validation that did not finish decides nothing,
@@ -1983,6 +2000,47 @@ ModelValidationResult ValidateAgainstModel(const std::vector<MergedReflection> &
 #endif
 }
 
+void WarmUpModelValidation(std::shared_ptr<const RugnuxModel> model, size_t nthreads, Logger logger) {
+#ifdef JFJOCH_USE_CUDA
+    if (!model || get_gpu_count() == 0)
+        return;
+    std::lock_guard lock(warm_up_m);
+    std::shared_future<void> previous = warm_up;
+    warm_up = std::async(std::launch::async, [model, nthreads, logger, previous]() mutable {
+        if (previous.valid())
+            previous.wait();
+        const auto start = std::chrono::steady_clock::now();
+        // Held for as long as it runs: work beside a run waits for it where the card is too small for both.
+        GPUWorkBeside beside;
+        Logger quiet = Logger::Buffered();
+        const double d_min = model->structure.resolution > 0.0 ? model->structure.resolution : 2.0;
+        std::vector<MergedReflection> fake = ModelReferenceIntensities(*model, std::nullopt, nullptr, d_min, quiet);
+        for (size_t i = 0; i < fake.size(); i++) {
+            fake[i].F = std::sqrt(fake[i].I);
+            fake[i].rfree_flag = i % 20 == 0;
+        }
+        const gemmi::UnitCell &c = model->structure.cell;
+        const UnitCell cell{static_cast<float>(c.a), static_cast<float>(c.b), static_cast<float>(c.c),
+                            static_cast<float>(c.alpha), static_cast<float>(c.beta), static_cast<float>(c.gamma)};
+        std::string failure;
+        try {
+            failure = Validate(fake, cell, *model, quiet, nullptr, true, nthreads, 0.0, {}, {}, nullptr, true,
+                               nullptr).failure_reason;
+        } catch (const std::exception &e) {
+            cuda_clear_error();
+            failure = e.what();
+        }
+        logger.Info("Model validation warm-up on the GPU: {:.2f} s{}",
+                    std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count(),
+                    failure.empty() ? "" : " (did not finish: " + failure + ")");
+    }).share();
+#else
+    (void) model;
+    (void) nthreads;
+    (void) logger;
+#endif
+}
+
 namespace {
 
 // The reindexing operator as it reads on Miller indices ("k,h,-l" rather than "y,x,-z").