From b0b217a795ded12d972e93a1788c550c81b1a854 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 21:21:42 +0200 Subject: [PATCH] Make the P1 cross-check merge while the model validation runs With --model, model validation and the P1 cross-check merge ran one after the other at the end of the run. Neither reads anything the other writes, so the first ValidateAgainstModel now runs on its own thread while the main thread makes the P1 merge (rsm->Run in P1); the merge's result is kept and reported at the cross-check, where it used to be made, through a new `already_run` argument of scale_and_merge. Why it is exact: the validation reads the adopted merge, the cell, the group and the wavelength, all captured before the thread starts, and nothing else. The scaling engine merges its own copy of the observations under the group set on the experiment (P1 for the merge, restored after, as at the cross-check), holds its own copy of the cell, reads the outcomes' lattices and mosaicities only at Ingest, and writes back per-image G, CC and mosaicity - which nothing between the validation and the cross-check reads (the unmerged MTZ build reads the mosaicity but replaces it after the merge, as before). Every relabelling the validation decides reaches the P1 merge afterwards through merge_to_written, exactly as when it was made later. The result fields scale_and_merge sets are still set at the cross-check, so the report and the finalist ledger read the same values. Without --model, or without a P1 cross-check, nothing changes. The validation logs into a Logger::Buffered() and the lines are replayed as one block when it is done, so the two do not interleave; the P1 merge's log lines now come before the validation's. Checked against rc173 (rugnux_r4int_77710e1ce, output-identical to it) on 8tyy, 9hnc, 8sa8, 6oel, 7ph1 with --model and on myob: p.hkl, p.mtz, p.cif, p_P1.mtz, p_unmerged.mtz, the maps and the placed model are byte-identical, and so is p_report.txt apart from date/command/timing lines. A last-digit flip of one image's bkg in p_plot.txt (8sa8, one of two runs) is image-loop nondeterminism the baseline shows too (8tyy, base vs base). Tail wall (sweep-quality line to unmerged MTZ written), paired back-to-back runs on a loaded box: 8tyy 110.3 -> 87.3 s, 6oel 77.8 -> 63.3 s, 8sa8 171.4 -> 154.6 s; unpaired 9hnc 368.8 -> 337.0 s, 7ph1 21.1 -> 15.7 s. The saving is the P1 merge's time; the validation still sets the floor. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- common/Logger.cpp | 26 ++++++++++++++++++++ common/Logger.h | 10 ++++++++ rugnux/Rugnux.cpp | 61 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/common/Logger.cpp b/common/Logger.cpp index 00d2f3041..df469dfc4 100644 --- a/common/Logger.cpp +++ b/common/Logger.cpp @@ -3,6 +3,7 @@ #include "spdlog/sinks/daily_file_sink.h" #include "spdlog/sinks/stdout_color_sinks.h" +#include "spdlog/sinks/base_sink.h" #include "spdlog/fmt/bin_to_hex.h" #include "Logger.h" @@ -33,6 +34,31 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) { variant); } +class LogBufferSink : public spdlog::sinks::base_sink { +public: + std::vector> lines; +protected: + void sink_it_(const spdlog::details::log_msg &msg) override { + lines.emplace_back(msg.level, std::string(msg.payload.data(), msg.payload.size())); + } + void flush_() override {} +}; + +Logger::Logger(std::shared_ptr sink) : buffer(std::move(sink)) { + spdlog_logger = std::make_shared("buffer", buffer); + spdlog_logger->set_level(spdlog::level::trace); // the logger it is replayed into filters +} + +Logger Logger::Buffered() { + return Logger(std::make_shared()); +} + +// Called once the work that logged is finished, so nothing is still writing to the buffer. +void Logger::ReplayInto(Logger &to) const { + for (const auto &[level, line] : buffer->lines) + to.spdlog_logger->log(level, line); +} + void Logger::ErrorException(const std::exception &e) { spdlog_logger->error(e.what()); } diff --git a/common/Logger.h b/common/Logger.h index 61aeae50a..b2f8b39bc 100644 --- a/common/Logger.h +++ b/common/Logger.h @@ -16,8 +16,12 @@ namespace spdlog { class logger; } +class LogBufferSink; + class Logger { std::shared_ptr spdlog_logger; + std::shared_ptr buffer; // only on a Buffered() logger + explicit Logger(std::shared_ptr sink); std::string service; std::string hostname; @@ -51,4 +55,10 @@ public: } Logger& Verbose(bool input); + + // A logger that keeps its lines instead of printing them, for work run beside other work that + // logs: ReplayInto hands them to `to` afterwards, in order and as one block, so the two do not + // interleave. + static Logger Buffered(); + void ReplayInto(Logger &to) const; }; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index c8119f214..26323110a 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -6704,15 +6704,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a // merge of its own - only the first merge, which the two-pass quality guard reads. + // already_run: the engine's result for this merge, when it was made earlier (see the P1 + // cross-check); it is then only reported, exactly as if it had been made here. auto scale_and_merge = [&](const std::string &label, bool for_search, - bool measure_cc = false) -> ScaleMergeResult { + bool measure_cc = false, + std::optional already_run = std::nullopt) + -> ScaleMergeResult { if (rsm) { phase("Scale/combine/merge (" + label + ")"); // The geometry pre-pass merges only to choose the space group and to give the quality // guard something to judge the second pass against; its reflections are never written, // so the parts of the merge that only fill in an output file are skipped there. - auto r = rsm->Run(for_search, /*full_stats=*/!geometry_prepass, - /*measure_cc_before_corrections=*/measure_cc); + auto r = already_run ? std::move(*already_run) + : rsm->Run(for_search, /*full_stats=*/!geometry_prepass, + /*measure_cc_before_corrections=*/measure_cc); result.error_model_isa = r.isa; result.error_model_isa_asymptotic = r.isa_asymptotic; result.error_model_isa_resolved = r.isa_resolved; @@ -8728,6 +8733,14 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b } } + // Whether the P1 cross-check dataset is made (see where it is written, below). + const bool p1_integration_complete = + search_space_group || (indexer && indexer->GetPredictionCentring() == 'P'); + const bool p1_crosscheck = result.consensus_cell && write_files && config_.write_merged + && !geometry_prepass && !superseded && config_.write_p1_crosscheck + && p1_integration_complete && is_rotation; + std::optional p1_merged_early; + // Model validation runs BEFORE the reflection files are written, because it is what settles the // frame they are written in: the enantiomorph, which merged intensities cannot choose, and - // with no reference MTZ - the alternative indexing. Both are relabelings of the same @@ -8743,13 +8756,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b std::vector report_shell_d_min; for (const auto &sh : sm.statistics.shells) report_shell_d_min.push_back(sh.d_min); - auto validation = - ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, - config_.output_prefix, logger, - data_sg ? &*data_sg : nullptr, - /*probe_indexing_ambiguity=*/config_.reference_data.empty(), - static_cast(config_.nthreads), - experiment_.GetWavelength_A(), report_shell_d_min); + const auto validate = [&, cell = *result.consensus_cell, + wavelength = experiment_.GetWavelength_A()](Logger &log) { + return ValidateAgainstModel(sm.merged, cell, config_.model_path, + config_.output_prefix, log, + data_sg ? &*data_sg : nullptr, + /*probe_indexing_ambiguity=*/config_.reference_data.empty(), + static_cast(config_.nthreads), + wavelength, report_shell_d_min); + }; + // The P1 cross-check merge and this validation read nothing the other writes, so they + // are made at the same time. The validation reads the merge above, the cell and the group, + // none of which the scaling engine touches; the engine merges its own copy of the + // observations, in P1, and writes back only per-image scale, CC and mosaicity, which + // nothing before the cross-check reads (the unmerged MTZ replaces the mosaicity it + // carries after the merge anyway). Every relabelling the validation decides reaches the + // P1 merge afterwards, through merge_to_written, as it did when it was made below. The + // validation's log lines are held and printed as one block once it is done. + ModelValidationResult validation; + if (p1_crosscheck && rsm) { + Logger held = Logger::Buffered(); + auto pending = std::async(std::launch::async, validate, std::ref(held)); + experiment_.SpaceGroupNumber(1); + p1_merged_early = rsm->Run(/*for_search=*/false, /*full_stats=*/true, + /*measure_cc_before_corrections=*/false); + experiment_.SetSpaceGroup(data_sg); + validation = pending.get(); + held.ReplayInto(logger); + } else + validation = validate(logger); // A model that was asked for and could not be used has to say so where anyone will see // it. Without this the run ends successfully with no R-free, no maps and nothing in the // report - indistinguishable from a run that was never given --model at all. @@ -8883,8 +8918,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // output (measured: same length, different bytes). The fix is to run this below the // unmerged export rather than here, which needs the merge lambda hoisted out of this // block; deferred, as rotation is what the online pipeline processes. - const bool p1_integration_complete = - search_space_group || (indexer && indexer->GetPredictionCentring() == 'P'); if (config_.write_p1_crosscheck && is_rotation && !p1_integration_complete) logger.Warning("No P1 cross-check dataset written: the fixed space group's centring " "absences were never predicted, so a P1 merge of this run would be " @@ -8902,7 +8935,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b : "a fixed centred space group's absences were never predicted, so a P1 " "merge would be missing whole centring classes"); - if (config_.write_p1_crosscheck && p1_integration_complete && is_rotation) { + if (p1_crosscheck) { // The whole GROUP, not its number: the search can adopt a non-reference setting, and // a number only ever names the reference one - so restoring through the number would // silently re-seat P 1 1 2(1) as P 1 2(1) 1 for everything written after this block, @@ -8938,7 +8971,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // Both the merge and the MTZ read the group from the experiment, so it is set for // the whole of it and restored after. experiment_.SpaceGroupNumber(1); - auto p1 = scale_and_merge("P1 cross-check", false); + auto p1 = scale_and_merge("P1 cross-check", false, false, std::move(p1_merged_early)); // The scaler still holds the observations in the indexing they were merged in, so every // relabelling since (the written setting, the model's indexing) is applied to this merge // too - or it would describe the dataset on other axes than the merged output beside it,