diff --git a/common/Logger.cpp b/common/Logger.cpp index 00d2f3041..df469dfc4 100644 --- a/common/Logger.cpp +++ b/common/Logger.cpp @@ -3,6 +3,7 @@ #include "spdlog/sinks/daily_file_sink.h" #include "spdlog/sinks/stdout_color_sinks.h" +#include "spdlog/sinks/base_sink.h" #include "spdlog/fmt/bin_to_hex.h" #include "Logger.h" @@ -33,6 +34,31 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) { variant); } +class LogBufferSink : public spdlog::sinks::base_sink { +public: + std::vector> lines; +protected: + void sink_it_(const spdlog::details::log_msg &msg) override { + lines.emplace_back(msg.level, std::string(msg.payload.data(), msg.payload.size())); + } + void flush_() override {} +}; + +Logger::Logger(std::shared_ptr sink) : buffer(std::move(sink)) { + spdlog_logger = std::make_shared("buffer", buffer); + spdlog_logger->set_level(spdlog::level::trace); // the logger it is replayed into filters +} + +Logger Logger::Buffered() { + return Logger(std::make_shared()); +} + +// Called once the work that logged is finished, so nothing is still writing to the buffer. +void Logger::ReplayInto(Logger &to) const { + for (const auto &[level, line] : buffer->lines) + to.spdlog_logger->log(level, line); +} + void Logger::ErrorException(const std::exception &e) { spdlog_logger->error(e.what()); } diff --git a/common/Logger.h b/common/Logger.h index 61aeae50a..b2f8b39bc 100644 --- a/common/Logger.h +++ b/common/Logger.h @@ -16,8 +16,12 @@ namespace spdlog { class logger; } +class LogBufferSink; + class Logger { std::shared_ptr spdlog_logger; + std::shared_ptr buffer; // only on a Buffered() logger + explicit Logger(std::shared_ptr sink); std::string service; std::string hostname; @@ -51,4 +55,10 @@ public: } Logger& Verbose(bool input); + + // A logger that keeps its lines instead of printing them, for work run beside other work that + // logs: ReplayInto hands them to `to` afterwards, in order and as one block, so the two do not + // interleave. + static Logger Buffered(); + void ReplayInto(Logger &to) const; }; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 6fcb5a959..519189f65 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -6727,15 +6727,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a // merge of its own - only the first merge, which the two-pass quality guard reads. + // already_run: the engine's result for this merge, when it was made earlier (see the P1 + // cross-check); it is then only reported, exactly as if it had been made here. auto scale_and_merge = [&](const std::string &label, bool for_search, - bool measure_cc = false) -> ScaleMergeResult { + bool measure_cc = false, + std::optional already_run = std::nullopt) + -> ScaleMergeResult { if (rsm) { phase("Scale/combine/merge (" + label + ")"); // The geometry pre-pass merges only to choose the space group and to give the quality // guard something to judge the second pass against; its reflections are never written, // so the parts of the merge that only fill in an output file are skipped there. - auto r = rsm->Run(for_search, /*full_stats=*/!geometry_prepass, - /*measure_cc_before_corrections=*/measure_cc); + auto r = already_run ? std::move(*already_run) + : rsm->Run(for_search, /*full_stats=*/!geometry_prepass, + /*measure_cc_before_corrections=*/measure_cc); result.error_model_isa = r.isa; result.error_model_isa_asymptotic = r.isa_asymptotic; result.error_model_isa_resolved = r.isa_resolved; @@ -8751,6 +8756,14 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b } } + // Whether the P1 cross-check dataset is made (see where it is written, below). + const bool p1_integration_complete = + search_space_group || (indexer && indexer->GetPredictionCentring() == 'P'); + const bool p1_crosscheck = result.consensus_cell && write_files && config_.write_merged + && !geometry_prepass && !superseded && config_.write_p1_crosscheck + && p1_integration_complete && is_rotation; + std::optional p1_merged_early; + // Model validation runs BEFORE the reflection files are written, because it is what settles the // frame they are written in: the enantiomorph, which merged intensities cannot choose, and - // with no reference MTZ - the alternative indexing. Both are relabelings of the same @@ -8766,13 +8779,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b std::vector report_shell_d_min; for (const auto &sh : sm.statistics.shells) report_shell_d_min.push_back(sh.d_min); - auto validation = - ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path, - config_.output_prefix, logger, - data_sg ? &*data_sg : nullptr, - /*probe_indexing_ambiguity=*/config_.reference_data.empty(), - static_cast(config_.nthreads), - experiment_.GetWavelength_A(), report_shell_d_min); + const auto validate = [&, cell = *result.consensus_cell, + wavelength = experiment_.GetWavelength_A()](Logger &log) { + return ValidateAgainstModel(sm.merged, cell, config_.model_path, + config_.output_prefix, log, + data_sg ? &*data_sg : nullptr, + /*probe_indexing_ambiguity=*/config_.reference_data.empty(), + static_cast(config_.nthreads), + wavelength, report_shell_d_min); + }; + // The P1 cross-check merge and this validation read nothing the other writes, so they + // are made at the same time. The validation reads the merge above, the cell and the group, + // none of which the scaling engine touches; the engine merges its own copy of the + // observations, in P1, and writes back only per-image scale, CC and mosaicity, which + // nothing before the cross-check reads (the unmerged MTZ replaces the mosaicity it + // carries after the merge anyway). Every relabelling the validation decides reaches the + // P1 merge afterwards, through merge_to_written, as it did when it was made below. The + // validation's log lines are held and printed as one block once it is done. + ModelValidationResult validation; + if (p1_crosscheck && rsm) { + Logger held = Logger::Buffered(); + auto pending = std::async(std::launch::async, validate, std::ref(held)); + experiment_.SpaceGroupNumber(1); + p1_merged_early = rsm->Run(/*for_search=*/false, /*full_stats=*/true, + /*measure_cc_before_corrections=*/false); + experiment_.SetSpaceGroup(data_sg); + validation = pending.get(); + held.ReplayInto(logger); + } else + validation = validate(logger); // A model that was asked for and could not be used has to say so where anyone will see // it. Without this the run ends successfully with no R-free, no maps and nothing in the // report - indistinguishable from a run that was never given --model at all. @@ -8906,8 +8941,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // output (measured: same length, different bytes). The fix is to run this below the // unmerged export rather than here, which needs the merge lambda hoisted out of this // block; deferred, as rotation is what the online pipeline processes. - const bool p1_integration_complete = - search_space_group || (indexer && indexer->GetPredictionCentring() == 'P'); if (config_.write_p1_crosscheck && is_rotation && !p1_integration_complete) logger.Warning("No P1 cross-check dataset written: the fixed space group's centring " "absences were never predicted, so a P1 merge of this run would be " @@ -8925,7 +8958,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b : "a fixed centred space group's absences were never predicted, so a P1 " "merge would be missing whole centring classes"); - if (config_.write_p1_crosscheck && p1_integration_complete && is_rotation) { + if (p1_crosscheck) { // The whole GROUP, not its number: the search can adopt a non-reference setting, and // a number only ever names the reference one - so restoring through the number would // silently re-seat P 1 1 2(1) as P 1 2(1) 1 for everything written after this block, @@ -8961,7 +8994,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // Both the merge and the MTZ read the group from the experiment, so it is set for // the whole of it and restored after. experiment_.SpaceGroupNumber(1); - auto p1 = scale_and_merge("P1 cross-check", false); + auto p1 = scale_and_merge("P1 cross-check", false, false, std::move(p1_merged_early)); // The scaler still holds the observations in the indexing they were merged in, so every // relabelling since (the written setting, the model's indexing) is applied to this merge // too - or it would describe the dataset on other axes than the merged output beside it,