Merge branch 'tail-p1-model' into rc173 (P1 cross-check merge beside model validation)

This commit is contained in:
leonarski_f committed 2026-09-27 21:22:12 +02:00
commit 5ed07eeb11
3 files changed
+83 -14

No files matched your search

+26
View File
@@ -3,6 +3,7 @@
#include "spdlog/sinks/daily_file_sink.h"
#include "spdlog/sinks/stdout_color_sinks.h"
#include "spdlog/sinks/base_sink.h"
#include "spdlog/fmt/bin_to_hex.h"
#include "Logger.h"
@@ -33,6 +34,31 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) {
variant);
}
class LogBufferSink : public spdlog::sinks::base_sink<std::mutex> {
public:
std::vector<std::pair<spdlog::level::level_enum, std::string>> lines;
protected:
void sink_it_(const spdlog::details::log_msg &msg) override {
lines.emplace_back(msg.level, std::string(msg.payload.data(), msg.payload.size()));
}
void flush_() override {}
};
Logger::Logger(std::shared_ptr<LogBufferSink> sink) : buffer(std::move(sink)) {
spdlog_logger = std::make_shared<spdlog::logger>("buffer", buffer);
spdlog_logger->set_level(spdlog::level::trace); // the logger it is replayed into filters
}
Logger Logger::Buffered() {
return Logger(std::make_shared<LogBufferSink>());
}
// Called once the work that logged is finished, so nothing is still writing to the buffer.
void Logger::ReplayInto(Logger &to) const {
for (const auto &[level, line] : buffer->lines)
to.spdlog_logger->log(level, line);
}
void Logger::ErrorException(const std::exception &e) {
spdlog_logger->error(e.what());
}
+10
View File
@@ -16,8 +16,12 @@ namespace spdlog {
class logger;
}
class LogBufferSink;
class Logger {
std::shared_ptr<spdlog::logger> spdlog_logger;
std::shared_ptr<LogBufferSink> buffer; // only on a Buffered() logger
explicit Logger(std::shared_ptr<LogBufferSink> sink);
std::string service;
std::string hostname;
@@ -51,4 +55,10 @@ public:
}
Logger& Verbose(bool input);
// A logger that keeps its lines instead of printing them, for work run beside other work that
// logs: ReplayInto hands them to `to` afterwards, in order and as one block, so the two do not
// interleave.
static Logger Buffered();
void ReplayInto(Logger &to) const;
};
+47 -14
View File
@@ -6727,15 +6727,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a
// merge of its own - only the first merge, which the two-pass quality guard reads.
// already_run: the engine's result for this merge, when it was made earlier (see the P1
// cross-check); it is then only reported, exactly as if it had been made here.
auto scale_and_merge = [&](const std::string &label, bool for_search,
bool measure_cc = false) -> ScaleMergeResult {
bool measure_cc = false,
std::optional<RotationScaleMerge::Result> already_run = std::nullopt)
-> ScaleMergeResult {
if (rsm) {
phase("Scale/combine/merge (" + label + ")");
// The geometry pre-pass merges only to choose the space group and to give the quality
// guard something to judge the second pass against; its reflections are never written,
// so the parts of the merge that only fill in an output file are skipped there.
auto r = rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
/*measure_cc_before_corrections=*/measure_cc);
auto r = already_run ? std::move(*already_run)
: rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
/*measure_cc_before_corrections=*/measure_cc);
result.error_model_isa = r.isa;
result.error_model_isa_asymptotic = r.isa_asymptotic;
result.error_model_isa_resolved = r.isa_resolved;
@@ -8751,6 +8756,14 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
}
}
// Whether the P1 cross-check dataset is made (see where it is written, below).
const bool p1_integration_complete =
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
const bool p1_crosscheck = result.consensus_cell && write_files && config_.write_merged
&& !geometry_prepass && !superseded && config_.write_p1_crosscheck
&& p1_integration_complete && is_rotation;
std::optional<RotationScaleMerge::Result> p1_merged_early;
// Model validation runs BEFORE the reflection files are written, because it is what settles the
// frame they are written in: the enantiomorph, which merged intensities cannot choose, and -
// with no reference MTZ - the alternative indexing. Both are relabelings of the same
@@ -8766,13 +8779,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
std::vector<float> report_shell_d_min;
for (const auto &sh : sm.statistics.shells)
report_shell_d_min.push_back(sh.d_min);
auto validation =
ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
config_.output_prefix, logger,
data_sg ? &*data_sg : nullptr,
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
static_cast<size_t>(config_.nthreads),
experiment_.GetWavelength_A(), report_shell_d_min);
const auto validate = [&, cell = *result.consensus_cell,
wavelength = experiment_.GetWavelength_A()](Logger &log) {
return ValidateAgainstModel(sm.merged, cell, config_.model_path,
config_.output_prefix, log,
data_sg ? &*data_sg : nullptr,
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
static_cast<size_t>(config_.nthreads),
wavelength, report_shell_d_min);
};
// The P1 cross-check merge and this validation read nothing the other writes, so they
// are made at the same time. The validation reads the merge above, the cell and the group,
// none of which the scaling engine touches; the engine merges its own copy of the
// observations, in P1, and writes back only per-image scale, CC and mosaicity, which
// nothing before the cross-check reads (the unmerged MTZ replaces the mosaicity it
// carries after the merge anyway). Every relabelling the validation decides reaches the
// P1 merge afterwards, through merge_to_written, as it did when it was made below. The
// validation's log lines are held and printed as one block once it is done.
ModelValidationResult validation;
if (p1_crosscheck && rsm) {
Logger held = Logger::Buffered();
auto pending = std::async(std::launch::async, validate, std::ref(held));
experiment_.SpaceGroupNumber(1);
p1_merged_early = rsm->Run(/*for_search=*/false, /*full_stats=*/true,
/*measure_cc_before_corrections=*/false);
experiment_.SetSpaceGroup(data_sg);
validation = pending.get();
held.ReplayInto(logger);
} else
validation = validate(logger);
// A model that was asked for and could not be used has to say so where anyone will see
// it. Without this the run ends successfully with no R-free, no maps and nothing in the
// report - indistinguishable from a run that was never given --model at all.
@@ -8906,8 +8941,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// output (measured: same length, different bytes). The fix is to run this below the
// unmerged export rather than here, which needs the merge lambda hoisted out of this
// block; deferred, as rotation is what the online pipeline processes.
const bool p1_integration_complete =
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
if (config_.write_p1_crosscheck && is_rotation && !p1_integration_complete)
logger.Warning("No P1 cross-check dataset written: the fixed space group's centring "
"absences were never predicted, so a P1 merge of this run would be "
@@ -8925,7 +8958,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
: "a fixed centred space group's absences were never predicted, so a P1 "
"merge would be missing whole centring classes");
if (config_.write_p1_crosscheck && p1_integration_complete && is_rotation) {
if (p1_crosscheck) {
// The whole GROUP, not its number: the search can adopt a non-reference setting, and
// a number only ever names the reference one - so restoring through the number would
// silently re-seat P 1 1 2(1) as P 1 2(1) 1 for everything written after this block,
@@ -8961,7 +8994,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// Both the merge and the MTZ read the group from the experiment, so it is set for
// the whole of it and restored after.
experiment_.SpaceGroupNumber(1);
auto p1 = scale_and_merge("P1 cross-check", false);
auto p1 = scale_and_merge("P1 cross-check", false, false, std::move(p1_merged_early));
// The scaler still holds the observations in the indexing they were merged in, so every
// relabelling since (the written setting, the model's indexing) is applied to this merge
// too - or it would describe the dataset on other axes than the merged output beside it,