Merge branch 'tail-p1-model' into rc173 (P1 cross-check merge beside model validation)
This commit is contained in:
commit
5ed07eeb11
3 files changed
+83
-14
No files matched your search
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "spdlog/sinks/daily_file_sink.h"
|
||||
#include "spdlog/sinks/stdout_color_sinks.h"
|
||||
#include "spdlog/sinks/base_sink.h"
|
||||
#include "spdlog/fmt/bin_to_hex.h"
|
||||
|
||||
#include "Logger.h"
|
||||
@@ -33,6 +34,31 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) {
|
||||
variant);
|
||||
}
|
||||
|
||||
class LogBufferSink : public spdlog::sinks::base_sink<std::mutex> {
|
||||
public:
|
||||
std::vector<std::pair<spdlog::level::level_enum, std::string>> lines;
|
||||
protected:
|
||||
void sink_it_(const spdlog::details::log_msg &msg) override {
|
||||
lines.emplace_back(msg.level, std::string(msg.payload.data(), msg.payload.size()));
|
||||
}
|
||||
void flush_() override {}
|
||||
};
|
||||
|
||||
Logger::Logger(std::shared_ptr<LogBufferSink> sink) : buffer(std::move(sink)) {
|
||||
spdlog_logger = std::make_shared<spdlog::logger>("buffer", buffer);
|
||||
spdlog_logger->set_level(spdlog::level::trace); // the logger it is replayed into filters
|
||||
}
|
||||
|
||||
Logger Logger::Buffered() {
|
||||
return Logger(std::make_shared<LogBufferSink>());
|
||||
}
|
||||
|
||||
// Called once the work that logged is finished, so nothing is still writing to the buffer.
|
||||
void Logger::ReplayInto(Logger &to) const {
|
||||
for (const auto &[level, line] : buffer->lines)
|
||||
to.spdlog_logger->log(level, line);
|
||||
}
|
||||
|
||||
void Logger::ErrorException(const std::exception &e) {
|
||||
spdlog_logger->error(e.what());
|
||||
}
|
||||
|
||||
@@ -16,8 +16,12 @@ namespace spdlog {
|
||||
class logger;
|
||||
}
|
||||
|
||||
class LogBufferSink;
|
||||
|
||||
class Logger {
|
||||
std::shared_ptr<spdlog::logger> spdlog_logger;
|
||||
std::shared_ptr<LogBufferSink> buffer; // only on a Buffered() logger
|
||||
explicit Logger(std::shared_ptr<LogBufferSink> sink);
|
||||
|
||||
std::string service;
|
||||
std::string hostname;
|
||||
@@ -51,4 +55,10 @@ public:
|
||||
}
|
||||
|
||||
Logger& Verbose(bool input);
|
||||
|
||||
// A logger that keeps its lines instead of printing them, for work run beside other work that
|
||||
// logs: ReplayInto hands them to `to` afterwards, in order and as one block, so the two do not
|
||||
// interleave.
|
||||
static Logger Buffered();
|
||||
void ReplayInto(Logger &to) const;
|
||||
};
|
||||
+47
-14
@@ -6727,15 +6727,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
|
||||
// measure_cc: whether anything reads the result's cc_half_before_corrections, which costs a
|
||||
// merge of its own - only the first merge, which the two-pass quality guard reads.
|
||||
// already_run: the engine's result for this merge, when it was made earlier (see the P1
|
||||
// cross-check); it is then only reported, exactly as if it had been made here.
|
||||
auto scale_and_merge = [&](const std::string &label, bool for_search,
|
||||
bool measure_cc = false) -> ScaleMergeResult {
|
||||
bool measure_cc = false,
|
||||
std::optional<RotationScaleMerge::Result> already_run = std::nullopt)
|
||||
-> ScaleMergeResult {
|
||||
if (rsm) {
|
||||
phase("Scale/combine/merge (" + label + ")");
|
||||
// The geometry pre-pass merges only to choose the space group and to give the quality
|
||||
// guard something to judge the second pass against; its reflections are never written,
|
||||
// so the parts of the merge that only fill in an output file are skipped there.
|
||||
auto r = rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
|
||||
/*measure_cc_before_corrections=*/measure_cc);
|
||||
auto r = already_run ? std::move(*already_run)
|
||||
: rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
|
||||
/*measure_cc_before_corrections=*/measure_cc);
|
||||
result.error_model_isa = r.isa;
|
||||
result.error_model_isa_asymptotic = r.isa_asymptotic;
|
||||
result.error_model_isa_resolved = r.isa_resolved;
|
||||
@@ -8751,6 +8756,14 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the P1 cross-check dataset is made (see where it is written, below).
|
||||
const bool p1_integration_complete =
|
||||
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
|
||||
const bool p1_crosscheck = result.consensus_cell && write_files && config_.write_merged
|
||||
&& !geometry_prepass && !superseded && config_.write_p1_crosscheck
|
||||
&& p1_integration_complete && is_rotation;
|
||||
std::optional<RotationScaleMerge::Result> p1_merged_early;
|
||||
|
||||
// Model validation runs BEFORE the reflection files are written, because it is what settles the
|
||||
// frame they are written in: the enantiomorph, which merged intensities cannot choose, and -
|
||||
// with no reference MTZ - the alternative indexing. Both are relabelings of the same
|
||||
@@ -8766,13 +8779,35 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
std::vector<float> report_shell_d_min;
|
||||
for (const auto &sh : sm.statistics.shells)
|
||||
report_shell_d_min.push_back(sh.d_min);
|
||||
auto validation =
|
||||
ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
|
||||
config_.output_prefix, logger,
|
||||
data_sg ? &*data_sg : nullptr,
|
||||
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
|
||||
static_cast<size_t>(config_.nthreads),
|
||||
experiment_.GetWavelength_A(), report_shell_d_min);
|
||||
const auto validate = [&, cell = *result.consensus_cell,
|
||||
wavelength = experiment_.GetWavelength_A()](Logger &log) {
|
||||
return ValidateAgainstModel(sm.merged, cell, config_.model_path,
|
||||
config_.output_prefix, log,
|
||||
data_sg ? &*data_sg : nullptr,
|
||||
/*probe_indexing_ambiguity=*/config_.reference_data.empty(),
|
||||
static_cast<size_t>(config_.nthreads),
|
||||
wavelength, report_shell_d_min);
|
||||
};
|
||||
// The P1 cross-check merge and this validation read nothing the other writes, so they
|
||||
// are made at the same time. The validation reads the merge above, the cell and the group,
|
||||
// none of which the scaling engine touches; the engine merges its own copy of the
|
||||
// observations, in P1, and writes back only per-image scale, CC and mosaicity, which
|
||||
// nothing before the cross-check reads (the unmerged MTZ replaces the mosaicity it
|
||||
// carries after the merge anyway). Every relabelling the validation decides reaches the
|
||||
// P1 merge afterwards, through merge_to_written, as it did when it was made below. The
|
||||
// validation's log lines are held and printed as one block once it is done.
|
||||
ModelValidationResult validation;
|
||||
if (p1_crosscheck && rsm) {
|
||||
Logger held = Logger::Buffered();
|
||||
auto pending = std::async(std::launch::async, validate, std::ref(held));
|
||||
experiment_.SpaceGroupNumber(1);
|
||||
p1_merged_early = rsm->Run(/*for_search=*/false, /*full_stats=*/true,
|
||||
/*measure_cc_before_corrections=*/false);
|
||||
experiment_.SetSpaceGroup(data_sg);
|
||||
validation = pending.get();
|
||||
held.ReplayInto(logger);
|
||||
} else
|
||||
validation = validate(logger);
|
||||
// A model that was asked for and could not be used has to say so where anyone will see
|
||||
// it. Without this the run ends successfully with no R-free, no maps and nothing in the
|
||||
// report - indistinguishable from a run that was never given --model at all.
|
||||
@@ -8906,8 +8941,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// output (measured: same length, different bytes). The fix is to run this below the
|
||||
// unmerged export rather than here, which needs the merge lambda hoisted out of this
|
||||
// block; deferred, as rotation is what the online pipeline processes.
|
||||
const bool p1_integration_complete =
|
||||
search_space_group || (indexer && indexer->GetPredictionCentring() == 'P');
|
||||
if (config_.write_p1_crosscheck && is_rotation && !p1_integration_complete)
|
||||
logger.Warning("No P1 cross-check dataset written: the fixed space group's centring "
|
||||
"absences were never predicted, so a P1 merge of this run would be "
|
||||
@@ -8925,7 +8958,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
: "a fixed centred space group's absences were never predicted, so a P1 "
|
||||
"merge would be missing whole centring classes");
|
||||
|
||||
if (config_.write_p1_crosscheck && p1_integration_complete && is_rotation) {
|
||||
if (p1_crosscheck) {
|
||||
// The whole GROUP, not its number: the search can adopt a non-reference setting, and
|
||||
// a number only ever names the reference one - so restoring through the number would
|
||||
// silently re-seat P 1 1 2(1) as P 1 2(1) 1 for everything written after this block,
|
||||
@@ -8961,7 +8994,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// Both the merge and the MTZ read the group from the experiment, so it is set for
|
||||
// the whole of it and restored after.
|
||||
experiment_.SpaceGroupNumber(1);
|
||||
auto p1 = scale_and_merge("P1 cross-check", false);
|
||||
auto p1 = scale_and_merge("P1 cross-check", false, false, std::move(p1_merged_early));
|
||||
// The scaler still holds the observations in the indexing they were merged in, so every
|
||||
// relabelling since (the written setting, the model's indexing) is applied to this merge
|
||||
// too - or it would describe the dataset on other axes than the merged output beside it,
|
||||
|
||||
Reference in new issue
Block a user