The rc184-2 receiver architecture is kept; what rc184 added since rc183 is carried into it: - JFJochReceiverRugnuxRotation: the live host budget is nine tenths of the machine's total less the image buffer and the margin, at most the machine's budget (was the budget less both); the pinned host pool is released after the end message, in MeasurementThread, not in RugnuxThread. - StillsGPUAnalysis: the escalating pre-pass is bounded by PREPASS_MAX_IMAGES (4 x PREPASS_IMAGES) as well as by half the image buffer, and frames arriving during a run do not make the next one larger than that. The other live-stills changes of rc184 (pre-pass holding until the geometry can be fitted, geometry fixed when it ends, fixed q range, no running refinement) were already in rc184-2. - IndexAndRefine, RotationIndexer, rugnux: rc184's versions, with rc184-2's receiver-owned indexer. - Tests: FPGAIntegrationTest.cpp stays removed (generator memory check is in MockAcquisitionDeviceTest, pixel-sum parity in ImagePreprocessorPixelSumTest); the FPGA live-stills tests of JFJochReceiverProcessingTest are covered by JFJochReceiverRugnuxTest (pre-pass cancel, ZeroMQ writer, writer down). - Docs: rc184's text plus rc184-2's receiver table, broker diagram, stream and test notes; NumberTextField and CHANGELOG as in rc184. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
589 lines
27 KiB
C++
589 lines
27 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "StillsGPUAnalysis.h"
|
|
|
|
#include <thread>
|
|
|
|
#include "../common/CUDAWrapper.h"
|
|
#include "../common/ThreadAffinity.h"
|
|
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
|
#include "../image_analysis/WriteReflections.h"
|
|
#include "../rugnux/RugnuxDefaults.h"
|
|
#include "../writer/RugnuxResultBlocks.h"
|
|
#include "RugnuxStructure.h"
|
|
|
|
StillsGPUAnalysis::StillsGPUAnalysis(const DiffractionExperiment &in_experiment, const PixelMask &in_mask,
|
|
const StartMessage &in_start, IndexerThreadPool *in_indexing_thread_pool,
|
|
int64_t nthreads, int64_t max_held, Logger &in_logger, StillsSink &in_sink,
|
|
RugnuxLiveStatus &in_status, bool in_merge,
|
|
std::shared_ptr<const RugnuxModel> in_model, double in_deadline_s)
|
|
: experiment(in_experiment), mask(in_mask), start(in_start), indexing_thread_pool(in_indexing_thread_pool),
|
|
logger(in_logger), sink(in_sink), status(in_status), merge_frames(in_merge),
|
|
hold_limit(std::max<int64_t>(1, max_held)), model(std::move(in_model)), deadline_s(in_deadline_s) {
|
|
live.images_needed = std::min({PREPASS_IMAGES, experiment.GetImageNum(), hold_limit});
|
|
live.file_beam_x_pxl = experiment.GetBeamX_pxl();
|
|
live.file_beam_y_pxl = experiment.GetBeamY_pxl();
|
|
live.file_distance_mm = experiment.GetDetectorDistance_mm();
|
|
status.Start(experiment.GetImageNum(),
|
|
static_cast<double>(experiment.GetImageNum())
|
|
* std::chrono::duration<double>(experiment.GetImageTime()).count(),
|
|
std::chrono::steady_clock::now());
|
|
status.Step("Pre-pass");
|
|
Publish();
|
|
prepass_future = std::async(std::launch::async, &StillsGPUAnalysis::PrePassThread, this);
|
|
for (int64_t i = 0; i < nthreads; i++)
|
|
analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysis::AnalysisThread, this));
|
|
logger.Info("GPU analysis: {} threads, at least the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES);
|
|
}
|
|
|
|
StillsGPUAnalysis::~StillsGPUAnalysis() {
|
|
try {
|
|
if (prepass_future.valid())
|
|
Finish();
|
|
} catch (const std::exception &e) {
|
|
logger.Error("GPU analysis: {}", e.what());
|
|
}
|
|
StopModel(true);
|
|
ReportOutcome();
|
|
}
|
|
|
|
void StillsGPUAnalysis::AnalysisError(const std::string &text) {
|
|
logger.Error("GPU analysis: {}", text);
|
|
std::unique_lock ul(error_mutex);
|
|
if (analysis_error.empty())
|
|
analysis_error = text;
|
|
}
|
|
|
|
void StillsGPUAnalysis::Publish() {
|
|
std::unique_lock ul(live_mutex);
|
|
if (live.prepass == RugnuxLivePrePass::Done || live.prepass == RugnuxLivePrePass::Failed) {
|
|
const double t = std::chrono::duration<double>(std::chrono::steady_clock::now() - ready_time).count();
|
|
if (t > 0 && live.images_analysed > 0)
|
|
live.analysis_rate_hz = static_cast<double>(live.images_analysed) / t;
|
|
}
|
|
status.Stills(live);
|
|
}
|
|
|
|
void StillsGPUAnalysis::Hold(const std::shared_ptr<HeldImage> &image) {
|
|
status.ImageReceived(std::chrono::steady_clock::now());
|
|
queue.PutBlocking(image);
|
|
{
|
|
std::unique_lock ul(prepass_mutex);
|
|
if (prepass_state != PrePassState::Over && static_cast<int64_t>(prepass_images.size()) < hold_limit) {
|
|
prepass_images[image->message.number] = image;
|
|
std::unique_lock ul_live(live_mutex);
|
|
live.images_held = static_cast<int64_t>(prepass_images.size());
|
|
}
|
|
}
|
|
prepass_cv.notify_all();
|
|
Publish();
|
|
}
|
|
|
|
void StillsGPUAnalysis::StartPrePass() {
|
|
{
|
|
std::unique_lock ul(prepass_mutex);
|
|
if (prepass_state == PrePassState::Holding)
|
|
prepass_state = PrePassState::Closing;
|
|
}
|
|
prepass_cv.notify_all();
|
|
}
|
|
|
|
void StillsGPUAnalysis::Finish() {
|
|
const auto finish_start = std::chrono::steady_clock::now();
|
|
StartPrePass();
|
|
prepass_future.get();
|
|
for (size_t i = 0; i < analysis_futures.size(); i++)
|
|
queue.PutBlocking(nullptr);
|
|
for (auto &f: analysis_futures)
|
|
f.get();
|
|
const auto last_image = status.LastImage();
|
|
Publish();
|
|
if (merge_future.valid()) {
|
|
status.Step("Merge");
|
|
merge_queue.Put(-1);
|
|
merge_future.get();
|
|
if (merged) {
|
|
PublishMerge(*merged, "the collection");
|
|
status.Headline(merge_experiment.GetSpaceGroupOrP1().xhm(), merged->merge.statistics.overall.d_min,
|
|
std::nullopt);
|
|
}
|
|
}
|
|
if (model_future.valid()) {
|
|
// The step on the merge of the collection, until the deadline; the provisional one running is cut, this starts from it
|
|
if (merged && model_left_out.empty()) {
|
|
const auto deadline = last_image.value_or(finish_start) + std::chrono::duration_cast<
|
|
std::chrono::steady_clock::duration>(std::chrono::duration<double>(deadline_s));
|
|
SubmitModel(*merged, "the collection", true);
|
|
StopModel(false);
|
|
if (deadline_s > 0 && model_future.wait_until(deadline) == std::future_status::timeout) {
|
|
{
|
|
std::unique_lock ul(model_mutex);
|
|
step_cut = true;
|
|
if (model_next)
|
|
model_next->cut = true;
|
|
}
|
|
logger.Warning("Model step cut at the deadline, {:.1f} s after the last image", deadline_s);
|
|
status.Cut("deadline");
|
|
}
|
|
model_future.get();
|
|
} else
|
|
StopModel(true);
|
|
std::unique_lock ul(model_mutex);
|
|
model_over = true;
|
|
}
|
|
if (setup_error) {
|
|
status.Finish(RugnuxLiveState::Failed, "GPU analysis setup failed");
|
|
std::rethrow_exception(setup_error);
|
|
}
|
|
// Reported by ReportOutcome, once everything is written - the end message and the writer go on meanwhile.
|
|
status.Step("Writing");
|
|
std::unique_lock ul(error_mutex);
|
|
if (analysis_error.empty())
|
|
outcome.emplace(RugnuxLiveState::Done, "");
|
|
else
|
|
outcome.emplace(RugnuxLiveState::Failed, analysis_error);
|
|
}
|
|
|
|
void StillsGPUAnalysis::ReportOutcome() {
|
|
if (!outcome)
|
|
return;
|
|
status.Finish(outcome->first, outcome->second);
|
|
outcome.reset();
|
|
}
|
|
|
|
void StillsGPUAnalysis::PrePassThread() {
|
|
const auto masked = [](const PixelMask &pixel_mask) {
|
|
return std::count_if(pixel_mask.GetMask().begin(), pixel_mask.GetMask().end(), [](uint32_t v) { return v != 0; });
|
|
};
|
|
setup.experiment = experiment;
|
|
setup.mask = mask;
|
|
const int64_t all_images = experiment.GetImageNum();
|
|
std::optional<StillsPrePassResult> result;
|
|
std::chrono::steady_clock::time_point prepass_start = std::chrono::steady_clock::now();
|
|
// With a cell, the pre-pass runs again on all that is held once it has doubled, until the fit has its frames, the
|
|
// end, or PREPASS_MAX_IMAGES
|
|
const int64_t most_held = std::min({all_images, hold_limit, PREPASS_MAX_IMAGES});
|
|
for (int64_t needed = std::min(PREPASS_IMAGES, most_held); ;) {
|
|
std::map<int64_t, std::shared_ptr<HeldImage>> sample;
|
|
bool last;
|
|
{
|
|
std::unique_lock ul(prepass_mutex);
|
|
prepass_cv.wait(ul, [&] {
|
|
return prepass_state != PrePassState::Holding || static_cast<int64_t>(prepass_images.size()) >= needed;
|
|
});
|
|
// Frames that arrived during the previous run do not make this one larger than the bound
|
|
for (const auto &[n, image]: prepass_images) {
|
|
if (static_cast<int64_t>(sample.size()) >= most_held)
|
|
break;
|
|
sample.emplace(n, image);
|
|
}
|
|
last = prepass_state != PrePassState::Holding || static_cast<int64_t>(sample.size()) >= most_held;
|
|
}
|
|
if (sample.empty())
|
|
break;
|
|
{
|
|
std::unique_lock ul(live_mutex);
|
|
live.prepass = RugnuxLivePrePass::Running;
|
|
}
|
|
Publish();
|
|
try {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
auto ret = RunStillsPrePass(start, sample, static_cast<int>(std::thread::hardware_concurrency()));
|
|
logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, "
|
|
"distance {:.2f} -> {:.2f} mm on {} frames, masked pixels {} -> {}", ret.images,
|
|
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count(),
|
|
experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), ret.beam_x_pxl, ret.beam_y_pxl,
|
|
experiment.GetDetectorDistance_mm(), ret.distance_mm, ret.geometry_frames,
|
|
masked(mask), masked(ret.mask));
|
|
result = std::move(ret);
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Pre-pass on {} images failed - {}", sample.size(), e.what());
|
|
}
|
|
if (last || !experiment.GetUnitCell()
|
|
|| (result && result->geometry_frames >= RUGNUX_REFINE_GEOMETRY_FRAMES))
|
|
break;
|
|
needed = std::min(2 * static_cast<int64_t>(sample.size()), most_held);
|
|
std::unique_lock ul(live_mutex);
|
|
live.prepass = RugnuxLivePrePass::Holding;
|
|
live.images_needed = needed;
|
|
}
|
|
{
|
|
// The frames are analysed from here on: their slots go back to the buffer once they are sent.
|
|
std::unique_lock ul(prepass_mutex);
|
|
prepass_state = PrePassState::Over;
|
|
prepass_images.clear();
|
|
}
|
|
const bool failed = !result;
|
|
int geometry_frames = 0;
|
|
if (result) {
|
|
setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm);
|
|
setup.mask = result->mask;
|
|
geometry_frames = result->geometry_frames;
|
|
const auto &result_mask = result->mask.GetMask();
|
|
std::unique_lock ul(live_mutex);
|
|
live.beam_x_pxl = result->beam_x_pxl;
|
|
live.beam_y_pxl = result->beam_y_pxl;
|
|
live.distance_mm = result->distance_mm;
|
|
live.shadow_pixels = std::count_if(result_mask.begin(), result_mask.end(), [](uint32_t v) {
|
|
return (v & (1u << PixelMask::BeamStopPixelBit)) != 0;
|
|
});
|
|
live.bad_pixels = std::count_if(result_mask.begin(), result_mask.end(), [](uint32_t v) {
|
|
return (v & (1u << PixelMask::HotPixelBit)) != 0;
|
|
});
|
|
} else
|
|
logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask");
|
|
try {
|
|
setup.mapping = std::make_unique<AzimuthalIntegrationMapping>(setup.experiment, setup.mask);
|
|
// Retained for the merge, which takes each image's reflections as soon as it is analysed.
|
|
indexer = std::make_unique<IndexAndRefine>(setup.experiment, indexing_thread_pool, merge_frames, true);
|
|
if (geometry_frames == 0) {
|
|
const std::string text = fmt::format(
|
|
"The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the "
|
|
"whole run is analysed at the distance in the header, {:.2f} mm, beam centre ({:.2f}, {:.2f}) px",
|
|
setup.experiment.GetDetectorDistance_mm(), setup.experiment.GetBeamX_pxl(),
|
|
setup.experiment.GetBeamY_pxl());
|
|
logger.Warning("{}", text);
|
|
status.Note(text);
|
|
}
|
|
if (merge_frames) {
|
|
merge_experiment = setup.experiment;
|
|
merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
|
|
merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
|
|
setup.experiment.GetUnitCell(), std::nullopt,
|
|
setup.experiment.IsDetectIceRings(),
|
|
SpotFindingSettings().ice_ring_width_Q_recipA, logger);
|
|
// The model's thread first: the merge thread hands it its merges
|
|
if (model)
|
|
model_future = std::async(std::launch::async, &StillsGPUAnalysis::ModelThread, this);
|
|
merge_future = std::async(std::launch::async, &StillsGPUAnalysis::MergeThread, this);
|
|
}
|
|
sink.StillsReady(setup);
|
|
} catch (const std::exception &e) {
|
|
// Nothing can be written without the start message: the frames are dropped, the collection is stopped
|
|
const auto *jfjoch_error = dynamic_cast<const JFJochException *>(&e);
|
|
const JFJochException error = jfjoch_error ? *jfjoch_error : JFJochException(JFJochExceptionCategory::WrongDAQState,
|
|
std::string("GPU analysis setup failed: ") + e.what());
|
|
logger.ErrorException(error);
|
|
setup_error = jfjoch_error ? std::current_exception() : std::make_exception_ptr(error);
|
|
sink.StillsSetupFailed(error);
|
|
prepass_done.count_down();
|
|
return;
|
|
}
|
|
{
|
|
std::unique_lock ul(live_mutex);
|
|
live.prepass = failed ? RugnuxLivePrePass::Failed : RugnuxLivePrePass::Done;
|
|
live.prepass_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - prepass_start).count();
|
|
ready_time = std::chrono::steady_clock::now();
|
|
}
|
|
status.Step("GPU analysis");
|
|
Publish();
|
|
prepass_done.count_down();
|
|
}
|
|
|
|
void StillsGPUAnalysis::AnalysisThread() {
|
|
try {
|
|
pin_gpu();
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Error pinning GPU {}", e.what());
|
|
}
|
|
|
|
prepass_done.wait();
|
|
|
|
if (setup_error) {
|
|
while (queue.GetBlocking()) {}
|
|
return;
|
|
}
|
|
|
|
// A thread whose analysis cannot be built still passes its frames on, unanalysed.
|
|
std::unique_ptr<MXAnalysisWithoutFPGA> analysis;
|
|
try {
|
|
analysis = std::make_unique<MXAnalysisWithoutFPGA>(setup.experiment, *setup.mapping, setup.mask,
|
|
*indexer, /*enable_fused_adaptive_gpu=*/true);
|
|
} catch (const std::exception &e) {
|
|
AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ")
|
|
+ e.what());
|
|
}
|
|
|
|
while (auto held = queue.GetBlocking()) {
|
|
DataMessage &message = held->message;
|
|
std::optional<AzimuthalIntegrationProfile> profile;
|
|
const auto start_time = std::chrono::steady_clock::now();
|
|
profile.emplace(*setup.mapping);
|
|
try {
|
|
if (analysis)
|
|
analysis->AnalyzeStreamImage(message, *profile, sink.AnalysisSettings());
|
|
} catch (const JFJochException &e) {
|
|
// The frame is still written, without its analysis.
|
|
logger.ErrorException(e);
|
|
} catch (const std::exception &e) {
|
|
AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what()));
|
|
}
|
|
message.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now()
|
|
- start_time).count();
|
|
try {
|
|
sink.SendAnalysedImage(message, *profile);
|
|
if (merge)
|
|
merge_queue.Put(message.number);
|
|
int64_t analysed;
|
|
{
|
|
std::unique_lock ul(live_mutex);
|
|
analysed = ++live.images_analysed;
|
|
if (message.indexing_result.value_or(false))
|
|
++live.images_indexed;
|
|
}
|
|
status.Progress(analysed, experiment.GetImageNum());
|
|
Publish();
|
|
} catch (const std::exception &e) {
|
|
// The thread goes on with the next frame, so that the queue is drained.
|
|
AnalysisError(fmt::format("image {} lost - {}", message.number, e.what()));
|
|
}
|
|
}
|
|
}
|
|
|
|
std::optional<UnitCell> StillsGPUAnalysis::GetConsensusUnitCell() const {
|
|
return indexer->GetConsensusUnitCell();
|
|
}
|
|
|
|
float StillsGPUAnalysis::GetBeamX_pxl() const {
|
|
return setup.experiment.GetBeamX_pxl();
|
|
}
|
|
|
|
float StillsGPUAnalysis::GetBeamY_pxl() const {
|
|
return setup.experiment.GetBeamY_pxl();
|
|
}
|
|
|
|
float StillsGPUAnalysis::GetDetectorDistance_mm() const {
|
|
return setup.experiment.GetDetectorDistance_mm();
|
|
}
|
|
|
|
const std::optional<StillsMergeOutput> &StillsGPUAnalysis::GetMerged() const {
|
|
return merged;
|
|
}
|
|
|
|
const DiffractionExperiment &StillsGPUAnalysis::GetMergeExperiment() const {
|
|
return merge_experiment;
|
|
}
|
|
|
|
const std::optional<ModelStepResult> &StillsGPUAnalysis::GetModel() const {
|
|
return model_final;
|
|
}
|
|
|
|
void StillsGPUAnalysis::MergeThread() {
|
|
constexpr int64_t PREPARE_EVERY = 200; // frames between two Prepare calls after a checkpoint
|
|
const int64_t images = experiment.GetImageNum();
|
|
// The cell merged in, as rugnux merges it: the consensus of the frames indexed by the first merge
|
|
bool cell_set = false;
|
|
const auto have_cell = [&] {
|
|
if (!cell_set)
|
|
if (const auto cell = indexer->GetConsensusUnitCell()) {
|
|
merge_experiment.SetUnitCell(*cell);
|
|
merge->Cell(*cell);
|
|
cell_set = true;
|
|
if (model) {
|
|
const ModelStepGroup group = ModelStepSpaceGroup(merge_experiment.GetGemmiSpaceGroup(), *cell, *model);
|
|
if (!group.space_group) {
|
|
model_left_out = "the model is left out: " + group.reason;
|
|
logger.Warning("{}", model_left_out);
|
|
status.Warning(model_left_out);
|
|
} else if (!merge_experiment.GetGemmiSpaceGroup()) {
|
|
merge_experiment.SetSpaceGroup(*group.space_group);
|
|
merge->SpaceGroup(*group.space_group);
|
|
logger.Info("Merged in the model's space group {}", group.space_group->xhm());
|
|
}
|
|
}
|
|
}
|
|
return cell_set;
|
|
};
|
|
size_t next = 0;
|
|
try {
|
|
const auto add = [&](int64_t n, IntegrationOutcome &&outcome) {
|
|
merge->Add(n, std::move(outcome));
|
|
if (next < CHECKPOINTS.size()
|
|
&& merge->Added() >= std::llround(CHECKPOINTS[next] * static_cast<double>(images))) {
|
|
if (have_cell()) {
|
|
StillsMergeOutput checkpoint = merge->Checkpoint(MERGE_THREADS);
|
|
const std::string title = fmt::format("{:.0f}% of the images (provisional)",
|
|
100.0 * CHECKPOINTS[next]);
|
|
PublishMerge(checkpoint, title);
|
|
if (model_future.valid() && model_left_out.empty())
|
|
SubmitModel(checkpoint, title, false);
|
|
}
|
|
++next;
|
|
} else if (next > 0 && merge->Added() % PREPARE_EVERY == 0)
|
|
merge->Prepare(MERGE_THREADS);
|
|
};
|
|
for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking())
|
|
add(n, indexer->TakeIntegrationOutcome(n));
|
|
if (have_cell()) {
|
|
merged = merge->Final(static_cast<int>(AvailableCpus()));
|
|
logger.Info("Stills merge after the last image: {:.2f} s", merged->seconds);
|
|
}
|
|
// The frames' reflections are not read again: what the model step and the end message need is the merge
|
|
merge.reset();
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Stills merge failed: {}", e.what());
|
|
merged.reset();
|
|
}
|
|
}
|
|
|
|
void StillsGPUAnalysis::PublishMerge(const StillsMergeOutput &out, const std::string &title) {
|
|
const auto &overall = out.merge.statistics.overall;
|
|
const std::string text = fmt::format("Merge of {}: {} crystals, {:.2f} A, CC1/2 {:.3f}, R_split {:.3f}, "
|
|
"completeness {:.1f}%, <I/sigma> {:.1f} ({:.1f} s)", title, out.images,
|
|
overall.d_min, overall.cc_half, overall.r_split,
|
|
overall.possible_unique_reflections > 0
|
|
? 100.0 * overall.unique_reflections / overall.possible_unique_reflections : 0.0,
|
|
overall.mean_i_over_sigma, out.seconds);
|
|
logger.Info("{}", text);
|
|
status.Note(text);
|
|
}
|
|
|
|
void StillsGPUAnalysis::SubmitModel(StillsMergeOutput &out, const std::string &title, bool final) {
|
|
ModelJob job;
|
|
// The merge of the collection is kept for the end message
|
|
job.merged = final ? out.merge.merged : std::move(out.merge.merged);
|
|
for (const auto &sh : out.merge.statistics.shells)
|
|
job.shells.push_back(sh.d_min);
|
|
job.title = title;
|
|
job.final = final;
|
|
{
|
|
std::unique_lock ul(model_mutex);
|
|
step_cut = true;
|
|
if (model_next)
|
|
logger.Info("Model step on the merge of {} left out: the merge of {} is there", model_next->title, title);
|
|
model_next = std::move(job);
|
|
}
|
|
model_cv.notify_all();
|
|
}
|
|
|
|
void StillsGPUAnalysis::Cut(const std::string &reason) {
|
|
if (!model)
|
|
return;
|
|
{
|
|
std::unique_lock ul(model_mutex);
|
|
if (model_cut || model_over)
|
|
return;
|
|
model_cut = true;
|
|
step_cut = true;
|
|
}
|
|
logger.Info("Model step cut: {}", reason);
|
|
status.Cut(reason);
|
|
}
|
|
|
|
void StillsGPUAnalysis::StopModel(bool cut) {
|
|
if (!model_future.valid())
|
|
return;
|
|
{
|
|
std::unique_lock ul(model_mutex);
|
|
model_stop = true;
|
|
if (cut) {
|
|
model_next.reset();
|
|
step_cut = true;
|
|
}
|
|
}
|
|
model_cv.notify_all();
|
|
if (cut)
|
|
model_future.get();
|
|
}
|
|
|
|
void StillsGPUAnalysis::ModelThread() {
|
|
try {
|
|
pin_gpu();
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Error pinning GPU {}", e.what());
|
|
}
|
|
while (true) {
|
|
ModelJob job;
|
|
{
|
|
std::unique_lock ul(model_mutex);
|
|
model_cv.wait(ul, [this] { return model_next || model_stop; });
|
|
if (!model_next)
|
|
return;
|
|
job = std::move(*model_next);
|
|
model_next.reset();
|
|
step_cut = model_cut || job.cut;
|
|
}
|
|
try {
|
|
ModelStepSettings settings;
|
|
// Beside the analysis on as many threads as a provisional merge, on every thread after the last image
|
|
settings.nthreads = job.final ? AvailableCpus() : MERGE_THREADS;
|
|
// The analysis's indexers stay on the card for the broker's life
|
|
settings.gpu_resident_bytes = indexing_thread_pool ? indexing_thread_pool->DeviceBytes() : 0;
|
|
settings.rfree_fraction = merge_experiment.GetScalingSettings().GetRfreeFraction();
|
|
settings.cut = &step_cut;
|
|
if (job.final)
|
|
settings.step = [this](const std::string &title) { status.Step(title); };
|
|
ModelStepResult result = RunModelStep(job.merged, merge_experiment.GetUnitCell().value(),
|
|
merge_experiment.GetSpaceGroupOrP1(), *model,
|
|
merge_experiment.GetWavelength_A(),
|
|
merge_experiment.GetScalingSettings().GetMergeFriedel(), job.shells,
|
|
settings, model_placed ? &*model_placed : nullptr, logger);
|
|
PublishModel(result, job.title);
|
|
if (result.placed_model)
|
|
model_placed = result;
|
|
if (job.final)
|
|
model_final = std::move(result);
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Model step on the merge of {} failed: {}", job.title, e.what());
|
|
}
|
|
}
|
|
}
|
|
|
|
void StillsGPUAnalysis::PublishModel(const ModelStepResult &result, const std::string &title) {
|
|
std::string text = fmt::format("Model on the merge of {}: ", title);
|
|
if (result.placement == "NOT_PLACED")
|
|
text += "not placed";
|
|
else
|
|
text += fmt::format("placed by {}{}", result.placement == "MR" ? "molecular replacement" : "its rigid body",
|
|
result.warm ? " (from the earlier placement)" : "");
|
|
if (result.validation && result.validation->ok)
|
|
text += fmt::format(", R-work {:.3f}, R-free {:.3f}", result.validation->r_work, result.validation->r_free);
|
|
if (result.refinement && result.refinement->ok)
|
|
text += fmt::format("; refined{} R-work {:.3f}, R-free {:.3f}, {} blobs", result.refinement->cancelled
|
|
? " (cut short)" : "", result.refinement->r_work, result.refinement->r_free,
|
|
result.refinement->blobs.size());
|
|
for (const auto &left_out : result.left_out)
|
|
text += "; " + left_out + " left out";
|
|
text += fmt::format(" ({:.1f} s placement, {:.1f} s refinement)", result.placement_s, result.refinement_s);
|
|
logger.Info("{}", text);
|
|
status.Note(text);
|
|
if (const auto live_model = LiveModel(ModelStepReport(result)))
|
|
status.Model(*live_model);
|
|
if (!result.placed_model)
|
|
return;
|
|
std::optional<PlacedModelOutput> refined;
|
|
if (result.refinement && result.refinement->ok && result.refinement->model)
|
|
refined = PlacedModelOutput{result.refinement->model, result.placed_model->cell, result.placed_model->space_group};
|
|
PublishStructures(status, *result.placed_model, result.maps.get(), refined ? &*refined : nullptr,
|
|
refined && result.refinement->maps ? result.refinement->maps.get() : nullptr);
|
|
}
|
|
|
|
void StillsGPUAnalysis::FillEndMessage(EndMessage &message) const {
|
|
message.unit_cell = GetConsensusUnitCell();
|
|
message.refined_beam_center_x = GetBeamX_pxl();
|
|
message.refined_beam_center_y = GetBeamY_pxl();
|
|
message.refined_detector_distance_mm = GetDetectorDistance_mm();
|
|
// In the model's frame where a model was placed against them
|
|
if (const auto &model_step = GetModel()) {
|
|
DiffractionExperiment model_experiment(GetMergeExperiment());
|
|
model_experiment.SetSpaceGroup(model_step->space_group);
|
|
message.merged = MergedMtzTable(model_step->merged, model_step->cell, model_experiment, OutputProvenance{});
|
|
if (model_step->maps)
|
|
message.map_coefficients = model_step->maps->coefficients;
|
|
if (model_step->placed_model)
|
|
message.placed_model = PlacedModelFromStructure(*model_step->placed_model->model, model_step->placed_model->cell,
|
|
model_step->placed_model->space_group.xhm());
|
|
if (const auto &refinement = model_step->refinement; refinement && refinement->ok && refinement->model) {
|
|
message.refined_model = PlacedModelFromStructure(*refinement->model, model_step->cell, model_step->space_group.xhm());
|
|
if (refinement->maps)
|
|
message.refined_map_coefficients = refinement->maps->coefficients;
|
|
message.blobs = DensityBlobsToMessage(refinement->blobs);
|
|
}
|
|
} else if (const auto &merge_output = GetMerged(); merge_output && GetMergeExperiment().GetUnitCell())
|
|
message.merged = MergedMtzTable(merge_output->merge.merged, *GetMergeExperiment().GetUnitCell(),
|
|
GetMergeExperiment(), OutputProvenance{});
|
|
if (message.merged)
|
|
DropResultBlocksThatDoNotFit(message, &status, logger);
|
|
}
|