mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 62 Wiki Activity

Remove the global (joint) scaling path

Browse Source 
Profiling showed the joint Ceres solve took ~120 s versus ~3.5 s for the
alternating per-image scaling loop (~35x) for no quality gain (HEWL
anomalous 0.54x vs 0.53x), so it is not worth keeping. Drop
GlobalScale.{h,cpp}, the jfjoch_process --global-scale flag, and
ScalingSettings::GlobalScaling.

While here, in the same scaling/process area: fold scale_fulls into
ScalingSettings (alongside combine_3d) so the CLI and experiment carry it
uniformly, add per-substep [timing] logging to the scaling/merge post-pass
(including the serial MergeAll vs parallel ScaleAllImages split), and carry
structured MergeStatistics + ISa in ProcessResult.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
leonarski_f
2026-06-25 14:22:10 +02:00
parent a37613690e
commit c84d581d3e
8 changed files with 71 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files
process/JFJochProcess.cpp
+47 -79
View File
@@ -27,7 +27,6 @@
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/GlobalScale.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
#include "../image_analysis/scale_merge/Combine3D.h"
@@ -57,69 +56,6 @@ namespace {
return ret;
}
// Global (joint) scaling: refine all per-image scales/mosaicities and the shared per-HKL
// intensities in one Ceres problem (GlobalScaleCeres), then write the result back into each
// reflection's image_scale_corr exactly as ScaleOnTheFly does - recompute the rotation
// partiality from the refined per-image mosaicity, then image_scale_corr = rlp/(partiality*G).
// The downstream rot3d combine and MergeOnTheFly are unchanged, so every metric stays
// comparable with the alternating path; only how G and mosaicity are found differs.
void RunGlobalScaling(const DiffractionExperiment &experiment,
std::vector<IntegrationOutcome> &outcomes, Logger &logger) {
const auto &s = experiment.GetScalingSettings();
const bool rotation = experiment.GetPartialityModel() == PartialityModel::Rotation;
std::vector<std::vector<Reflection>> observations;
observations.reserve(outcomes.size());
for (const auto &o: outcomes)
observations.push_back(o.reflections);
GlobalScaleOptions opt;
opt.merge_friedel = s.GetMergeFriedel();
if (const auto sg = experiment.GetGemmiSpaceGroup())
opt.space_group = *sg;
opt.d_min_limit_A = s.GetHighResolutionLimit_A().value_or(0.0);
opt.mosaicity_init_deg = s.GetDefaultMosaicity();
// The joint problem is large (per-image G + mosaicity + all Itrue + wedge); give the solver
// enough room to converge rather than the per-image default.
opt.max_solver_time_s = 300.0;
opt.max_num_iterations = 50;
double wedge = 0.0;
if (rotation) {
opt.partiality_model = GlobalScaleOptions::PartialityModel::Rotation;
wedge = experiment.GetRotationWedgeForScaling().value_or(0.0);
opt.wedge_deg = wedge;
opt.mosaicity_min_deg = s.GetMinMosaicity();
opt.mosaicity_max_deg = s.GetMaxMosaicity();
} else {
opt.partiality_model = GlobalScaleOptions::PartialityModel::Fixed;
}
const auto result = GlobalScaleCeres(observations, opt);
const double half_wedge = wedge / 2.0;
for (size_t i = 0; i < outcomes.size(); ++i) {
const double G = result.image_scale_g[i];
const double mos = result.mosaicity_deg[i];
for (auto &r: outcomes[i].reflections) {
if (rotation && std::isfinite(r.delta_phi_deg) && std::isfinite(r.zeta) && mos > 1e-6) {
const double c1 = r.zeta / std::sqrt(2.0);
r.partiality = static_cast<float>(
(std::erf((r.delta_phi_deg + half_wedge) * c1 / mos)
- std::erf((r.delta_phi_deg - half_wedge) * c1 / mos)) / 2.0);
}
const double denom = r.partiality * G;
r.image_scale_corr = (std::isfinite(r.rlp) && std::isfinite(denom) && denom > 0.0)
? static_cast<float>(r.rlp / denom) : NAN;
}
if (std::isfinite(G))
outcomes[i].image_scale_g = static_cast<float>(G);
if (rotation && std::isfinite(mos))
outcomes[i].mosaicity_deg = static_cast<float>(mos);
}
logger.Info("Global scaling complete ({} images)", outcomes.size());
}
// XDS-order scaling. The rot3d combine emits fulls with partiality == 1 (image_scale_corr == 1),
// so they were only ever scaled as per-frame *partials* upstream - their per-frame scale is
// entangled with the rocking-curve/partiality model. This refits a per-frame scale directly on
@@ -457,8 +393,21 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
// Scaling and merging (full analysis only).
if (full && !cancelled_ && result.indexing_rate.has_value() && result.indexing_rate > 0
&& (config_.run_scaling || !config_.reference_data.empty())) {
if (observer)
observer->OnPhase("Scaling and merging");
// Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
// bar reflects what is happening instead of freezing on one label. Also time each phase
// (logged on transition) so the bottlenecks are visible.
auto t_phase = std::chrono::steady_clock::now();
std::string prev_phase;
auto phase = [&](const std::string &p) {
const auto now = std::chrono::steady_clock::now();
if (!prev_phase.empty())
logger.Info("[timing] {}: {:.2f} s", prev_phase,
std::chrono::duration<double>(now - t_phase).count());
t_phase = now;
prev_phase = p;
if (observer) observer->OnPhase(p);
};
phase("Scaling and merging");
const bool pixel_refine_path =
experiment_.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine;
@@ -466,31 +415,41 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
// ScaleOnTheFly is only for the classical, no-reference path; with a reference (or
// PixelRefine) each image is already scaled, so we merge directly.
if (config_.reference_data.empty() && !pixel_refine_path) {
if (config_.global_scaling) {
logger.Info("Running global (joint) scaling ...");
RunGlobalScaling(experiment_, indexer->GetIntegrationOutcome(), logger);
} else {
logger.Info("Running scaling ...");
ScalingResult scale_result(0);
for (int i = 0; i < config_.scaling_iter; i++) {
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
scale_result = indexer->ScaleAllImages(merge_result);
}
logger.Info("Running scaling ...");
ScalingResult scale_result(0);
double t_merge_all = 0.0, t_scale = 0.0;
for (int i = 0; i < config_.scaling_iter; i++) {
phase("Scaling images (iteration " + std::to_string(i + 1) + "/"
+ std::to_string(config_.scaling_iter) + ")");
const auto a = std::chrono::steady_clock::now();
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
const auto b = std::chrono::steady_clock::now();
scale_result = indexer->ScaleAllImages(merge_result);
const auto c = std::chrono::steady_clock::now();
t_merge_all += std::chrono::duration<double>(b - a).count();
t_scale += std::chrono::duration<double>(c - b).count();
}
logger.Info("[timing] scaling loop ({} iter): MergeAll(serial) {:.2f} s, ScaleAllImages(parallel) {:.2f} s",
config_.scaling_iter, t_merge_all, t_scale);
}
// -P rot3d: weight-sum each reflection's per-frame partials into one full before merging, so
// the error model sees counting statistics (high ISa) instead of rocking-curve slicing scatter.
const bool rot3d = experiment_.GetScalingSettings().GetCombine3D();
std::vector<IntegrationOutcome> combined;
if (rot3d)
if (rot3d) {
phase("Combining 3D partials");
combined = CombineRotationObservations(indexer->GetIntegrationOutcome(), experiment_, &logger,
config_.observation_dump_path);
if (rot3d && config_.scale_fulls)
}
if (rot3d && experiment_.GetScalingSettings().GetScaleFulls()) {
phase("Scaling fulls (XDS order)");
ScaleFulls(experiment_, combined, static_cast<int>(config_.scaling_iter), config_.nthreads, logger);
}
const std::vector<IntegrationOutcome> &merge_input =
rot3d ? combined : indexer->GetIntegrationOutcome();
phase("Merging");
MergeOnTheFly merge_engine(experiment_);
if (result.consensus_cell.has_value())
merge_engine.ReferenceCell(*result.consensus_cell);
@@ -503,6 +462,7 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
merge_engine.AddImage(outcome);
auto merged_reflections = merge_engine.ExportReflections();
phase("Computing statistics");
auto merged_statistics = merge_engine.MergeStats(merged_reflections, merge_input,
config_.reference_data);
logger.Info("Merge complete ({} unique reflections)", merged_reflections.size());
@@ -523,8 +483,16 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
stats_text << merged_statistics;
result.merge_statistics_text = stats_text.str();
if (result.consensus_cell && write_files)
result.has_merge_statistics = true;
result.merge_statistics = merged_statistics;
result.error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
result.has_reference = !config_.reference_data.empty();
if (result.consensus_cell && write_files) {
phase("Writing reflections");
WriteReflections(merged_reflections, *result.consensus_cell, experiment_, config_.output_prefix);
}
phase(""); // flush the last phase's timing
}
if (writer) {