stills merge 2.3-2.5x faster, output unchanged: parallel observation passes, hash accumulator, no amplitudes on scaling references
The stills scale/merge after the image loop was almost entirely serial: - every scaling-reference merge (MergeAll: the per-image scaling reference and the two partiality post-refinement references) ran French-Wilson single-threaded on its export, about 3 s each, and assigned free-R flags - neither is read by a reference, which uses I only. MergeAll now exports intensities alone (MergeOnTheFly::ExportIntensities). - the merge accumulated into a std::map; it is now an unordered_map exported in key order, which is the std::map order. - AddImage, the error-model collection and the R_meas pass of MergeStats did the per-observation work (systematic-absence test, ASU key, filters, error-model sigma, outlier test, merged-I lookup) in one serial loop. They now reduce a block of images in parallel and consume it in image order (ReduceImagesInOrder), so every floating-point sum, half-set choice and hash-map insertion is made in the serial order. Measured on a 25k-frame XFEL stills run (box loaded, load 9-14): merge P1 19.0 -> 7.1 s, final-group merge 27.7 -> 13.2 s; on a second run 23.4 -> 7.7 s and 29.7 -> 13.6 s. mtz/unmerged mtz data, hkl, cif and report identical on both runs, mx and --mode scale. New test MergeOnTheFly_AddImagesIsTheSerialMerge checks the parallel merge against the serial loop across a block boundary. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include "../image_analysis/bragg_integration/CalcISigma.h"
|
||||
#include "../image_analysis/scale_merge/ErrorModel.h"
|
||||
@@ -585,3 +586,67 @@ TEST_CASE("Global Wilson B over a widened window, with no upper cap", "[wilson]"
|
||||
CHECK(1.0 / (3.9 * 3.9) - 1.0 / (low_res.d_low_A * low_res.d_low_A)
|
||||
== Catch::Approx(1.0 / 9.0 - 1.0 / 16.0).epsilon(0.01));
|
||||
}
|
||||
|
||||
// The parallel AddImages is the serial AddImage loop, bit for bit: the per-observation work is spread
|
||||
// over the workers, but every sum and every half-set choice is still made in image order. More images
|
||||
// than one of its blocks, so the block boundary is crossed.
|
||||
TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") {
|
||||
DiffractionExperiment x;
|
||||
x.SetSpaceGroup(TestSpaceGroup());
|
||||
|
||||
std::mt19937 rng(7);
|
||||
std::uniform_int_distribution<int> index(-6, 6);
|
||||
std::uniform_real_distribution<float> unit(0.0f, 1.0f);
|
||||
std::vector<IntegrationOutcome> outcomes(5000);
|
||||
for (auto &o: outcomes) {
|
||||
for (int j = 0; j < 20; ++j) {
|
||||
Reflection r{};
|
||||
r.h = index(rng);
|
||||
r.k = index(rng);
|
||||
r.l = index(rng);
|
||||
if (r.h == 0 && r.k == 0 && r.l == 0)
|
||||
continue;
|
||||
r.d = 20.0f / std::sqrt(static_cast<float>(r.h * r.h + r.k * r.k + r.l * r.l));
|
||||
r.I = 100.0f * unit(rng) - 5.0f;
|
||||
r.sigma = 1.0f + 5.0f * unit(rng);
|
||||
r.var_bkg = r.sigma * r.sigma * unit(rng);
|
||||
r.prescaling_corr = 1.0f;
|
||||
r.partiality = 0.5f + 0.5f * unit(rng);
|
||||
r.image_scale_corr = 0.5f + unit(rng);
|
||||
r.on_ice_ring = unit(rng) < 0.05f;
|
||||
o.reflections.push_back(r);
|
||||
}
|
||||
}
|
||||
|
||||
MergeOnTheFly serial(x);
|
||||
serial.RefineErrorModel(outcomes);
|
||||
for (size_t i = 0; i < outcomes.size(); ++i)
|
||||
serial.AddImage(outcomes[i], static_cast<int64_t>(i));
|
||||
const auto serial_merged = serial.ExportReflections();
|
||||
|
||||
MergeOnTheFly parallel(x);
|
||||
parallel.Threads(8);
|
||||
parallel.RefineErrorModel(outcomes);
|
||||
parallel.AddImages(outcomes);
|
||||
const auto parallel_merged = parallel.ExportReflections();
|
||||
|
||||
REQUIRE(serial.ErrorModelActive());
|
||||
CHECK(serial.ErrorModelA() == parallel.ErrorModelA());
|
||||
CHECK(serial.ErrorModelB() == parallel.ErrorModelB());
|
||||
REQUIRE(serial_merged.size() == parallel_merged.size());
|
||||
for (size_t i = 0; i < serial_merged.size(); ++i) {
|
||||
CHECK(serial_merged[i].h == parallel_merged[i].h);
|
||||
CHECK(serial_merged[i].k == parallel_merged[i].k);
|
||||
CHECK(serial_merged[i].l == parallel_merged[i].l);
|
||||
CHECK(serial_merged[i].I == parallel_merged[i].I);
|
||||
CHECK(serial_merged[i].sigma == parallel_merged[i].sigma);
|
||||
for (int half = 0; half < 2; ++half) // NaN where a half is empty, on both sides
|
||||
CHECK(std::memcmp(&serial_merged[i].I_half[half], ¶llel_merged[i].I_half[half], sizeof(float)) == 0);
|
||||
}
|
||||
|
||||
const auto serial_stats = serial.MergeStats(serial_merged, outcomes);
|
||||
const auto parallel_stats = parallel.MergeStats(parallel_merged, outcomes);
|
||||
CHECK(serial_stats.overall.total_observations == parallel_stats.overall.total_observations);
|
||||
CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half);
|
||||
CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user