leonarski_fandClaude Opus 4.8
c5cfdcef7e
Rotation scaling modelled dose-dependent scattering power with a single global relative-B decay slope only (RefineDecay); per-frame refinement was scale-only. Add a per-batch relative Debye-Waller B, both as an opt-in correction and as an always-on measurement. - FitRelativeBCurve / SolveCurvatureSmoothedB: per rotation-range batch, fit the s^2 slope of ln(Iref/Iobs) that the resolution-flat per-frame G leaves, regularised by a second-difference (curvature) penalty so a genuine smooth relative-B is kept while resolution-correlated noise (which an unconstrained per-batch fit would chase into an oscillating curve) is suppressed. - Correction (--relative-b[=deg], default off): applied after RefineDecay, gated by a physical peak-to-peak floor and cross-validated by ASU-group parity (a per-batch parameter owns whole frames, so it is scored on held-out equivalents, not held-out frames). On a battery re-scaling identical stored reflections it is a no-op on well-behaved data and a small gain on radiation-damaged rotation data; never a regression. - Radiation-damage monitor (always on for rotation, report-only): measure the relative-B change from the pristine low-dose start of the run to its end (a whole-run-mean reference inverts on damaged data) plus the per-batch curve, and report them on the command line (in the radiation-damage report and the merge statistics) and in the mmCIF (_reflns.jfjoch_radiation_damage_ relative_B and a per-batch loop). A large magnitude flags radiation damage, complementing the existing per-image CC / mosaicity read. - --scale now reproduces the de-novo rotation merge (it previously omitted scale-fulls and smooth-G, silently using a weaker model), and honours --relative-b, so an offline re-scale matches the full pipeline. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1454 lines
85 KiB
C++
1454 lines
85 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "Rugnux.h"
|
|
#include "ModelValidation.h"
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <functional>
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <numeric>
|
|
#include <set>
|
|
#include <sstream>
|
|
|
|
#include "../reader/JFJochHDF5Reader.h"
|
|
#include "../common/Logger.h"
|
|
#include "../common/AzimuthalIntegrationMapping.h"
|
|
#include "../common/AzimuthalIntegrationProfile.h"
|
|
#include "../common/CUDAWrapper.h"
|
|
#include "../common/time_utc.h"
|
|
#include "../writer/FileWriter.h"
|
|
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
|
#include "../image_analysis/IndexAndRefine.h"
|
|
#include "../image_analysis/geom_refinement/GeometryRefiner.h"
|
|
#include "../image_analysis/indexing/IndexerThreadPool.h"
|
|
#include "../image_analysis/azint/AzIntEngineCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
|
|
#include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
|
|
#include "../image_analysis/scale_merge/Merge.h"
|
|
#include "../image_analysis/scale_merge/RfreeFlags.h"
|
|
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
|
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
|
|
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
|
|
#include "../image_analysis/scale_merge/ScalingResult.h"
|
|
#include "../image_analysis/scale_merge/SearchSpaceGroup.h"
|
|
#include "../image_analysis/geom_refinement/PostRefine.h"
|
|
#include "../image_analysis/lattice_search/LatticeSearch.h"
|
|
#include "../image_analysis/scale_merge/TwinningAnalysis.h"
|
|
#include "../image_analysis/scale_merge/HKLKey.h"
|
|
#include "../image_analysis/WriteReflections.h"
|
|
#include "../image_analysis/bragg_integration/CalcISigma.h"
|
|
#include "../common/Definitions.h"
|
|
#include "../common/CorrelationCoefficient.h"
|
|
#include <array>
|
|
#include <map>
|
|
|
|
namespace {
|
|
// Pick up to requested_images ordinals spread evenly across [0, images_to_process) for the
|
|
// first pass of two-pass rotation indexing.
|
|
std::vector<int> select_equally_spaced_image_ordinals(int images_to_process, int requested_images) {
|
|
std::vector<int> ret;
|
|
if (images_to_process <= 0 || requested_images <= 0)
|
|
return ret;
|
|
|
|
const int n = std::min(images_to_process, requested_images);
|
|
if (n == 1) {
|
|
ret.push_back(0);
|
|
return ret;
|
|
}
|
|
|
|
std::set<int> unique_ordinals;
|
|
for (int i = 0; i < n; i++)
|
|
unique_ordinals.insert(static_cast<int>(
|
|
std::llround(static_cast<double>(i) * static_cast<double>(images_to_process - 1) /
|
|
static_cast<double>(n - 1))));
|
|
|
|
ret.assign(unique_ordinals.begin(), unique_ordinals.end());
|
|
return ret;
|
|
}
|
|
|
|
}
|
|
|
|
Rugnux::Rugnux(JFJochHDF5Reader &reader, DiffractionExperiment experiment,
|
|
PixelMask pixel_mask, ProcessConfig config)
|
|
: reader_(reader), experiment_(std::move(experiment)),
|
|
pixel_mask_(std::move(pixel_mask)), config_(std::move(config)) {}
|
|
|
|
void Rugnux::RefineStillsGeometry(int start_image, int end_image, int images_to_process,
|
|
RugnuxObserver *observer) {
|
|
Logger logger("Rugnux");
|
|
|
|
// Rotation has its own two-pass; this is stills only.
|
|
if (experiment_.IsRotationIndexing()) {
|
|
logger.Warning("--refine-geometry is stills-only; rotation data uses two-pass indexing. Ignoring.");
|
|
return;
|
|
}
|
|
// The bundle anchors a known cell; without one there is nothing to refine against.
|
|
const auto cell = experiment_.GetUnitCell();
|
|
if (!cell.has_value()) {
|
|
logger.Warning("--refine-geometry needs a reference unit cell (-C / -S / reference MTZ). Skipping.");
|
|
return;
|
|
}
|
|
|
|
const int refine_frames = std::max(1, config_.refine_geometry.value());
|
|
if (observer)
|
|
observer->OnPhase("Geometry refinement (first pass)");
|
|
|
|
const auto dataset = reader_.GetDataset();
|
|
|
|
// Index a spread sample large enough to yield ~refine_frames strong frames even at a low hit rate.
|
|
const int sample_budget = std::min(images_to_process, std::max(refine_frames * 50, 8000));
|
|
const std::vector<int> sample = select_equally_spaced_image_ordinals(images_to_process, sample_budget);
|
|
|
|
// First-pass engines, built from the current (nominal) geometry: an index-only pass (no
|
|
// integration) to obtain each frame's spots + assigned HKL + orientation.
|
|
AzimuthalIntegrationMapping mapping(experiment_, pixel_mask_);
|
|
IndexerThreadPool pool(experiment_.GetIndexingSettings());
|
|
IndexAndRefine indexer(experiment_, &pool, /*retain_outcomes=*/false);
|
|
|
|
auto pass_settings = config_.spot_finding;
|
|
pass_settings.enable = true;
|
|
pass_settings.indexing = true;
|
|
pass_settings.quick_integration = false;
|
|
|
|
std::vector<GeomRefineFrame> frames;
|
|
std::mutex frames_mutex;
|
|
std::atomic<int> next_i = 0;
|
|
|
|
auto worker = [&]() {
|
|
pin_gpu(); // round-robin per worker thread; must precede engine construction
|
|
MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, indexer);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
|
|
while (!cancelled_) {
|
|
const int idx = next_i.fetch_add(1);
|
|
if (idx >= static_cast<int>(sample.size())) break;
|
|
const int ordinal = sample[idx];
|
|
const int image_idx = start_image + ordinal * config_.stride;
|
|
|
|
std::shared_ptr<JFJochReaderRawImage> img;
|
|
try {
|
|
img = reader_.GetRawImage(image_idx);
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("Geometry refinement: failed to load image {}: {}", image_idx, e.what());
|
|
continue;
|
|
}
|
|
if (!img) continue;
|
|
|
|
DataMessage msg{};
|
|
msg.image = img->image;
|
|
msg.number = ordinal;
|
|
msg.original_number = image_idx;
|
|
if (dataset->efficiency.size() > image_idx)
|
|
msg.image_collection_efficiency = dataset->efficiency[image_idx];
|
|
|
|
try {
|
|
analysis.Analyze(msg, profile, pass_settings);
|
|
} catch (const std::exception &e) {
|
|
continue;
|
|
}
|
|
if (!msg.indexing_result.value_or(false) || !msg.indexing_lattice.has_value())
|
|
continue;
|
|
|
|
GeomRefineFrame f;
|
|
f.lattice = *msg.indexing_lattice;
|
|
for (const auto &s : msg.spots)
|
|
if (s.indexed && s.lattice == 0)
|
|
f.spots.push_back(GeomRefineSpot{s.x, s.y,
|
|
static_cast<int32_t>(s.h), static_cast<int32_t>(s.k), static_cast<int32_t>(s.l)});
|
|
if (f.spots.size() >= 6) {
|
|
const std::unique_lock ul(frames_mutex);
|
|
frames.push_back(std::move(f));
|
|
}
|
|
}
|
|
};
|
|
|
|
std::vector<std::future<void>> futures;
|
|
futures.reserve(config_.nthreads);
|
|
for (int i = 0; i < config_.nthreads; ++i)
|
|
futures.push_back(std::async(std::launch::async, worker));
|
|
for (auto &fut : futures)
|
|
fut.get();
|
|
|
|
if (cancelled_)
|
|
return;
|
|
|
|
// Keep the strongest frames (most indexed spots) for the bundle - the true cell indexes many
|
|
// spots per frame, and orientation diversity comes for free from independent serial stills.
|
|
constexpr int MIN_STRONG_SPOTS = 20;
|
|
std::vector<GeomRefineFrame> strong;
|
|
for (auto &f : frames)
|
|
if (static_cast<int>(f.spots.size()) >= MIN_STRONG_SPOTS)
|
|
strong.push_back(std::move(f));
|
|
std::sort(strong.begin(), strong.end(),
|
|
[](const GeomRefineFrame &a, const GeomRefineFrame &b) { return a.spots.size() > b.spots.size(); });
|
|
if (static_cast<int>(strong.size()) > refine_frames)
|
|
strong.resize(refine_frames);
|
|
|
|
logger.Info("Geometry refinement: indexed {} of {} sampled frames, bundling {} strong frames (>= {} spots)",
|
|
frames.size(), sample.size(), strong.size(), MIN_STRONG_SPOTS);
|
|
|
|
GeometryRefinerSettings gr_settings;
|
|
gr_settings.crystal_system = experiment_.GetCrystalSystem();
|
|
gr_settings.num_threads = config_.nthreads;
|
|
|
|
const GeometryRefinerResult r = RefineGlobalGeometry(
|
|
experiment_.GetDiffractionGeometry(), *cell, strong, gr_settings);
|
|
if (!r.ok) {
|
|
logger.Warning("Geometry refinement did not run (too few strong frames / did not converge); "
|
|
"keeping the input geometry");
|
|
return;
|
|
}
|
|
|
|
logger.Info("Geometry refinement: beam ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, distance {:.4f} -> {:.4f} mm",
|
|
experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(), r.beam_x_px, r.beam_y_px,
|
|
experiment_.GetDetectorDistance_mm(), r.distance_mm);
|
|
logger.Info("Geometry refinement: cell a/b/c {:.3f}/{:.3f}/{:.3f} -> {:.3f}/{:.3f}/{:.3f} A "
|
|
"(median residual {:.3f} px, {} frames, {} spots)",
|
|
cell->a, cell->b, cell->c, r.cell.a, r.cell.b, r.cell.c,
|
|
r.median_residual_px, r.frames_used, r.spots_used);
|
|
|
|
experiment_.BeamX_pxl(r.beam_x_px).BeamY_pxl(r.beam_y_px).DetectorDistance_mm(r.distance_mm);
|
|
experiment_.SetUnitCell(r.cell);
|
|
}
|
|
|
|
ProcessResult Rugnux::Run(RugnuxObserver *observer) {
|
|
// Rotation two-pass geometry post-refinement: the first pass integrates at the header geometry and
|
|
// post-refines the detector geometry (distance + beam from the observed spot positions, cell scale + axis
|
|
// from phi_obs); the second pass re-indexes and re-integrates with the refined geometry. BOTH passes are
|
|
// written out - "<prefix>_01_*" is the first (header-geometry) pass and "<prefix>_02_*" the second (refined
|
|
// geometry) - so the two can be compared and the better one kept, and the ice-ring flags / geometry stored
|
|
// in each master reflect the geometry that pass actually used.
|
|
if (config_.mode == ProcessMode::FullAnalysis && config_.rotation_postrefine_geometry
|
|
&& experiment_.IsRotationIndexing()) {
|
|
Logger logger("Rugnux");
|
|
const std::string base_prefix = config_.output_prefix;
|
|
const auto gonio_snapshot = experiment_.GetGoniometer();
|
|
prepass_detector_geometry_.reset();
|
|
prepass_sg_reindexed_ = false;
|
|
prepass_merge_sg_.reset();
|
|
|
|
logger.Info("Rotation two-pass geometry post-refinement: first pass (header geometry) -> {}_01_*",
|
|
base_prefix);
|
|
config_.output_prefix = base_prefix + "_01";
|
|
auto pass1 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/true);
|
|
if (cancelled_) { config_.output_prefix = base_prefix; return pass1; }
|
|
if (gonio_snapshot) experiment_.Goniometer(*gonio_snapshot); // undo the pre-pass goniometer shift
|
|
|
|
// Apply the post-refined detector geometry for the second pass.
|
|
if (prepass_detector_geometry_) {
|
|
const auto &g = *prepass_detector_geometry_;
|
|
experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2]);
|
|
}
|
|
// Space group: the second pass RE-INDEXES DE NOVO (clear the group here) so the indexer's pseudo-
|
|
// symmetry safeguards recover the true cell - reusing pass-1's group in the indexer forces build_sr
|
|
// onto a wrong / doubled cell (a huge oblique cell collapses; a pseudo-centred cell doubles). Pass-1's
|
|
// group is instead reinstated for the MERGE ONLY (RunPipeline, before RotationScaleMerge), so the
|
|
// symmetry stays fixed with no re-search / no flip - EXCEPT a reindex-derived group, whose conventional
|
|
// setting the de-novo primitive frame does not share, which is left to re-search.
|
|
experiment_.SpaceGroupNumber(std::nullopt);
|
|
prepass_merge_sg_ = prepass_sg_reindexed_ ? std::nullopt : pass1.space_group_number;
|
|
|
|
logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry) -> {}_02_*",
|
|
base_prefix);
|
|
config_.output_prefix = base_prefix + "_02";
|
|
auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
|
config_.output_prefix = base_prefix;
|
|
return pass2;
|
|
}
|
|
return RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
|
}
|
|
|
|
ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
|
|
Logger logger("Rugnux");
|
|
ProcessResult result;
|
|
|
|
const auto dataset = reader_.GetDataset();
|
|
if (!dataset)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"No experiment dataset found in the input file");
|
|
|
|
if (config_.stride <= 0)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image stride must be positive");
|
|
|
|
const auto total_images_in_file = static_cast<int>(reader_.GetNumberOfImages());
|
|
int end_image = config_.end_image;
|
|
if (end_image < 0 || end_image > total_images_in_file)
|
|
end_image = total_images_in_file;
|
|
const int start_image = config_.start_image;
|
|
const int images_to_process = (end_image - start_image) / config_.stride;
|
|
if (images_to_process <= 0) {
|
|
logger.Warning("No images to process (start {}, end {}, stride {}, total {})",
|
|
start_image, end_image, config_.stride, total_images_in_file);
|
|
return result;
|
|
}
|
|
|
|
const bool full = (config_.mode == ProcessMode::FullAnalysis);
|
|
const bool write_files = write_output && !config_.output_prefix.empty();
|
|
|
|
// Output/runtime invariants. Algorithm settings (indexing, scaling, integration, polarization,
|
|
// space group, unit cell, ...) are configured on experiment_ by the caller.
|
|
experiment_.BitDepthImage(32).PixelSigned(true);
|
|
experiment_.Mode(DetectorMode::Standard);
|
|
experiment_.OverwriteExistingFiles(true);
|
|
// Offline processing: the output prefix is a local path the operator chose (rugnux -o or the
|
|
// viewer's "next to the input file"), so use the trusted setter that permits an absolute path.
|
|
experiment_.FilePrefixTrusted(config_.output_prefix.empty() ? "output" : config_.output_prefix);
|
|
experiment_.SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
|
|
experiment_.ImagesPerTrigger(images_to_process);
|
|
experiment_.NumTriggers(1);
|
|
if (full)
|
|
experiment_.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
|
|
// The pipeline indexes images 0..N-1 within this run; if we process a sub-range/strided
|
|
// selection, shift the goniometer so local index i maps to the angle of original image
|
|
// start+i*stride (keeping the per-image rotation wedge), otherwise rotation angles would be
|
|
// wrong for any start_image != 0.
|
|
if (const auto g = experiment_.GetGoniometer();
|
|
g.has_value() && (start_image != 0 || config_.stride != 1)) {
|
|
const float incr = g->GetIncrement_deg();
|
|
GoniometerAxis shifted(g->GetName(),
|
|
g->GetStart_deg() + incr * static_cast<float>(start_image),
|
|
incr * static_cast<float>(config_.stride),
|
|
g->GetAxis(), g->GetHelicalStep());
|
|
shifted.ScreeningWedge(g->GetScreeningWedge().value_or(incr));
|
|
experiment_.Goniometer(shifted);
|
|
}
|
|
|
|
// Stills global geometry-refinement first pass: bundle-adjust the shared beam/distance/cell from a
|
|
// sample of strongly-indexed frames and apply it to experiment_, so everything below (azimuthal
|
|
// mapping, the main index+integrate pass, scaling) uses the refined geometry. Rotation is skipped
|
|
// inside (it has its own two-pass).
|
|
if (full && config_.refine_geometry.has_value())
|
|
RefineStillsGeometry(start_image, end_image, images_to_process, observer);
|
|
|
|
AzimuthalIntegrationMapping mapping(experiment_, pixel_mask_);
|
|
|
|
JFJochReceiverPlots plots;
|
|
plots.Setup(experiment_, mapping);
|
|
|
|
// Output file (NXmxIntegrated master that links back to the original images).
|
|
StartMessage start_message;
|
|
experiment_.FillMessage(start_message);
|
|
start_message.arm_date = dataset->arm_date;
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
start_message.az_int_bin_to_two_theta = mapping.GetBinToTwoTheta();
|
|
start_message.az_int_q_bin_count = mapping.GetQBinCount();
|
|
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
|
|
if (mapping.GetAzimuthalBinCount() > 1)
|
|
start_message.az_int_bin_to_phi = mapping.GetBinToPhi();
|
|
start_message.pixel_mask["default"] = pixel_mask_.GetMask(experiment_);
|
|
if (full) {
|
|
start_message.rois = experiment_.ROI().ExportMetadata();
|
|
if (!experiment_.ROI().empty())
|
|
start_message.roi_map = experiment_.ExportROIMap();
|
|
start_message.max_spot_count = experiment_.GetMaxSpotCount();
|
|
}
|
|
start_message.master_suffix = "process";
|
|
start_message.file_format = FileWriterFormat::NXmxIntegrated;
|
|
start_message.write_master_file = true;
|
|
start_message.write_images = false;
|
|
start_message.hdf5_source_data = reader_.GetHDF5DataSource(start_image, images_to_process);
|
|
|
|
std::unique_ptr<FileWriter> writer;
|
|
if (write_files && config_.write_process_h5)
|
|
writer = std::make_unique<FileWriter>(start_message, /*check_overwrite_at_start=*/true,
|
|
/*trusted_path=*/true);
|
|
|
|
logger.Info("Processing {} images (range {}-{}, stride {}) using {} threads [{}]",
|
|
images_to_process, start_image, end_image, config_.stride, config_.nthreads,
|
|
full ? "full analysis" : "azimuthal integration");
|
|
if (observer)
|
|
observer->OnPhase(full ? "Full analysis" : "Azimuthal integration");
|
|
|
|
// Full-analysis shared engines.
|
|
std::unique_ptr<IndexerThreadPool> indexer_pool;
|
|
std::unique_ptr<IndexAndRefine> indexer;
|
|
if (full) {
|
|
auto indexing_settings = experiment_.GetIndexingSettings();
|
|
// Two-pass rotation fires the FFT indexer only twice - the two first-pass schemes, run in
|
|
// parallel (the main per-image loop then uses the forced lattice, no FFT). Size the pool to
|
|
// those two so a rotation run doesn't pay to initialise (cuFFT plan + GPU alloc) indexers it
|
|
// never uses. Stills fire the FFT once per image across all workers, so keep the default there.
|
|
if (config_.rotation_indexing && config_.two_pass_rotation && experiment_.IsRotationIndexing())
|
|
indexing_settings.IndexingThreads(2);
|
|
indexer_pool = std::make_unique<IndexerThreadPool>(indexing_settings);
|
|
indexer = std::make_unique<IndexAndRefine>(experiment_, indexer_pool.get());
|
|
if (!config_.reference_data.empty())
|
|
indexer->ReferenceIntensities(config_.reference_data);
|
|
// Second pass of the rotation two-pass: predict with the smoothed mosaicity fitted in the pre-pass.
|
|
if (!geometry_prepass && !prepass_mosaicity_.empty())
|
|
indexer->SetPredictionMosaicityOverride(prepass_mosaicity_);
|
|
}
|
|
|
|
const auto start_time = std::chrono::steady_clock::now();
|
|
|
|
// First pass of two-pass rotation indexing (full analysis only).
|
|
if (full && config_.forced_rotation_lattice.has_value()) {
|
|
indexer->ForceRotationIndexerLattice(*config_.forced_rotation_lattice);
|
|
logger.Info("Rotation indexer lattice forced externally - skipping first pass");
|
|
} else if (full && config_.rotation_indexing && config_.two_pass_rotation) {
|
|
if (observer)
|
|
observer->OnPhase("Rotation indexing (first pass)");
|
|
|
|
// Mid-exposure goniometer angle for an image ordinal (matches the per-image path).
|
|
auto rot_angle = [&](int ordinal) -> std::optional<float> {
|
|
if (const auto g = experiment_.GetGoniometer())
|
|
return g->GetAngle_deg(static_cast<float>(ordinal)) + g->GetWedge_deg() / 2.0f;
|
|
return std::nullopt;
|
|
};
|
|
|
|
// Reuse stored spots only if the file actually has them: a plain DECTRIS dataset carries no
|
|
// spot-finding results, so "reusing" would leave every frame with zero spots and fail
|
|
// indexing - find them instead. (--redo-rotation-spots already forces finding.)
|
|
const bool reuse_spots = config_.reuse_rotation_spots && reader_.HasStoredSpots();
|
|
if (config_.reuse_rotation_spots && !reuse_spots)
|
|
logger.Info("Dataset has no stored spots - running spot finding for first-pass rotation indexing");
|
|
|
|
// Spots for a first-pass image ordinal, cached (a frame is reused across schemes and the
|
|
// validation set; re-finding is expensive on the --redo-rotation-spots path). Accessed only
|
|
// from single-threaded sections (the scheme feed and the validation loop), so no locking.
|
|
std::map<int, std::vector<SpotToSave>> spot_cache;
|
|
auto get_spots = [&](int ordinal) -> const std::vector<SpotToSave> & {
|
|
auto it = spot_cache.find(ordinal);
|
|
if (it != spot_cache.end())
|
|
return it->second;
|
|
const int image_idx = start_image + ordinal * config_.stride;
|
|
std::vector<SpotToSave> spots;
|
|
try {
|
|
if (reuse_spots) {
|
|
spots = reader_.ReadSpots(image_idx);
|
|
} else if (auto img = reader_.GetRawImage(image_idx)) {
|
|
DataMessage m{};
|
|
m.number = ordinal;
|
|
m.original_number = image_idx;
|
|
MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, *indexer);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
auto first_pass = config_.spot_finding;
|
|
first_pass.indexing = false;
|
|
first_pass.quick_integration = false;
|
|
m.image = img->image;
|
|
if (dataset->efficiency.size() > image_idx)
|
|
m.image_collection_efficiency = dataset->efficiency[image_idx];
|
|
analysis.Analyze(m, profile, first_pass);
|
|
spots = std::move(m.spots);
|
|
}
|
|
} catch (const std::exception &e) {
|
|
logger.Warning("First-pass spot read failed for image {}: {}", image_idx, e.what());
|
|
}
|
|
return spot_cache.emplace(ordinal, std::move(spots)).first->second;
|
|
};
|
|
|
|
// How many validation frames a candidate global lattice actually indexes per image, scored on
|
|
// the *real* per-image path (force the candidate, then run the same indexing+refinement the
|
|
// main loop uses). A cell that merely fits the accumulated first-pass cloud (a spurious
|
|
// sub-lattice from an under-sampled scheme) indexes few frames, while the true cell indexes
|
|
// many - this is the discriminator that lets the best scheme win.
|
|
const std::vector<int> validation = select_equally_spaced_image_ordinals(
|
|
images_to_process, std::min(images_to_process, 60));
|
|
auto validation_settings = config_.spot_finding;
|
|
validation_settings.indexing = true;
|
|
validation_settings.quick_integration = false;
|
|
auto count_indexed = [&](IndexAndRefine &idx, const RotationIndexerResult &r) -> int {
|
|
idx.ForceRotationIndexerResult(r);
|
|
int count = 0;
|
|
for (const int ordinal : validation) {
|
|
DataMessage m{};
|
|
m.number = ordinal;
|
|
m.spots = get_spots(ordinal);
|
|
if (idx.IndexFrameOnly(m, validation_settings))
|
|
count++;
|
|
}
|
|
return count;
|
|
};
|
|
|
|
// Feed one first-pass scheme (a set of image ordinals) into its own rotation indexer, ready to
|
|
// be indexed. Spots are pulled from the single-threaded cache here; the FFT + refinement runs
|
|
// separately (RunIndexing) so the two schemes' indexing can overlap.
|
|
auto feed_scheme = [&](IndexerThreadPool &pool, const std::vector<int> &ordinals) {
|
|
auto ri = std::make_unique<RotationIndexer>(experiment_, pool);
|
|
for (const int ordinal : ordinals) {
|
|
if (cancelled_ || ri->AccumulationFull())
|
|
break;
|
|
ri->ProcessImage(ordinal, get_spots(ordinal), rot_angle(ordinal));
|
|
}
|
|
return ri;
|
|
};
|
|
|
|
// Sampling schemes, tried in order of preference. We index each and keep the one that indexes
|
|
// the most frames; the schemes cover complementary failure modes and no single one is best for
|
|
// every crystal:
|
|
// - "spread": images spread over the whole rotation - full angular range, coarse stride; the
|
|
// long-standing default, best for typical crystals.
|
|
// - "wedge": consecutive frames from the clean start - native stride keeps each reflection's
|
|
// rocking curve continuous across frames, so the FFT can trace a long axis whose fine
|
|
// reciprocal spacing the coarse spread cannot resolve (e.g. an F-cubic cell viewed down
|
|
// [111]); bounded by the accumulation cap to the least radiation-damaged early wedge.
|
|
std::vector<int> all_consecutive(images_to_process);
|
|
std::iota(all_consecutive.begin(), all_consecutive.end(), 0);
|
|
const std::vector<int> spread_ordinals = select_equally_spaced_image_ordinals(
|
|
images_to_process, config_.rotation_indexing_image_count);
|
|
const std::vector<std::pair<std::string, std::vector<int>>> schemes = {
|
|
{"spread", spread_ordinals},
|
|
{"wedge", all_consecutive},
|
|
};
|
|
|
|
struct FirstPass { std::optional<RotationIndexerResult> result; int score = -1; std::string name; double vol = 0.0; };
|
|
|
|
// Feed both schemes (single-threaded spot cache), run their RunIndexing() concurrently, then keep
|
|
// the lattice that indexes the most validation frames on the real per-image path.
|
|
auto pick_best = [&](IndexerThreadPool &pool, IndexAndRefine &idx) -> FirstPass {
|
|
std::vector<std::unique_ptr<RotationIndexer>> ris;
|
|
ris.reserve(schemes.size());
|
|
for (const auto &[name, ordinals] : schemes) {
|
|
if (cancelled_)
|
|
break;
|
|
ris.push_back(feed_scheme(pool, ordinals));
|
|
}
|
|
std::vector<std::future<void>> index_futures;
|
|
index_futures.reserve(ris.size());
|
|
for (auto &ri : ris) {
|
|
RotationIndexer *rp = ri.get();
|
|
index_futures.push_back(std::async(std::launch::async, [rp] { rp->RunIndexing(); }));
|
|
}
|
|
for (auto &f : index_futures)
|
|
f.get();
|
|
|
|
FirstPass bp;
|
|
for (size_t i = 0; i < ris.size(); i++) {
|
|
if (cancelled_)
|
|
break;
|
|
const auto result = ris[i]->GetLattice();
|
|
if (!result.has_value())
|
|
continue;
|
|
const int score = count_indexed(idx, *result);
|
|
const double vol = std::abs(result->lattice.CalcVolume());
|
|
const std::string &name = schemes[i].first;
|
|
logger.Info("First-pass scheme '{}': indexes {}/{} validation frames", name, score,
|
|
static_cast<int>(validation.size()));
|
|
|
|
// A later scheme wins if it indexes clearly more frames (>10%).
|
|
const bool clearly_more = static_cast<float>(score) > bp.score * 1.1f + 0.5f;
|
|
|
|
// Integer-supercell tie-break. The validation-frame count saturates - a spurious axis
|
|
// multiple (2x/3x...) indexes every frame its true cell does, so both schemes reach the
|
|
// same frame total and the count alone cannot tell them apart; the default then keeps
|
|
// whichever ran first. When the two schemes tie on frames but their cell volumes are
|
|
// related by an integer factor >=2, the larger cell is that spurious supercell and the
|
|
// smaller is the true reduced cell: take it, regardless of scheme order. A near-integer
|
|
// ratio separates a real axis multiplication from a centering coincidence (a rhombohedral
|
|
// H cell vs its C2 sub-cell is 1.5x, non-integer, and is correctly left alone).
|
|
bool integer_subcell = false;
|
|
if (bp.result.has_value() && !clearly_more && vol > 1.0 && bp.vol > 1.0) {
|
|
const bool tied = static_cast<float>(score) >= bp.score * 0.9f - 0.5f;
|
|
const double ratio = (vol < bp.vol) ? bp.vol / vol : vol / bp.vol;
|
|
const double nearest = std::round(ratio);
|
|
const bool integer_multiple = nearest >= 2.0 && std::abs(ratio - nearest) < 0.15;
|
|
integer_subcell = tied && integer_multiple && vol < bp.vol;
|
|
}
|
|
|
|
if (!bp.result.has_value() || clearly_more || integer_subcell) {
|
|
if (integer_subcell)
|
|
logger.Info("Scheme '{}' cell (vol {:.0f}) is a {:.0f}x sub-cell of '{}' (vol {:.0f}) at "
|
|
"equal frame count - adopting the smaller true cell", name, vol,
|
|
std::round(bp.vol / vol), bp.name, bp.vol);
|
|
bp.score = score;
|
|
bp.vol = vol;
|
|
bp.name = name;
|
|
bp.result = result;
|
|
}
|
|
}
|
|
return bp;
|
|
};
|
|
|
|
FirstPass best = pick_best(*indexer_pool, *indexer);
|
|
|
|
// Long-axis rescue. When the de-novo cell indexes few validation frames, a long, finely-spaced
|
|
// axis was likely lost: the unconstrained FFT either collapsed it to a short sub-multiple or let
|
|
// a denser supercell over-fit the accumulated cloud (a small global-orientation error throws the
|
|
// many high-order reflections off along the fine axis, so the true cell scores worst on the raw
|
|
// cloud). Recover the true metric with a COARSE-resolution pass - only low-order reflections,
|
|
// where the fine axis stays robust - then RE-INDEX at full resolution constrained by that cell as
|
|
// a reference (the -C path): the reference filter drops the collapsed/supercell candidates and
|
|
// refines an accurate global lattice. Only runs after a poor standard pass, so well-indexing
|
|
// crystals never pay for it and their result is untouched.
|
|
auto max_axis = [](const RotationIndexerResult &r) {
|
|
const auto uc = r.lattice.GetUnitCell();
|
|
return std::max({uc.a, uc.b, uc.c});
|
|
};
|
|
if (!cancelled_ && best.result.has_value()
|
|
&& best.score < 0.5 * static_cast<double>(validation.size())) {
|
|
auto coarse_settings = experiment_.GetIndexingSettings();
|
|
coarse_settings.FFT_HighResolution_A(3.5f); // low-order reflections only -> robust long axis
|
|
coarse_settings.IndexingThreads(2);
|
|
IndexerThreadPool coarse_pool(coarse_settings);
|
|
// Coarse first pass: keep the recovered cell with the LONGEST axis directly. Its full-
|
|
// resolution per-frame validation would be low (the coarse cell is metrically right but
|
|
// imprecise), so do NOT score it here - it is only the reference for the constrained re-index.
|
|
std::vector<std::unique_ptr<RotationIndexer>> cris;
|
|
cris.reserve(schemes.size());
|
|
for (const auto &[name, ordinals] : schemes) {
|
|
if (cancelled_) break;
|
|
cris.push_back(feed_scheme(coarse_pool, ordinals));
|
|
}
|
|
std::vector<std::future<void>> cfut;
|
|
cfut.reserve(cris.size());
|
|
for (auto &ri : cris) {
|
|
RotationIndexer *rp = ri.get();
|
|
cfut.push_back(std::async(std::launch::async, [rp] { rp->RunIndexing(); }));
|
|
}
|
|
for (auto &f : cfut) f.get();
|
|
std::optional<RotationIndexerResult> coarse_ref;
|
|
for (auto &ri : cris) {
|
|
auto r = ri->GetLattice();
|
|
if (r.has_value() && (!coarse_ref.has_value() || max_axis(*r) > max_axis(*coarse_ref)))
|
|
coarse_ref = r;
|
|
}
|
|
// A recovered axis at least 30% longer than the standard pass found signals a rescued metric.
|
|
if (!cancelled_ && coarse_ref.has_value()
|
|
&& max_axis(*coarse_ref) > 1.3 * max_axis(*best.result)) {
|
|
logger.Info("Long-axis rescue: coarse pass recovered a {:.0f} A axis (was {:.0f} A); "
|
|
"re-indexing constrained by that cell", max_axis(*coarse_ref), max_axis(*best.result));
|
|
experiment_.SetUnitCell(coarse_ref->lattice.GetUnitCell());
|
|
const FirstPass constrained = pick_best(*indexer_pool, *indexer);
|
|
experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo
|
|
if (constrained.result.has_value() && constrained.score > best.score)
|
|
best = constrained;
|
|
}
|
|
}
|
|
|
|
if (!cancelled_) {
|
|
if (!best.result.has_value())
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Two-pass rotation indexing failed");
|
|
indexer->ForceRotationIndexerResult(*best.result);
|
|
logger.Info("Two-pass rotation indexing found lattice (scheme '{}': {}/{} validation frames)",
|
|
best.name, best.score, static_cast<int>(validation.size()));
|
|
}
|
|
}
|
|
|
|
// Main per-image loop, spread over N worker threads pulling from a shared counter. HDF5 reads
|
|
// are serialized by the global hdf5_mutex; the analysis runs in parallel.
|
|
std::atomic<int> next_ordinal = 0;
|
|
std::atomic<int> finished_count = 0;
|
|
std::atomic<uint64_t> total_uncompressed_bytes = 0;
|
|
|
|
auto azint_worker = [&]() {
|
|
std::vector<uint8_t> decompression_buffer;
|
|
ImagePreprocessorCPU preprocessor(experiment_, pixel_mask_);
|
|
ImagePreprocessorBuffer buffer(experiment_.GetPixelsNum());
|
|
AzIntEngineCPU azint(mapping);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
|
|
while (!cancelled_) {
|
|
const int ordinal = next_ordinal.fetch_add(1);
|
|
const int image_idx = start_image + ordinal * config_.stride;
|
|
if (image_idx >= end_image) break;
|
|
|
|
std::shared_ptr<JFJochReaderRawImage> img;
|
|
try {
|
|
img = reader_.GetRawImage(image_idx);
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Failed to load image {}: {}", image_idx, e.what());
|
|
continue;
|
|
}
|
|
if (!img) continue;
|
|
|
|
DataMessage msg{};
|
|
msg.image = img->image;
|
|
msg.number = ordinal;
|
|
msg.original_number = image_idx;
|
|
if (dataset->efficiency.size() > image_idx)
|
|
msg.image_collection_efficiency = dataset->efficiency[image_idx];
|
|
total_uncompressed_bytes += msg.image.GetUncompressedSize();
|
|
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
try {
|
|
const uint8_t *image_ptr = msg.image.GetUncompressedPtr(decompression_buffer);
|
|
preprocessor.Analyze(buffer, image_ptr, msg.image.GetMode());
|
|
azint.Run(buffer, profile);
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Error integrating image {}: {}", image_idx, e.what());
|
|
continue;
|
|
}
|
|
msg.azint_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now() - t0).count();
|
|
msg.processing_time_s = msg.azint_time_s;
|
|
msg.az_int_profile = profile.GetResult();
|
|
msg.az_int_profile_count = profile.GetPixelCount();
|
|
msg.az_int_profile_std = profile.GetStd();
|
|
msg.bkg_estimate = profile.GetBkgEstimate(mapping.Settings());
|
|
msg.ice_ring_score = profile.GetIceRingScore(mapping.Settings(),
|
|
config_.spot_finding.ice_ring_width_Q_recipA);
|
|
msg.run_number = experiment_.GetRunNumber();
|
|
msg.run_name = experiment_.GetRunName();
|
|
|
|
plots.Add(msg, profile);
|
|
if (writer) writer->Write(msg);
|
|
if (observer) observer->OnImageProcessed(msg);
|
|
const int done = finished_count.fetch_add(1) + 1;
|
|
if (observer) observer->OnProgress(done, images_to_process);
|
|
}
|
|
};
|
|
|
|
auto full_worker = [&]() {
|
|
pin_gpu(); // round-robin per worker thread; must precede engine construction
|
|
MXAnalysisWithoutFPGA analysis(experiment_, mapping, pixel_mask_, *indexer);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
|
|
while (!cancelled_) {
|
|
const int ordinal = next_ordinal.fetch_add(1);
|
|
const int image_idx = start_image + ordinal * config_.stride;
|
|
if (image_idx >= end_image) break;
|
|
|
|
std::shared_ptr<JFJochReaderRawImage> img;
|
|
try {
|
|
img = reader_.GetRawImage(image_idx);
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Failed to load image {}: {}", image_idx, e.what());
|
|
continue;
|
|
}
|
|
if (!img) continue;
|
|
|
|
DataMessage msg{};
|
|
msg.image = img->image;
|
|
msg.number = ordinal;
|
|
msg.original_number = image_idx;
|
|
if (dataset->efficiency.size() > image_idx)
|
|
msg.image_collection_efficiency = dataset->efficiency[image_idx];
|
|
total_uncompressed_bytes += msg.image.GetUncompressedSize();
|
|
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
try {
|
|
analysis.Analyze(msg, profile, config_.spot_finding);
|
|
} catch (const std::exception &e) {
|
|
logger.Error("Error analyzing image {}: {}", image_idx, e.what());
|
|
continue;
|
|
}
|
|
msg.processing_time_s = std::chrono::duration<float>(std::chrono::steady_clock::now() - t0).count();
|
|
msg.run_number = experiment_.GetRunNumber();
|
|
msg.run_name = experiment_.GetRunName();
|
|
|
|
plots.Add(msg, profile);
|
|
if (writer) writer->Write(msg);
|
|
if (observer) observer->OnImageProcessed(msg);
|
|
const int done = finished_count.fetch_add(1) + 1;
|
|
if (observer) observer->OnProgress(done, images_to_process);
|
|
}
|
|
};
|
|
|
|
if (observer)
|
|
observer->OnPhase("Processing images");
|
|
|
|
std::function<void()> worker = full ? std::function<void()>(full_worker)
|
|
: std::function<void()>(azint_worker);
|
|
std::vector<std::future<void> > futures;
|
|
futures.reserve(config_.nthreads);
|
|
for (int i = 0; i < config_.nthreads; ++i)
|
|
futures.push_back(std::async(std::launch::async, worker));
|
|
for (auto &f: futures)
|
|
f.get();
|
|
|
|
result.cancelled = cancelled_;
|
|
result.images_processed = finished_count.load();
|
|
result.mean_processing_time = plots.GetMeanProcessingTime();
|
|
result.indexing_rate = plots.GetIndexingRate();
|
|
|
|
// End message (also written to the file).
|
|
EndMessage end_msg;
|
|
end_msg.max_image_number = result.images_processed;
|
|
end_msg.images_collected_count = result.images_processed;
|
|
end_msg.images_sent_to_write_count = result.images_processed;
|
|
end_msg.end_date = time_UTC(std::chrono::system_clock::now());
|
|
end_msg.run_number = experiment_.GetRunNumber();
|
|
end_msg.run_name = experiment_.GetRunName();
|
|
end_msg.bkg_estimate = plots.GetBkgEstimate();
|
|
end_msg.az_int_result["dataset"] = plots.GetAzIntProfile();
|
|
end_msg.indexing_rate = result.indexing_rate;
|
|
|
|
if (full && !cancelled_) {
|
|
if (const auto rot = indexer->FinalizeRotationIndexing(); rot.has_value()) {
|
|
end_msg.rotation_lattice = rot->lattice;
|
|
// Write the refined geometry (not the nominal StartMessage values) into the _process.h5.
|
|
end_msg.refined_beam_center_x = rot->geom.GetBeamX_pxl();
|
|
end_msg.refined_beam_center_y = rot->geom.GetBeamY_pxl();
|
|
end_msg.refined_poni_rot1 = rot->geom.GetPoniRot1_rad();
|
|
end_msg.refined_poni_rot2 = rot->geom.GetPoniRot2_rad();
|
|
end_msg.refined_poni_rot3 = rot->geom.GetPoniRot3_rad();
|
|
if (rot->axis)
|
|
end_msg.refined_rotation_axis = rot->axis->GetAxis();
|
|
end_msg.rotation_lattice_type = LatticeMessage{
|
|
.centering = rot->search_result.centering,
|
|
.niggli_class = rot->search_result.niggli_class,
|
|
.crystal_system = rot->search_result.system
|
|
};
|
|
result.rotation_lattice_found = true;
|
|
}
|
|
result.consensus_cell = indexer->GetConsensusUnitCell();
|
|
end_msg.unit_cell = result.consensus_cell;
|
|
}
|
|
|
|
// Scaling and merging (full analysis only).
|
|
if (full && !cancelled_ && result.indexing_rate.has_value() && result.indexing_rate > 0
|
|
&& (config_.run_scaling || !config_.reference_data.empty())) {
|
|
// Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
|
|
// bar reflects what is happening instead of freezing on one label.
|
|
auto phase = [&](const std::string &p) {
|
|
if (observer) observer->OnPhase(p);
|
|
};
|
|
phase("Scaling and merging");
|
|
|
|
// Ice-ring handling (--detect-ice-rings): flag reflections sitting on a hexagonal-ice powder
|
|
// ring. Their integrated intensity is contaminated by the strong, variable ice background, so
|
|
// they drag the per-image scale fit. Flag them here so scaling (the per-image G fit and the
|
|
// fulls refit) skips them, while the combine, the merge and the statistics keep them - dropping
|
|
// them outright guts low/mid-resolution completeness on crystals that merge them fine.
|
|
if (experiment_.IsDetectIceRings()) {
|
|
const float ice_width = config_.spot_finding.ice_ring_width_Q_recipA;
|
|
size_t total = 0, flagged = 0;
|
|
for (auto &outcome : indexer->GetIntegrationOutcome()) {
|
|
for (auto &r : outcome.reflections) {
|
|
++total;
|
|
r.on_ice_ring = IsOnIceRing(r.d, ice_width);
|
|
if (r.on_ice_ring)
|
|
++flagged;
|
|
}
|
|
}
|
|
logger.Info("Ice-ring handling: flagged {} of {} reflections on ice rings (half-width {:.3f} A^-1); "
|
|
"excluded from scaling, kept for merging", flagged, total, ice_width);
|
|
}
|
|
|
|
// Scale the images and merge. Factored so it can run twice: first in P1 to give the
|
|
// space-group search a merged dataset, then again in the determined space group so the
|
|
// scaling sees symmetry equivalents and the final statistics are in the right symmetry.
|
|
// (With a user-fixed space group it simply runs once, already in that symmetry.)
|
|
struct ScaleMergeResult {
|
|
std::vector<MergedReflection> merged;
|
|
MergeStatistics statistics;
|
|
};
|
|
// The reference path computes each image's G once (per-image scaling against the
|
|
// reference); the scaling loop below is skipped, so G is stable across the two passes.
|
|
// Smoothing it more than once would compound the correction, so do it only on the first
|
|
// pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly.
|
|
// Ice rings masked from the merge by the CC1/2 test after the final merge; empty until then,
|
|
// at which point a re-merge applies them.
|
|
std::vector<char> masked_ice_rings;
|
|
|
|
// Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single
|
|
// allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame
|
|
// partials, scales the fulls and merges; the whole hot path runs on the GPU when one is present).
|
|
// Ingested once here (after ice flagging) and reused across both space-group passes; a fixed
|
|
// (forced) mosaicity is handled by the recompute. The decay + absorption correction surfaces run as
|
|
// post-scale-fulls stages (on by default, see ScalingSettings::CorrectionSurfaces); external-
|
|
// reference scaling, the stills -B (per-image B-factor) and wedge refinement are unsupported here.
|
|
// Two-pass second pass: if the de-novo indexer DEMOTED to a triclinic (primitive) cell while the reused
|
|
// merge space group is higher-symmetry (a huge oblique cell whose constrained centred refine was
|
|
// ill-posed, so the pseudo-symmetry guard kept the primitive), the reflections are in the primitive
|
|
// frame but the merge expects the group's CONVENTIONAL setting - reindex them into it (the same
|
|
// primitive->conventional change of basis the space-group search applies to a demoted result), else the
|
|
// merge folds the wrong equivalents. No-op when the indexer already produced the group's setting.
|
|
if (prepass_merge_sg_ && end_msg.rotation_lattice.has_value() && end_msg.rotation_lattice_type.has_value()
|
|
&& end_msg.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Triclinic) {
|
|
const auto *msg_sg = gemmi::find_spacegroup_by_number(static_cast<int>(*prepass_merge_sg_));
|
|
if (msg_sg && msg_sg->crystal_system() != gemmi::CrystalSystem::Triclinic) {
|
|
const auto cand = LatticeSearch(*end_msg.rotation_lattice);
|
|
// Exact integer reindex prim->conv: P[i][j] = conv_real[i] . rot_reciprocal[j] (as the
|
|
// centred-lattice test), robust to the refined cell not being exactly LatticeSearch's Niggli cell.
|
|
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
|
|
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
|
|
end_msg.rotation_lattice->Cstar()};
|
|
gemmi::Mat33 reindex;
|
|
double reindex_res = 0.0;
|
|
for (int i = 0; i < 3; ++i)
|
|
for (int j = 0; j < 3; ++j) {
|
|
const double v = cv[i] * rs[j];
|
|
reindex.a[i][j] = std::round(v);
|
|
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
|
|
}
|
|
// Only when LatticeSearch recovers the reused group's own metric (system + centring): otherwise
|
|
// the reused group and the indexed metric disagree and reindexing would be wrong - leave it.
|
|
if (cand.system == msg_sg->crystal_system() && cand.centering == msg_sg->centring_type()
|
|
&& reindex_res < 0.1) {
|
|
for (auto &io : indexer->GetIntegrationOutcome()) {
|
|
io.latt = io.latt.Multiply(reindex);
|
|
for (auto &r : io.reflections) {
|
|
const double h = r.h, k = r.k, l = r.l;
|
|
r.h = static_cast<int32_t>(std::lround(reindex.a[0][0]*h + reindex.a[0][1]*k + reindex.a[0][2]*l));
|
|
r.k = static_cast<int32_t>(std::lround(reindex.a[1][0]*h + reindex.a[1][1]*k + reindex.a[1][2]*l));
|
|
r.l = static_cast<int32_t>(std::lround(reindex.a[2][0]*h + reindex.a[2][1]*k + reindex.a[2][2]*l));
|
|
}
|
|
}
|
|
result.consensus_cell = cand.conventional.GetUnitCell();
|
|
end_msg.unit_cell = result.consensus_cell;
|
|
end_msg.rotation_lattice = end_msg.rotation_lattice->Multiply(reindex);
|
|
end_msg.rotation_lattice_type = LatticeMessage{ .centering = cand.centering,
|
|
.niggli_class = end_msg.rotation_lattice_type->niggli_class, .crystal_system = cand.system };
|
|
logger.Info("Two-pass: reindexed the de-novo primitive cell into the space-group {} "
|
|
"conventional setting for the merge", static_cast<int>(*prepass_merge_sg_));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Two-pass second pass only: the indexer above ran DE NOVO (Run() cleared the space group so its
|
|
// pseudo-symmetry safeguards find the true cell); now reinstate pass-1's space group for the merge,
|
|
// exactly as a user -S would, so the symmetry is fixed without a re-search. No-op on pass 1 / single pass.
|
|
if (prepass_merge_sg_)
|
|
experiment_.SpaceGroupNumber(*prepass_merge_sg_);
|
|
|
|
const auto &rot_ss = experiment_.GetScalingSettings();
|
|
const bool is_rotation = experiment_.IsRotationIndexing(); // rotation indexing -> rotation scaling/merge
|
|
std::optional<RotationScaleMerge> rsm;
|
|
if (is_rotation) {
|
|
if (rot_ss.GetRefineB()
|
|
|| experiment_.GetRefineRotationWedgeInScaling()
|
|
|| rot_ss.GetRotationWedgeForScaling().has_value())
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Rotation scaling/merging (RotationScaleMerge) does not support "
|
|
"B-factor refinement or wedge refinement");
|
|
// A reference MTZ is allowed for rotation: it fixes the space group / cell (on the CLI) and
|
|
// resolves the indexing ambiguity (below), but is NOT used to scale - the rotation merge stays
|
|
// self-consistent.
|
|
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
|
static_cast<int>(config_.scaling_iter),
|
|
config_.spot_finding.ice_ring_width_Q_recipA,
|
|
config_.nthreads, logger, config_.observation_dump_path);
|
|
rsm->Ingest();
|
|
}
|
|
|
|
auto scale_and_merge = [&](const std::string &label, bool for_search) -> ScaleMergeResult {
|
|
if (rsm) {
|
|
phase("Scale/combine/merge (" + label + ")");
|
|
auto r = rsm->Run(for_search, masked_ice_rings);
|
|
result.error_model_isa = r.isa;
|
|
return ScaleMergeResult{std::move(r.merged), std::move(r.statistics)};
|
|
}
|
|
// Stills (rotation goes through RotationScaleMerge above): self-scale each image against the
|
|
// running merge with ScaleOnTheFly (fixed partiality), then merge directly. This runs even
|
|
// with an external reference: the reference is used only to break the per-image indexing
|
|
// ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor (scaling each
|
|
// image against a foreign dataset injects cross-dataset systematics and is a worse reference
|
|
// than the data's own merge). One pass: the per-image scale G is the exact one-pass solution,
|
|
// so iterating would only rebuild the reference from the freshly-scaled (noisy) data and
|
|
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
|
|
{
|
|
phase("Scaling images (" + label + ")");
|
|
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
|
|
indexer->ScaleAllImages(merge_result);
|
|
}
|
|
const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
|
|
|
|
phase("Merging");
|
|
MergeOnTheFly merge_engine(experiment_);
|
|
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
|
|
// intensities and the error model, so the space-group search sees clean data; the final
|
|
// in-symmetry merge keeps them so completeness is not lost.
|
|
merge_engine.ExcludeIceRings(for_search);
|
|
merge_engine.MaskIceRings(masked_ice_rings, config_.spot_finding.ice_ring_width_Q_recipA);
|
|
if (result.consensus_cell.has_value())
|
|
merge_engine.ReferenceCell(*result.consensus_cell);
|
|
// Optional detector-plane modulation (flat-field) correction for stills, folded into each
|
|
// reflection's scale before the error model and merge (never for the P1 search pass). The
|
|
// rotation path fits its own modulation inside RotationScaleMerge.
|
|
if (experiment_.GetScalingSettings().GetStillsModulation() && !for_search) {
|
|
const double mod_gain = merge_engine.RefineModulation(indexer->GetIntegrationOutcome());
|
|
if (mod_gain > 0.0)
|
|
logger.Info("Stills modulation: detector-frame 16x16 surface applied "
|
|
"(cross-validated, held-out gain {:.1f}%)", 100.0 * mod_gain);
|
|
else
|
|
logger.Info("Stills modulation: no cross-validated gain (skipped)");
|
|
}
|
|
merge_engine.RefineErrorModel(merge_input);
|
|
if (merge_engine.ErrorModelActive())
|
|
logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f} chi2={:.2f}", merge_engine.ErrorModelA(),
|
|
merge_engine.ErrorModelB(),
|
|
merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0,
|
|
merge_engine.ErrorModelChi2());
|
|
for (size_t i = 0; i < merge_input.size(); ++i)
|
|
merge_engine.AddImage(merge_input[i], static_cast<int64_t>(i));
|
|
|
|
ScaleMergeResult out;
|
|
out.merged = merge_engine.ExportReflections();
|
|
|
|
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
|
|
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
|
|
// range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A
|
|
// manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the
|
|
// space-group search still sees the full range. (Rotation is cut inside RotationScaleMerge.)
|
|
const auto &cut_ss = experiment_.GetScalingSettings();
|
|
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
|
|
out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
|
|
cut_ss.GetResolutionCCTarget(), for_search, logger);
|
|
|
|
phase("Computing statistics");
|
|
out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min);
|
|
result.error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
|
|
logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label);
|
|
return out;
|
|
};
|
|
|
|
// First pass: P1 when searching, or directly the user-fixed space group.
|
|
const auto initial_sg = experiment_.GetGemmiSpaceGroup();
|
|
auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value());
|
|
|
|
// Rotation two-pass geometry pre-pass: this first scale/merge has now fitted a frame-order-smoothed
|
|
// mosaicity and written it back onto the per-frame outcomes. Capture it for the second pass's Bragg
|
|
// prediction (so prediction uses the smoothed value, not a fresh per-image estimate) and post-refine +
|
|
// apply the detector geometry from this integration. The pre-pass then CONTINUES through the space-
|
|
// group search (below) so the second pass can reuse the same space group and lattice - re-searching at
|
|
// the slightly changed geometry could flip a borderline determination and mix the two passes.
|
|
if (geometry_prepass && rsm && experiment_.IsRotationIndexing()) {
|
|
const auto &outcomes = indexer->GetIntegrationOutcome();
|
|
prepass_mosaicity_.assign(outcomes.size(), NAN);
|
|
for (size_t o = 0; o < outcomes.size(); ++o)
|
|
if (outcomes[o].mosaicity_deg) prepass_mosaicity_[o] = *outcomes[o].mosaicity_deg;
|
|
if (result.consensus_cell.has_value()) {
|
|
if (const auto rot = indexer->FinalizeRotationIndexing(); rot && rot->axis) {
|
|
PostRefineSettings prs;
|
|
prs.crystal_system = rot->search_result.system;
|
|
prs.num_threads = static_cast<int>(config_.nthreads);
|
|
prs.refine_geometry = true;
|
|
const auto pr = PostRefineRotationGeometry(outcomes, *rot->axis,
|
|
experiment_.GetDiffractionGeometry(), rot->lattice, prs, logger);
|
|
if (pr.ok && pr.detector_refined) {
|
|
// Store, don't apply: this pass continues to write its own (header-geometry) output;
|
|
// Run() applies the refined geometry before the second pass.
|
|
prepass_detector_geometry_ = std::array<float, 3>{
|
|
static_cast<float>(pr.beam_x_after_px),
|
|
static_cast<float>(pr.beam_y_after_px),
|
|
static_cast<float>(pr.distance_after_mm)};
|
|
logger.Info("Two-pass: post-refined detector geometry (distance {:.3f} mm, "
|
|
"beam {:.2f},{:.2f} px) will drive the second integration pass",
|
|
pr.distance_after_mm, pr.beam_x_after_px, pr.beam_y_after_px);
|
|
} else {
|
|
logger.Info("Two-pass: geometry post-refine committed no detector change "
|
|
"- the second pass reproduces the first");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// For a de-novo search, gate the P1 scaling+merge (the pass that feeds the space-group search) at
|
|
// <I/sigma> >= 1 over thin resolution shells: noise-dominated high-res shells otherwise mislead
|
|
// the determination, where a single artifact (a zinger the box sum integrates whole, or a
|
|
// weak-reflection profile-fit runaway) has an astronomical resolution-normalised E and wrecks the
|
|
// intensity-correlation / second-moment statistics. The final in-symmetry merge keeps the full
|
|
// range (only this P1 search pass is cut); a manual --scaling-high-resolution, if coarser, wins.
|
|
double d_min_search = 0.0;
|
|
if (!initial_sg.has_value()) {
|
|
std::vector<std::pair<float, float>> rs; // (d, I/sigma) over the full P1 merge
|
|
rs.reserve(sm.merged.size());
|
|
for (const auto &m : sm.merged)
|
|
if (std::isfinite(m.I) && std::isfinite(m.sigma) && m.sigma > 0.0f &&
|
|
std::isfinite(m.d) && m.d > 0.0f)
|
|
rs.emplace_back(m.d, m.I / m.sigma);
|
|
if (rs.size() >= 400) {
|
|
std::sort(rs.begin(), rs.end(),
|
|
[](const auto &a, const auto &b) { return a.first > b.first; }); // low -> high res
|
|
const int bins = std::clamp(static_cast<int>(rs.size() / 100), 1, 40);
|
|
const size_t per = (rs.size() + bins - 1) / static_cast<size_t>(bins);
|
|
for (size_t b = 0; b * per < rs.size(); ++b) {
|
|
const size_t lo = b * per, hi = std::min(rs.size(), lo + per);
|
|
double sum = 0.0;
|
|
for (size_t j = lo; j < hi; ++j) sum += rs[j].second;
|
|
if (sum / static_cast<double>(hi - lo) < 1.0) {
|
|
// Cut the noise-dominated high-res shells, but only when a good low-res region
|
|
// exists above the cutoff. If even the lowest-res shell (b == 0) is below the
|
|
// threshold - as happens for very weak data whose I/sigma is floored by the
|
|
// error model although CC1/2 is still high - there is no clean/noise boundary,
|
|
// and cutting here would discard essentially all reflections and leave the
|
|
// re-merge with no resolution range. Leave d_min_search at 0 (no cut) instead.
|
|
if (b > 0) d_min_search = rs[lo].first;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
const double d_manual = experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0);
|
|
if (rsm && d_min_search > d_manual) {
|
|
logger.Info("Space-group search: re-scaling the P1 pass at {:.2f} A (<I/sigma> >= 1)", d_min_search);
|
|
rsm->SetDMinLimit(d_min_search);
|
|
sm = scale_and_merge("P1, space-group search", true);
|
|
rsm->SetDMinLimit(d_manual > 0.0 ? std::optional<double>(d_manual) : std::nullopt);
|
|
}
|
|
}
|
|
|
|
std::ostringstream stats_text;
|
|
// Set when the space group was determined via a change of basis (the intensity-based centred-lattice
|
|
// test): its conventional setting does not match the indexer's primitive frame, so the two-pass must
|
|
// NOT reuse it - the second pass re-searches (and re-runs the same test) instead.
|
|
bool sg_reindexed = false;
|
|
if (!experiment_.GetGemmiSpaceGroup().has_value()) {
|
|
SearchSpaceGroupOptions sg_opts;
|
|
sg_opts.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
|
|
sg_opts.d_min_limit_A = std::max<double>(
|
|
d_min_search, experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0));
|
|
// Constrain the search to subgroups of the lattice (metric) symmetry found by rotation
|
|
// indexing. Centering is not constrained here - it is determined from the absences.
|
|
if (end_msg.rotation_lattice_type.has_value())
|
|
sg_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
|
|
auto sg_search = SearchSpaceGroup(sm.merged, sg_opts);
|
|
|
|
// Miller-index reindex under a change of basis: (h,k,l)_conv = reindex * (h,k,l)_prim.
|
|
const auto reindex_hkl = [](auto &r, const gemmi::Mat33 &m) {
|
|
const double h = r.h, k = r.k, l = r.l;
|
|
r.h = static_cast<int32_t>(std::lround(m.a[0][0] * h + m.a[0][1] * k + m.a[0][2] * l));
|
|
r.k = static_cast<int32_t>(std::lround(m.a[1][0] * h + m.a[1][1] * k + m.a[1][2] * l));
|
|
r.l = static_cast<int32_t>(std::lround(m.a[2][0] * h + m.a[2][1] * k + m.a[2][2] * l));
|
|
};
|
|
|
|
// Intensity-based centred-lattice test (XDS CORRECT / POINTLESS style). When the indexer
|
|
// demoted to a primitive/triclinic cell but the metric is pseudo-symmetric, the point-group
|
|
// search above tested no operators (its candidate set is bounded by the indexer's lattice).
|
|
// A genuinely centred lattice and a merely pseudo-symmetric triclinic one are indistinguishable
|
|
// from spot POSITIONS - the 2-fold is an INTENSITY property - so decide it here, after P1
|
|
// integration: reindex the P1-merged data to the metric candidate's conventional setting and
|
|
// re-run the search. The I(h) vs I(Rh) correlation confirms a real centred lattice (high CC ->
|
|
// commit the centring) and rejects a pseudo-symmetric triclinic (low CC -> stays P1, no reindex).
|
|
std::optional<gemmi::Mat33> commit_reindex;
|
|
std::optional<UnitCell> commit_cell;
|
|
std::optional<CrystalLattice> commit_lattice;
|
|
std::optional<LatticeMessage> commit_lattice_type;
|
|
// Only a demoted (triclinic) indexing result reaches here with a primitive cell - that is both
|
|
// the case the point-group search could not test (a triclinic holohedry offers no operators)
|
|
// and the case where LatticeSearch's Niggli reduction is valid (it assumes a primitive input).
|
|
const bool searched_trivial = !sg_search.best_space_group || sg_search.best_space_group->number == 1;
|
|
const bool demoted_triclinic = end_msg.rotation_lattice_type.has_value()
|
|
&& end_msg.rotation_lattice_type->crystal_system == gemmi::CrystalSystem::Triclinic;
|
|
if (searched_trivial && demoted_triclinic && end_msg.rotation_lattice.has_value()) {
|
|
// The metric only generates the candidate; the intensity CC gate below makes the decision
|
|
// (a false pseudo-symmetry is rejected by the correlation).
|
|
const auto cand = LatticeSearch(*end_msg.rotation_lattice);
|
|
// Exact integer reindex from the indexed setting straight to the candidate conventional
|
|
// setting: P[i][j] = conv_real[i] . rot_reciprocal[j]. Derived from the two lattices, so it
|
|
// holds even if the refined indexed cell is not exactly LatticeSearch's Niggli cell (whose
|
|
// primitive_reduced->conventional reindex would otherwise be applied in the wrong setting).
|
|
const Coord cv[3] = {cand.conventional.Vec0(), cand.conventional.Vec1(), cand.conventional.Vec2()};
|
|
const Coord rs[3] = {end_msg.rotation_lattice->Astar(), end_msg.rotation_lattice->Bstar(),
|
|
end_msg.rotation_lattice->Cstar()};
|
|
gemmi::Mat33 reindex;
|
|
double reindex_res = 0.0;
|
|
for (int i = 0; i < 3; ++i)
|
|
for (int j = 0; j < 3; ++j) {
|
|
const double v = cv[i] * rs[j];
|
|
reindex.a[i][j] = std::round(v);
|
|
reindex_res = std::max(reindex_res, std::fabs(v - reindex.a[i][j]));
|
|
}
|
|
if (cand.system != gemmi::CrystalSystem::Triclinic && reindex_res < 0.1) {
|
|
std::vector<MergedReflection> merged_c = sm.merged;
|
|
for (auto &m : merged_c)
|
|
reindex_hkl(m, reindex);
|
|
SearchSpaceGroupOptions o2 = sg_opts;
|
|
o2.lattice_system = cand.system;
|
|
const auto s2 = SearchSpaceGroup(merged_c, o2);
|
|
if (s2.best_space_group.has_value() && s2.best_space_group->number > 1) {
|
|
// The centring cannot be confirmed from absences here (integrated in the primitive
|
|
// cell, so the centring-absent reflections do not exist) - it is metric-determined.
|
|
// Among the point-group-equivalent candidates the intensities cannot separate
|
|
// (e.g. P2 / P21 / C2), pick the one matching the metric candidate's centring.
|
|
gemmi::SpaceGroup chosen = *s2.best_space_group;
|
|
if (chosen.centring_type() != cand.centering)
|
|
for (const auto &alt : s2.alternatives)
|
|
if (alt.centring_type() == cand.centering) { chosen = alt; break; }
|
|
// Commit only if the adopted centring matches the metric candidate whose conventional
|
|
// cell we reindex into - otherwise the space group and the written cell disagree.
|
|
if (chosen.centring_type() == cand.centering) {
|
|
sg_search = s2;
|
|
sg_search.best_space_group = chosen;
|
|
commit_reindex = reindex;
|
|
commit_cell = cand.conventional.GetUnitCell();
|
|
commit_lattice = end_msg.rotation_lattice->Multiply(reindex);
|
|
commit_lattice_type = LatticeMessage{ .centering = cand.centering,
|
|
.niggli_class = end_msg.rotation_lattice_type->niggli_class,
|
|
.crystal_system = cand.system };
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Adopt the determined space group and re-scale/merge in it, so scaling uses symmetry
|
|
// equivalents and the statistics come out in the right symmetry. P1 stands when nothing
|
|
// is determined.
|
|
if (sg_search.best_space_group.has_value()) {
|
|
const auto &sg = *sg_search.best_space_group;
|
|
logger.Info("Adopting space group {} (number {})", sg.short_name(), sg.number);
|
|
// A reindex is committed only when the intensity re-test confirmed a higher symmetry in a
|
|
// centred setting: bring the integrated reflections + cell into that setting and re-ingest
|
|
// the rotation merge, so the final scaling/merging folds the equivalents correctly.
|
|
if (commit_reindex) {
|
|
sg_reindexed = true;
|
|
for (auto &io : indexer->GetIntegrationOutcome()) {
|
|
io.latt = io.latt.Multiply(*commit_reindex);
|
|
for (auto &r : io.reflections)
|
|
reindex_hkl(r, *commit_reindex);
|
|
}
|
|
result.consensus_cell = commit_cell;
|
|
end_msg.unit_cell = commit_cell;
|
|
// Keep the lattice metadata (hence the master-file UB matrix and indexed-lattice
|
|
// vectors) in the same conventional setting as the cell, reflections and space group.
|
|
end_msg.rotation_lattice = commit_lattice;
|
|
end_msg.rotation_lattice_type = commit_lattice_type;
|
|
if (rsm) {
|
|
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
|
static_cast<int>(config_.scaling_iter),
|
|
config_.spot_finding.ice_ring_width_Q_recipA,
|
|
config_.nthreads, logger, config_.observation_dump_path);
|
|
rsm->Ingest();
|
|
}
|
|
}
|
|
experiment_.SpaceGroupNumber(sg.number);
|
|
end_msg.space_group_number = sg.number;
|
|
result.space_group_number = sg.number;
|
|
phase("Re-scaling in space group " + sg.short_name());
|
|
sm = scale_and_merge(sg.short_name(), false);
|
|
}
|
|
result.space_group_search = sg_search;
|
|
} else {
|
|
// A space group was fixed by the user; surface it so the viewer/CLI can still show it.
|
|
result.space_group_number = experiment_.GetSpaceGroupNumber();
|
|
}
|
|
|
|
// Rotation two-pass geometry pre-pass: the determined space group is now fixed on experiment_, so the
|
|
// second pass REUSES it instead of re-searching at the slightly changed geometry (which could flip a
|
|
// borderline determination and mix the two passes). The lattice is deliberately NOT forced - the second
|
|
// pass re-indexes at the refined geometry so the CELL comes out self-consistent with it (forcing the
|
|
// pre-pass lattice would pin the nominal cell and undo the geometry refinement). EXCEPTION: a
|
|
// reindex-derived space group (the centred-lattice test) is in a conventional setting the indexer's
|
|
// primitive frame does not share, so reusing it would break the second pass's indexing - remember it so
|
|
// the caller clears it and lets the second pass re-search. Unlike a throwaway pre-pass, this one now
|
|
// CONTINUES to the write below so its own (header-geometry) result is saved as the first pass.
|
|
if (geometry_prepass)
|
|
prepass_sg_reindexed_ = sg_reindexed;
|
|
|
|
// Reference-based indexing-ambiguity resolution (rotation). When a reference MTZ is supplied and
|
|
// the crystal has an indexing ambiguity (merohedral: lattice symmetry higher than the Laue group),
|
|
// pick the reindexing that best correlates with the reference intensities and re-merge in it. The
|
|
// twin-law reindex is metric-preserving, so only the hkl labels change (the cell is unchanged).
|
|
// Stills resolve the ambiguity per image with ReindexAmbiguityResolver, not here.
|
|
if (rsm && !config_.reference_data.empty() && result.consensus_cell
|
|
&& experiment_.GetSpaceGroupNumber().has_value()) {
|
|
const int sg_num = static_cast<int>(*experiment_.GetSpaceGroupNumber());
|
|
const auto choice = ChooseReindex(
|
|
sm.merged, *result.consensus_cell, sg_num,
|
|
[&](const std::vector<MergedReflection> &m) {
|
|
return ReferenceIntensityCC(m, config_.reference_data, sg_num);
|
|
});
|
|
if (!choice.is_identity) {
|
|
logger.Info("Reference: resolved indexing ambiguity by reindexing to match the reference "
|
|
"(CC {:.3f}, vs {:.3f} unchanged)", choice.score, choice.identity_score);
|
|
for (auto &io : indexer->GetIntegrationOutcome())
|
|
for (auto &r : io.reflections) {
|
|
const gemmi::Op::Miller h = choice.op.apply_to_hkl({{r.h, r.k, r.l}});
|
|
r.h = h[0]; r.k = h[1]; r.l = h[2];
|
|
}
|
|
rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
|
|
static_cast<int>(config_.scaling_iter),
|
|
config_.spot_finding.ice_ring_width_Q_recipA,
|
|
config_.nthreads, logger, config_.observation_dump_path);
|
|
rsm->Ingest();
|
|
phase("Re-merging in the reference indexing");
|
|
sm = scale_and_merge(experiment_.GetGemmiSpaceGroup()->short_name(), false);
|
|
}
|
|
}
|
|
|
|
// Ice-ring CC1/2 mask: a hexagonal-ice ring whose merged half-set CC1/2 collapses well below its
|
|
// resolution shoulders has been decorrelated by ice (its Bragg is unrecoverable) - drop it and
|
|
// re-merge. Weak/absent rings track their neighbours and stay, so completeness on clean crystals
|
|
// is untouched. Uses the ring band (+/- ice width in q=2pi/d) vs the shoulders either side.
|
|
if (experiment_.IsDetectIceRings() && !sm.merged.empty()) {
|
|
constexpr float two_pi = 6.283185307f;
|
|
const float w = config_.spot_finding.ice_ring_width_Q_recipA;
|
|
std::vector<char> mask(ICE_RING_RES_A.size(), 0);
|
|
for (size_t i = 0; i < ICE_RING_RES_A.size(); ++i) {
|
|
const float q_ring = two_pi / ICE_RING_RES_A[i];
|
|
CorrelationCoefficient ring, shoulder;
|
|
size_t n_ring = 0, n_shoulder = 0;
|
|
for (const auto &m : sm.merged) {
|
|
if (!(m.d > 0.0f) || !std::isfinite(m.I_half[0]) || !std::isfinite(m.I_half[1]))
|
|
continue;
|
|
const float dq = std::fabs(two_pi / m.d - q_ring);
|
|
if (dq < w) { ring.Add(m.I_half[0], m.I_half[1]); ++n_ring; }
|
|
else if (dq < 3.0f * w) { shoulder.Add(m.I_half[0], m.I_half[1]); ++n_shoulder; }
|
|
}
|
|
if (n_ring >= 20 && n_shoulder >= 20 && shoulder.GetCC() > 0.5
|
|
&& ring.GetCC() < shoulder.GetCC() - 0.05) {
|
|
mask[i] = 1;
|
|
logger.Info("Ice-ring mask: {:.2f} A ring CC1/2 {:.3f} << shoulders {:.3f}; masked from merge",
|
|
ICE_RING_RES_A[i], ring.GetCC(), shoulder.GetCC());
|
|
}
|
|
}
|
|
if (std::any_of(mask.begin(), mask.end(), [](char c) { return c != 0; })) {
|
|
masked_ice_rings = std::move(mask);
|
|
const auto final_sg = experiment_.GetGemmiSpaceGroup();
|
|
sm = scale_and_merge(final_sg ? final_sg->short_name() : "P1", false);
|
|
}
|
|
}
|
|
|
|
const auto twin_sg_number = experiment_.GetSpaceGroupNumber();
|
|
const gemmi::SpaceGroup *twin_sg = twin_sg_number
|
|
? gemmi::find_spacegroup_by_number(twin_sg_number.value()) : nullptr;
|
|
result.twinning = AnalyzeTwinning(sm.merged, twin_sg);
|
|
stats_text << TwinningAnalysisToText(result.twinning) << "\n";
|
|
|
|
// Indexing-ambiguity (alternative-indexing) advisory. When the lattice metric symmetry exceeds
|
|
// the Laue symmetry the crystal can be validly indexed in several hands related by twin-law
|
|
// operators. For an obvious merohedral case (P3/P4/P6...) users expect this; but a PSEUDO-merohedral
|
|
// metric (e.g. a C2 crystal whose beta makes it pseudo-F-orthorhombic) is easy to miss, so surface it. The
|
|
// reindex operator is applied directly to (h,k,l), so its triplet reads as an h,k,l transform.
|
|
if (result.consensus_cell && twin_sg_number) {
|
|
const auto twin_ops = ReindexAmbiguityOperators(*result.consensus_cell,
|
|
static_cast<int>(*twin_sg_number), 2.0);
|
|
if (!twin_ops.empty()) {
|
|
std::string laws;
|
|
for (const auto &op : twin_ops) {
|
|
std::string t = op.triplet();
|
|
std::replace(t.begin(), t.end(), 'x', 'h');
|
|
std::replace(t.begin(), t.end(), 'y', 'k');
|
|
std::replace(t.begin(), t.end(), 'z', 'l');
|
|
laws += (laws.empty() ? "" : " ; ") + t;
|
|
}
|
|
if (config_.reference_data.empty()) {
|
|
const std::string msg = fmt::format(
|
|
"Indexing ambiguity: this cell / space group admits alternative indexing "
|
|
"(reindex operator(s): {}). {} are indexed in one hand at random, and rugnux can only "
|
|
"break this against an external reference. WITHOUT one the merge mixes the hands and "
|
|
"CC1/2 is degraded - supply a reference (-z reference.mtz or --model) to resolve it.",
|
|
laws, experiment_.IsRotationIndexing() ? "Lattices" : "Serial-stills crystals");
|
|
logger.Warning("{}", msg);
|
|
stats_text << " !! " << msg << "\n\n";
|
|
} else {
|
|
logger.Info("Indexing ambiguity present (reindex operator(s): {}); resolved against the "
|
|
"supplied reference.", laws);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Dataset-wide Wilson B-factor estimate (like XDS's WILSON LINE B). Diagnostic only - it is not
|
|
// fed back into scaling; it just lands in the printed statistics, the mmCIF, and the log.
|
|
{
|
|
const GlobalWilsonB wilson = CalcGlobalWilsonB(sm.merged);
|
|
sm.statistics.wilson_b = wilson.b;
|
|
sm.statistics.wilson_b_correlation = wilson.correlation;
|
|
if (std::isfinite(wilson.b) && wilson.b > 0.0)
|
|
logger.Info("Wilson B-factor estimate: {:.2f} A^2 (correlation {:.3f}, {} shells)",
|
|
wilson.b, wilson.correlation, wilson.n_shells);
|
|
}
|
|
|
|
stats_text << sm.statistics;
|
|
result.merge_statistics_text = stats_text.str();
|
|
result.has_merge_statistics = true;
|
|
result.merge_statistics = sm.statistics;
|
|
result.has_reference = !config_.reference_data.empty();
|
|
|
|
// Inherit the campaign's shared R-free test set from the reference MTZ (overriding the
|
|
// per-hkl hash that the merge assigned), so every dataset flags the same free reflections.
|
|
if (config_.reference_has_free_flags && !config_.reference_data.empty() && !sm.merged.empty()) {
|
|
const auto sg = experiment_.GetSpaceGroupNumber().value_or(1);
|
|
const size_t matched = ApplyReferenceFreeFlags(sm.merged, static_cast<int32_t>(sg),
|
|
config_.reference_data);
|
|
logger.Info("R-free flags: inherited the reference test set ({} of {} merged reflections matched)",
|
|
matched, sm.merged.size());
|
|
}
|
|
|
|
// Radiation-damage report (rotation): the per-image scale CC-to-merge and mosaicity across the
|
|
// sweep (the per-image scaling the rotation merge already fits, binned by frame = dose). A
|
|
// per-image CC that falls, and/or a mosaicity that rises, with frame number is the classic
|
|
// radiation-damage signature - a data-quality-vs-dose read complementary to the fitted decay
|
|
// correction. The full per-image table is written to <prefix>_scaling.txt for detail.
|
|
if (experiment_.IsRotationIndexing()) {
|
|
const auto &outs = indexer->GetIntegrationOutcome();
|
|
const int nf = static_cast<int>(outs.size());
|
|
constexpr int nb = 10;
|
|
std::array<double, nb> cc_sum{}, mos_sum{};
|
|
std::array<int, nb> cc_n{}, mos_n{};
|
|
for (int f = 0; f < nf; ++f) {
|
|
const int b = nf > 1 ? std::min(nb - 1, f * nb / nf) : 0;
|
|
if (outs[f].image_scale_cc && std::isfinite(*outs[f].image_scale_cc)) {
|
|
cc_sum[b] += *outs[f].image_scale_cc; cc_n[b]++;
|
|
}
|
|
if (outs[f].mosaicity_deg && std::isfinite(*outs[f].mosaicity_deg)) {
|
|
mos_sum[b] += *outs[f].mosaicity_deg; mos_n[b]++;
|
|
}
|
|
}
|
|
int tot = 0;
|
|
for (int n : cc_n) tot += n;
|
|
if (tot >= 20) {
|
|
std::ostringstream os;
|
|
os << fmt::format("Radiation-damage report (per-image scale over {} frames):\n", nf);
|
|
os << " dose <CC to merge> <mosaicity deg>\n";
|
|
for (int b = 0; b < nb; ++b) {
|
|
if (cc_n[b] == 0 && mos_n[b] == 0) continue;
|
|
os << fmt::format(" {:3d}-{:3d}% {:>7} {:>7}\n", b * 10, (b + 1) * 10,
|
|
cc_n[b] ? fmt::format("{:.3f}", cc_sum[b] / cc_n[b]) : std::string("-"),
|
|
mos_n[b] ? fmt::format("{:.3f}", mos_sum[b] / mos_n[b]) : std::string("-"));
|
|
}
|
|
if (cc_n[0] > 0 && cc_n[nb - 1] > 0) {
|
|
const double cc0 = cc_sum[0] / cc_n[0], cc9 = cc_sum[nb - 1] / cc_n[nb - 1];
|
|
os << fmt::format(" => per-image CC to merge {:.3f} (first 10%) -> {:.3f} (last 10%){}",
|
|
cc0, cc9,
|
|
cc9 < cc0 - 0.05 ? " (falling: radiation damage / crystal decay)" : "");
|
|
}
|
|
// Relative B-factor across the sweep (measured before any decay correction, against the
|
|
// low-dose start): the resolution-dependent complement to the CC/mosaicity read above. A large
|
|
// magnitude flags radiation damage; positive is the typical direction (high-res fades with dose).
|
|
if (std::isfinite(sm.statistics.radiation_damage_delta_b)) {
|
|
os << fmt::format("\n => relative B-factor change over run = {:+.2f} A^2 (first->last){}",
|
|
sm.statistics.radiation_damage_delta_b,
|
|
std::fabs(sm.statistics.radiation_damage_delta_b) > 5.0
|
|
? " (significant dose-dependent scaling: radiation damage)" : "");
|
|
os << fmt::format("\n per-batch relative-B (A^2, {:.0f} deg/batch):",
|
|
sm.statistics.radiation_damage_batch_deg);
|
|
for (float bb : sm.statistics.radiation_damage_b_batch)
|
|
os << fmt::format(" {:.1f}", bb);
|
|
}
|
|
logger.Info("{}", os.str());
|
|
}
|
|
}
|
|
|
|
if (result.consensus_cell && write_files && config_.write_merged) {
|
|
phase("Writing reflections");
|
|
WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics,
|
|
result.error_model_isa > 0 ? fmt::format("{:.2f}", result.error_model_isa) : "?",
|
|
result.twinning, config_.output_prefix);
|
|
// Per-image scaling table (G, B-factor, mosaicity, wedge, CC) for inspection / XDS
|
|
// comparison. The offline self-scaling result is otherwise not exposed (process.h5's
|
|
// per-image arrays are only filled on the online per-image path). Sourced from the
|
|
// partials, which carry the first-pass per-image scale.
|
|
ScalingResult(indexer->GetIntegrationOutcome()).SaveToFile(config_.output_prefix);
|
|
}
|
|
|
|
if (result.consensus_cell && write_files && !config_.model_path.empty()) {
|
|
phase("Validating against model");
|
|
const auto data_sg = experiment_.GetSpaceGroupNumber();
|
|
// With a reference MTZ the merohedral indexing was already resolved against it (rotation
|
|
// merge / stills scaling), so trust that; only probe indexing by R-free when model-only.
|
|
ValidateAgainstModel(sm.merged, *result.consensus_cell, config_.model_path,
|
|
config_.output_prefix, logger,
|
|
data_sg ? std::optional<int>(static_cast<int>(*data_sg)) : std::nullopt,
|
|
/*probe_indexing_ambiguity=*/config_.reference_data.empty());
|
|
}
|
|
}
|
|
|
|
if (writer) {
|
|
writer->WriteHDF5(end_msg);
|
|
writer->Finalize();
|
|
result.written_master_path = config_.output_prefix + "_process.h5";
|
|
}
|
|
|
|
if (observer)
|
|
observer->OnPhase(cancelled_ ? "Cancelled" : "Done");
|
|
|
|
// Total wall time for the whole run, including scaling/merging/writing above (not just the
|
|
// per-image pass), so the reported rate reflects everything rugnux did.
|
|
result.processing_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - start_time).count();
|
|
if (result.processing_time_s > 0.0) {
|
|
result.frame_rate_hz = static_cast<double>(result.images_processed) / result.processing_time_s;
|
|
result.throughput_MBs = static_cast<double>(total_uncompressed_bytes) / (result.processing_time_s * 1e6);
|
|
}
|
|
|
|
logger.Info("{} {} images in {:.2f} s ({:.2f} Hz)", cancelled_ ? "Cancelled after" : "Processed",
|
|
result.images_processed, result.processing_time_s, result.frame_rate_hz);
|
|
return result;
|
|
}
|