From db8ae0452e39609a842c59e8093ed6bc1ee439ca Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 14 Jul 2026 10:06:07 +0200 Subject: [PATCH] rugnux: fix --scale on a self-contained _process.h5 (reflections + error model) Offline --scale re-scales the reflections stored in a _process.h5 without re-integration, but was broken for a self-contained integrated snapshot: 1. HDF5MetadataSource::ReadReflections fell back per-image to the linked source pixel files for every non-indexed frame (a snapshot's master holds /entry/reflections only for the sparse indexed images). With the raw _data_NNNNNN.h5 absent or not co-located this threw an HDF5 error; when present it needlessly reopened multi-GB files thousands of times. Decide once whether the master is the authoritative reflection store and, if so, never fall back - a missing per-image group just means the frame has none. Legacy/VDS acquisitions (no /entry/reflections in the master) still resolve reflections lazily to the source data files. 2. The scale-only stills branch never fit the (a,b) error model, so it merged with the identity model - far worse intensities than the full pipeline (lyso8 CC1/2 76%->21%). Fit RefineErrorModel and honour --reject-outliers, mirroring Rugnux.cpp. Round-trip validated: full merge == integrate(--no-merge) + --scale (identical error model a=0.793 b=2.287, matching unique reflection counts), and --scale now reads reflections in ~2.5s with no raw-file access. Co-Authored-By: Claude Opus 4.8 (1M context) --- reader/HDF5MetadataSource.cpp | 9 ++++++++- rugnux/rugnux_cli.cpp | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/reader/HDF5MetadataSource.cpp b/reader/HDF5MetadataSource.cpp index 95bfc19a..40469725 100644 --- a/reader/HDF5MetadataSource.cpp +++ b/reader/HDF5MetadataSource.cpp @@ -1099,6 +1099,13 @@ std::vector HDF5MetadataSource::ReadReflections(size_t start std::vector ret; ret.reserve(end_image_val - start_image + 1); + // A self-contained integrated _process.h5 keeps all reflections in this master (one group per + // indexed image), so a missing per-image group means that image simply has none - never fall + // back to the linked source pixel files (which may be absent, and never hold a snapshot's + // reflections). A legacy/VDS acquisition has no /entry/reflections in the master and resolves + // reflections lazily from the source data files instead. + const bool master_reflections_authoritative = master_file->Exists("/entry/reflections"); + for (size_t img = start_image; img <= end_image_val; img++) { IntegrationOutcome outcome; @@ -1112,7 +1119,7 @@ std::vector HDF5MetadataSource::ReadReflections(size_t start HDF5ReadOnlyFile *meta_file = master_file.get(); size_t meta_image_id = img; std::string refl_group = fmt::format("/entry/reflections/image_{:06d}", img); - if (!master_file->Exists(refl_group)) { + if (!master_reflections_authoritative && !master_file->Exists(refl_group)) { const auto loc = ResolveMeta(static_cast(img)); meta_file = loc.file.get(); meta_image_id = loc.local_index; diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 33d55ebc..e9f86dc1 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -986,6 +986,9 @@ int main(int argc, char **argv) { scaling_settings.MinCapturedFraction(min_captured_fraction_arg.value_or( (experiment.GetGoniometer().has_value() && !force_still) ? 0.7 : 0.0)); scaling_settings.MinCCForImage(min_image_cc / 100.0); // --min-image-cc is percent; the setting is a fraction + scaling_settings.OutlierRejectNsigma( + outlier_reject_nsigma.value_or( + (experiment.GetGoniometer().has_value() && !force_still) ? REJECT_OUTLIERS_DEFAULT_NSIGMA : 0.0)); if (intensity_format) scaling_settings.FileFormat(intensity_format.value()); experiment.ImportScalingSettings(scaling_settings); @@ -1031,6 +1034,15 @@ int main(int argc, char **argv) { } MergeOnTheFly merge_engine(experiment); merge_engine.ReferenceCell(experiment.GetUnitCell()); + // Fit the (a, b) error model from symmetry-mate scatter before merging, exactly as the full + // pipeline does (Rugnux.cpp). Without this the offline --scale merge would use the identity + // model and produce much worse stills intensities (no (b*I)^2 systematic term, no sigma floor). + merge_engine.RefineErrorModel(reflections); + if (merge_engine.ErrorModelActive()) + logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f} chi2={:.2f}", merge_engine.ErrorModelA(), + merge_engine.ErrorModelB(), + merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0, + merge_engine.ErrorModelChi2()); for (size_t i = 0; i < reflections.size(); ++i) merge_engine.AddImage(reflections[i], static_cast(i)); merged_reflections = merge_engine.ExportReflections();