v1.0.0-rc.156 (#66)

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use. * jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output. * jfjoch_process: Add EXPERIMENTAL ice-ring detection (--detect-ice-rings) that excludes ice reflections from scaling. * Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don't fit rather than aborting), and add the jfjoch_recompress tool. * jfjoch_viewer: Report "Multiple lattices detected" and grey out "Analyze dataset" on a live connection. * jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata. * CI: Build Windows (CUDA and non-CUDA) installers.Reviewed-on: #66 Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-07-03 19:18:56 +02:00
parent 54c667190f
commit d6389e12da
295 changed files with 9853 additions and 3031 deletions
@@ -27,10 +27,16 @@
 #include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
 #include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
 #include "../image_analysis/scale_merge/Merge.h"
-#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
+#include "../image_analysis/scale_merge/RotationScaleMerge.h"
+#include "../image_analysis/scale_merge/ScalingResult.h"
 #include "../image_analysis/scale_merge/SearchSpaceGroup.h"
-#include "../image_analysis/scale_merge/Combine3D.h"
+#include "../image_analysis/scale_merge/TwinningAnalysis.h"
+#include "../image_analysis/scale_merge/HKLKey.h"
 #include "../image_analysis/WriteReflections.h"
+#include "../common/Definitions.h"
+#include "../common/CorrelationCoefficient.h"
+#include <array>
+#include <map>

 namespace {
    // Pick up to requested_images ordinals spread evenly across [0, images_to_process) for the
@@ -56,26 +62,6 @@ namespace {
        return ret;
    }

-    // XDS-order scaling. The rot3d combine emits fulls with partiality == 1 (image_scale_corr == 1),
-    // so they were only ever scaled as per-frame *partials* upstream - their per-frame scale is
-    // entangled with the rocking-curve/partiality model. This refits a per-frame scale directly on
-    // the complete reflections with the Unity model (no partiality term, G*Itrue - I_full), the way
-    // XDS/DIALS scale 3D-integrated fulls. A pure post-correction: it updates image_scale_corr on the
-    // fulls (1 -> 1/G) without re-combining.
-    void ScaleFulls(const DiffractionExperiment &experiment,
-                    std::vector<IntegrationOutcome> &fulls, int scaling_iter,
-                    size_t nthreads, Logger &logger) {
-        DiffractionExperiment unity = experiment;
-        ScalingSettings ss = unity.GetScalingSettings();
-        ss.SetPartialityModel(PartialityModel::Unity);
-        unity.ImportScalingSettings(ss);
-
-        for (int i = 0; i < scaling_iter; i++) {
-            const auto reference = MergeAll(unity, fulls, false);
-            ScaleOnTheFly(unity, reference).Scale(fulls, nthreads);
-        }
-        logger.Info("Scaled fulls (XDS order, Unity model)");
-    }
 }

 JFJochProcess::JFJochProcess(JFJochHDF5Reader &reader, DiffractionExperiment experiment,
@@ -166,7 +152,7 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
    start_message.hdf5_source_data = reader_.GetHDF5DataSource(start_image, images_to_process);

    std::unique_ptr<FileWriter> writer;
-    if (write_files)
+    if (write_files && config_.write_process_h5)
        writer = std::make_unique<FileWriter>(start_message);

    logger.Info("Processing {} images (range {}-{}, stride {}) using {} threads [{}]",
@@ -284,6 +270,8 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
            msg.az_int_profile_count = profile.GetPixelCount();
            msg.az_int_profile_std = profile.GetStd();
            msg.bkg_estimate = profile.GetBkgEstimate(mapping.Settings());
+            msg.ice_ring_score = profile.GetIceRingScore(mapping.Settings(),
+                                                         config_.spot_finding.ice_ring_width_Q_recipA);
            msg.run_number = experiment_.GetRunNumber();
            msg.run_name = experiment_.GetRunName();

@@ -409,85 +397,199 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
        };
        phase("Scaling and merging");

-        // ScaleOnTheFly self-scaling is only for the no-reference path; with a reference each image
-        // is already scaled against it during the per-image pass, so we merge directly.
-        if (config_.reference_data.empty()) {
-            logger.Info("Running scaling ...");
-            ScalingResult scale_result(0);
-            double t_merge_all = 0.0, t_scale = 0.0;
-            for (int i = 0; i < config_.scaling_iter; i++) {
-                phase("Scaling images (iteration " + std::to_string(i + 1) + "/"
-                      + std::to_string(config_.scaling_iter) + ")");
-                const auto a = std::chrono::steady_clock::now();
-                auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
-                const auto b = std::chrono::steady_clock::now();
-                scale_result = indexer->ScaleAllImages(merge_result);
-                const auto c = std::chrono::steady_clock::now();
-                t_merge_all += std::chrono::duration<double>(b - a).count();
-                t_scale += std::chrono::duration<double>(c - b).count();
+        // Ice-ring handling (--detect-ice-rings): flag reflections sitting on a hexagonal-ice powder
+        // ring. Their integrated intensity is contaminated by the strong, variable ice background, so
+        // they drag the per-image scale fit. Flag them here so scaling (the per-image G fit and the
+        // fulls refit) skips them, while the combine, the merge and the statistics keep them - dropping
+        // them outright guts low/mid-resolution completeness on crystals that merge them fine.
+        if (experiment_.IsDetectIceRings()) {
+            const float ice_width = config_.spot_finding.ice_ring_width_Q_recipA;
+            size_t total = 0, flagged = 0;
+            for (auto &outcome : indexer->GetIntegrationOutcome()) {
+                for (auto &r : outcome.reflections) {
+                    ++total;
+                    r.on_ice_ring = IsOnIceRing(r.d, ice_width);
+                    if (r.on_ice_ring)
+                        ++flagged;
+                }
            }
-            logger.Info("[timing] scaling loop ({} iter): MergeAll(serial) {:.2f} s, ScaleAllImages(parallel) {:.2f} s",
-                        config_.scaling_iter, t_merge_all, t_scale);
+            logger.Info("Ice-ring handling: flagged {} of {} reflections on ice rings (half-width {:.3f} A^-1); "
+                        "excluded from scaling, kept for merging", flagged, total, ice_width);
        }

-        // -P rot3d: weight-sum each reflection's per-frame partials into one full before merging, so
-        // the error model sees counting statistics (high ISa) instead of rocking-curve slicing scatter.
-        const bool rot3d = experiment_.GetScalingSettings().GetCombine3D();
-        std::vector<IntegrationOutcome> combined;
-        if (rot3d) {
-            phase("Combining 3D partials");
-            combined = CombineRotationObservations(indexer->GetIntegrationOutcome(), experiment_, &logger,
-                                                   config_.observation_dump_path);
-        }
-        if (rot3d && experiment_.GetScalingSettings().GetScaleFulls()) {
-            phase("Scaling fulls (XDS order)");
-            ScaleFulls(experiment_, combined, static_cast<int>(config_.scaling_iter), config_.nthreads, logger);
-        }
-        const std::vector<IntegrationOutcome> &merge_input =
-            rot3d ? combined : indexer->GetIntegrationOutcome();
+        // Scale the images and merge. Factored so it can run twice: first in P1 to give the
+        // space-group search a merged dataset, then again in the determined space group so the
+        // scaling sees symmetry equivalents and the final statistics are in the right symmetry.
+        // (With a user-fixed space group it simply runs once, already in that symmetry.)
+        struct ScaleMergeResult {
+            std::vector<MergedReflection> merged;
+            MergeStatistics statistics;
+        };
+        // The reference path computes each image's G once (per-image scaling against the
+        // reference); the scaling loop below is skipped, so G is stable across the two passes.
+        // Smoothing it more than once would compound the correction, so do it only on the first
+        // pass. The no-reference path recomputes G from scratch each pass and re-smooths correctly.
+        // Ice rings masked from the merge by the CC1/2 test after the final merge; empty until then,
+        // at which point a re-merge applies them.
+        std::vector<char> masked_ice_rings;

-        phase("Merging");
-        MergeOnTheFly merge_engine(experiment_);
-        if (result.consensus_cell.has_value())
-            merge_engine.ReferenceCell(*result.consensus_cell);
-        merge_engine.RefineErrorModel(merge_input);
-        if (merge_engine.ErrorModelActive())
-            logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f}", merge_engine.ErrorModelA(),
-                        merge_engine.ErrorModelB(),
-                        merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0);
-        for (const auto &outcome: merge_input)
-            merge_engine.AddImage(outcome);
+        // Rotation self-scaling + 3D combine + merge is done by the dedicated RotationScaleMerge (a single
+        // allocate-once engine that recomputes partiality from the fitted mosaicity, combines the per-frame
+        // partials, scales the fulls and merges; the whole hot path runs on the GPU when one is present).
+        // Ingested once here (after ice flagging) and reused across both space-group passes; a fixed
+        // (forced) mosaicity is handled by the recompute. It does NOT support external-reference scaling,
+        // B-factor refinement, an absorption surface, or wedge refinement - reject those combinations.
+        const auto &rot_ss = experiment_.GetScalingSettings();
+        const bool is_rotation = experiment_.IsRotationIndexing();   // rotation indexing -> rotation scaling/merge
+        std::optional<RotationScaleMerge> rsm;
+        if (is_rotation) {
+            if (!config_.reference_data.empty() || rot_ss.GetRefineB() || rot_ss.GetAbsorptionIter() > 0
+                || experiment_.GetRefineRotationWedgeInScaling()
+                || rot_ss.GetRotationWedgeForScaling().has_value())
+                throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                      "Rotation scaling/merging (RotationScaleMerge) does not support "
+                                      "reference scaling, B-factor refinement, absorption surface or "
+                                      "wedge refinement");
+            rsm.emplace(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
+                        static_cast<int>(config_.scaling_iter),
+                        config_.spot_finding.ice_ring_width_Q_recipA,
+                        config_.nthreads, logger, config_.observation_dump_path);
+            rsm->Ingest();
+        }

-        auto merged_reflections = merge_engine.ExportReflections();
-        phase("Computing statistics");
-        auto merged_statistics = merge_engine.MergeStats(merged_reflections, merge_input,
-                                                         config_.reference_data);
-        logger.Info("Merge complete ({} unique reflections)", merged_reflections.size());
+        auto scale_and_merge = [&](const std::string &label, bool for_search) -> ScaleMergeResult {
+            if (rsm) {
+                logger.Info("Scaling+merging ({}) via RotationScaleMerge", label);
+                phase("Scale/combine/merge (" + label + ")");
+                auto r = rsm->Run(for_search, masked_ice_rings);
+                result.error_model_isa = r.isa;
+                return ScaleMergeResult{std::move(r.merged), std::move(r.statistics)};
+            }
+            // Stills (rotation goes through RotationScaleMerge above): self-scale each image against the
+            // running merge with ScaleOnTheFly (fixed partiality), then merge directly. With an external
+            // reference each image is already scaled against it during the per-image pass, so skip.
+            if (config_.reference_data.empty()) {
+                logger.Info("Running scaling ({}) ...", label);
+                for (int i = 0; i < config_.scaling_iter; i++) {
+                    phase("Scaling images (" + label + ", iteration " + std::to_string(i + 1) + "/"
+                          + std::to_string(config_.scaling_iter) + ")");
+                    auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
+                    indexer->ScaleAllImages(merge_result);
+                }
+            }
+            const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
+
+            phase("Merging");
+            MergeOnTheFly merge_engine(experiment_);
+            // For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
+            // intensities and the error model, so the space-group search sees clean data; the final
+            // in-symmetry merge keeps them so completeness is not lost.
+            merge_engine.ExcludeIceRings(for_search);
+            merge_engine.MaskIceRings(masked_ice_rings, config_.spot_finding.ice_ring_width_Q_recipA);
+            if (result.consensus_cell.has_value())
+                merge_engine.ReferenceCell(*result.consensus_cell);
+            merge_engine.RefineErrorModel(merge_input);
+            if (merge_engine.ErrorModelActive())
+                logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f} chi2={:.2f}", merge_engine.ErrorModelA(),
+                            merge_engine.ErrorModelB(),
+                            merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0,
+                            merge_engine.ErrorModelChi2());
+            for (size_t i = 0; i < merge_input.size(); ++i)
+                merge_engine.AddImage(merge_input[i], static_cast<int64_t>(i));
+
+            ScaleMergeResult out;
+            out.merged = merge_engine.ExportReflections();
+            phase("Computing statistics");
+            out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data);
+            result.error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
+            logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label);
+            return out;
+        };
+
+        // First pass: P1 when searching, or directly the user-fixed space group.
+        const auto initial_sg = experiment_.GetGemmiSpaceGroup();
+        auto sm = scale_and_merge(initial_sg ? initial_sg->short_name() : "P1", !initial_sg.has_value());

        std::ostringstream stats_text;
        if (!experiment_.GetGemmiSpaceGroup().has_value()) {
            SearchSpaceGroupOptions sg_opts;
-            sg_opts.crystal_system.reset();
-            sg_opts.centering = '\0';
            sg_opts.merge_friedel = experiment_.GetScalingSettings().GetMergeFriedel();
            sg_opts.d_min_limit_A = experiment_.GetScalingSettings().GetHighResolutionLimit_A().value_or(0.0);
-            sg_opts.min_operator_cc = 0.80;
-            sg_opts.min_pairs_per_operator = 20;
-            sg_opts.min_total_compared = 100;
-            sg_opts.test_systematic_absences = true;
-            stats_text << SearchSpaceGroupResultToText(SearchSpaceGroup(merged_reflections, sg_opts)) << "\n\n";
-        }
-        stats_text << merged_statistics;
-        result.merge_statistics_text = stats_text.str();
+            // Constrain the search to subgroups of the lattice (metric) symmetry found by rotation
+            // indexing. Centering is not constrained here - it is determined from the absences.
+            if (end_msg.rotation_lattice_type.has_value())
+                sg_opts.lattice_system = end_msg.rotation_lattice_type->crystal_system;
+            const auto sg_search = SearchSpaceGroup(sm.merged, sg_opts);
+            stats_text << SearchSpaceGroupResultToText(sg_search) << "\n\n";

+            // Adopt the determined space group and re-scale/merge in it, so scaling uses symmetry
+            // equivalents and the statistics come out in the right symmetry. P1 stands when nothing
+            // is determined.
+            if (sg_search.best_space_group.has_value()) {
+                const auto &sg = *sg_search.best_space_group;
+                logger.Info("Adopting space group {} (number {})", sg.short_name(), sg.number);
+                experiment_.SpaceGroupNumber(sg.number);
+                end_msg.space_group_number = sg.number;
+                result.space_group_number = sg.number;
+                phase("Re-scaling in space group " + sg.short_name());
+                sm = scale_and_merge(sg.short_name(), false);
+            }
+        }
+
+        // Ice-ring CC1/2 mask: a hexagonal-ice ring whose merged half-set CC1/2 collapses well below its
+        // resolution shoulders has been decorrelated by ice (its Bragg is unrecoverable) - drop it and
+        // re-merge. Weak/absent rings track their neighbours and stay, so completeness on clean crystals
+        // is untouched. Uses the ring band (+/- ice width in q=2pi/d) vs the shoulders either side.
+        if (experiment_.IsDetectIceRings() && !sm.merged.empty()) {
+            constexpr float two_pi = 6.283185307f;
+            const float w = config_.spot_finding.ice_ring_width_Q_recipA;
+            std::vector<char> mask(ICE_RING_RES_A.size(), 0);
+            for (size_t i = 0; i < ICE_RING_RES_A.size(); ++i) {
+                const float q_ring = two_pi / ICE_RING_RES_A[i];
+                CorrelationCoefficient ring, shoulder;
+                size_t n_ring = 0, n_shoulder = 0;
+                for (const auto &m : sm.merged) {
+                    if (!(m.d > 0.0f) || !std::isfinite(m.I_half[0]) || !std::isfinite(m.I_half[1]))
+                        continue;
+                    const float dq = std::fabs(two_pi / m.d - q_ring);
+                    if (dq < w) { ring.Add(m.I_half[0], m.I_half[1]); ++n_ring; }
+                    else if (dq < 3.0f * w) { shoulder.Add(m.I_half[0], m.I_half[1]); ++n_shoulder; }
+                }
+                if (n_ring >= 20 && n_shoulder >= 20 && shoulder.GetCC() > 0.5
+                    && ring.GetCC() < shoulder.GetCC() - 0.05) {
+                    mask[i] = 1;
+                    logger.Info("Ice-ring mask: {:.2f} A ring CC1/2 {:.3f} << shoulders {:.3f}; masked from merge",
+                                ICE_RING_RES_A[i], ring.GetCC(), shoulder.GetCC());
+                }
+            }
+            if (std::any_of(mask.begin(), mask.end(), [](char c) { return c != 0; })) {
+                masked_ice_rings = std::move(mask);
+                const auto final_sg = experiment_.GetGemmiSpaceGroup();
+                sm = scale_and_merge(final_sg ? final_sg->short_name() : "P1", false);
+            }
+        }
+
+        const auto twin_sg_number = experiment_.GetSpaceGroupNumber();
+        const gemmi::SpaceGroup *twin_sg = twin_sg_number
+            ? gemmi::find_spacegroup_by_number(twin_sg_number.value()) : nullptr;
+        result.twinning = AnalyzeTwinning(sm.merged, twin_sg);
+        stats_text << TwinningAnalysisToText(result.twinning) << "\n";
+        stats_text << sm.statistics;
+        result.merge_statistics_text = stats_text.str();
        result.has_merge_statistics = true;
-        result.merge_statistics = merged_statistics;
-        result.error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
+        result.merge_statistics = sm.statistics;
        result.has_reference = !config_.reference_data.empty();

        if (result.consensus_cell && write_files) {
            phase("Writing reflections");
-            WriteReflections(merged_reflections, *result.consensus_cell, experiment_, config_.output_prefix);
+            WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics,
+                             result.error_model_isa > 0 ? fmt::format("{:.2f}", result.error_model_isa) : "?",
+                             config_.output_prefix);
+            // Per-image scaling table (G, B-factor, mosaicity, wedge, CC) for inspection / XDS
+            // comparison. The offline self-scaling result is otherwise not exposed (process.h5's
+            // per-image arrays are only filled on the online per-image path). Sourced from the
+            // partials, which carry the first-pass per-image scale.
+            ScalingResult(indexer->GetIntegrationOutcome()).SaveToFile(config_.output_prefix);
        }
        phase("");   // flush the last phase's timing
    }