v1.0.0-rc.155 (#65)

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

* jfjoch_process: Remove pixelrefine option (replaced with ProfileIntegrate2D)
* jfjoch_viewer: Some graphical improvements.
* jfjoch_viewer: Simplify und unify data analysis settings.
* jfjoch_writer: Add TCP keepalive to increase robustness if jfjoch_broker "dies" in the middle of data acquisition.

Reviewed-on: #65

process/JFJochProcess.cpp

@@ -27,7 +27,9 @@
 #include "../image_analysis/image_preprocessing/ImagePreprocessorCPU.h"
 #include "../image_analysis/image_preprocessing/ImagePreprocessorBuffer.h"
 #include "../image_analysis/scale_merge/Merge.h"
+#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
 #include "../image_analysis/scale_merge/SearchSpaceGroup.h"
+#include "../image_analysis/scale_merge/Combine3D.h"
 #include "../image_analysis/WriteReflections.h"
 
 namespace {
@@ -53,6 +55,27 @@ namespace {
         ret.assign(unique_ordinals.begin(), unique_ordinals.end());
         return ret;
     }
+
+    // XDS-order scaling. The rot3d combine emits fulls with partiality == 1 (image_scale_corr == 1),
+    // so they were only ever scaled as per-frame *partials* upstream - their per-frame scale is
+    // entangled with the rocking-curve/partiality model. This refits a per-frame scale directly on
+    // the complete reflections with the Unity model (no partiality term, G*Itrue - I_full), the way
+    // XDS/DIALS scale 3D-integrated fulls. A pure post-correction: it updates image_scale_corr on the
+    // fulls (1 -> 1/G) without re-combining.
+    void ScaleFulls(const DiffractionExperiment &experiment,
+                    std::vector<IntegrationOutcome> &fulls, int scaling_iter,
+                    size_t nthreads, Logger &logger) {
+        DiffractionExperiment unity = experiment;
+        ScalingSettings ss = unity.GetScalingSettings();
+        ss.SetPartialityModel(PartialityModel::Unity);
+        unity.ImportScalingSettings(ss);
+
+        for (int i = 0; i < scaling_iter; i++) {
+            const auto reference = MergeAll(unity, fulls, false);
+            ScaleOnTheFly(unity, reference).Scale(fulls, nthreads);
+        }
+        logger.Info("Scaled fulls (XDS order, Unity model)");
+    }
 }
 
 JFJochProcess::JFJochProcess(JFJochHDF5Reader &reader, DiffractionExperiment experiment,
@@ -370,32 +393,74 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
     // Scaling and merging (full analysis only).
     if (full && !cancelled_ && result.indexing_rate.has_value() && result.indexing_rate > 0
         && (config_.run_scaling || !config_.reference_data.empty())) {
-        if (observer)
-            observer->OnPhase("Scaling and merging");
+        // Scaling/merging is a long post-pass; report each sub-step as a phase so the GUI progress
+        // bar reflects what is happening instead of freezing on one label. Also time each phase
+        // (logged on transition) so the bottlenecks are visible.
+        auto t_phase = std::chrono::steady_clock::now();
+        std::string prev_phase;
+        auto phase = [&](const std::string &p) {
+            const auto now = std::chrono::steady_clock::now();
+            if (!prev_phase.empty())
+                logger.Info("[timing] {}: {:.2f} s", prev_phase,
+                            std::chrono::duration<double>(now - t_phase).count());
+            t_phase = now;
+            prev_phase = p;
+            if (observer) observer->OnPhase(p);
+        };
+        phase("Scaling and merging");
 
-        const bool pixel_refine_path =
-            experiment_.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine;
-
-        // ScaleOnTheFly is only for the classical, no-reference path; with a reference (or
-        // PixelRefine) each image is already scaled, so we merge directly.
-        if (config_.reference_data.empty() && !pixel_refine_path) {
+        // ScaleOnTheFly self-scaling is only for the no-reference path; with a reference each image
+        // is already scaled against it during the per-image pass, so we merge directly.
+        if (config_.reference_data.empty()) {
             logger.Info("Running scaling ...");
             ScalingResult scale_result(0);
+            double t_merge_all = 0.0, t_scale = 0.0;
             for (int i = 0; i < config_.scaling_iter; i++) {
+                phase("Scaling images (iteration " + std::to_string(i + 1) + "/"
+                      + std::to_string(config_.scaling_iter) + ")");
+                const auto a = std::chrono::steady_clock::now();
                 auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), false);
+                const auto b = std::chrono::steady_clock::now();
                 scale_result = indexer->ScaleAllImages(merge_result);
+                const auto c = std::chrono::steady_clock::now();
+                t_merge_all += std::chrono::duration<double>(b - a).count();
+                t_scale += std::chrono::duration<double>(c - b).count();
             }
+            logger.Info("[timing] scaling loop ({} iter): MergeAll(serial) {:.2f} s, ScaleAllImages(parallel) {:.2f} s",
+                        config_.scaling_iter, t_merge_all, t_scale);
         }
 
+        // -P rot3d: weight-sum each reflection's per-frame partials into one full before merging, so
+        // the error model sees counting statistics (high ISa) instead of rocking-curve slicing scatter.
+        const bool rot3d = experiment_.GetScalingSettings().GetCombine3D();
+        std::vector<IntegrationOutcome> combined;
+        if (rot3d) {
+            phase("Combining 3D partials");
+            combined = CombineRotationObservations(indexer->GetIntegrationOutcome(), experiment_, &logger,
+                                                   config_.observation_dump_path);
+        }
+        if (rot3d && experiment_.GetScalingSettings().GetScaleFulls()) {
+            phase("Scaling fulls (XDS order)");
+            ScaleFulls(experiment_, combined, static_cast<int>(config_.scaling_iter), config_.nthreads, logger);
+        }
+        const std::vector<IntegrationOutcome> &merge_input =
+            rot3d ? combined : indexer->GetIntegrationOutcome();
+
+        phase("Merging");
         MergeOnTheFly merge_engine(experiment_);
         if (result.consensus_cell.has_value())
             merge_engine.ReferenceCell(*result.consensus_cell);
-        merge_engine.RefineErrorModel(indexer->GetIntegrationOutcome());
-        for (const auto &outcome: indexer->GetIntegrationOutcome())
+        merge_engine.RefineErrorModel(merge_input);
+        if (merge_engine.ErrorModelActive())
+            logger.Info("Error model: a={:.3f} b={:.3f} ISa={:.1f}", merge_engine.ErrorModelA(),
+                        merge_engine.ErrorModelB(),
+                        merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0);
+        for (const auto &outcome: merge_input)
             merge_engine.AddImage(outcome);
 
         auto merged_reflections = merge_engine.ExportReflections();
-        auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer->GetIntegrationOutcome(),
+        phase("Computing statistics");
+        auto merged_statistics = merge_engine.MergeStats(merged_reflections, merge_input,
                                                          config_.reference_data);
         logger.Info("Merge complete ({} unique reflections)", merged_reflections.size());
 
@@ -415,8 +480,16 @@ ProcessResult JFJochProcess::Run(JFJochProcessObserver *observer) {
         stats_text << merged_statistics;
         result.merge_statistics_text = stats_text.str();
 
-        if (result.consensus_cell && write_files)
+        result.has_merge_statistics = true;
+        result.merge_statistics = merged_statistics;
+        result.error_model_isa = merge_engine.ErrorModelB() > 0 ? 1.0 / merge_engine.ErrorModelB() : 0.0;
+        result.has_reference = !config_.reference_data.empty();
+
+        if (result.consensus_cell && write_files) {
+            phase("Writing reflections");
             WriteReflections(merged_reflections, *result.consensus_cell, experiment_, config_.output_prefix);
+        }
+        phase("");   // flush the last phase's timing
     }
 
     if (writer) {