Live stills: the frames are analysed with the pre-pass's settings, bounded by iterations

The per-image analysis of a live serial collection ran on the broker's bare experiment and took only the
beam centre, distance and mask from the pre-pass, so it lost what the pre-pass itself integrated with:
the polarization (no Lp correction at all), the stills integration radii and the background ring's
radial correction as the pre-scan measured them, and the run's rocking width - every frame predicted
through twice its own profile radius, which on the same frames integrated 13% fewer reflections than
the run's window and left the merge thousands of far-tail observations (13723 under a partiality of
0.001 against 77 offline). The pre-pass result now carries them (StillsPrePassResult), the analysis
experiment imports them, the indexer gets the window and the live merge the run width.

The per-frame refinement was bounded by the wall clock (real_time), which the code itself documents as
making the answer depend on machine load: the same 1122 frames gave 521 to 567 crystals from run to
run. The frames wait in the buffer, so it is bounded by the iteration caps as offline (50 and 25
iterations, LMSolver); measured offline that refinement takes 0.2 ms a frame against the 40 ms budget.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK

receiver/StillsGPUAnalysis.cpp

@@ -218,6 +218,9 @@ void StillsGPUAnalysis::PrePassThread() {
     int geometry_frames = 0;
     if (result) {
         setup.experiment.BeamX_pxl(result->beam_x_pxl).BeamY_pxl(result->beam_y_pxl).DetectorDistance_mm(result->distance_mm);
+        // The frames are integrated as the pre-pass integrated its own: its polarization and background ring
+        setup.experiment.PolarizationFactor(result->polarization);
+        setup.experiment.ImportBraggIntegrationSettings(result->integration);
         setup.mask = result->mask;
         geometry_frames = result->geometry_frames;
         const auto &result_mask = result->mask.GetMask();
@@ -235,8 +238,13 @@ void StillsGPUAnalysis::PrePassThread() {
         logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask");
     try {
         setup.mapping = std::make_unique<AzimuthalIntegrationMapping>(setup.experiment, setup.mask);
-        // Retained for the merge, which takes each image's reflections as soon as it is analysed.
-        indexer = std::make_unique<IndexAndRefine>(setup.experiment, indexing_thread_pool, merge_frames, true);
+        // Retained for the merge, which takes each image's reflections as soon as it is analysed. The frames
+        // wait in the buffer, so the per-frame refinement is bounded by iterations as offline, not by the
+        // clock: the same frames give the same lattices whatever else the machine is doing.
+        indexer = std::make_unique<IndexAndRefine>(setup.experiment, indexing_thread_pool, merge_frames, false);
+        // Every frame predicts through the window of the rocking width the pre-pass measured, as offline
+        if (result)
+            indexer->SetStillsWindow(result->window);
         if (geometry_frames == 0) {
             const std::string text = fmt::format(
                     "The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the "
@@ -250,7 +258,8 @@ void StillsGPUAnalysis::PrePassThread() {
             merge_experiment = setup.experiment;
             merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false));
             merge = std::make_unique<StillsLiveMerge>(merge_experiment, experiment.GetImageNum(),
-                                                      setup.experiment.GetUnitCell(), std::nullopt,
+                                                      setup.experiment.GetUnitCell(),
+                                                      result ? result->run_width : std::nullopt,
                                                       setup.experiment.IsDetectIceRings(),
                                                       SpotFindingSettings().ice_ring_width_Q_recipA, logger);
             // The model's thread first: the merge thread hands it its merges

receiver/StillsPrePass.cpp

@@ -60,6 +60,10 @@ StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<i
         .distance_mm = result.used_distance_mm,
         .mask = rugnux.GetPixelMask(),
         .images = images.size(),
-        .geometry_frames = result.stills_geometry_frames
+        .geometry_frames = result.stills_geometry_frames,
+        .polarization = rugnux.GetExperiment().GetPolarizationFactor(),
+        .integration = rugnux.GetExperiment().GetBraggIntegrationSettings(),
+        .run_width = rugnux.GetStillsRunWidth(),
+        .window = rugnux.GetStillsWindow()
     };
 }

receiver/StillsPrePass.h

@@ -7,9 +7,12 @@
 #include <memory>
 #include <vector>
 
+#include "../common/BraggIntegrationSettings.h"
 #include "../common/JFJochMessages.h"
 #include "../common/PixelMask.h"
+#include "../common/StillsRunWidth.h"
 #include "../common/ZeroCopyReturnValue.h"
+#include "../image_analysis/IndexAndRefine.h"   // StillsWindow
 
 // An image held until analysed, pointing into `slot`, which goes back unsent with the last copy. The slot is
 // forgotten then, so that the copy sent with the analysis is the only one the image buffer finds.
@@ -33,8 +36,16 @@ struct StillsPrePassResult {
     PixelMask mask;
     size_t images = 0;
     int geometry_frames = 0;   // the indexed frames the distance and centre were refined on; 0: kept as stated
+    // What the pre-pass integrated its frames with, for the frames after it: the polarization, the integration
+    // radii and background ring as the pre-scan measured them, and the run's rocking width with the prediction
+    // window made from it (empty where the pre-pass could not measure it).
+    std::optional<float> polarization;
+    BraggIntegrationSettings integration;
+    std::optional<StillsRunWidth> run_width;
+    std::optional<StillsWindow> window;
 };
 
-// The first frames of a serial collection through rugnux: beam centre, distance (with a cell) and mask it decides
+// The first frames of a serial collection through rugnux: beam centre, distance (with a cell), mask and
+// integration settings it decides
 StillsPrePassResult RunStillsPrePass(const StartMessage &start, const std::map<int64_t, std::shared_ptr<HeldImage>> &images,
                                      int nthreads);

rugnux/Rugnux.h

@@ -33,6 +33,7 @@
 #include "../image_analysis/scale_merge/TwinningAnalysis.h"   // TwinningAnalysisResult
 #include "../image_analysis/scale_merge/CentreOfSymmetry.h"   // CentreOfSymmetryResult
 #include "../image_analysis/scale_merge/StillsPartialityRefine.h"   // StillsPartialityRefine::RunWidth
+#include "../image_analysis/IndexAndRefine.h"   // StillsWindow
 #include "../image_analysis/scale_merge/TranslationalNCS.h"   // TranslationalNCSResult
 #include "../image_analysis/scale_merge/IceRingVerdict.h"   // IceRingZoneVerdict
 #include "../image_analysis/scale_merge/IceRingIntensity.h"
@@ -1824,6 +1825,7 @@ class Rugnux {
     // Stills: the rocking width measured before the image pass, on a sample of its frames, which every
     // frame is then predicted through (ImagePass) and the partialities are computed from (ScaleMerge).
     std::optional<StillsPartialityRefine::RunWidth> stills_run_width_;
+    std::optional<StillsWindow> stills_window_;   // ... and the window made from it
     // --export-observations: the written stills merge's verdict on every observation, its resolution cut and
     // the rocking curve its partialities were computed from.
     std::vector<std::vector<ObservationVerdict>> observation_verdicts_;
@@ -1974,6 +1976,11 @@ public:
     // The mask the run ended with: the input's, plus the beam-stop shadow and the defective pixels the
     // pre-scan found.
     const PixelMask &GetPixelMask() const { return pixel_mask_; }
+    // The experiment the run ended with: the geometry and the settings its frames were integrated at.
+    const DiffractionExperiment &GetExperiment() const { return experiment_; }
+    // Stills: the rocking width the run measured and the prediction window it made from it (ImagePass).
+    const std::optional<StillsRunWidth> &GetStillsRunWidth() const { return stills_run_width_; }
+    const std::optional<StillsWindow> &GetStillsWindow() const { return stills_window_; }
 
     // Request cancellation; safe to call from any thread (the worker loop checks between images).
     // fast_live makes it a cut: a pass that is past its images still merges and makes its outputs, the

rugnux/RugnuxImagePass.cpp

@@ -271,6 +271,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
                                && experiment_.GetScalingSettings().GetStillsPartialityRefine();
     if (stills_window) {
         stills_run_width_.reset();
+        stills_window_.reset();
         indexer->SetStillsWindow(std::nullopt);
         const int n_sample = std::min(images_to_process,
                                       std::max(STILLS_WINDOW_MIN_FRAMES, images_to_process / STILLS_WINDOW_FRACTION));
@@ -462,11 +463,12 @@ void Rugnux::ImagePass(PipelineLocals &p) {
             outcomes[o] = IntegrationOutcome{};
         if (stills_run_width_) {
             const auto k = static_cast<float>(config_.stills_window_sigmas);
-            indexer->SetStillsWindow(StillsWindow{.c0 = k * static_cast<float>(stills_run_width_->gamma0),
-                                                  .ang = k * static_cast<float>(stills_run_width_->gamma_e),
-                                                  .bw = k * static_cast<float>(stills_run_width_->bandwidth_sigma),
-                                                  .q_scale = static_cast<float>(stills_run_width_->radial_scale),
-                                                  .radial = k * k * static_cast<float>(radial_spread2)});
+            stills_window_ = StillsWindow{.c0 = k * static_cast<float>(stills_run_width_->gamma0),
+                                          .ang = k * static_cast<float>(stills_run_width_->gamma_e),
+                                          .bw = k * static_cast<float>(stills_run_width_->bandwidth_sigma),
+                                          .q_scale = static_cast<float>(stills_run_width_->radial_scale),
+                                          .radial = k * k * static_cast<float>(radial_spread2)};
+            indexer->SetStillsWindow(stills_window_);
             Note(fmt::format("prediction window {:.1f} x the rocking width measured on {} frames, centred at "
                              "radial scale {:+.2e}", config_.stills_window_sigmas, window_frames.size(), radial_scale - 1.0));
             Note(fmt::format("spots placed with their excitation error as by a radial spread of {:.1e}",