Merge broker-rugnux-live into rc176: rugnux in jfjoch_broker as a live thread during a DECTRIS rotation collection (off by default), the rugnux library without file I/O (results in ProcessResult::outputs, the CLI writes them through JFJochRugnuxOutput), END carrying the merged table, map coefficients and placed model

Branch reviewed at e16db07ff (review: owner's decisions accepted - _maps.mtz instead of .ccp4 in the
viewer, -o "" computes everything incl. MR/validation, Ctrl+C/error paths may leave fewer files, the
writer links gemmi/zlib) and merged at 55184eb09 with its review fixes (END size guard, frame drop as
in Lite, outputs released before re-runs, pool priority, docs, frontend gating).

Conflicts: rugnux/Rugnux.h (kept rc176's host_memory_budget_bytes, output_prefix removed as on the
branch), rugnux/RugnuxPipeline.h (MtzTable future + rc176's GPUMerge pointer), MolecularReplacement.h/.cpp
(rc176's GPU cost model, packed transform and split functions with the branch's RugnuxModel signature;
search_model with rc176's other_budgets, result.model_name). tests/RugnuxLargeTest.cpp: rc176's
host-budget test writes its files through WriteRugnuxOutputs now that the library writes none.
Generated code (broker/gen, frontend client, Python client docs, Redoc) regenerated from the merged
jfjoch_api.yaml with update_version.sh (version unchanged).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

image_analysis/scale_merge/RotationScaleMerge.cpp

@@ -12,7 +12,7 @@
 #include <array>
 #include <cmath>
 #include <cstdint>
-#include <fstream>
+#include <sstream>
 #include <future>
 #include <limits>
 #include <numeric>
@@ -303,10 +303,10 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
                                        std::optional<UnitCell> reference_cell,
                                        int scaling_iterations,
                                        size_t nthreads, Logger &logger,
-                                       std::string observation_dump_path)
+                                       std::function<void(const std::string &)> observation_dump)
     : x(experiment), partials_out(partial_outcomes), reference_cell(std::move(reference_cell)),
       nthreads(nthreads == 0 ? AvailableCpus() : nthreads), logger(&logger),
-      observation_dump_path(std::move(observation_dump_path)) {
+      observation_dump(std::move(observation_dump)) {
     const auto s = x.GetScalingSettings();
     min_partiality = s.GetMinPartiality();
     d_min_limit = s.GetHighResolutionLimit_A();
@@ -401,7 +401,7 @@ void RotationScaleMerge::IngestHost() {
     // only the ingest arrays depends on whether the device pipeline will be active.
     gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
     gpu_active_ = gpu_->Available() && !scale_on_cpu_;
-    resident_ingest = gpu_active_ && observation_dump_path.empty();
+    resident_ingest = gpu_active_ && !observation_dump;
 #endif
 
     frame_cell_ok.assign(n_frames, 1);
@@ -4440,7 +4440,7 @@ void RotationScaleMerge::Combine() {
     // Combine one raw-hkl run into fulls, appended to `out`. Independent per run (the events of one hkl
     // touch no shared state), so runs parallelise cleanly. Returns the number of usable partials seen.
     // `dump` (serial path only) writes each emitted full for the diagnostic observation dump.
-    auto process_rawrun = [&](int r, std::vector<Obs> &out, std::ofstream *dump, int64_t &n_overloaded) -> size_t {
+    auto process_rawrun = [&](int r, std::vector<Obs> &out, std::ostream *dump, int64_t &n_overloaded) -> size_t {
         prefetch_run(r + COMBINE_PREFETCH_RUNS);
         const int lo = rawrun_start[r], hi = rawrun_start[r] + rawrun_count[r];
         std::vector<int32_t> ev;   // usable perm-indices of this raw hkl, in image-number order
@@ -4583,11 +4583,10 @@ void RotationScaleMerge::Combine() {
     size_t n_used = 0;
     std::vector<std::vector<Obs>> part;   // the fulls of consecutive runs, in run order
 
-    if (!observation_dump_path.empty() || nthreads <= 1) {
+    if (observation_dump || nthreads <= 1) {
         // Serial (diagnostic dump needs a single writer).
-        std::ofstream dump;
-        if (!observation_dump_path.empty()) {
-            dump.open(observation_dump_path);
+        std::ostringstream dump;
+        if (observation_dump) {
             // Columns are enough to rescale/merge the combined fulls in an external program: peak_frame
             // is a per-full batch (ZD) and px/py the predicted detector position (XD/YD).
             dump << "# h k l I sigma d n_frames captured_fraction peak_frame px py\n";
@@ -4595,7 +4594,9 @@ void RotationScaleMerge::Combine() {
         part.resize(1);
         part[0].reserve(n_run);
         for (int r = 0; r < n_run; ++r)
-            n_used += process_rawrun(r, part[0], dump.is_open() ? &dump : nullptr, fulls_dropped_overloaded);
+            n_used += process_rawrun(r, part[0], observation_dump ? &dump : nullptr, fulls_dropped_overloaded);
+        if (observation_dump)
+            observation_dump(dump.str());
     } else {
         // Parallel over contiguous rawrun chunks; the per-thread fulls are taken in run order below, so
         // the result is deterministic.
@@ -6876,7 +6877,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
     // scale-fulls loop below. Only the diagnostic dump (serial, one writer) falls back to CPU. The
     // decay/absorption correction surfaces run on the downloaded fulls (px/py is carried through the
     // combine) and their corrected corr is re-uploaded before the resident merge - so they stay on the GPU.
-    if (gpu_active_ && observation_dump_path.empty()) {
+    if (gpu_active_ && !observation_dump) {
         // The smoothed corr is already resident (scaling + smooth-G ran on the device, no round-trip).
         const int nf = gpu_->Combine(rawrun_group.data(), min_partiality, capture_uncertainty_coeff,
                                      min_captured_fraction, max_frame_gap, fulls_dropped_overloaded);