Merge broker-rugnux-live into rc176: rugnux in jfjoch_broker as a live thread during a DECTRIS rotation collection (off by default), the rugnux library without file I/O (results in ProcessResult::outputs, the CLI writes them through JFJochRugnuxOutput), END carrying the merged table, map coefficients and placed model

Branch reviewed at e16db07ff (review: owner's decisions accepted - _maps.mtz instead of .ccp4 in the
viewer, -o "" computes everything incl. MR/validation, Ctrl+C/error paths may leave fewer files, the
writer links gemmi/zlib) and merged at 55184eb09 with its review fixes (END size guard, frame drop as
in Lite, outputs released before re-runs, pool priority, docs, frontend gating).

Conflicts: rugnux/Rugnux.h (kept rc176's host_memory_budget_bytes, output_prefix removed as on the
branch), rugnux/RugnuxPipeline.h (MtzTable future + rc176's GPUMerge pointer), MolecularReplacement.h/.cpp
(rc176's GPU cost model, packed transform and split functions with the branch's RugnuxModel signature;
search_model with rc176's other_budgets, result.model_name). tests/RugnuxLargeTest.cpp: rc176's
host-budget test writes its files through WriteRugnuxOutputs now that the library writes none.
Generated code (broker/gen, frontend client, Python client docs, Redoc) regenerated from the merged
jfjoch_api.yaml with update_version.sh (version unchanged).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

image_analysis/WriteReflections.cpp

@@ -7,6 +7,7 @@
 #include "scale_merge/TwinningAnalysis.h"
 #include "bragg_integration/SystematicAbsence.h"
 #include "../common/ParallelFor.h"
+#include "../writer/WriteRugnuxFiles.h"
 
 #include <algorithm>
 #include <charconv>
@@ -134,16 +135,16 @@ std::string GitDescription() {
 // (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each
 // continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is
 // never cut just after a space: the space starts the next one, behind its '+'.
-void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) {
-    const auto add = [&mtz](const std::string &label, const std::string &value) {
+void AddProvenanceHistory(std::vector<std::string> &history, const OutputProvenance &provenance) {
+    const auto add = [&history](const std::string &label, const std::string &value) {
         if (value.empty())
             return;
         std::string rest = label + value;
-        while (!rest.empty() && mtz.history.size() < 30) {
+        while (!rest.empty() && history.size() < 30) {
             size_t n = std::min<size_t>(80, rest.size());
             while (n > 1 && n < rest.size() && rest[n - 1] == ' ')
                 --n;
-            mtz.history.push_back(rest.substr(0, n));
+            history.push_back(rest.substr(0, n));
             rest = n < rest.size() ? "+" + rest.substr(n) : std::string();
         }
     };
@@ -619,81 +620,58 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
     out.close();
 }
 
-void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
-                         const UnitCell &unitCell,
-                         const DiffractionExperiment &experiment,
-                         const OutputProvenance &provenance,
-                         const std::string &filename) {
-    gemmi::Mtz mtz;
-
-    // Optional but recommended metadata
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
-
+MtzTable MergedMtzTable(const std::vector<MergedReflection> &reflections,
+                        const UnitCell &unitCell,
+                        const DiffractionExperiment &experiment,
+                        const OutputProvenance &provenance) {
+    MtzTable m;
+    m.unit_cell = unitCell;
+    m.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
     // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention).
-    mtz.title = "Rugnux merged reflections";
-    mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-
-    // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the
-    // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is
-    // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that
-    // takes f'/f'' from the file would phase on.
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset& ds = mtz.add_dataset("native");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
-
-    const int dataset_id = ds.id;
+    m.title = "Rugnux merged reflections";
+    m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
+    AddProvenanceHistory(m.history, provenance);
+    // HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the reserved base dataset
+    // as far as CCP4's mtzlib is concerned, and the wavelength on it is dropped - mtzinfo then reports the
+    // Cu K-alpha default 1.54187 A, which is what a program that takes f'/f'' from the file would phase on.
+    m.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                  {"native", experiment.GetSampleName(), "native", experiment.GetWavelength_A()}};
 
     // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for
     // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly.
     bool has_anom = true;
     const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
+    std::vector<std::pair<const char *, char>> columns{{"H", 'H'}, {"K", 'H'}, {"L", 'H'},
+                                                       {"IMEAN", 'J'}, {"SIGIMEAN", 'Q'}};
+    if (has_anom)
+        columns.insert(columns.end(), {{"I(+)", 'K'}, {"SIGI(+)", 'M'}, {"I(-)", 'K'}, {"SIGI(-)", 'M'}});
+    columns.insert(columns.end(), {{"F", 'F'}, {"SIGF", 'Q'}});   // F: the French-Wilson amplitude
+    if (has_anom)
+        columns.insert(columns.end(), {{"F(+)", 'G'}, {"SIGF(+)", 'L'}, {"F(-)", 'G'}, {"SIGF(-)", 'L'}});
+    columns.push_back({"FreeR_flag", 'I'});
+    for (const auto &[label, type] : columns)
+        m.columns.push_back({label, type, type == 'H' ? 0 : 1, {}});
+    for (const auto &row : out_rows) {
+        std::vector<float> v{static_cast<float>(row.h), static_cast<float>(row.k), static_cast<float>(row.l),
+                             row.Imean, row.sImean};
+        if (has_anom)
+            v.insert(v.end(), {row.Ip, row.sIp, row.Im, row.sIm});
+        v.insert(v.end(), {row.Fmean, row.sFmean});
+        if (has_anom)
+            v.insert(v.end(), {row.Fp, row.sFp, row.Fm, row.sFm});
+        v.push_back(static_cast<float>(row.rfree));
+        for (size_t c = 0; c < v.size(); c++)
+            m.columns[c].values.push_back(v[c]);
+    }
+    return m;
+}
 
-    mtz.add_column("IMEAN", 'J', dataset_id, -1, false);
-    mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("I(+)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false);
-        mtz.add_column("I(-)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false);
-    }
-    mtz.add_column("F", 'F', dataset_id, -1, false);       // French-Wilson amplitude
-    mtz.add_column("SIGF", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("F(+)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false);
-        mtz.add_column("F(-)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false);
-    }
-    mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false);
-
-    mtz.nreflections = static_cast<int>(out_rows.size());
-    mtz.data.reserve(out_rows.size() * 16);
-    for (const auto& row : out_rows) {
-        mtz.data.push_back(static_cast<float>(row.h));
-        mtz.data.push_back(static_cast<float>(row.k));
-        mtz.data.push_back(static_cast<float>(row.l));
-        mtz.data.push_back(row.Imean);
-        mtz.data.push_back(row.sImean);
-        if (has_anom) {
-            mtz.data.push_back(row.Ip);
-            mtz.data.push_back(row.sIp);
-            mtz.data.push_back(row.Im);
-            mtz.data.push_back(row.sIm);
-        }
-        mtz.data.push_back(row.Fmean);
-        mtz.data.push_back(row.sFmean);
-        if (has_anom) {
-            mtz.data.push_back(row.Fp);
-            mtz.data.push_back(row.sFp);
-            mtz.data.push_back(row.Fm);
-            mtz.data.push_back(row.sFm);
-        }
-        mtz.data.push_back(static_cast<float>(row.rfree));
-    }
-    mtz.write_to_file(filename);
+void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
+                         const UnitCell &unitCell,
+                         const DiffractionExperiment &experiment,
+                         const OutputProvenance &provenance,
+                         const std::string &filename) {
+    WriteMtz(MergedMtzTable(reflections, unitCell, experiment, provenance), filename);
 }
 
 namespace {
@@ -946,22 +924,22 @@ std::vector<std::vector<Reflection>> SumRockingEvents(const std::vector<Integrat
 
 } // namespace
 
-gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
-                       const UnitCell &unitCell,
-                       const DiffractionExperiment &experiment,
-                       bool sum_partials,
-                       const OutputProvenance &provenance,
-                       size_t nthreads) {
-    gemmi::Mtz mtz;
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
+MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
+                          const UnitCell &unitCell,
+                          const DiffractionExperiment &experiment,
+                          bool sum_partials,
+                          const OutputProvenance &provenance,
+                          size_t nthreads) {
+    const gemmi::SpaceGroup *spacegroup = &experiment.GetSpaceGroupOrP1();
+    MtzTable mtz;
+    mtz.space_group_name = spacegroup->xhm();
+    mtz.unit_cell = unitCell;
     mtz.title = "Rugnux unmerged observations";
     mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
+    AddProvenanceHistory(mtz.history, provenance);
+    mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                    {"unmerged", experiment.GetSampleName(), "unmerged", experiment.GetWavelength_A()}};
+    const int dataset_id = 1;
 
     // Every data column belongs to HKL_base and only the batches carry a dataset id, as in a
     // POINTLESS file. Down to FLAG this is POINTLESS's own column set, plus QE and FLIGHT; the four after it
@@ -981,22 +959,11 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // rises above 1 toward the detector edge where a longer crossing makes the sensor more
     // efficient. DIALS stores the un-normalised absorbed fraction instead; the two differ by the
     // per-dataset constant QE(0), which is an overall scale.
-    mtz.add_column("M/ISYM", 'Y', 0, -1, false);
-    mtz.add_column("BATCH", 'B', 0, -1, false);
-    mtz.add_column("I", 'J', 0, -1, false);
-    mtz.add_column("SIGI", 'Q', 0, -1, false);
-    mtz.add_column("FRACTIONCALC", 'R', 0, -1, false);
-    mtz.add_column("XDET", 'R', 0, -1, false);
-    mtz.add_column("YDET", 'R', 0, -1, false);
-    mtz.add_column("ROT", 'R', 0, -1, false);
-    mtz.add_column("LP", 'R', 0, -1, false);
-    mtz.add_column("QE", 'R', 0, -1, false);
-    mtz.add_column("FLIGHT", 'R', 0, -1, false);
-    mtz.add_column("FLAG", 'I', 0, -1, false);
-    mtz.add_column("DELPHI", 'R', 0, -1, false);
-    mtz.add_column("ZETA", 'R', 0, -1, false);
-    mtz.add_column("BGMEAN", 'R', 0, -1, false);
-    mtz.add_column("BGVAR", 'R', 0, -1, false);
+    for (const auto &[label, type] : std::vector<std::pair<const char *, char>>{
+             {"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'},
+             {"FRACTIONCALC", 'R'}, {"XDET", 'R'}, {"YDET", 'R'}, {"ROT", 'R'}, {"LP", 'R'}, {"QE", 'R'},
+             {"FLIGHT", 'R'}, {"FLAG", 'I'}, {"DELPHI", 'R'}, {"ZETA", 'R'}, {"BGMEAN", 'R'}, {"BGVAR", 'R'}})
+        mtz.columns.push_back({label, type, 0, {}});
 
     const auto gon = experiment.GetGoniometer();
     // A scanning goniometer means rugnux integrated a rocking curve image by image, so each
@@ -1025,7 +992,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // None of the three is a scale, and a program that reads this file has no way to recompute them. The
     // partiality is NOT divided out - that IS a scale, FRACTIONCALC carries it, and every program
     // this file is for wants to handle it its own way.
-    gemmi::UnmergedHklMover hkl_mover(mtz.spacegroup);
+    gemmi::UnmergedHklMover hkl_mover(spacegroup);
     // The batch headers are written for every image the observations span, not only for the images
     // that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that
     // indexed nothing leaves a hole that cuts the sweep in two: measured on a 360 deg sweep with 67
@@ -1040,7 +1007,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // roughly halve AIMLESS's <I/sigma>. Screw and glide absences ARE written: they are the evidence
     // the space group was chosen on, so a program reading this file can check that choice, and
     // removing them turns its test into an assumption. XDS and DIALS draw the line in the same place.
-    const char centering = mtz.spacegroup ? mtz.spacegroup->hm[0] : 'P';
+    const char centering = spacegroup->hm[0];
 
     // On stills only the crystals the merge takes are written: one whose cell is not the run's was indexed
     // in another lattice (on few spots, typically an axis multiple), so its indices do not name the
@@ -1080,13 +1047,14 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
                     add_row(r);
     }
 
-    // Every row has its own slot, so they are filled on several workers.
+    // Every row has its own slot in each column, so they are filled on several workers.
     const size_t w = mtz.columns.size();
-    mtz.data.resize(rows.size() * w);
+    for (auto &c : mtz.columns)
+        c.values.resize(rows.size());
     ParallelChunks(static_cast<int>(rows.size()), ThreadsForWork(rows.size(), nthreads), [&](int lo, int hi) {
+        std::vector<float> row(w);
         for (int i = lo; i < hi; ++i) {
             const Reflection &r = *rows[i];
-            float *row = &mtz.data[static_cast<size_t>(i) * w];
             std::array<int, 3> hkl{r.h, r.k, r.l};
             const int isym = hkl_mover.move_to_asu(hkl);
             row[0] = static_cast<float>(hkl[0]);
@@ -1113,9 +1081,10 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
             row[16] = r.zeta;
             row[17] = r.bkg;
             row[18] = r.var_bkg;
+            for (size_t c = 0; c < w; c++)
+                mtz.columns[c].values[i] = row[c];
         }
     });
-    mtz.nreflections = static_cast<int>(mtz.data.size() / mtz.columns.size());
 
     // The batch header's orientation matrix is the crystal at rotation angle zero - each batch's own
     // PHISTT is applied on top of it - but the lattice stored with an outcome is the crystal as it
@@ -1146,7 +1115,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     gemmi::Mtz::Batch batch;
     batch.title = "Rugnux";
     batch.axes.emplace_back("PHI");
-    batch.set_dataset_id(ds.id);
+    batch.set_dataset_id(dataset_id);
     batch.ints[12] = 1;   // ncryst
     batch.ints[14] = partials ? 1 : 2;   // ldtype: oscillation data (2D spots) / area detector (3D)
     batch.ints[15] = 1;   // jsaxs: the goniostat scan axis
@@ -1183,38 +1152,41 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
         batch.number = number;
         batch.floats[36] = phi_start_deg(static_cast<float>(number - 1));   // phistt
         batch.floats[37] = batch.floats[36] + wedge_deg;                    // phiend
-        mtz.batches.push_back(batch);
+        mtz.batches.push_back({batch.number, batch.title, {batch.ints.begin(), batch.ints.end()}, batch.floats,
+                               batch.axes});
     }
 
     // By H K L M/ISYM BATCH, the order POINTLESS leaves an unmerged file in. This is Mtz::sort(5) -
     // a stable sort on those five columns - done on several workers: rows equal in all five keep the
     // order they were added in, so with the row number as the last key the order is total and the
     // parallel sort returns the same rows in the same order.
-    if (!mtz.has_data())
-        mtz.sort(5);   // no rows: its own "No data." failure, as before
+    if (rows.empty())
+        gemmi::fail("No data.");   // as gemmi's own sort has it
     {
-        std::vector<int> order(mtz.nreflections);
+        std::vector<int> order(rows.size());
         std::iota(order.begin(), order.end(), 0);
         ParallelSort(order.begin(), order.end(), nthreads, [&](int i, int j) {
-            const float *a = &mtz.data[static_cast<size_t>(i) * w], *b = &mtz.data[static_cast<size_t>(j) * w];
-            for (int n = 0; n < 5; ++n)
-                if (a[n] != b[n])
-                    return a[n] < b[n];
+            for (int n = 0; n < 5; ++n) {
+                const float a = mtz.columns[n].values[i], b = mtz.columns[n].values[j];
+                if (a != b)
+                    return a < b;
+            }
             return i < j;
         });
-        std::vector<float> sorted(mtz.data.size());
-        ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
-            for (int i = lo; i < hi; ++i)
-                std::memcpy(&sorted[static_cast<size_t>(i) * w], &mtz.data[static_cast<size_t>(order[i]) * w],
-                            w * sizeof(float));
-        });
-        mtz.data.swap(sorted);
-        mtz.sort_order = {{1, 2, 3, 4, 5}};
+        for (auto &c : mtz.columns) {
+            std::vector<float> sorted(order.size());
+            ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
+                for (int i = lo; i < hi; ++i)
+                    sorted[i] = c.values[order[i]];
+            });
+            c.values.swap(sorted);
+        }
+        mtz.sort_order = {1, 2, 3, 4, 5};
     }
     return mtz;
 }
 
-void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector<IntegrationOutcome> &outcomes) {
+void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes) {
     // The reflecting range of the same image the orientation matrix is taken from.
     std::optional<float> mosaicity_deg;
     for (const auto &outcome : outcomes) {
@@ -1234,9 +1206,9 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
                                  const OutputProvenance &provenance,
                                  const std::string &filename,
                                  size_t nthreads) {
-    gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
+    MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
     SetUnmergedMtzMosaicity(mtz, outcomes);
-    mtz.write_to_file(filename);
+    WriteMtz(mtz, filename);
 }
 
 void WriteReflections(const std::vector<MergedReflection> &reflections,