v1.0.0-rc.176 (#86)

* jfjoch_broker: Optional live Rugnux analysis of DECTRIS rotation collections inside the broker, with merged data, maps and the placed model written beside the master file, the rotation lattice in the scan result, and a "Rugnux live" tab and dashboard row in the web frontend.
* Rugnux: Faster processing, with molecular replacement and model validation spread over several GPUs and run on the Apple GPU on macOS.
* Rugnux: The polarization stated in miniCBF and d*TREK headers is used, twinned crystals' space groups are decided more reliably, and a run that finds no lattice still reports every image.

Reviewed-on: #86
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

image_analysis/WriteReflections.cpp

@@ -7,8 +7,10 @@
 #include "scale_merge/TwinningAnalysis.h"
 #include "bragg_integration/SystematicAbsence.h"
 #include "../common/ParallelFor.h"
+#include "../writer/WriteRugnuxFiles.h"
 
 #include <algorithm>
+#include <charconv>
 #include <cmath>
 #include <cctype>
 #include <cstring>
@@ -45,16 +47,28 @@ std::string CurrentDateISO() {
     return buf;
 }
 
+/// The C locale's "%.*f" of a finite double. std::to_chars where the standard library has it for
+/// floating point: the same digits - both round the exact binary value, halfway cases to even - at a
+/// quarter of snprintf's time, and with no locale to consult. snprintf elsewhere (libc++ before macOS
+/// 13.3). 512 characters hold any finite double.
+std::string FixedDigits(double val, int decimals) {
+    char buf[512];
+#ifdef __cpp_lib_to_chars
+    const auto r = std::to_chars(buf, buf + sizeof(buf), val, std::chars_format::fixed, decimals);
+    return std::string(buf, r.ptr);
+#else
+    const int n = std::snprintf(buf, sizeof(buf), "%.*f", decimals, val);
+    return std::string(buf, n);
+#endif
+}
+
 /// Format a double with given decimal places; returns "?" for non-finite.
-/// snprintf rather than an ostringstream: the reflection loop below calls this twelve times per
-/// reflection, and building a stream (and its locale) per call dominated the time spent writing a
-/// merged file. Same digits - both go through the C locale's %.*f.
+/// Not an ostringstream: the reflection loop below calls this twelve times per reflection, and
+/// building a stream (and its locale) per call dominated the time spent writing a merged file.
 std::string Fmt(double val, int decimals = 4) {
     if (!std::isfinite(val))
         return "?";
-    char buf[512];
-    const int n = std::snprintf(buf, sizeof(buf), "%.*f", decimals, val);
-    return std::string(buf, n);
+    return FixedDigits(val, decimals);
 }
 
 /// Quote a CIF string value; returns "?" for empty.
@@ -121,16 +135,16 @@ std::string GitDescription() {
 // (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each
 // continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is
 // never cut just after a space: the space starts the next one, behind its '+'.
-void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) {
-    const auto add = [&mtz](const std::string &label, const std::string &value) {
+void AddProvenanceHistory(std::vector<std::string> &history, const OutputProvenance &provenance) {
+    const auto add = [&history](const std::string &label, const std::string &value) {
         if (value.empty())
             return;
         std::string rest = label + value;
-        while (!rest.empty() && mtz.history.size() < 30) {
+        while (!rest.empty() && history.size() < 30) {
             size_t n = std::min<size_t>(80, rest.size());
             while (n > 1 && n < rest.size() && rest[n - 1] == ' ')
                 --n;
-            mtz.history.push_back(rest.substr(0, n));
+            history.push_back(rest.substr(0, n));
             rest = n < rest.size() ? "+" + rest.substr(n) : std::string();
         }
     };
@@ -606,81 +620,58 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
     out.close();
 }
 
-void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
-                         const UnitCell &unitCell,
-                         const DiffractionExperiment &experiment,
-                         const OutputProvenance &provenance,
-                         const std::string &filename) {
-    gemmi::Mtz mtz;
-
-    // Optional but recommended metadata
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
-
+MtzTable MergedMtzTable(const std::vector<MergedReflection> &reflections,
+                        const UnitCell &unitCell,
+                        const DiffractionExperiment &experiment,
+                        const OutputProvenance &provenance) {
+    MtzTable m;
+    m.unit_cell = unitCell;
+    m.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
     // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention).
-    mtz.title = "Rugnux merged reflections";
-    mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-
-    // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the
-    // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is
-    // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that
-    // takes f'/f'' from the file would phase on.
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset& ds = mtz.add_dataset("native");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
-
-    const int dataset_id = ds.id;
+    m.title = "Rugnux merged reflections";
+    m.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
+    AddProvenanceHistory(m.history, provenance);
+    // HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the reserved base dataset
+    // as far as CCP4's mtzlib is concerned, and the wavelength on it is dropped - mtzinfo then reports the
+    // Cu K-alpha default 1.54187 A, which is what a program that takes f'/f'' from the file would phase on.
+    m.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                  {"native", experiment.GetSampleName(), "native", experiment.GetWavelength_A()}};
 
     // One row per reflection in the CCP4 anomalous layout (IMEAN + I(+)/I(-), and the same split for
     // the French-Wilson amplitude), which aimless / ctruncate / mtz2sca / ANODE read directly.
     bool has_anom = true;
     const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
+    std::vector<std::pair<const char *, char>> columns{{"H", 'H'}, {"K", 'H'}, {"L", 'H'},
+                                                       {"IMEAN", 'J'}, {"SIGIMEAN", 'Q'}};
+    if (has_anom)
+        columns.insert(columns.end(), {{"I(+)", 'K'}, {"SIGI(+)", 'M'}, {"I(-)", 'K'}, {"SIGI(-)", 'M'}});
+    columns.insert(columns.end(), {{"F", 'F'}, {"SIGF", 'Q'}});   // F: the French-Wilson amplitude
+    if (has_anom)
+        columns.insert(columns.end(), {{"F(+)", 'G'}, {"SIGF(+)", 'L'}, {"F(-)", 'G'}, {"SIGF(-)", 'L'}});
+    columns.push_back({"FreeR_flag", 'I'});
+    for (const auto &[label, type] : columns)
+        m.columns.push_back({label, type, type == 'H' ? 0 : 1, {}});
+    for (const auto &row : out_rows) {
+        std::vector<float> v{static_cast<float>(row.h), static_cast<float>(row.k), static_cast<float>(row.l),
+                             row.Imean, row.sImean};
+        if (has_anom)
+            v.insert(v.end(), {row.Ip, row.sIp, row.Im, row.sIm});
+        v.insert(v.end(), {row.Fmean, row.sFmean});
+        if (has_anom)
+            v.insert(v.end(), {row.Fp, row.sFp, row.Fm, row.sFm});
+        v.push_back(static_cast<float>(row.rfree));
+        for (size_t c = 0; c < v.size(); c++)
+            m.columns[c].values.push_back(v[c]);
+    }
+    return m;
+}
 
-    mtz.add_column("IMEAN", 'J', dataset_id, -1, false);
-    mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("I(+)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(+)", 'M', dataset_id, -1, false);
-        mtz.add_column("I(-)", 'K', dataset_id, -1, false);
-        mtz.add_column("SIGI(-)", 'M', dataset_id, -1, false);
-    }
-    mtz.add_column("F", 'F', dataset_id, -1, false);       // French-Wilson amplitude
-    mtz.add_column("SIGF", 'Q', dataset_id, -1, false);
-    if (has_anom) {
-        mtz.add_column("F(+)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(+)", 'L', dataset_id, -1, false);
-        mtz.add_column("F(-)", 'G', dataset_id, -1, false);
-        mtz.add_column("SIGF(-)", 'L', dataset_id, -1, false);
-    }
-    mtz.add_column("FreeR_flag", 'I', dataset_id, -1, false);
-
-    mtz.nreflections = static_cast<int>(out_rows.size());
-    mtz.data.reserve(out_rows.size() * 16);
-    for (const auto& row : out_rows) {
-        mtz.data.push_back(static_cast<float>(row.h));
-        mtz.data.push_back(static_cast<float>(row.k));
-        mtz.data.push_back(static_cast<float>(row.l));
-        mtz.data.push_back(row.Imean);
-        mtz.data.push_back(row.sImean);
-        if (has_anom) {
-            mtz.data.push_back(row.Ip);
-            mtz.data.push_back(row.sIp);
-            mtz.data.push_back(row.Im);
-            mtz.data.push_back(row.sIm);
-        }
-        mtz.data.push_back(row.Fmean);
-        mtz.data.push_back(row.sFmean);
-        if (has_anom) {
-            mtz.data.push_back(row.Fp);
-            mtz.data.push_back(row.sFp);
-            mtz.data.push_back(row.Fm);
-            mtz.data.push_back(row.sFm);
-        }
-        mtz.data.push_back(static_cast<float>(row.rfree));
-    }
-    mtz.write_to_file(filename);
+void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
+                         const UnitCell &unitCell,
+                         const DiffractionExperiment &experiment,
+                         const OutputProvenance &provenance,
+                         const std::string &filename) {
+    WriteMtz(MergedMtzTable(reflections, unitCell, experiment, provenance), filename);
 }
 
 namespace {
@@ -691,11 +682,7 @@ namespace {
             if (v.size() < width) s.append(width - v.size(), ' ');
             s.append(v);
         };
-        const auto num2 = [](double v) {
-            char b[64];
-            const int n = std::snprintf(b, sizeof b, "%.2f", v);
-            return std::string(b, static_cast<size_t>(std::clamp(n, 0, static_cast<int>(sizeof b) - 1)));
-        };
+        const auto num2 = [](double v) { return FixedDigits(v, 2); };
         column(std::to_string(h), 4);
         column(std::to_string(k), 4);
         column(std::to_string(l), 4);
@@ -937,22 +924,22 @@ std::vector<std::vector<Reflection>> SumRockingEvents(const std::vector<Integrat
 
 } // namespace
 
-gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
-                       const UnitCell &unitCell,
-                       const DiffractionExperiment &experiment,
-                       bool sum_partials,
-                       const OutputProvenance &provenance,
-                       size_t nthreads) {
-    gemmi::Mtz mtz;
-    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
-    mtz.set_cell_for_all(unitCell);
+MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
+                          const UnitCell &unitCell,
+                          const DiffractionExperiment &experiment,
+                          bool sum_partials,
+                          const OutputProvenance &provenance,
+                          size_t nthreads) {
+    const gemmi::SpaceGroup *spacegroup = &experiment.GetSpaceGroupOrP1();
+    MtzTable mtz;
+    mtz.space_group_name = spacegroup->xhm();
+    mtz.unit_cell = unitCell;
     mtz.title = "Rugnux unmerged observations";
     mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
-    AddProvenanceHistory(mtz, provenance);
-    mtz.add_base();                 // the HKL_base dataset and the H K L columns
-    gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged");
-    ds.crystal_name = experiment.GetSampleName();
-    ds.wavelength = experiment.GetWavelength_A();
+    AddProvenanceHistory(mtz.history, provenance);
+    mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
+                    {"unmerged", experiment.GetSampleName(), "unmerged", experiment.GetWavelength_A()}};
+    const int dataset_id = 1;
 
     // Every data column belongs to HKL_base and only the batches carry a dataset id, as in a
     // POINTLESS file. Down to FLAG this is POINTLESS's own column set, plus QE and FLIGHT; the four after it
@@ -972,22 +959,11 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // rises above 1 toward the detector edge where a longer crossing makes the sensor more
     // efficient. DIALS stores the un-normalised absorbed fraction instead; the two differ by the
     // per-dataset constant QE(0), which is an overall scale.
-    mtz.add_column("M/ISYM", 'Y', 0, -1, false);
-    mtz.add_column("BATCH", 'B', 0, -1, false);
-    mtz.add_column("I", 'J', 0, -1, false);
-    mtz.add_column("SIGI", 'Q', 0, -1, false);
-    mtz.add_column("FRACTIONCALC", 'R', 0, -1, false);
-    mtz.add_column("XDET", 'R', 0, -1, false);
-    mtz.add_column("YDET", 'R', 0, -1, false);
-    mtz.add_column("ROT", 'R', 0, -1, false);
-    mtz.add_column("LP", 'R', 0, -1, false);
-    mtz.add_column("QE", 'R', 0, -1, false);
-    mtz.add_column("FLIGHT", 'R', 0, -1, false);
-    mtz.add_column("FLAG", 'I', 0, -1, false);
-    mtz.add_column("DELPHI", 'R', 0, -1, false);
-    mtz.add_column("ZETA", 'R', 0, -1, false);
-    mtz.add_column("BGMEAN", 'R', 0, -1, false);
-    mtz.add_column("BGVAR", 'R', 0, -1, false);
+    for (const auto &[label, type] : std::vector<std::pair<const char *, char>>{
+             {"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'},
+             {"FRACTIONCALC", 'R'}, {"XDET", 'R'}, {"YDET", 'R'}, {"ROT", 'R'}, {"LP", 'R'}, {"QE", 'R'},
+             {"FLIGHT", 'R'}, {"FLAG", 'I'}, {"DELPHI", 'R'}, {"ZETA", 'R'}, {"BGMEAN", 'R'}, {"BGVAR", 'R'}})
+        mtz.columns.push_back({label, type, 0, {}});
 
     const auto gon = experiment.GetGoniometer();
     // A scanning goniometer means rugnux integrated a rocking curve image by image, so each
@@ -1016,7 +992,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // None of the three is a scale, and a program that reads this file has no way to recompute them. The
     // partiality is NOT divided out - that IS a scale, FRACTIONCALC carries it, and every program
     // this file is for wants to handle it its own way.
-    gemmi::UnmergedHklMover hkl_mover(mtz.spacegroup);
+    gemmi::UnmergedHklMover hkl_mover(spacegroup);
     // The batch headers are written for every image the observations span, not only for the images
     // that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that
     // indexed nothing leaves a hole that cuts the sweep in two: measured on a 360 deg sweep with 67
@@ -1031,7 +1007,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     // roughly halve AIMLESS's <I/sigma>. Screw and glide absences ARE written: they are the evidence
     // the space group was chosen on, so a program reading this file can check that choice, and
     // removing them turns its test into an assumption. XDS and DIALS draw the line in the same place.
-    const char centering = mtz.spacegroup ? mtz.spacegroup->hm[0] : 'P';
+    const char centering = spacegroup->hm[0];
 
     // On stills only the crystals the merge takes are written: one whose cell is not the run's was indexed
     // in another lattice (on few spots, typically an axis multiple), so its indices do not name the
@@ -1071,13 +1047,14 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
                     add_row(r);
     }
 
-    // Every row has its own slot, so they are filled on several workers.
+    // Every row has its own slot in each column, so they are filled on several workers.
     const size_t w = mtz.columns.size();
-    mtz.data.resize(rows.size() * w);
+    for (auto &c : mtz.columns)
+        c.values.resize(rows.size());
     ParallelChunks(static_cast<int>(rows.size()), ThreadsForWork(rows.size(), nthreads), [&](int lo, int hi) {
+        std::vector<float> row(w);
         for (int i = lo; i < hi; ++i) {
             const Reflection &r = *rows[i];
-            float *row = &mtz.data[static_cast<size_t>(i) * w];
             std::array<int, 3> hkl{r.h, r.k, r.l};
             const int isym = hkl_mover.move_to_asu(hkl);
             row[0] = static_cast<float>(hkl[0]);
@@ -1104,9 +1081,10 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
             row[16] = r.zeta;
             row[17] = r.bkg;
             row[18] = r.var_bkg;
+            for (size_t c = 0; c < w; c++)
+                mtz.columns[c].values[i] = row[c];
         }
     });
-    mtz.nreflections = static_cast<int>(mtz.data.size() / mtz.columns.size());
 
     // The batch header's orientation matrix is the crystal at rotation angle zero - each batch's own
     // PHISTT is applied on top of it - but the lattice stored with an outcome is the crystal as it
@@ -1137,7 +1115,7 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
     gemmi::Mtz::Batch batch;
     batch.title = "Rugnux";
     batch.axes.emplace_back("PHI");
-    batch.set_dataset_id(ds.id);
+    batch.set_dataset_id(dataset_id);
     batch.ints[12] = 1;   // ncryst
     batch.ints[14] = partials ? 1 : 2;   // ldtype: oscillation data (2D spots) / area detector (3D)
     batch.ints[15] = 1;   // jsaxs: the goniostat scan axis
@@ -1174,38 +1152,41 @@ gemmi::Mtz UnmergedMtz(const std::vector<IntegrationOutcome> &outcomes,
         batch.number = number;
         batch.floats[36] = phi_start_deg(static_cast<float>(number - 1));   // phistt
         batch.floats[37] = batch.floats[36] + wedge_deg;                    // phiend
-        mtz.batches.push_back(batch);
+        mtz.batches.push_back({batch.number, batch.title, {batch.ints.begin(), batch.ints.end()}, batch.floats,
+                               batch.axes});
     }
 
     // By H K L M/ISYM BATCH, the order POINTLESS leaves an unmerged file in. This is Mtz::sort(5) -
     // a stable sort on those five columns - done on several workers: rows equal in all five keep the
     // order they were added in, so with the row number as the last key the order is total and the
     // parallel sort returns the same rows in the same order.
-    if (!mtz.has_data())
-        mtz.sort(5);   // no rows: its own "No data." failure, as before
+    if (rows.empty())
+        gemmi::fail("No data.");   // as gemmi's own sort has it
     {
-        std::vector<int> order(mtz.nreflections);
+        std::vector<int> order(rows.size());
         std::iota(order.begin(), order.end(), 0);
         ParallelSort(order.begin(), order.end(), nthreads, [&](int i, int j) {
-            const float *a = &mtz.data[static_cast<size_t>(i) * w], *b = &mtz.data[static_cast<size_t>(j) * w];
-            for (int n = 0; n < 5; ++n)
-                if (a[n] != b[n])
-                    return a[n] < b[n];
+            for (int n = 0; n < 5; ++n) {
+                const float a = mtz.columns[n].values[i], b = mtz.columns[n].values[j];
+                if (a != b)
+                    return a < b;
+            }
             return i < j;
         });
-        std::vector<float> sorted(mtz.data.size());
-        ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
-            for (int i = lo; i < hi; ++i)
-                std::memcpy(&sorted[static_cast<size_t>(i) * w], &mtz.data[static_cast<size_t>(order[i]) * w],
-                            w * sizeof(float));
-        });
-        mtz.data.swap(sorted);
-        mtz.sort_order = {{1, 2, 3, 4, 5}};
+        for (auto &c : mtz.columns) {
+            std::vector<float> sorted(order.size());
+            ParallelChunks(static_cast<int>(order.size()), ThreadsForWork(order.size(), nthreads), [&](int lo, int hi) {
+                for (int i = lo; i < hi; ++i)
+                    sorted[i] = c.values[order[i]];
+            });
+            c.values.swap(sorted);
+        }
+        mtz.sort_order = {1, 2, 3, 4, 5};
     }
     return mtz;
 }
 
-void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector<IntegrationOutcome> &outcomes) {
+void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes) {
     // The reflecting range of the same image the orientation matrix is taken from.
     std::optional<float> mosaicity_deg;
     for (const auto &outcome : outcomes) {
@@ -1225,9 +1206,9 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
                                  const OutputProvenance &provenance,
                                  const std::string &filename,
                                  size_t nthreads) {
-    gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
+    MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
     SetUnmergedMtzMosaicity(mtz, outcomes);
-    mtz.write_to_file(filename);
+    WriteMtz(mtz, filename);
 }
 
 void WriteReflections(const std::vector<MergedReflection> &reflections,