v1.0.0-rc.166 (#76)

* `rugnux --mode calibration` writes `<prefix>.json` beside the `.poni`, whose `dataset_settings` member is a `jfjoch_broker` `dataset_settings` body as it stands.
* `rugnux` and `jfjoch_viewer` read PILATUS miniCBF sweeps natively, without conversion.
* Masters written by other facilities open, including Eiger 1.x and third-party NXmx variants.
* `rugnux` measures the beam centre on every run, and indexes with it when the file's value indexes nothing.
* A detector swung out on a 2theta arm is placed where the file says it stands, and the calibration can hold the tilt fixed.
* `rugnux` writes the unmerged MTZ by default, and a P1 merge beside it, so a wrong space group can be re-merged without reprocessing.
* Significant improvements to symmetry handling in `rugnux`: the lattice, the point group, the setting and the systematic absences.
* The `rugnux` report gives the resolution the CC1/2 fit reached, beside the range the reflections were written to.
* The `rugnux` report gives the twinning statistics measured before the space group was decided, beside the ones measured after.
* The `rugnux` report gives the strong-direction diffraction limit, and warns when CC1/2 is not monotone with resolution.
* `rugnux` ranks screw axes on the evidence their absences carry, rather than on how many control reflections a candidate happens to have.
* Twinning is no longer reported when the L-test contradicts it.
* The `rugnux` report gives the detector tilt, the measured tilt and the direct beam beside the beam centre, and a post-refined beam centre is judged against the run's own measurement rather than the file's.
* `--no-refine-tilt` holds the detector tilt at the value in the file, instead of zeroing it, when the calibration starts from the spots.
* The `jfjoch_viewer` grid scan view draws the cells in the proportion of the scan steps, so the map has the shape of the scanned area.

Reviewed-on: #76
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

image_analysis/WriteReflections.cpp

@@ -12,7 +12,6 @@
 #include <cmath>
 #include <cstdio>
 #include <map>
-#include <set>
 #include <tuple>
 #include <fstream>
 #include <iomanip>
@@ -75,7 +74,14 @@ struct MergedOutRow {
     int h = 0, k = 0, l = 0;
     float Imean = NAN, sImean = NAN, Ip = NAN, sIp = NAN, Im = NAN, sIm = NAN;
     float Fmean = NAN, sFmean = NAN, Fp = NAN, sFp = NAN, Fm = NAN, sFm = NAN;
-    int rfree = 0;
+    // The value written into FreeR_flag: 0 for the TEST set, 1 for the working set. That is the CCP4
+    // convention the column NAME belongs to - CCP4's own freerflag writes 0..19 with 0 as the ~5%
+    // test bin, and REFMAC's default FREE 0 reads it that way, so the opposite (phenix/CNS) numbering
+    // under this name stops REFMAC dead: "more than half of reflections are in free R set", then
+    // "Cannot switch free R flag", exit 1. phenix auto-detects either numbering with equal confidence
+    // (measured on both), and rugnux's own reference-MTZ reader already takes 0 as the test set, so
+    // this is the numbering that works everywhere and the one the rest of the code assumes.
+    int rfree = 1;
 };
 
 std::vector<MergedOutRow> BuildMergedRows(const std::vector<MergedReflection> &reflections,
@@ -96,24 +102,24 @@ std::vector<MergedOutRow> BuildMergedRows(const std::vector<MergedReflection> &r
         for (const auto& r : reflections)
             out_rows.push_back({r.h, r.k, r.l, r.I, r.sigma, r.I_plus, r.sigma_plus, r.I_minus,
                                 r.sigma_minus, r.F, r.sigmaF, r.F_plus, r.sigmaF_plus, r.F_minus,
-                                r.sigmaF_minus, r.rfree_flag ? 1 : 0});
+                                r.sigmaF_minus, r.rfree_flag ? 0 : 1});
     } else {
         // Anomalous: group the two mates by their (shared) Friedel-merged ASU representative, then form
         // IMEAN / F as their inverse-variance Friedel mean. A single generator gives both the group key
         // (its hkl, identical for +hkl and -hkl) and which mate this row is (.plus).
-        const HKLKeyGenerator key_gen(false, experiment.GetSpaceGroupNumber().value_or(1));
+        const HKLKeyGenerator key_gen(false, experiment.GetSpaceGroupOrP1());
         struct AnomRow {
             int h = 0, k = 0, l = 0;
             float Ip = NAN, sIp = NAN, Im = NAN, sIm = NAN;
             float Fp = NAN, sFp = NAN, Fm = NAN, sFm = NAN;
-            int rfree = 0;
+            int rfree = 1;
         };
         std::map<std::tuple<int, int, int>, AnomRow> rows;
         for (const auto& r : reflections) {
             const HKLKey key = key_gen(r);
             AnomRow& row = rows[{key.h, key.k, key.l}];
             row.h = key.h; row.k = key.k; row.l = key.l;
-            row.rfree = r.rfree_flag ? 1 : 0;
+            row.rfree = r.rfree_flag ? 0 : 1;
             if (key.plus) { row.Ip = r.I; row.sIp = r.sigma; row.Fp = r.F; row.sFp = r.sigmaF; }
             else          { row.Im = r.I; row.sIm = r.sigma; row.Fm = r.F; row.sFm = r.sigmaF; }
         }
@@ -186,13 +192,11 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
     out << "_cell.angle_beta   " << Fmt(unitCell.beta, 2) << "\n";
     out << "_cell.angle_gamma  " << Fmt(unitCell.gamma, 2) << "\n";
 
-    auto *sg = gemmi::find_spacegroup_by_number(experiment.GetSpaceGroupNumber().value_or(1));
-    if (sg == nullptr)
-        throw std::runtime_error("WriteMmcifReflections: invalid space group number");
+    const gemmi::SpaceGroup &sg = experiment.GetSpaceGroupOrP1();
 
     // ---------- _symmetry ----------
-    out << "_symmetry.space_group_name_H-M  " << CifStr(sg->hm) << "\n";
-    out << "_symmetry.Int_Tables_number      " << sg->number << "\n";
+    out << "_symmetry.space_group_name_H-M  " << CifStr(sg.hm) << "\n";
+    out << "_symmetry.Int_Tables_number      " << sg.number << "\n";
     out << "#\n";
 
     // ---------- _diffrn_source / _diffrn_detector ----------
@@ -378,7 +382,6 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
     out << "_refln.pdbx_F_plus_sigma\n";
     out << "_refln.pdbx_F_minus\n";
     out << "_refln.pdbx_F_minus_sigma\n";
-    out << "_refln.status_free\n";
     out << "_refln.status\n";
 
     // One row per unique reflection, twelve formatted floats each - tens of megabytes on a crowded
@@ -418,8 +421,10 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
                     column(Fmt(r.sigmaF_plus, 4), 14);
                     column(Fmt(r.F_minus, 4), 14);
                     column(Fmt(r.sigmaF_minus, 4), 14);
-                    s.push_back(r.rfree_flag ? '1' : '0');
-                    s.append(" o\n");   // 'o' = observed
+                    // PDBx/mmCIF _refln.status: 'f' marks the free (test) set, 'o' the working
+                    // set. This is the only place the free set is carried in the file - cif2mtz and
+                    // gemmi both read it, and neither knows any other reflection-level free-set item.
+                    s.append(r.rfree_flag ? "f\n" : "o\n");
                 }
             }
         });
@@ -439,15 +444,18 @@ void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
     gemmi::Mtz mtz;
 
     // Optional but recommended metadata
-    mtz.spacegroup = gemmi::find_spacegroup_by_number(
-        experiment.GetSpaceGroupNumber().value_or(1));
+    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
     mtz.set_cell_for_all(unitCell);
 
     // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention).
     mtz.title = "Rugnux merged reflections";
     mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
 
-    // Add dataset
+    // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the
+    // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is
+    // dropped - mtzinfo then reports the Cu K-alpha default 1.54187 A, which is what a program that
+    // takes f'/f'' from the file would phase on.
+    mtz.add_base();                 // the HKL_base dataset and the H K L columns
     gemmi::Mtz::Dataset& ds = mtz.add_dataset("native");
     ds.crystal_name = experiment.GetSampleName();
     ds.wavelength = experiment.GetWavelength_A();
@@ -459,9 +467,6 @@ void WriteMtzReflections(const std::vector<MergedReflection> &reflections,
     bool has_anom = true;
     const std::vector<MergedOutRow> out_rows = BuildMergedRows(reflections, experiment, has_anom);
 
-    mtz.add_column("H", 'H', dataset_id, -1, false);
-    mtz.add_column("K", 'H', dataset_id, -1, false);
-    mtz.add_column("L", 'H', dataset_id, -1, false);
     mtz.add_column("IMEAN", 'J', dataset_id, -1, false);
     mtz.add_column("SIGIMEAN", 'Q', dataset_id, -1, false);
     if (has_anom) {
@@ -606,15 +611,26 @@ float DetectorY(const Reflection &r) { return std::isfinite(r.observed_y) ? r.ob
 // 8-41% of events have parts summing to zero or less and 10-21% of the intensity-weighted centroids
 // fall outside the event's own frame range. Partialities are the rocking curve's own weights and are
 // positive by construction.
-// min_partiality is the combine's own floor on the assembled full (--min-partiality, default 0.02,
-// 0 = off). An event that caught a thousandth of its rocking curve is not a measurement of that
-// reflection - rugnux does not merge one either - and writing it as a full hands the reading program
-// a whole observation whose intensity is noise and whose sigma is small, so it is weighted heavily.
-// On a crystal whose rocking curves span twenty frames those events are 7% of the file and cost
-// POINTLESS the point group. Everything above the floor is written with its honest FRACTIONCALC,
-// truncated edge-of-sweep events included, for the reader to cut where it wants.
+// The floor is the SAME pair the 3D combine applies to the same event - --min-partiality (default
+// 0.02) and --min-captured-fraction (default 0.7 on rotation) - so what is exported as a full is
+// exactly what rugnux merges as a full. The captured-fraction floor is what makes the FULL
+// declaration true: the batch header says LDTYPE=2 and M/ISYM carries M=0, AIMLESS reads that as
+// "all runs have only fulls" and ignores FRACTIONCALC entirely, so an event that caught a twentieth
+// of its rocking curve does not enter the merge at a twentieth of its intensity - it enters whole,
+// with a small sigma, and is weighted heavily. There is no cut for the reader to make: the column
+// that would let it make one is one the reading program has been told to ignore. Measured, dropping
+// those events moves AIMLESS's Rmerge at 1.8 A from 1.353 to 0.694 and CC(1/2) from 0.985 to 0.993.
+//
+// FRACTIONCALC itself is written as summed, above 1 included. Each part's partiality is the erf pair
+// the predictor computed on THAT frame, from that frame's own refined lattice and its own mosaicity
+// estimate, so the parts of one event do not tile the rocking curve exactly - measured, the offset
+// from one part to the next steps by the wedge to within 12% of it, and the sums land in a peak at
+// 1.000 whose 95th percentile is 1.09. It is the honest estimate of a captured fraction and it is
+// not the number rugnux scales on (the merge recomputes every partiality from a frame-order-smoothed
+// mosaicity, RotationScaleMerge::SmoothMosaicity), so clamping it to 1 would hide the spread and buy
+// a reading program nothing.
 std::vector<Reflection> SumRockingEvents(const std::vector<IntegrationOutcome> &outcomes,
-                                         double min_partiality) {
+                                         double min_partiality, double min_captured_fraction) {
     constexpr float MAX_FRAME_GAP = 2.0f;   // == RotationScaleMerge's: what makes one rocking event
 
     // The sort key travels with the part instead of being read back through the pointer, the way the
@@ -668,7 +684,7 @@ std::vector<Reflection> SumRockingEvents(const std::vector<IntegrationOutcome> &
         }
         Reflection full = *parts[i].r;
         i = j;
-        if (sum_p < min_partiality)
+        if (sum_p < min_partiality || sum_p < min_captured_fraction)
             continue;
 
         // The Lorentz-polarization factor is applied by the writer, which multiplies I by rlp, so
@@ -698,7 +714,7 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
                                  bool sum_partials,
                                  const std::string &filename) {
     gemmi::Mtz mtz;
-    mtz.spacegroup = gemmi::find_spacegroup_by_number(experiment.GetSpaceGroupNumber().value_or(1));
+    mtz.spacegroup = &experiment.GetSpaceGroupOrP1();
     mtz.set_cell_for_all(unitCell);
     mtz.title = "Rugnux unmerged observations";
     mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction");
@@ -751,7 +767,13 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
     // way to recover it. The partiality is NOT divided out - that is a scale, FRACTIONCALC carries
     // it, and every program this file is for wants to handle it its own way.
     gemmi::UnmergedHklMover hkl_mover(mtz.spacegroup);
-    std::set<int> batch_numbers;
+    // The batch headers are written for every image the observations span, not only for the images
+    // that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that
+    // indexed nothing leaves a hole that cuts the sweep in two: measured on a 360 deg sweep with 67
+    // unindexed images, AIMLESS made 23 runs of it and its scale model diverged - CC(1/2) 0.811,
+    // Rmeas overflowed to -1266, and the result no longer correlated with rugnux's own merge
+    // (Pearson 0.0018). Filling the gaps gives one run and the numbers the sweep deserves.
+    int first_batch = 0, last_batch = -1;
 
     // Lattice-centring absences are integrated on purpose - prediction runs in P so the space-group
     // search can confirm or disprove the centring - but they are not written here. POINTLESS reads
@@ -769,7 +791,9 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
         // A summed full's image_number is its rocking-curve centroid, so this is the batch the
         // reflection is centred on - which is what a batch means for a full everywhere else.
         const int batch = 1 + static_cast<int>(std::lround(r.image_number));
-        batch_numbers.insert(batch);
+        if (last_batch < first_batch) { first_batch = batch; last_batch = batch; }
+        first_batch = std::min(first_batch, batch);
+        last_batch = std::max(last_batch, batch);
         mtz.data.push_back(static_cast<float>(hkl[0]));
         mtz.data.push_back(static_cast<float>(hkl[1]));
         mtz.data.push_back(static_cast<float>(hkl[2]));
@@ -790,7 +814,8 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
     };
     if (scanning && sum_partials) {
         for (const auto &r : SumRockingEvents(outcomes,
-                                              experiment.GetScalingSettings().GetMinPartiality()))
+                                              experiment.GetScalingSettings().GetMinPartiality(),
+                                              experiment.GetScalingSettings().GetMinCapturedFraction()))
             add_row(r);
     } else {
         for (const auto &outcome : outcomes)
@@ -864,7 +889,7 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
     batch.floats[115] = 1.0f;
     batch.floats[116] = static_cast<float>(experiment.GetYPixelsNum());
 
-    for (const int number : batch_numbers) {
+    for (int number = first_batch; number <= last_batch; number++) {
         batch.number = number;
         batch.floats[36] = phi_start_deg(static_cast<float>(number - 1));   // phistt
         batch.floats[37] = batch.floats[36] + wedge_deg;                    // phiend