v1.0.0-rc.166 (#76)

* `rugnux --mode calibration` writes `<prefix>.json` beside the `.poni`, whose `dataset_settings` member is a `jfjoch_broker` `dataset_settings` body as it stands.
* `rugnux` and `jfjoch_viewer` read PILATUS miniCBF sweeps natively, without conversion.
* Masters written by other facilities open, including Eiger 1.x and third-party NXmx variants.
* `rugnux` measures the beam centre on every run, and indexes with it when the file's value indexes nothing.
* A detector swung out on a 2theta arm is placed where the file says it stands, and the calibration can hold the tilt fixed.
* `rugnux` writes the unmerged MTZ by default, and a P1 merge beside it, so a wrong space group can be re-merged without reprocessing.
* Significant improvements to symmetry handling in `rugnux`: the lattice, the point group, the setting and the systematic absences.
* The `rugnux` report gives the resolution the CC1/2 fit reached, beside the range the reflections were written to.
* The `rugnux` report gives the twinning statistics measured before the space group was decided, beside the ones measured after.
* The `rugnux` report gives the strong-direction diffraction limit, and warns when CC1/2 is not monotone with resolution.
* `rugnux` ranks screw axes on the evidence their absences carry, rather than on how many control reflections a candidate happens to have.
* Twinning is no longer reported when the L-test contradicts it.
* The `rugnux` report gives the detector tilt, the measured tilt and the direct beam beside the beam centre, and a post-refined beam centre is judged against the run's own measurement rather than the file's.
* `--no-refine-tilt` holds the detector tilt at the value in the file, instead of zeroing it, when the calibration starts from the spots.
* The `jfjoch_viewer` grid scan view draws the cells in the proportion of the scan steps, so the map has the shape of the scanned area.

Reviewed-on: #76
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

image_analysis/geom_refinement/RingOptimizer.cpp

@@ -1,8 +1,10 @@
 // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
 // SPDX-License-Identifier: GPL-3.0-only
 
+#include <cmath>
 #include <algorithm>
 
+#include "../../common/DetectorOrientation.h"
 #include "../../common/JFJochMath.h"
 #include "RingOptimizer.h"
 #include "ceres/ceres.h"
@@ -10,11 +12,18 @@
 struct RingResidual {
     RingResidual(double x, double y, double lambda,
                  double pixel_size,
-                 double expected_q)
+                 double expected_q,
+                 const DetectorOrientation &orientation)
             : obs_x(x), obs_y(y),
               lambda(lambda),
               pixel_size(pixel_size),
-              expected_len_recip_sq(expected_q * expected_q / (4.0 * PI * PI)) {}
+              expected_len_recip_sq(expected_q * expected_q / (4.0 * PI * PI)) {
+        const RotMatrix delta = orientation.Matrix();
+        det_m00 = delta.Column(0).x;
+        det_m01 = delta.Column(1).x;
+        det_m10 = delta.Column(0).y;
+        det_m11 = delta.Column(1).y;
+    }
 
     template<typename T>
     bool operator()(const T* const center_x, const T* const center_y,
@@ -22,8 +31,13 @@ struct RingResidual {
                     const T* const rot2, T* residual) const {
 
         // Calculate lab coordinates from observed pixel coordinates
-        T x_lab = (T(obs_x) - center_x[0]) * T(pixel_size); // convert to mm
-        T y_lab = (T(obs_y) - center_y[0]) * T(pixel_size);
+        T u_lab = (T(obs_x) - center_x[0]) * T(pixel_size); // convert to mm
+        T v_lab = (T(obs_y) - center_y[0]) * T(pixel_size);
+        // The discrete image orientation, which turns the offset from the PONI before the tilt acts.
+        // Identity unless a detector says otherwise. It cannot change a ring's radius, but it does
+        // change which way the tilt tips the ring, which is exactly what this fits.
+        T x_lab = det_m00 * u_lab + det_m01 * v_lab;
+        T y_lab = det_m10 * u_lab + det_m11 * v_lab;
         T z_lab = distance[0];
 
 // Apply rotations around y and x axes
@@ -53,11 +67,94 @@ struct RingResidual {
     const double lambda;
     const double pixel_size;
     const double expected_len_recip_sq;
+    double det_m00, det_m01, det_m10, det_m11;
 };
 
-RingOptimizer::RingOptimizer(const DiffractionGeometry& geom) : reference(geom) {}
+RingOptimizer::RingOptimizer(const DiffractionGeometry& geom, bool refine_tilt)
+        : reference(geom), refine_tilt(refine_tilt) {}
+
+namespace {
+
+// Covariance of the converged fit, scaled to the residual scatter this fit actually left. Ceres hands
+// back the bare (J^T J)^-1 of an unweighted problem, which carries the shape of the correlations but
+// not their size; multiplying by chi2 per degree of freedom is what turns it into a sigma. The
+// residual is in q^2, so the scale is in q^2 too - and it cancels, because (J^T J)^-1 is in
+// parameter^2 per residual^2. The sigmas therefore come out in pixels, mm and radians whatever the
+// residual is measured in, which is why this does not need to wait on the residual being restated.
+void ComputeUncertainty(ceres::Problem &problem, const ceres::Solver::Summary &summary,
+                        const std::vector<const double *> &free_blocks,
+                        double *const centre_x, double *const centre_y, double *const distance,
+                        double *const rot1, double *const rot2,
+                        RingFitUncertainty &unc) {
+    const int p = static_cast<int>(free_blocks.size());
+    const int n = static_cast<int>(summary.num_residuals);
+    unc.free_parameters = p;
+    if (n <= p)
+        return;
+
+    // DENSE_SVD rather than the SPARSE_QR default: five parameters is not a sparse problem, and SVD is
+    // the one that survives a near-degenerate normal matrix long enough to report how degenerate it is.
+    ceres::Covariance::Options options;
+    options.algorithm_type = ceres::DENSE_SVD;
+    options.num_threads = 1;
+    ceres::Covariance covariance(options);
+
+    std::vector<std::pair<const double *, const double *>> pairs;
+    for (const double *a : free_blocks)
+        for (const double *b : free_blocks)
+            pairs.emplace_back(a, b);
+    // A failure here is not an error to report upwards: it means the problem is rank-deficient at the
+    // solution, i.e. some direction in parameter space costs the fit nothing at all. valid = false
+    // carries that, and it is the strongest statement this struct can make.
+    if (!covariance.Compute(pairs, &problem))
+        return;
+
+    std::vector<double> cov(static_cast<size_t>(p) * p);
+    if (!covariance.GetCovarianceMatrix(free_blocks, cov.data()))
+        return;
+
+    // 2 * final_cost is sum of squared residuals - Ceres' cost is half of it.
+    unc.chi2_per_dof = 2.0 * summary.final_cost / static_cast<double>(n - p);
+    for (auto &v : cov)
+        v *= unc.chi2_per_dof;
+
+    const auto index_of = [&](const double *block) {
+        for (int i = 0; i < p; ++i)
+            if (free_blocks[i] == block) return i;
+        return -1;
+    };
+    const auto sigma = [&](const double *block) {
+        const int i = index_of(block);
+        return i < 0 ? 0.0 : std::sqrt(std::max(0.0, cov[i * p + i]));
+    };
+    const auto corr = [&](const double *a, const double *b) {
+        const int i = index_of(a), j = index_of(b);
+        if (i < 0 || j < 0) return 0.0;
+        const double d = std::sqrt(cov[i * p + i] * cov[j * p + j]);
+        return d > 0.0 ? cov[i * p + j] / d : 0.0;
+    };
+
+    unc.sigma_beam_x_pxl = sigma(centre_x);
+    unc.sigma_beam_y_pxl = sigma(centre_y);
+    unc.sigma_distance_mm = sigma(distance);
+    unc.sigma_rot1_rad = sigma(rot1);
+    unc.sigma_rot2_rad = sigma(rot2);
+    unc.corr_beam_x_rot1 = corr(centre_x, rot1);
+    unc.corr_beam_y_rot2 = corr(centre_y, rot2);
+    unc.valid = true;
+}
+
+}   // namespace
+
+DiffractionGeometry RingOptimizer::Run(const std::vector<RingOptimizerInput> &input,
+                                       RingFitUncertainty *unc) {
+    // Cleared on the way in. Run writes *unc only where the solve is usable and the covariance
+    // conditions, so on any other path it would keep whatever the PREVIOUS fit left there - and
+    // CalibrateFromSpots runs this twice, once with the tilt free and again with it pinned. A failed
+    // second fit would then hand back the declined first fit's sigmas, valid flag and all.
+    if (unc)
+        *unc = RingFitUncertainty{};
 
-DiffractionGeometry RingOptimizer::Run(const std::vector<RingOptimizerInput> &input) {
     // Initial guess for the parameters
     double center_x = reference.GetBeamX_pxl();
     double center_y = reference.GetBeamY_pxl();
@@ -74,7 +171,13 @@ DiffractionGeometry RingOptimizer::Run(const std::vector<RingOptimizerInput> &in
                         new RingResidual(pt.x, pt.y,
                                          reference.GetWavelength_A(),
                                          reference.GetPixelSize_mm(),
-                                         pt.q_expected)),
+                                         pt.q_expected,
+                                         reference.GetOrientation())),
+                // No robust loss. Tried and measured on the LaB6 distance series: a Cauchy loss scaled
+                // to the previous pass's median residual moved nothing (rms 0.415 -> 0.421 at 110 mm, no
+                // case improved, every direct beam within 0.06 px) - the ring points are per-sector
+                // peaks that already had to stand 3 sigma clear of their own background, so there are no
+                // gross outliers left for it to reject.
                 nullptr,
                 &center_x,
                 &center_y,
@@ -92,7 +195,8 @@ DiffractionGeometry RingOptimizer::Run(const std::vector<RingOptimizerInput> &in
             && std::all_of(input.begin(), input.end(), [&](const RingOptimizerInput &p) {
                    return p.q_expected == input.front().q_expected;
                });
-    if (single_ring) {
+    const bool tilt_free = !(single_ring || !refine_tilt);
+    if (!tilt_free) {
         problem.SetParameterBlockConstant(&rot1);
         problem.SetParameterBlockConstant(&rot2);
     }
@@ -109,6 +213,16 @@ DiffractionGeometry RingOptimizer::Run(const std::vector<RingOptimizerInput> &in
     // Run optimization
     ceres::Solve(options, &problem, &summary);
 
+    if (unc && summary.IsSolutionUsable()) {
+        std::vector<const double *> free_blocks = {&center_x, &center_y, &distance};
+        if (tilt_free) {
+            free_blocks.push_back(&rot1);
+            free_blocks.push_back(&rot2);
+        }
+        ComputeUncertainty(problem, summary, free_blocks,
+                           &center_x, &center_y, &distance, &rot1, &rot2, *unc);
+    }
+
     // A failed fit must not move the detector geometry. Both callers assign the result straight
     // back over the geometry they passed in, so handing back the reference leaves the calibration
     // where it was instead of committing a diverged beam centre and distance.