RESULT-CHANGING (owner decision): stills tilt post-refinement solved directly instead of by Ceres

Each crystal's (psi_x, psi_y) tilt was refined by a Ceres problem rebuilt for every inner
iteration: one auto-differentiated residual block and one Cauchy loss per reflection. With the
scaling reference cleaned up it was 70% of the stills scale/merge. The same objective - Cauchy
on the sigma-weighted intensity residuals plus the Gaussian prior on the tilt, the tilt bounded -
is now minimised by a small Levenberg-Marquardt on the rho'-reweighted 2x2 normal equations,
the tilt applied to all reflections through one rotation matrix and the Jacobian taken by
central differences.

It reaches the same answer to optimizer precision: on the new synthetic test
(StillsPartialityRefine_RecoversTheTilt) partiality error 0.0559 -> 0.03019 with Ceres and
0.03016 with this, mean tilt 0.02065 vs 0.02067 deg; on two 25k-frame XFEL runs (--mode scale)
CC1/2, I/sigma, R_pim and the CC against a deposited lysozyme model agree to 0.0005, mean
|dpsi| 0.023 deg on both. Scale+merge 6.9 -> 3.0 s (--mode scale, loaded box).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-14 11:59:09 +02:00
1 parent 4936f9b408
commit 3c4d730f89
2 files changed
+205 -67

No files matched your search

@@ -3,13 +3,14 @@
#include "StillsPartialityRefine.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <cmath>
#include <future>
#include <thread>
#include <vector>
#include <ceres/ceres.h>
#include <ceres/rotation.h>
#include "Merge.h"
@@ -101,44 +102,114 @@ namespace {
return G;
}
// Ceres residual: refine the orientation tilt (psi_x, psi_y) holding the scale G fixed. The tilt
// rotates the base reciprocal vector q; partiality follows analytically. Residual is the intensity
// mismatch weighted by 1/sigma, exactly matching ScaleOnTheFly's intensity-space objective.
struct PsiResidual {
double qx, qy, qz;
// The rotation by the angle-axis vector (psi_x, psi_y, 0) as a matrix (Rodrigues), so that one tilt
// is applied to all the reflections of a crystal by a matrix product each.
std::array<double, 9> TiltMatrix(double psi_x, double psi_y) {
const double theta = std::sqrt(psi_x * psi_x + psi_y * psi_y);
if (theta < 1e-15)
return {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0};
const double kx = psi_x / theta, ky = psi_y / theta;
const double c = std::cos(theta), s = std::sin(theta), t = 1.0 - c;
return {c + t * kx * kx, t * kx * ky, s * ky,
t * kx * ky, c + t * ky * ky, -s * kx,
-s * ky, s * kx, c};
}
// ComputeP with the tilt given as its rotation matrix.
struct TiltModel {
double s0x, s0y, s0z, inv_lambda;
double gamma0, gamma_e, bw;
double G, inv_corr, Iref, Iobs, weight;
template<typename T>
bool operator()(const T *const psi, T *residual) const {
T q[3] = {T(qx), T(qy), T(qz)};
T aa[3] = {psi[0], psi[1], T(0.0)};
T qr[3];
ceres::AngleAxisRotatePoint(aa, q, qr);
const T Sx = qr[0] + T(s0x), Sy = qr[1] + T(s0y), Sz = qr[2] + T(s0z);
const T de = ceres::sqrt(Sx * Sx + Sy * Sy + Sz * Sz) - T(inv_lambda);
const T dstar = ceres::sqrt(qr[0] * qr[0] + qr[1] * qr[1] + qr[2] * qr[2]);
const T sig_ang = T(gamma_e) * dstar;
const T sbw = T(bw) * ceres::abs(qr[2]);
const T sig2 = T(gamma0) * T(gamma0) + sig_ang * sig_ang + sbw * sbw;
const T p = ceres::exp(T(-0.5) * de * de / sig2);
residual[0] = T(weight) * (T(G) * p * T(inv_corr) * T(Iref) - T(Iobs));
return true;
double P(const std::array<double, 9> &R, const FitObs &o) const {
const double qx = R[0] * o.qx + R[1] * o.qy + R[2] * o.qz;
const double qy = R[3] * o.qx + R[4] * o.qy + R[5] * o.qz;
const double qz = R[6] * o.qx + R[7] * o.qy + R[8] * o.qz;
const double de = VecLen(qx + s0x, qy + s0y, qz + s0z) - inv_lambda;
const double sig_ang = gamma_e * VecLen(qx, qy, qz);
const double sbw = bw * std::fabs(qz);
const double sig2 = gamma0 * gamma0 + sig_ang * sig_ang + sbw * sbw;
if (!(sig2 > 0.0))
return 1.0;
return std::exp(-0.5 * de * de / sig2);
}
};
// Gaussian prior N(0, sigma_prior^2) on the tilt. The data residuals above are (model-obs)/sigma, a
// proper chi^2, so the MAP prior residual is simply dpsi/sigma_prior - no scale calibration needed.
struct PsiPrior {
double inv_sigma;
template<typename T>
bool operator()(const T *const psi, T *residual) const {
residual[0] = T(inv_sigma) * psi[0];
residual[1] = T(inv_sigma) * psi[1];
return true;
// Refine the orientation tilt (psi_x, psi_y) holding the scale G fixed: minimise
// sum_i rho(r_i^2) / 2 + |psi / sigma_prior|^2 / 2, r_i = weight_i (G p_i(psi) Iref_i / corr_i - Iobs_i),
// with the Cauchy rho(s) = k^2 log(1 + s / k^2) - the intensity-space objective of ScaleOnTheFly, and
// the Gaussian prior a proper MAP term since r_i is already in sigma units. Levenberg-Marquardt on the
// normal equations reweighted by rho', the Jacobian by central differences, every trial tilt held inside
// the bound. Returns false when the cost cannot be evaluated, leaving psi as it came in.
bool RefineTilt(const std::vector<FitObs> &obs, const TiltModel &model, double G, double robust_k,
double inv_sigma_prior, double max_tilt, double psi[2]) {
constexpr int MAX_ITERATIONS = 25;
constexpr double STEP = 1e-7; // rad, for the central differences
const double k2 = robust_k * robust_k;
const double prior2 = inv_sigma_prior * inv_sigma_prior;
auto cost = [&](const double p[2]) {
const auto R = TiltMatrix(p[0], p[1]);
double c = 0.5 * prior2 * (p[0] * p[0] + p[1] * p[1]);
for (const auto &o: obs) {
const double r = o.weight * (G * model.P(R, o) * o.inv_corr * o.Iref - o.Iobs);
c += 0.5 * k2 * std::log1p(r * r / k2);
}
return c;
};
double current = cost(psi);
if (!std::isfinite(current))
return false;
double lambda = 1e-4;
for (int iter = 0; iter < MAX_ITERATIONS; ++iter) {
const auto R = TiltMatrix(psi[0], psi[1]);
const auto Rxp = TiltMatrix(psi[0] + STEP, psi[1]), Rxm = TiltMatrix(psi[0] - STEP, psi[1]);
const auto Ryp = TiltMatrix(psi[0], psi[1] + STEP), Rym = TiltMatrix(psi[0], psi[1] - STEP);
double A00 = prior2, A01 = 0.0, A11 = prior2;
double b0 = prior2 * psi[0], b1 = prior2 * psi[1];
for (const auto &o: obs) {
const double a = o.weight * G * o.inv_corr * o.Iref;
const double r = a * model.P(R, o) - o.weight * o.Iobs;
const double jx = a * (model.P(Rxp, o) - model.P(Rxm, o)) / (2.0 * STEP);
const double jy = a * (model.P(Ryp, o) - model.P(Rym, o)) / (2.0 * STEP);
const double w = 1.0 / (1.0 + r * r / k2); // rho'
A00 += w * jx * jx;
A01 += w * jx * jy;
A11 += w * jy * jy;
b0 += w * jx * r;
b1 += w * jy * r;
}
bool accepted = false;
double trial[2] = {psi[0], psi[1]};
double trial_cost = current;
while (lambda < 1e10) {
const double M00 = A00 * (1.0 + lambda), M11 = A11 * (1.0 + lambda);
const double det = M00 * M11 - A01 * A01;
if (!(det > 0.0))
break;
trial[0] = std::clamp(psi[0] - (M11 * b0 - A01 * b1) / det, -max_tilt, max_tilt);
trial[1] = std::clamp(psi[1] - (M00 * b1 - A01 * b0) / det, -max_tilt, max_tilt);
trial_cost = cost(trial);
if (std::isfinite(trial_cost) && trial_cost < current) {
accepted = true;
lambda = std::max(lambda / 10.0, 1e-12);
break;
}
lambda *= 10.0;
}
if (!accepted)
break;
const double step = std::hypot(trial[0] - psi[0], trial[1] - psi[1]);
const double decrease = current - trial_cost;
psi[0] = trial[0];
psi[1] = trial[1];
current = trial_cost;
if (decrease <= 1e-6 * current || step <= 1e-10)
break;
}
};
return true;
}
}
StillsPartialityRefine::StillsPartialityRefine(const DiffractionExperiment &x)
@@ -238,6 +309,8 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
double psi[2] = {0.0, 0.0};
double G = 1.0;
const TiltModel model{.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda,
.gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw};
const bool refine_tilt = obs.size() >= settings_.min_reflections;
const int inner = refine_tilt ? settings_.inner_iterations : 1;
@@ -258,42 +331,11 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
if (!refine_tilt)
break;
// (2) Refine the tilt holding G fixed.
ceres::Problem problem;
for (const auto &o: obs) {
auto *cost = new ceres::AutoDiffCostFunction<PsiResidual, 1, 2>(new PsiResidual{
.qx = o.qx, .qy = o.qy, .qz = o.qz,
.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda,
.gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw,
.G = G, .inv_corr = o.inv_corr, .Iref = o.Iref, .Iobs = o.Iobs, .weight = o.weight});
problem.AddResidualBlock(cost, new ceres::CauchyLoss(settings_.robust_k), psi);
}
if (settings_.prior_sigma_deg > 0.0) {
const double inv_sigma = kRadToDeg / settings_.prior_sigma_deg; // 1 / sigma_prior (rad)
problem.AddResidualBlock(new ceres::AutoDiffCostFunction<PsiPrior, 2, 2>(
new PsiPrior{inv_sigma}), nullptr, psi);
}
problem.SetParameterLowerBound(psi, 0, -settings_.max_tilt_rad);
problem.SetParameterUpperBound(psi, 0, settings_.max_tilt_rad);
problem.SetParameterLowerBound(psi, 1, -settings_.max_tilt_rad);
problem.SetParameterUpperBound(psi, 1, settings_.max_tilt_rad);
ceres::Solver::Options options;
options.linear_solver_type = ceres::DENSE_QR;
options.minimizer_progress_to_stdout = false;
options.num_threads = 1;
options.max_num_iterations = 25;
ceres::Solver::Summary summary;
const double psi_before[2] = {psi[0], psi[1]};
ceres::Solve(options, &problem, &summary);
// A solve that failed leaves whatever the minimiser last wrote in psi, and that tilt would go
// straight onto every reflection's partiality below. Keep the tilt this crystal came in with
// and stop refining it instead - G alone is still a usable model.
if (!summary.IsSolutionUsable()) {
psi[0] = psi_before[0];
psi[1] = psi_before[1];
// (2) Refine the tilt holding G fixed. A cost that cannot be evaluated keeps the tilt this crystal
// came in with and stops refining it - G alone is still a usable model.
const double inv_sigma_prior = settings_.prior_sigma_deg > 0.0 ? kRadToDeg / settings_.prior_sigma_deg : 0.0;
if (!RefineTilt(obs, model, G, settings_.robust_k, inv_sigma_prior, settings_.max_tilt_rad, psi))
break;
}
}
// Keep what the crystal came in with. This refinement is on by default, so a crystal the tilt model
+96
View File
@@ -10,6 +10,8 @@
#include "../image_analysis/scale_merge/HKLKey.h"
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/ResolutionCutoff.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
#include "gemmi/reciproc.hpp"
TEST_CASE("HKLKey_NoSG_noMergeFriedel") {
@@ -650,3 +652,97 @@ TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") {
CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half);
CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas);
}
// Stills partiality post-refinement on noiseless synthetic crystals: each crystal's true orientation is
// tilted from the stored one by up to 0.05 deg, its intensities are the true partials of a fixed set of
// full intensities, and its stored partialities those of the stored orientation. The refinement has to
// move the partialities towards the true ones. (The Ceres solve this replaced gave the same numbers: error
// 0.0559 -> 0.0302, mean tilt 0.0206 deg.)
TEST_CASE("StillsPartialityRefine_RecoversTheTilt") {
constexpr float CELL = 50.0f, WAVELENGTH = 1.0f, D_MIN = 2.0f;
constexpr double GAMMA0 = 0.001, ACCEPT = 1.7 * GAMMA0; // RLP width; |dist_ewald| accepted at prediction
DiffractionExperiment x;
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
DiffractionGeometry geom;
geom.Wavelength_A(WAVELENGTH);
const Coord S0 = geom.GetScatteringVector();
std::mt19937 rng(11);
std::normal_distribution<double> gauss(0.0, 1.0);
std::uniform_real_distribution<double> unit(0.0, 1.0);
auto full_I = [](int h, int k, int l) { // a fixed, Friedel-symmetric pseudo-random full intensity
const uint64_t key = HKLKeyGenerator(true, gemmi::get_spacegroup_by_name("P 1"))(h, k, l).pack();
return 10.0 + 990.0 * static_cast<double>((key * 0x9e3779b97f4a7c15ULL) >> 40) / static_cast<double>(1ULL << 24);
};
auto rotate = [](const double R[9], const Coord &v) {
return Coord(static_cast<float>(R[0] * v.x + R[1] * v.y + R[2] * v.z),
static_cast<float>(R[3] * v.x + R[4] * v.y + R[5] * v.z),
static_cast<float>(R[6] * v.x + R[7] * v.y + R[8] * v.z));
};
auto rotation = [](double ax, double ay, double az, double angle, double R[9]) { // Rodrigues
const double n = std::sqrt(ax * ax + ay * ay + az * az);
ax /= n; ay /= n; az /= n;
const double c = std::cos(angle), s = std::sin(angle), t = 1.0 - c;
const double M[9] = {c + t * ax * ax, t * ax * ay - s * az, t * ax * az + s * ay,
t * ax * ay + s * az, c + t * ay * ay, t * ay * az - s * ax,
t * ax * az - s * ay, t * ay * az + s * ax, c + t * az * az};
std::copy(M, M + 9, R);
};
auto partiality = [&](const Coord &q) {
const double de = (q + S0).Length() - 1.0 / WAVELENGTH;
return std::exp(-0.5 * de * de / (GAMMA0 * GAMMA0));
};
std::vector<IntegrationOutcome> outcomes(150);
std::vector<std::vector<double>> true_p(outcomes.size());
const int hmax = static_cast<int>(CELL / D_MIN);
for (size_t c = 0; c < outcomes.size(); ++c) {
double R[9], T[9];
rotation(gauss(rng), gauss(rng), gauss(rng), 2.0 * M_PI * unit(rng), R);
const double tilt = (0.02 + 0.03 * unit(rng)) * M_PI / 180.0, phi = 2.0 * M_PI * unit(rng);
rotation(std::cos(phi), std::sin(phi), 0.0, tilt, T); // about an axis normal to the beam
auto &o = outcomes[c];
o.geom = geom;
o.latt = CrystalLattice(rotate(R, Coord(CELL, 0, 0)), rotate(R, Coord(0, CELL, 0)),
rotate(R, Coord(0, 0, CELL)));
const double G = 0.5 + unit(rng);
for (int h = -hmax; h <= hmax; ++h)
for (int k = -hmax; k <= hmax; ++k)
for (int l = -hmax; l <= hmax; ++l) {
const Coord q = o.latt.Astar() * h + o.latt.Bstar() * k + o.latt.Cstar() * l;
const float dstar = q.Length();
if (dstar == 0.0f || dstar > 1.0f / D_MIN)
continue;
if (std::fabs((q + S0).Length() - 1.0 / WAVELENGTH) > ACCEPT)
continue;
const double p_true = partiality(rotate(T, q));
Reflection r{};
r.h = h; r.k = k; r.l = l;
r.d = 1.0f / dstar;
r.I = static_cast<float>(G * p_true * full_I(h, k, l));
r.sigma = static_cast<float>(1.0 + 0.01 * r.I);
r.prescaling_corr = 1.0f;
r.partiality = static_cast<float>(partiality(q));
r.image_scale_corr = 1.0f / r.partiality;
o.reflections.push_back(r);
true_p[c].push_back(p_true);
}
REQUIRE(o.reflections.size() >= 40);
}
auto mean_abs_error = [&]() {
double sum = 0.0;
size_t n = 0;
for (size_t c = 0; c < outcomes.size(); ++c)
for (size_t i = 0; i < outcomes[c].reflections.size(); ++i, ++n)
sum += std::fabs(outcomes[c].reflections[i].partiality - true_p[c][i]);
return sum / static_cast<double>(n);
};
const double before = mean_abs_error();
ScaleOnTheFly(x, MergeAll(x, outcomes, 4, std::nullopt)).Scale(outcomes, 4);
const double mean_tilt = StillsPartialityRefine(x).Run(outcomes, 4, std::nullopt);
const double after = mean_abs_error();
CHECK(mean_tilt > 0.015);
CHECK(mean_tilt < 0.06);
CHECK(after < 0.6 * before); // measured 0.54: the prior holds the larger tilts back
}