From 50f953bcc575f55165c401a427abe9a4dba8dc0c Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 09:41:37 +0200 Subject: [PATCH 001/228] PixelRefine: Work in progress --- image_analysis/CMakeLists.txt | 1 + .../pixel_refinement/CMakeLists.txt | 2 + .../pixel_refinement/PixelRefine.cpp | 133 ++++++++++++++++++ image_analysis/pixel_refinement/PixelRefine.h | 36 +++++ 4 files changed, 172 insertions(+) create mode 100644 image_analysis/pixel_refinement/CMakeLists.txt create mode 100644 image_analysis/pixel_refinement/PixelRefine.cpp create mode 100644 image_analysis/pixel_refinement/PixelRefine.h diff --git a/image_analysis/CMakeLists.txt b/image_analysis/CMakeLists.txt index 9572d0b0..913870c5 100644 --- a/image_analysis/CMakeLists.txt +++ b/image_analysis/CMakeLists.txt @@ -50,5 +50,6 @@ ADD_SUBDIRECTORY(lattice_search) ADD_SUBDIRECTORY(scale_merge) ADD_SUBDIRECTORY(image_preprocessing) ADD_SUBDIRECTORY(azint) +ADD_SUBDIRECTORY(pixel_refinement) TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge gemmi) diff --git a/image_analysis/pixel_refinement/CMakeLists.txt b/image_analysis/pixel_refinement/CMakeLists.txt new file mode 100644 index 00000000..5d5e3c6c --- /dev/null +++ b/image_analysis/pixel_refinement/CMakeLists.txt @@ -0,0 +1,2 @@ +ADD_LIBRARY(JFJochPixelRefine PixelRefine.cpp PixelRefine.h) +TARGET_LINK_LIBRARIES(JFJochPixelRefine JFJochBraggPrediction JFJochCommon ceres_static) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp new file mode 100644 index 00000000..1e547b08 --- /dev/null +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "PixelRefine.h" + +#include +#include + + + + +struct PixelRefineResidual { + ScalingPostRefResidual(int32_t h, int32_t k, int32_t l, + double x, double y, + double Itrue, + const DiffractionGeometry &geometry, + const CrystalLattice &lattice) + : + integration_center_x(r.predicted_x), + integration_center_y(r.predicted_y), + inv_lambda(SafeInv(geometry.GetWavelength_A(), 0.0)), + pixel_size(geometry.GetPixelSize_mm()), + det_dist_mm(geometry.GetDetectorDistance_mm()), + beam_x(geometry.GetBeamX_pxl()), + beam_y(geometry.GetBeamY_pxl()), + exp_h(r.h), + exp_k(r.k), + exp_l(r.l), + Astar(lattice.Astar()), + Bstar(lattice.Bstar()), + Cstar(lattice.Cstar()), + c1(std::cos(geometry.GetPoniRot1_rad())), + s1(std::sin(geometry.GetPoniRot1_rad())), + c2(std::cos(geometry.GetPoniRot2_rad())), + s2(std::sin(geometry.GetPoniRot2_rad())) { + } + + template + T CalcPartiality(const T *const R, + const T *const beam_corr, + const T *const p0) const { + // Detector coordinates in mm + const T det_x = (T(integration_center_x) - beam_x - beam_corr[0]) * T(pixel_size); + const T det_y = (T(integration_center_y) - beam_y - beam_corr[1]) * T(pixel_size); + const T det_z = T(det_dist_mm); + + // Apply Ry(rot1) first: rotate around Y + const T t1_x = T(c1) * det_x + T(s1) * det_z; + const T t1_y = det_y; + const T t1_z = T(-s1) * det_x + T(c1) * det_z; + + // Then apply Rx(-rot2): rotate around X + const T x = t1_x; + const T y = T(c2) * t1_y + T(s2) * t1_z; + const T z = - T(s2) * t1_y + T(c2) * t1_z; + + // convert to recip space + const T lab_norm = ceres::sqrt(x * x + y * y + z * z); + const T inv_norm = T(1) / lab_norm; + + T recip_obs[3]; + recip_obs[0] = x * inv_norm * inv_lambda; + recip_obs[1] = y * inv_norm * inv_lambda; + recip_obs[2] = (z * inv_norm - T(1.0)) * inv_lambda; + + const Eigen::Matrix e_obs_recip(recip_obs[0], recip_obs[1], recip_obs[2]); + + const T astar_unrot[3] = {T(Astar.x), T(Astar.y), T(Astar.z)}; + const T bstar_unrot[3] = {T(Bstar.x), T(Bstar.y), T(Bstar.z)}; + const T cstar_unrot[3] = {T(Cstar.x), T(Cstar.y), T(Cstar.z)}; + + T astar_rot[3], bstar_rot[3], cstar_rot[3]; + + ceres::AngleAxisRotatePoint(p0, astar_unrot, astar_rot); + ceres::AngleAxisRotatePoint(p0, bstar_unrot, bstar_rot); + ceres::AngleAxisRotatePoint(p0, cstar_unrot, cstar_rot); + + const Eigen::Matrix e_pred_recip(T(exp_h) * astar_rot[0] + T(exp_k) * bstar_rot[0] + T(exp_l) * cstar_rot[0], + T(exp_h) * astar_rot[1] + T(exp_k) * bstar_rot[1] + T(exp_l) * cstar_rot[1], + T(exp_h) * astar_rot[2] + T(exp_k) * bstar_rot[2] + T(exp_l) * cstar_rot[2] + ); + + // Ewald sphere centre is at -k_i = (0, 0, -inv_lambda) + // Radial direction: outward normal at g_hkl + const Eigen::Matrix S_pred( + e_pred_recip[0], + e_pred_recip[1], + e_pred_recip[2] + T(inv_lambda) // g_hkl + k_i + ); + const T S_pred_norm = S_pred.norm(); + if (S_pred_norm < T(1e-10)) + return T(0); + + const Eigen::Matrix n_radial = S_pred / S_pred_norm; + + const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; + const T eps_radial = delta_q.dot(n_radial); + const Eigen::Matrix dq_tang = delta_q - eps_radial * n_radial; + const T eps_tangential_sq = dq_tang.squaredNorm(); // guaranteed ≥ 0 + // ───────────────────────────────────────────────────────────── + + return ceres::exp(- eps_radial * eps_radial / (R[0] * R[0]) - eps_tangential_sq / (R[1] * R[1])); + } + + template + bool operator()(const T *const G, + const T *const B, + const T *const R, + const T *const beam_corr, + const T *const p0, + T *residual) const { + if (R[0] < T(1e-10) || R[1] < T(1e-10)) + return false; + + const T B_term = ceres::exp(B[0] * T(b_resolution_coeff)); + + const T partiality = CalcPartiality(R, beam_corr, p0); + residual[0] = (G[0] * partiality * B_term * T(lp) * T(Itrue) + - T(Iobs)) * T(weight); + return true; + } + + const double integration_center_x, integration_center_y; + const double inv_lambda; + const double pixel_size; + const double det_dist_mm; + const double beam_x, beam_y; + const double exp_h; + const double exp_k; + const double exp_l; + const Coord Astar, Bstar, Cstar; + const double c1,s1,c2,s2; +}; \ No newline at end of file diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h new file mode 100644 index 00000000..1610ea9c --- /dev/null +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include "../bragg_prediction/BraggPrediction.h" +#include "../common/DiffractionExperiment.h" +#include "../common/AzimuthalIntegrationMapping.h" + +struct PixelRefineData { + DiffractionGeometry geom; + CrystalLattice latt; + gemmi::CrystalSystem crystal_system = gemmi::CrystalSystem::Triclinic; + char centering = 'P'; + + double B_factor = 0.0; + double scale_factor = 1.0; + double R[2] = {0.001, 0.001}; + + bool refine_beam_center = false; + bool refine_unit_cell = false; +}; + +class PixelRefine { + BraggPrediction &prediction; + const AzimuthalIntegrationMapping &mapping; + const size_t xpixel, ypixel; + +public: + PixelRefine(const DiffractionExperiment &experiment, + const AzimuthalIntegrationMapping &mapping; + BraggPrediction &prediction); + + template + void Run(const T *image, PixelRefineData &data); +}; -- 2.52.0 From 0d6b2787673e8e7ac9c4d741a44218d441293137 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 11:11:35 +0200 Subject: [PATCH 002/228] LatticeReduction: Move outside of XtalOptimizer --- image_analysis/geom_refinement/CMakeLists.txt | 2 + .../geom_refinement/LatticeReduction.cpp | 257 +++++++++++++++++ .../geom_refinement/LatticeReduction.h | 19 ++ .../geom_refinement/XtalOptimizer.cpp | 260 +----------------- .../geom_refinement/XtalOptimizer.h | 13 - tests/CMakeLists.txt | 1 + tests/LatticeReductionTest.cpp | 225 +++++++++++++++ tests/XtalOptimizerTest.cpp | 213 -------------- 8 files changed, 505 insertions(+), 485 deletions(-) create mode 100644 image_analysis/geom_refinement/LatticeReduction.cpp create mode 100644 image_analysis/geom_refinement/LatticeReduction.h create mode 100644 tests/LatticeReductionTest.cpp diff --git a/image_analysis/geom_refinement/CMakeLists.txt b/image_analysis/geom_refinement/CMakeLists.txt index a81b8b54..51df98fa 100644 --- a/image_analysis/geom_refinement/CMakeLists.txt +++ b/image_analysis/geom_refinement/CMakeLists.txt @@ -6,6 +6,8 @@ ADD_LIBRARY(JFJochGeomRefinement STATIC AssignSpotsToRings.h XtalOptimizer.cpp XtalOptimizer.h + LatticeReduction.cpp + LatticeReduction.h ) TARGET_LINK_LIBRARIES(JFJochGeomRefinement Ceres::ceres Eigen3::Eigen JFJochCommon) diff --git a/image_analysis/geom_refinement/LatticeReduction.cpp b/image_analysis/geom_refinement/LatticeReduction.cpp new file mode 100644 index 00000000..7da10261 --- /dev/null +++ b/image_analysis/geom_refinement/LatticeReduction.cpp @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "LatticeReduction.h" + +#include + +void LatticeToRodriguesAndLengths_GS(const CrystalLattice &latt, + double rod[3], + double lengths[3]) { + // Load lattice columns + const Coord a = latt.Vec0(); + const Coord b = latt.Vec1(); + const Coord c = latt.Vec2(); + + Eigen::Vector3d A(a[0], a[1], a[2]); + Eigen::Vector3d B(b[0], b[1], b[2]); + Eigen::Vector3d C(c[0], c[1], c[2]); + + // Lengths = column norms (orthorhombic assumption) + lengths[0] = A.norm(); + lengths[1] = B.norm(); + lengths[2] = C.norm(); + + auto safe_unit = [](const Eigen::Vector3d &v, double eps = 1e-15) -> Eigen::Vector3d { + double n = v.norm(); + return (n > eps) ? (v / n) : Eigen::Vector3d(1.0, 0.0, 0.0); + }; + + // Gram–Schmidt with original order: x from A, y from B orthogonalized vs x + Eigen::Vector3d e1 = safe_unit(A); + Eigen::Vector3d y = B - (e1.dot(B)) * e1; + Eigen::Vector3d e2 = safe_unit(y); + + // z from cross to ensure right-handed basis + Eigen::Vector3d e3 = e1.cross(e2); + double n3 = e3.norm(); + if (n3 < 1e-15) { + // Degenerate case: B nearly collinear with A → use C instead + y = C - (e1.dot(C)) * e1; + e2 = safe_unit(y); + e3 = e1.cross(e2); + n3 = e3.norm(); + if (n3 < 1e-15) { + // Still degenerate: pick any perpendicular to e1 + e2 = safe_unit((std::abs(e1.x()) < 0.9) + ? Eigen::Vector3d::UnitX().cross(e1) + : Eigen::Vector3d::UnitY().cross(e1)); + e3 = e1.cross(e2); + } + } else { + e3 /= n3; + } + + Eigen::Matrix3d R; + R.col(0) = e1; + R.col(1) = e2; + R.col(2) = e3; + + // Convert rotation to Rodrigues (axis * angle) + Eigen::AngleAxisd aa(R); + Eigen::Vector3d r = aa.angle() * aa.axis(); + rod[0] = r.x(); + rod[1] = r.y(); + rod[2] = r.z(); +} + +void LatticeToRodriguesAndLengths_Hex(const CrystalLattice &latt, double rod[3], double ac[3]) { + const Coord a = latt.Vec0(); + const Coord b = latt.Vec1(); + const Coord c = latt.Vec2(); + + Eigen::Vector3d A(a[0], a[1], a[2]); + Eigen::Vector3d B(b[0], b[1], b[2]); + Eigen::Vector3d C(c[0], c[1], c[2]); + + const double a_len = A.norm(); + const double b_len = B.norm(); + const double c_len = C.norm(); + + ac[0] = (a_len + b_len) / 2.0; + ac[1] = (a_len + b_len) / 2.0; + ac[2] = c_len; + + Eigen::Vector3d e1; + Eigen::Vector3d e3; + + if (a_len > 0.0) + e1 = A / a_len; + else + e1 = Eigen::Vector3d::UnitX(); + + if (c_len > 0.0) + e3 = C / c_len; + else + e3 = Eigen::Vector3d::UnitZ(); + + Eigen::Vector3d e2 = e3.cross(e1); + if (e2.norm() < 1e-15) { + e2 = (std::abs(e1.x()) < 0.9) + ? Eigen::Vector3d::UnitX().cross(e1) + : Eigen::Vector3d::UnitY().cross(e1); + } + e2.normalize(); + e3 = e1.cross(e2).normalized(); + + Eigen::Matrix3d R; + R.col(0) = e1; + R.col(1) = e2; + R.col(2) = e3; + + Eigen::AngleAxisd aa(R); + Eigen::Vector3d r = aa.angle() * aa.axis(); + rod[0] = r.x(); + rod[1] = r.y(); + rod[2] = r.z(); +} + +// Extract rotation (Rodrigues), lengths (a,b,c) and beta (rad) for monoclinic (unique axis b). +// Frame choice: e2 aligned with b; e1 from a projected orthogonal to e2; e3 = e1 x e2. +void LatticeToRodriguesLengthsBeta_Mono(const CrystalLattice &latt, + double rod[3], + double lengths[3], + double &beta_rad) { + const Coord a = latt.Vec0(); + const Coord b = latt.Vec1(); + const Coord c = latt.Vec2(); + + const Eigen::Vector3d A(a[0], a[1], a[2]); + const Eigen::Vector3d Bv(b[0], b[1], b[2]); + const Eigen::Vector3d C(c[0], c[1], c[2]); + + // Unit cell lengths + const double a_len = A.norm(); + const double b_len = Bv.norm(); + const double c_len = C.norm(); + + lengths[0] = a_len; + lengths[1] = b_len; + lengths[2] = c_len; + + // Monoclinic beta = angle(a, c) + double cos_beta = 0.0; + if (a_len > 1e-15 && c_len > 1e-15) { + cos_beta = A.dot(C) / (a_len * c_len); + cos_beta = std::clamp(cos_beta, -1.0, 1.0); + } + + beta_rad = std::acos(cos_beta); + + // Protect against singular construction + const double sin_beta = std::max(std::abs(std::sin(beta_rad)), 1e-12); + + // Canonical monoclinic basis: + // + // B = + // [ 1 0 cos(beta) ] + // [ 0 1 0 ] + // [ 0 0 sin(beta) ] + // + Eigen::Matrix3d Bmono = Eigen::Matrix3d::Zero(); + Bmono(0,0) = 1.0; + Bmono(1,1) = 1.0; + Bmono(0,2) = std::cos(beta_rad); + Bmono(2,2) = sin_beta; + + // Scale by lengths + Eigen::DiagonalMatrix D(a_len, b_len, c_len); + + // Ideal body-frame lattice + const Eigen::Matrix3d M = Bmono * D; + + // Observed lattice + Eigen::Matrix3d L; + L.col(0) = A; + L.col(1) = Bv; + L.col(2) = C; + + // Estimate rotation: + // R ≈ L * M^{-1} + Eigen::Matrix3d R_est = L * M.inverse(); + + // Project to nearest proper rotation matrix + Eigen::JacobiSVD svd(R_est, Eigen::ComputeFullU | Eigen::ComputeFullV); + + Eigen::Matrix3d R = svd.matrixU() * svd.matrixV().transpose(); + + // Enforce det(R)=+1 + if (R.determinant() < 0.0) { + Eigen::Matrix3d U = svd.matrixU(); + U.col(2) *= -1.0; + R = U * svd.matrixV().transpose(); + } + + // Rodrigues vector + Eigen::AngleAxisd aa(R); + const Eigen::Vector3d r = aa.angle() * aa.axis(); + + rod[0] = r.x(); + rod[1] = r.y(); + rod[2] = r.z(); +} + +static inline Eigen::Matrix3d B_from_angles(double alpha_rad, double beta_rad, double gamma_rad) { + const double ca = std::cos(alpha_rad); + const double cb = std::cos(beta_rad); + const double cg = std::cos(gamma_rad); + const double sg = std::sin(gamma_rad); + + Eigen::Matrix3d B = Eigen::Matrix3d::Identity(); + + // a along x, b in x-y, c general + B(0, 0) = 1.0; B(1, 0) = 0.0; B(2, 0) = 0.0; + B(0, 1) = cg; B(1, 1) = sg; B(2, 1) = 0.0; + + // c vector components (standard crystallography construction) + const double cx = cb; + const double cy = (ca - cb * cg) / sg; + const double cz = std::sqrt(std::max(0.0, 1.0 - cx * cx - cy * cy)); + + B(0, 2) = cx; + B(1, 2) = cy; + B(2, 2) = cz; + + return B; +} + +CrystalLattice AngleAxisAndCellToLattice(const double rod[3], + const double lengths[3], + double alpha_rad, + double beta_rad, + double gamma_rad) { + const Eigen::Vector3d r(rod[0], rod[1], rod[2]); + const double angle = r.norm(); + + Eigen::Matrix3d R = Eigen::Matrix3d::Identity(); + if (angle > 0.0) + R = Eigen::AngleAxisd(angle, r / angle).toRotationMatrix(); + + const Eigen::DiagonalMatrix D(lengths[0], lengths[1], lengths[2]); + const Eigen::Matrix3d B = B_from_angles(alpha_rad, beta_rad, gamma_rad); + + // IMPORTANT convention: L = R * B * D (scale columns by lengths) + const Eigen::Matrix3d latt = R * B * D; + + return CrystalLattice(Coord(latt(0, 0), latt(1, 0), latt(2, 0)), + Coord(latt(0, 1), latt(1, 1), latt(2, 1)), + Coord(latt(0, 2), latt(1, 2), latt(2, 2))); +} + + +CrystalLattice AngleAxisAndLengthsToLattice(const double rod[3], const double lengths[3], bool hex) { + return AngleAxisAndCellToLattice(rod, lengths, + /*alpha=*/M_PI / 2.0, + /*beta =*/M_PI / 2.0, + /*gamma=*/hex ? 2.0 * M_PI / 3.0 : M_PI / 2.0); +} diff --git a/image_analysis/geom_refinement/LatticeReduction.h b/image_analysis/geom_refinement/LatticeReduction.h new file mode 100644 index 00000000..bcf0d3d3 --- /dev/null +++ b/image_analysis/geom_refinement/LatticeReduction.h @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include "../common/CrystalLattice.h" + +void LatticeToRodriguesAndLengths_GS(const CrystalLattice &latt, double rod[3], double lengths[3]); +void LatticeToRodriguesAndLengths_Hex(const CrystalLattice &latt, double rod[3], double ac[3]); +void LatticeToRodriguesLengthsBeta_Mono(const CrystalLattice &latt, + double rod[3], + double lengths[3], + double &beta_rad); + +CrystalLattice AngleAxisAndCellToLattice(const double rod[3], + const double lengths[3], + double alpha_rad, + double beta_rad, + double gamma_rad); diff --git a/image_analysis/geom_refinement/XtalOptimizer.cpp b/image_analysis/geom_refinement/XtalOptimizer.cpp index cefb24cc..03d5ce08 100644 --- a/image_analysis/geom_refinement/XtalOptimizer.cpp +++ b/image_analysis/geom_refinement/XtalOptimizer.cpp @@ -6,6 +6,7 @@ #include "XtalOptimizer.h" #include "ceres/ceres.h" #include "ceres/rotation.h" +#include "LatticeReduction.h" struct XtalResidual { XtalResidual(double x, double y, @@ -244,265 +245,6 @@ struct RotationNormRegularizer { const double weight; }; -inline void LatticeToRodriguesAndLengths_GS(const CrystalLattice &latt, - double rod[3], - double lengths[3]) { - // Load lattice columns - const Coord a = latt.Vec0(); - const Coord b = latt.Vec1(); - const Coord c = latt.Vec2(); - - Eigen::Vector3d A(a[0], a[1], a[2]); - Eigen::Vector3d B(b[0], b[1], b[2]); - Eigen::Vector3d C(c[0], c[1], c[2]); - - // Lengths = column norms (orthorhombic assumption) - lengths[0] = A.norm(); - lengths[1] = B.norm(); - lengths[2] = C.norm(); - - auto safe_unit = [](const Eigen::Vector3d &v, double eps = 1e-15) -> Eigen::Vector3d { - double n = v.norm(); - return (n > eps) ? (v / n) : Eigen::Vector3d(1.0, 0.0, 0.0); - }; - - // Gram–Schmidt with original order: x from A, y from B orthogonalized vs x - Eigen::Vector3d e1 = safe_unit(A); - Eigen::Vector3d y = B - (e1.dot(B)) * e1; - Eigen::Vector3d e2 = safe_unit(y); - - // z from cross to ensure right-handed basis - Eigen::Vector3d e3 = e1.cross(e2); - double n3 = e3.norm(); - if (n3 < 1e-15) { - // Degenerate case: B nearly collinear with A → use C instead - y = C - (e1.dot(C)) * e1; - e2 = safe_unit(y); - e3 = e1.cross(e2); - n3 = e3.norm(); - if (n3 < 1e-15) { - // Still degenerate: pick any perpendicular to e1 - e2 = safe_unit((std::abs(e1.x()) < 0.9) - ? Eigen::Vector3d::UnitX().cross(e1) - : Eigen::Vector3d::UnitY().cross(e1)); - e3 = e1.cross(e2); - } - } else { - e3 /= n3; - } - - Eigen::Matrix3d R; - R.col(0) = e1; - R.col(1) = e2; - R.col(2) = e3; - - // Convert rotation to Rodrigues (axis * angle) - Eigen::AngleAxisd aa(R); - Eigen::Vector3d r = aa.angle() * aa.axis(); - rod[0] = r.x(); - rod[1] = r.y(); - rod[2] = r.z(); -} - -void LatticeToRodriguesAndLengths_Hex(const CrystalLattice &latt, double rod[3], double ac[3]) { - const Coord a = latt.Vec0(); - const Coord b = latt.Vec1(); - const Coord c = latt.Vec2(); - - Eigen::Vector3d A(a[0], a[1], a[2]); - Eigen::Vector3d B(b[0], b[1], b[2]); - Eigen::Vector3d C(c[0], c[1], c[2]); - - const double a_len = A.norm(); - const double b_len = B.norm(); - const double c_len = C.norm(); - - ac[0] = (a_len + b_len) / 2.0; - ac[1] = (a_len + b_len) / 2.0; - ac[2] = c_len; - - Eigen::Vector3d e1; - Eigen::Vector3d e3; - - if (a_len > 0.0) - e1 = A / a_len; - else - e1 = Eigen::Vector3d::UnitX(); - - if (c_len > 0.0) - e3 = C / c_len; - else - e3 = Eigen::Vector3d::UnitZ(); - - Eigen::Vector3d e2 = e3.cross(e1); - if (e2.norm() < 1e-15) { - e2 = (std::abs(e1.x()) < 0.9) - ? Eigen::Vector3d::UnitX().cross(e1) - : Eigen::Vector3d::UnitY().cross(e1); - } - e2.normalize(); - e3 = e1.cross(e2).normalized(); - - Eigen::Matrix3d R; - R.col(0) = e1; - R.col(1) = e2; - R.col(2) = e3; - - Eigen::AngleAxisd aa(R); - Eigen::Vector3d r = aa.angle() * aa.axis(); - rod[0] = r.x(); - rod[1] = r.y(); - rod[2] = r.z(); -} - -// Extract rotation (Rodrigues), lengths (a,b,c) and beta (rad) for monoclinic (unique axis b). -// Frame choice: e2 aligned with b; e1 from a projected orthogonal to e2; e3 = e1 x e2. -void LatticeToRodriguesLengthsBeta_Mono(const CrystalLattice &latt, - double rod[3], - double lengths[3], - double &beta_rad) { - const Coord a = latt.Vec0(); - const Coord b = latt.Vec1(); - const Coord c = latt.Vec2(); - - const Eigen::Vector3d A(a[0], a[1], a[2]); - const Eigen::Vector3d Bv(b[0], b[1], b[2]); - const Eigen::Vector3d C(c[0], c[1], c[2]); - - // Unit cell lengths - const double a_len = A.norm(); - const double b_len = Bv.norm(); - const double c_len = C.norm(); - - lengths[0] = a_len; - lengths[1] = b_len; - lengths[2] = c_len; - - // Monoclinic beta = angle(a, c) - double cos_beta = 0.0; - if (a_len > 1e-15 && c_len > 1e-15) { - cos_beta = A.dot(C) / (a_len * c_len); - cos_beta = std::clamp(cos_beta, -1.0, 1.0); - } - - beta_rad = std::acos(cos_beta); - - // Protect against singular construction - const double sin_beta = std::max(std::abs(std::sin(beta_rad)), 1e-12); - - // Canonical monoclinic basis: - // - // B = - // [ 1 0 cos(beta) ] - // [ 0 1 0 ] - // [ 0 0 sin(beta) ] - // - Eigen::Matrix3d Bmono = Eigen::Matrix3d::Zero(); - Bmono(0,0) = 1.0; - Bmono(1,1) = 1.0; - Bmono(0,2) = std::cos(beta_rad); - Bmono(2,2) = sin_beta; - - // Scale by lengths - Eigen::DiagonalMatrix D(a_len, b_len, c_len); - - // Ideal body-frame lattice - const Eigen::Matrix3d M = Bmono * D; - - // Observed lattice - Eigen::Matrix3d L; - L.col(0) = A; - L.col(1) = Bv; - L.col(2) = C; - - // Estimate rotation: - // R ≈ L * M^{-1} - Eigen::Matrix3d R_est = L * M.inverse(); - - // Project to nearest proper rotation matrix - Eigen::JacobiSVD svd(R_est, Eigen::ComputeFullU | Eigen::ComputeFullV); - - Eigen::Matrix3d R = svd.matrixU() * svd.matrixV().transpose(); - - // Enforce det(R)=+1 - if (R.determinant() < 0.0) { - Eigen::Matrix3d U = svd.matrixU(); - U.col(2) *= -1.0; - R = U * svd.matrixV().transpose(); - } - - // Rodrigues vector - Eigen::AngleAxisd aa(R); - const Eigen::Vector3d r = aa.angle() * aa.axis(); - - rod[0] = r.x(); - rod[1] = r.y(); - rod[2] = r.z(); -} - -static inline Eigen::Matrix3d B_from_angles(double alpha_rad, double beta_rad, double gamma_rad) { - const double ca = std::cos(alpha_rad); - const double cb = std::cos(beta_rad); - const double cg = std::cos(gamma_rad); - const double sg = std::sin(gamma_rad); - - Eigen::Matrix3d B = Eigen::Matrix3d::Identity(); - - // a along x, b in x-y, c general - B(0, 0) = 1.0; B(1, 0) = 0.0; B(2, 0) = 0.0; - B(0, 1) = cg; B(1, 1) = sg; B(2, 1) = 0.0; - - // c vector components (standard crystallography construction) - const double cx = cb; - const double cy = (ca - cb * cg) / sg; - const double cz = std::sqrt(std::max(0.0, 1.0 - cx * cx - cy * cy)); - - B(0, 2) = cx; - B(1, 2) = cy; - B(2, 2) = cz; - - return B; -} - -CrystalLattice AngleAxisAndCellToLattice(const double rod[3], - const double lengths[3], - double alpha_rad, - double beta_rad, - double gamma_rad) { - const Eigen::Vector3d r(rod[0], rod[1], rod[2]); - const double angle = r.norm(); - - Eigen::Matrix3d R = Eigen::Matrix3d::Identity(); - if (angle > 0.0) - R = Eigen::AngleAxisd(angle, r / angle).toRotationMatrix(); - - const Eigen::DiagonalMatrix D(lengths[0], lengths[1], lengths[2]); - const Eigen::Matrix3d B = B_from_angles(alpha_rad, beta_rad, gamma_rad); - - // IMPORTANT convention: L = R * B * D (scale columns by lengths) - const Eigen::Matrix3d latt = R * B * D; - - return CrystalLattice(Coord(latt(0, 0), latt(1, 0), latt(2, 0)), - Coord(latt(0, 1), latt(1, 1), latt(2, 1)), - Coord(latt(0, 2), latt(1, 2), latt(2, 2))); -} - - -CrystalLattice AngleAxisAndLengthsToLattice(const double rod[3], const double lengths[3], bool hex) { - if (!hex) { - return AngleAxisAndCellToLattice(rod, lengths, - /*alpha=*/M_PI / 2.0, - /*beta =*/M_PI / 2.0, - /*gamma=*/M_PI / 2.0); - } - - // Hexagonal: caller must already enforce a=b in `lengths`. - return AngleAxisAndCellToLattice(rod, lengths, - /*alpha=*/M_PI / 2.0, - /*beta =*/M_PI / 2.0, - /*gamma=*/2.0 * M_PI / 3.0); -} - bool XtalOptimizerInternal(XtalOptimizerData &data, const std::vector> &spots, const float tolerance) { diff --git a/image_analysis/geom_refinement/XtalOptimizer.h b/image_analysis/geom_refinement/XtalOptimizer.h index c472e8ab..7dbaae47 100644 --- a/image_analysis/geom_refinement/XtalOptimizer.h +++ b/image_analysis/geom_refinement/XtalOptimizer.h @@ -43,19 +43,6 @@ struct XtalOptimizerData { std::optional angle_axis; }; -void LatticeToRodriguesAndLengths_GS(const CrystalLattice &latt, double rod[3], double lengths[3]); -void LatticeToRodriguesAndLengths_Hex(const CrystalLattice &latt, double rod[3], double ac[3]); -void LatticeToRodriguesLengthsBeta_Mono(const CrystalLattice &latt, - double rod[3], - double lengths[3], - double &beta_rad); - -CrystalLattice AngleAxisAndCellToLattice(const double rod[3], - const double lengths[3], - double alpha_rad, - double beta_rad, - double gamma_rad); - bool XtalOptimizer(XtalOptimizerData &data, const std::vector> &spots); bool XtalOptimizerRotationOnly(XtalOptimizerData &data, const std::vector &spots, float tolerance); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f4868b12..751e8f64 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -71,6 +71,7 @@ ADD_EXECUTABLE(jfjoch_test UnitCellTest.cpp CCTest.cpp MultiLatticeSearchTest.cpp + LatticeReductionTest.cpp ) target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochWriter diff --git a/tests/LatticeReductionTest.cpp b/tests/LatticeReductionTest.cpp new file mode 100644 index 00000000..8ad8aa93 --- /dev/null +++ b/tests/LatticeReductionTest.cpp @@ -0,0 +1,225 @@ +// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include + +#include "../image_analysis/geom_refinement/LatticeReduction.h" + + +namespace { + Eigen::Vector3d to_eigen(const Coord& v) { + return {v[0], v[1], v[2]}; + } + + double angle_rad(const Eigen::Vector3d& a, const Eigen::Vector3d& b) { + const double na = a.norm(); + const double nb = b.norm(); + if (na == 0.0 || nb == 0.0) + return 0.0; + double c = a.dot(b) / (na * nb); + c = std::max(-1.0, std::min(1.0, c)); + return std::acos(c); + } + + // Compare two lattices up to a global rotation: compare Gram matrices G = L^T L (rotation-invariant). + Eigen::Matrix3d gram(const CrystalLattice& latt) { + const Eigen::Vector3d A = to_eigen(latt.Vec0()); + const Eigen::Vector3d B = to_eigen(latt.Vec1()); + const Eigen::Vector3d C = to_eigen(latt.Vec2()); + Eigen::Matrix3d G; + G(0,0) = A.dot(A); G(0,1) = A.dot(B); G(0,2) = A.dot(C); + G(1,0) = B.dot(A); G(1,1) = B.dot(B); G(1,2) = B.dot(C); + G(2,0) = C.dot(A); G(2,1) = C.dot(B); G(2,2) = C.dot(C); + return G; + } + + void check_gram_close(const CrystalLattice& a, + const CrystalLattice& b, + double abs_eps, + double rel_eps) { + const Eigen::Matrix3d Ga = gram(a); + const Eigen::Matrix3d Gb = gram(b); + + for (int r = 0; r < 3; ++r) { + for (int c = 0; c < 3; ++c) { + const double va = Ga(r, c); + const double vb = Gb(r, c); + + // Scale for relative error; avoid blowing up around zero. + const double scale = std::max({1.0, std::abs(va), std::abs(vb)}); + const double tol = std::max(abs_eps, rel_eps * scale); + + INFO("G(" << r << "," << c << ") va=" << va << " vb=" << vb + << " scale=" << scale << " tol=" << tol); + + CHECK(va == Catch::Approx(vb).margin(tol)); + } + } + } +} // namespace + + +TEST_CASE("LatticeToRodrigues") { + double rod[3]; + double lengths[3]; + + CrystalLattice latt_i(40,50,80,90,90,90); + + LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); + CHECK(lengths[0] == Catch::Approx(40.0)); + CHECK(lengths[1] == Catch::Approx(50.0)); + CHECK(lengths[2] == Catch::Approx(80.0)); + CHECK(fabs(rod[0]) < 0.001); + CHECK(fabs(rod[1]) < 0.001); + CHECK(fabs(rod[2]) < 0.001); + + auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); + + CHECK(latt_o.Vec0().Length() == Catch::Approx(40.0)); + CHECK(latt_o.Vec1().Length() == Catch::Approx(50.0)); + CHECK(latt_o.Vec2().Length() == Catch::Approx(80.0)); +} + + +TEST_CASE("LatticeToRodrigues_irregular") { + double rod[3]; + double lengths[3]; + + CrystalLattice latt_i(Coord(40,0,0), + Coord(0, 50 / sqrt(2), -50 / sqrt(2)), + Coord(0, 80 / sqrt(2), 80 / sqrt(2))); + + LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); + CHECK(lengths[0] == Catch::Approx(40.0)); + CHECK(lengths[1] == Catch::Approx(50.0)); + CHECK(lengths[2] == Catch::Approx(80.0)); + + auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); + + CHECK(latt_o.Vec0().Length() == Catch::Approx(40.0)); + CHECK(latt_o.Vec1().Length() == Catch::Approx(50.0)); + CHECK(latt_o.Vec2().Length() == Catch::Approx(80.0)); +} + +TEST_CASE("LatticeToRodrigues_Hex") { + double rod[3]; + double lengths[3]; + + Coord a = Coord(40,0,0); + Coord b = Coord(40 / 2, 40 * sqrt(3)/ 2.0, 0); + Coord c = Coord(0, 0, 70); + + RotMatrix R(1.0, Coord(0,1,1)); + CrystalLattice latt_i(R*a,R*b,R*c); + + LatticeToRodriguesAndLengths_Hex(latt_i, rod, lengths); + CHECK(lengths[0] == Catch::Approx(40.0)); + CHECK(lengths[1] == Catch::Approx(40.0)); + CHECK(lengths[2] == Catch::Approx(70.0)); + + auto latt_o = AngleAxisAndCellToLattice(rod, lengths,M_PI / 2.0, M_PI / 2.0, 2.0 * M_PI / 3.0); + auto uc_o = latt_o.GetUnitCell(); + CHECK(uc_o.a == Catch::Approx(40.0)); + CHECK(uc_o.b == Catch::Approx(40.0)); + CHECK(uc_o.c == Catch::Approx(70.0)); + CHECK(uc_o.alpha == Catch::Approx(90.0)); + CHECK(uc_o.beta == Catch::Approx(90.0)); + CHECK(uc_o.gamma == Catch::Approx(120.0)); +} + + +TEST_CASE("LatticeReduction Lattice param roundtrip (GS) preserves Gram matrix") { + // Non-orthogonal, irregular basis, but still a valid lattice + CrystalLattice latt_i(Coord(40, 1, 2), + Coord( 3, 50, -4), + Coord(-5, 6, 80)); + + double rod[3]{}, lengths[3]{}; + LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); + auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); + + // This parametrization only keeps "lengths + rotation", i.e. it cannot reproduce shear. + // So we *do not* compare Gram matrices here. + // Instead, sanity-check: reconstructed vectors have the requested lengths. + CHECK(latt_o.Vec0().Length() == Catch::Approx(lengths[0]).margin(1e-9)); + CHECK(latt_o.Vec1().Length() == Catch::Approx(lengths[1]).margin(1e-9)); + CHECK(latt_o.Vec2().Length() == Catch::Approx(lengths[2]).margin(1e-9)); +} + +TEST_CASE("LatticeReduction Lattice param roundtrip (Hex) preserves unit cell") { + Coord a = Coord(40, 0, 0); + Coord b = Coord(40 / 2.0, 40 * std::sqrt(3) / 2.0, 0); + Coord c = Coord(0, 0, 70); + + // Apply an arbitrary rotation to ensure the rod extraction is meaningful + RotMatrix R(1.0, Coord(0, 1, 1)); + CrystalLattice latt_i(R * a, R * b, R * c); + + double rod[3]{}, ac[3]{}; + LatticeToRodriguesAndLengths_Hex(latt_i, rod, ac); + auto latt_o = AngleAxisAndCellToLattice(rod, ac, M_PI/2, M_PI/2, 2*M_PI/3); + + auto uc_o = latt_o.GetUnitCell(); + CHECK(uc_o.a == Catch::Approx(40.0).margin(1e-6)); + CHECK(uc_o.b == Catch::Approx(40.0).margin(1e-6)); + CHECK(uc_o.c == Catch::Approx(70.0).margin(1e-6)); + CHECK(uc_o.alpha == Catch::Approx(90.0).margin(1e-6)); + CHECK(uc_o.beta == Catch::Approx(90.0).margin(1e-6)); + CHECK(uc_o.gamma == Catch::Approx(120.0).margin(1e-6)); +} + +TEST_CASE("LatticeReduction Monoclinic param roundtrip") { + struct Case { double beta_deg; }; + const std::vector cases = { + {60.0}, {75.0}, {115.0}, {130.0} + }; + + for (const auto& cs : cases) { + INFO("beta_deg=" << cs.beta_deg); + + // Start from a clean monoclinic cell in its conventional setting (unique axis b). + CrystalLattice latt0(50, 60, 70, 90, cs.beta_deg, 90); + + // Now apply a TRUE global rotation: rotate each basis vector (left-multiply). + RotMatrix R(0.7, Coord(0.3, 0.9, 0.1)); + CrystalLattice latt_i(R * latt0.Vec0(), + R * latt0.Vec1(), + R * latt0.Vec2()); + + double rod[3]{}, lengths[3]{}, beta_rad = 0.0; + LatticeToRodriguesLengthsBeta_Mono(latt_i, rod, lengths, beta_rad); + + // Basic sanity + CHECK(lengths[0] == Catch::Approx(50.0).margin(1e-6)); + CHECK(lengths[1] == Catch::Approx(60.0).margin(1e-6)); + CHECK(lengths[2] == Catch::Approx(70.0).margin(1e-6)); + CHECK(beta_rad * 180.0 / M_PI == Catch::Approx(cs.beta_deg).margin(1e-6)); + + auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, beta_rad, M_PI/2); + + // Rotation-invariant check: Gram matrices match. + check_gram_close(latt_i, latt_o, /*abs_eps=*/5e-4, /*rel_eps=*/1e-10); + + // Also check the unit-cell angles we expect for monoclinic(unique b). + auto uc_o = latt_o.GetUnitCell(); + CHECK(uc_o.alpha == Catch::Approx(90.0).margin(1e-4)); + CHECK(uc_o.gamma == Catch::Approx(90.0).margin(1e-4)); + CHECK(uc_o.beta == Catch::Approx(cs.beta_deg).margin(1e-4)); + } +} + +TEST_CASE("LatticeReduction monoclinic") { + // This isolates only the beta definition, independent of other choices. + CrystalLattice latt_i(50, 60, 70, 90, 130, 90); + + const Eigen::Vector3d A = to_eigen(latt_i.Vec0()); + const Eigen::Vector3d C = to_eigen(latt_i.Vec2()); + const double beta_geom = angle_rad(A, C); + + double rod[3]{}, lengths[3]{}, beta_rad = 0.0; + LatticeToRodriguesLengthsBeta_Mono(latt_i, rod, lengths, beta_rad); + + CHECK(beta_rad == Catch::Approx(beta_geom).margin(1e-12)); +} diff --git a/tests/XtalOptimizerTest.cpp b/tests/XtalOptimizerTest.cpp index 8b90983b..5a809a87 100644 --- a/tests/XtalOptimizerTest.cpp +++ b/tests/XtalOptimizerTest.cpp @@ -489,73 +489,6 @@ TEST_CASE("XtalOptimizer_monoclinic") { CHECK(fabs(uc_o.gamma - 90) < 0.05); } -TEST_CASE("LatticeToRodrigues") { - double rod[3]; - double lengths[3]; - - CrystalLattice latt_i(40,50,80,90,90,90); - - LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); - CHECK(lengths[0] == Catch::Approx(40.0)); - CHECK(lengths[1] == Catch::Approx(50.0)); - CHECK(lengths[2] == Catch::Approx(80.0)); - CHECK(fabs(rod[0]) < 0.001); - CHECK(fabs(rod[1]) < 0.001); - CHECK(fabs(rod[2]) < 0.001); - - auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); - - CHECK(latt_o.Vec0().Length() == Catch::Approx(40.0)); - CHECK(latt_o.Vec1().Length() == Catch::Approx(50.0)); - CHECK(latt_o.Vec2().Length() == Catch::Approx(80.0)); -} - -TEST_CASE("LatticeToRodrigues_irregular") { - double rod[3]; - double lengths[3]; - - CrystalLattice latt_i(Coord(40,0,0), - Coord(0, 50 / sqrt(2), -50 / sqrt(2)), - Coord(0, 80 / sqrt(2), 80 / sqrt(2))); - - LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); - CHECK(lengths[0] == Catch::Approx(40.0)); - CHECK(lengths[1] == Catch::Approx(50.0)); - CHECK(lengths[2] == Catch::Approx(80.0)); - - auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); - - CHECK(latt_o.Vec0().Length() == Catch::Approx(40.0)); - CHECK(latt_o.Vec1().Length() == Catch::Approx(50.0)); - CHECK(latt_o.Vec2().Length() == Catch::Approx(80.0)); -} - -TEST_CASE("LatticeToRodrigues_Hex") { - double rod[3]; - double lengths[3]; - - Coord a = Coord(40,0,0); - Coord b = Coord(40 / 2, 40 * sqrt(3)/ 2.0, 0); - Coord c = Coord(0, 0, 70); - - RotMatrix R(1.0, Coord(0,1,1)); - CrystalLattice latt_i(R*a,R*b,R*c); - - LatticeToRodriguesAndLengths_Hex(latt_i, rod, lengths); - CHECK(lengths[0] == Catch::Approx(40.0)); - CHECK(lengths[1] == Catch::Approx(40.0)); - CHECK(lengths[2] == Catch::Approx(70.0)); - - auto latt_o = AngleAxisAndCellToLattice(rod, lengths,M_PI / 2.0, M_PI / 2.0, 2.0 * M_PI / 3.0); - auto uc_o = latt_o.GetUnitCell(); - CHECK(uc_o.a == Catch::Approx(40.0)); - CHECK(uc_o.b == Catch::Approx(40.0)); - CHECK(uc_o.c == Catch::Approx(70.0)); - CHECK(uc_o.alpha == Catch::Approx(90.0)); - CHECK(uc_o.beta == Catch::Approx(90.0)); - CHECK(uc_o.gamma == Catch::Approx(120.0)); -} - TEST_CASE("XtalOptimizer_rotation") { // Geometry DiffractionExperiment exp_i; @@ -736,149 +669,3 @@ TEST_CASE("XtalOptimizer_refine_rotation_axis") { // --- helpers for lattice sanity tests --- #include - -namespace { -Eigen::Vector3d to_eigen(const Coord& v) { - return {v[0], v[1], v[2]}; -} - -double angle_rad(const Eigen::Vector3d& a, const Eigen::Vector3d& b) { - const double na = a.norm(); - const double nb = b.norm(); - if (na == 0.0 || nb == 0.0) - return 0.0; - double c = a.dot(b) / (na * nb); - c = std::max(-1.0, std::min(1.0, c)); - return std::acos(c); -} - -// Compare two lattices up to a global rotation: compare Gram matrices G = L^T L (rotation-invariant). -Eigen::Matrix3d gram(const CrystalLattice& latt) { - const Eigen::Vector3d A = to_eigen(latt.Vec0()); - const Eigen::Vector3d B = to_eigen(latt.Vec1()); - const Eigen::Vector3d C = to_eigen(latt.Vec2()); - Eigen::Matrix3d G; - G(0,0) = A.dot(A); G(0,1) = A.dot(B); G(0,2) = A.dot(C); - G(1,0) = B.dot(A); G(1,1) = B.dot(B); G(1,2) = B.dot(C); - G(2,0) = C.dot(A); G(2,1) = C.dot(B); G(2,2) = C.dot(C); - return G; -} - - void check_gram_close(const CrystalLattice& a, - const CrystalLattice& b, - double abs_eps, - double rel_eps) { - const Eigen::Matrix3d Ga = gram(a); - const Eigen::Matrix3d Gb = gram(b); - - for (int r = 0; r < 3; ++r) { - for (int c = 0; c < 3; ++c) { - const double va = Ga(r, c); - const double vb = Gb(r, c); - - // Scale for relative error; avoid blowing up around zero. - const double scale = std::max({1.0, std::abs(va), std::abs(vb)}); - const double tol = std::max(abs_eps, rel_eps * scale); - - INFO("G(" << r << "," << c << ") va=" << va << " vb=" << vb - << " scale=" << scale << " tol=" << tol); - - CHECK(va == Catch::Approx(vb).margin(tol)); - } - } -} -} // namespace - -TEST_CASE("XtalOptimizer Lattice param roundtrip (GS) preserves Gram matrix") { - // Non-orthogonal, irregular basis, but still a valid lattice - CrystalLattice latt_i(Coord(40, 1, 2), - Coord( 3, 50, -4), - Coord(-5, 6, 80)); - - double rod[3]{}, lengths[3]{}; - LatticeToRodriguesAndLengths_GS(latt_i, rod, lengths); - auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, M_PI/2, M_PI/2); - - // This parametrization only keeps "lengths + rotation", i.e. it cannot reproduce shear. - // So we *do not* compare Gram matrices here. - // Instead, sanity-check: reconstructed vectors have the requested lengths. - CHECK(latt_o.Vec0().Length() == Catch::Approx(lengths[0]).margin(1e-9)); - CHECK(latt_o.Vec1().Length() == Catch::Approx(lengths[1]).margin(1e-9)); - CHECK(latt_o.Vec2().Length() == Catch::Approx(lengths[2]).margin(1e-9)); -} - -TEST_CASE("XtalOptimizer Lattice param roundtrip (Hex) preserves unit cell") { - Coord a = Coord(40, 0, 0); - Coord b = Coord(40 / 2.0, 40 * std::sqrt(3) / 2.0, 0); - Coord c = Coord(0, 0, 70); - - // Apply an arbitrary rotation to ensure the rod extraction is meaningful - RotMatrix R(1.0, Coord(0, 1, 1)); - CrystalLattice latt_i(R * a, R * b, R * c); - - double rod[3]{}, ac[3]{}; - LatticeToRodriguesAndLengths_Hex(latt_i, rod, ac); - auto latt_o = AngleAxisAndCellToLattice(rod, ac, M_PI/2, M_PI/2, 2*M_PI/3); - - auto uc_o = latt_o.GetUnitCell(); - CHECK(uc_o.a == Catch::Approx(40.0).margin(1e-6)); - CHECK(uc_o.b == Catch::Approx(40.0).margin(1e-6)); - CHECK(uc_o.c == Catch::Approx(70.0).margin(1e-6)); - CHECK(uc_o.alpha == Catch::Approx(90.0).margin(1e-6)); - CHECK(uc_o.beta == Catch::Approx(90.0).margin(1e-6)); - CHECK(uc_o.gamma == Catch::Approx(120.0).margin(1e-6)); -} - -TEST_CASE("XtalOptimizer Monoclinic param roundtrip") { - struct Case { double beta_deg; }; - const std::vector cases = { - {60.0}, {75.0}, {115.0}, {130.0} - }; - - for (const auto& cs : cases) { - INFO("beta_deg=" << cs.beta_deg); - - // Start from a clean monoclinic cell in its conventional setting (unique axis b). - CrystalLattice latt0(50, 60, 70, 90, cs.beta_deg, 90); - - // Now apply a TRUE global rotation: rotate each basis vector (left-multiply). - RotMatrix R(0.7, Coord(0.3, 0.9, 0.1)); - CrystalLattice latt_i(R * latt0.Vec0(), - R * latt0.Vec1(), - R * latt0.Vec2()); - - double rod[3]{}, lengths[3]{}, beta_rad = 0.0; - LatticeToRodriguesLengthsBeta_Mono(latt_i, rod, lengths, beta_rad); - - // Basic sanity - CHECK(lengths[0] == Catch::Approx(50.0).margin(1e-6)); - CHECK(lengths[1] == Catch::Approx(60.0).margin(1e-6)); - CHECK(lengths[2] == Catch::Approx(70.0).margin(1e-6)); - CHECK(beta_rad * 180.0 / M_PI == Catch::Approx(cs.beta_deg).margin(1e-6)); - - auto latt_o = AngleAxisAndCellToLattice(rod, lengths, M_PI/2, beta_rad, M_PI/2); - - // Rotation-invariant check: Gram matrices match. - check_gram_close(latt_i, latt_o, /*abs_eps=*/5e-4, /*rel_eps=*/1e-10); - - // Also check the unit-cell angles we expect for monoclinic(unique b). - auto uc_o = latt_o.GetUnitCell(); - CHECK(uc_o.alpha == Catch::Approx(90.0).margin(1e-4)); - CHECK(uc_o.gamma == Catch::Approx(90.0).margin(1e-4)); - CHECK(uc_o.beta == Catch::Approx(cs.beta_deg).margin(1e-4)); - } -} - -TEST_CASE("XtalOptimizer Monoclinic beta geometry: extracted beta equals angle(a,c)") { - // This isolates only the beta definition, independent of other choices. - CrystalLattice latt_i(50, 60, 70, 90, 130, 90); - - const Eigen::Vector3d A = to_eigen(latt_i.Vec0()); - const Eigen::Vector3d C = to_eigen(latt_i.Vec2()); - const double beta_geom = angle_rad(A, C); - - double rod[3]{}, lengths[3]{}, beta_rad = 0.0; - LatticeToRodriguesLengthsBeta_Mono(latt_i, rod, lengths, beta_rad); - - CHECK(beta_rad == Catch::Approx(beta_geom).margin(1e-12)); -} -- 2.52.0 From 9a991b6614d81cb5cf7e5c46753e4687d228f325 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 11:59:48 +0200 Subject: [PATCH 003/228] PixelRefine: Work in progress --- .../pixel_refinement/CMakeLists.txt | 2 +- .../pixel_refinement/PixelRefine.cpp | 317 ++++++++++++++---- image_analysis/pixel_refinement/PixelRefine.h | 14 +- 3 files changed, 257 insertions(+), 76 deletions(-) diff --git a/image_analysis/pixel_refinement/CMakeLists.txt b/image_analysis/pixel_refinement/CMakeLists.txt index 5d5e3c6c..5eb253dc 100644 --- a/image_analysis/pixel_refinement/CMakeLists.txt +++ b/image_analysis/pixel_refinement/CMakeLists.txt @@ -1,2 +1,2 @@ ADD_LIBRARY(JFJochPixelRefine PixelRefine.cpp PixelRefine.h) -TARGET_LINK_LIBRARIES(JFJochPixelRefine JFJochBraggPrediction JFJochCommon ceres_static) +TARGET_LINK_LIBRARIES(JFJochPixelRefine JFJochBraggPrediction JFJochCommon JFJochScaleMerge ceres_static) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 1e547b08..fe1d2089 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -7,78 +7,177 @@ #include - - -struct PixelRefineResidual { - ScalingPostRefResidual(int32_t h, int32_t k, int32_t l, - double x, double y, - double Itrue, - const DiffractionGeometry &geometry, - const CrystalLattice &lattice) - : - integration_center_x(r.predicted_x), - integration_center_y(r.predicted_y), - inv_lambda(SafeInv(geometry.GetWavelength_A(), 0.0)), - pixel_size(geometry.GetPixelSize_mm()), - det_dist_mm(geometry.GetDetectorDistance_mm()), - beam_x(geometry.GetBeamX_pxl()), - beam_y(geometry.GetBeamY_pxl()), - exp_h(r.h), - exp_k(r.k), - exp_l(r.l), - Astar(lattice.Astar()), - Bstar(lattice.Bstar()), - Cstar(lattice.Cstar()), - c1(std::cos(geometry.GetPoniRot1_rad())), - s1(std::sin(geometry.GetPoniRot1_rad())), - c2(std::cos(geometry.GetPoniRot2_rad())), - s2(std::sin(geometry.GetPoniRot2_rad())) { +struct PixelResidual { + // Assume that Itrue and Ibkg are already corrected with solid angle and polarization correction + PixelResidual(double x, double y, + double Itrue, double Iobs, + double Ibkg, double Ibkg_sigma, + double lambda, + double pixel_size, + double angle_rad, + double exp_h, double exp_k, + double exp_l, + gemmi::CrystalSystem symmetry) + : Itrue(Itrue), Iobs(Iobs), + Ibkg(Ibkg), Ibkg_sigma(Ibkg_sigma), obs_x(x), + obs_y(y), + inv_lambda(1.0 / lambda), + pixel_size(pixel_size), + exp_h(exp_h), + exp_k(exp_k), + exp_l(exp_l), + angle_rad(angle_rad), + symmetry(symmetry) { + if (std::fabs(lambda) < 1e-6) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Lambda cannot be close to zero"); } - template - T CalcPartiality(const T *const R, - const T *const beam_corr, - const T *const p0) const { + template + bool operator()(const T *const beam, + const T *const distance_mm, + const T *const detector_rot, + const T *const rotation_axis, + const T *const p0, + const T *const p1, + const T *const p2, + const T *const scale_factor, + const T *const B, + const T *const R, + T *residual) const { + // PyFAI convention (left-handed for rot1/rot2): + // poni_rot = Rz(-rot3) * Rx(-rot2) * Ry(+rot1) + // detector_rot[0] = rot1, detector_rot[1] = rot2 (rot3 = 0 assumed) + + const T rot1 = detector_rot[0]; + const T rot2 = detector_rot[1]; + + // Ry(+rot1): rotation around Y-axis + const T c1 = ceres::cos(rot1); + const T s1 = ceres::sin(rot1); + + // Rx(-rot2): rotation around X-axis with inverted sign (PyFAI left-handed) + const T c2 = ceres::cos(rot2); + const T s2 = ceres::sin(rot2); + // Detector coordinates in mm - const T det_x = (T(integration_center_x) - beam_x - beam_corr[0]) * T(pixel_size); - const T det_y = (T(integration_center_y) - beam_y - beam_corr[1]) * T(pixel_size); - const T det_z = T(det_dist_mm); + const T det_x = (T(obs_x) - beam[0]) * T(pixel_size); + const T det_y = (T(obs_y) - beam[1]) * T(pixel_size); + const T det_z = T(distance_mm[0]); // Apply Ry(rot1) first: rotate around Y - const T t1_x = T(c1) * det_x + T(s1) * det_z; + const T t1_x = c1 * det_x + s1 * det_z; const T t1_y = det_y; - const T t1_z = T(-s1) * det_x + T(c1) * det_z; + const T t1_z = -s1 * det_x + c1 * det_z; // Then apply Rx(-rot2): rotate around X const T x = t1_x; - const T y = T(c2) * t1_y + T(s2) * t1_z; - const T z = - T(s2) * t1_y + T(c2) * t1_z; + const T y = c2 * t1_y + s2 * t1_z; + const T z = -s2 * t1_y + c2 * t1_z; // convert to recip space const T lab_norm = ceres::sqrt(x * x + y * y + z * z); const T inv_norm = T(1) / lab_norm; + T recip_raw[3]; + recip_raw[0] = x * inv_norm * T(inv_lambda); + recip_raw[1] = y * inv_norm * T(inv_lambda); + recip_raw[2] = (z * inv_norm - T(1.0)) * T(inv_lambda); + + // Apply goniometer "back-to-start" rotation: + // brings observed reciprocal from image orientation into reference crystal frame + const T aa_back[3] = { + T(angle_rad) * rotation_axis[0], + T(angle_rad) * rotation_axis[1], + T(angle_rad) * rotation_axis[2] + }; + T recip_obs[3]; - recip_obs[0] = x * inv_norm * inv_lambda; - recip_obs[1] = y * inv_norm * inv_lambda; - recip_obs[2] = (z * inv_norm - T(1.0)) * inv_lambda; + ceres::AngleAxisRotatePoint(aa_back, recip_raw, recip_obs); const Eigen::Matrix e_obs_recip(recip_obs[0], recip_obs[1], recip_obs[2]); - const T astar_unrot[3] = {T(Astar.x), T(Astar.y), T(Astar.z)}; - const T bstar_unrot[3] = {T(Bstar.x), T(Bstar.y), T(Bstar.z)}; - const T cstar_unrot[3] = {T(Cstar.x), T(Cstar.y), T(Cstar.z)}; + // Build unit cell lengths and B (convention: columns are a, b, c prior to global rotation) + Eigen::Matrix e_uc_len = Eigen::Matrix::Zero(); + Eigen::Matrix B = Eigen::Matrix::Identity(); - T astar_rot[3], bstar_rot[3], cstar_rot[3]; + if (symmetry == gemmi::CrystalSystem::Hexagonal) { + e_uc_len << p1[0], p1[0], p1[2]; + B(0, 1) = T(-0.5); // cos(120) + B(1, 1) = T(sqrt(3.0) / 2.0); // sin(120) + } else if (symmetry == gemmi::CrystalSystem::Orthorhombic) { + e_uc_len << p1[0], p1[1], p1[2]; + } else if (symmetry == gemmi::CrystalSystem::Tetragonal) { + e_uc_len << p1[0], p1[0], p1[2]; + } else if (symmetry == gemmi::CrystalSystem::Cubic) { + e_uc_len << p1[0], p1[0], p1[0]; + } else if (symmetry == gemmi::CrystalSystem::Monoclinic) { + // Unique axis b: alpha = gamma = 90°, beta free (angle between a and c) + e_uc_len << p1[0], p1[1], p1[2]; + B(0, 2) = ceres::cos(p2[0]); + B(2, 2) = ceres::sin(p2[0]); + } else { + // Triclinic: p1 = (a,b,c), p2 = (alpha, beta, gamma) in radians + const T ca = ceres::cos(p2[0]); + const T cb = ceres::cos(p2[1]); + const T cg = ceres::cos(p2[2]); + const T sg = ceres::sin(p2[2]); - ceres::AngleAxisRotatePoint(p0, astar_unrot, astar_rot); - ceres::AngleAxisRotatePoint(p0, bstar_unrot, bstar_rot); - ceres::AngleAxisRotatePoint(p0, cstar_unrot, cstar_rot); + e_uc_len << p1[0], p1[1], p1[2]; - const Eigen::Matrix e_pred_recip(T(exp_h) * astar_rot[0] + T(exp_k) * bstar_rot[0] + T(exp_l) * cstar_rot[0], - T(exp_h) * astar_rot[1] + T(exp_k) * bstar_rot[1] + T(exp_l) * cstar_rot[1], - T(exp_h) * astar_rot[2] + T(exp_k) * bstar_rot[2] + T(exp_l) * cstar_rot[2] - ); + B(0, 0) = T(1); + B(1, 0) = T(0); + B(2, 0) = T(0); + B(0, 1) = cg; + B(1, 1) = sg; + B(2, 1) = T(0); + + // c vector components: + const T cx = cb; + const T cy = (ca - cb * cg) / sg; + const T v = T(1) - cx * cx - cy * cy; + const T cz = (v >= T(0)) ? ceres::sqrt(v) : T(0); + + B(0, 2) = cx; + B(1, 2) = cy; + B(2, 2) = cz; + } + + // Build unrotated direct lattice columns: (B * D), then rotate them by p0. + // This avoids AngleAxisToRotationMatrix + matrix multiplications. + const T L0 = e_uc_len[0]; + const T L1 = e_uc_len[1]; + const T L2 = e_uc_len[2]; + + T col0_unrot[3] = {B(0, 0) * L0, B(1, 0) * L0, B(2, 0) * L0}; + T col1_unrot[3] = {B(0, 1) * L1, B(1, 1) * L1, B(2, 1) * L1}; + T col2_unrot[3] = {B(0, 2) * L2, B(1, 2) * L2, B(2, 2) * L2}; + + T col0_rot[3], col1_rot[3], col2_rot[3]; + ceres::AngleAxisRotatePoint(p0, col0_unrot, col0_rot); + ceres::AngleAxisRotatePoint(p0, col1_unrot, col1_rot); + ceres::AngleAxisRotatePoint(p0, col2_unrot, col2_rot); + + const Eigen::Matrix A(col0_rot[0], col0_rot[1], col0_rot[2]); + const Eigen::Matrix Bv(col1_rot[0], col1_rot[1], col1_rot[2]); + const Eigen::Matrix C(col2_rot[0], col2_rot[1], col2_rot[2]); + + const Eigen::Matrix BxC = Bv.cross(C); + const Eigen::Matrix CxA = C.cross(A); + const Eigen::Matrix AxB = A.cross(Bv); + + const T V = A.dot(BxC); + const T invV = T(1) / V; + + const Eigen::Matrix Astar = BxC * invV; + const Eigen::Matrix Bstar = CxA * invV; + const Eigen::Matrix Cstar = AxB * invV; + + const T h = T(exp_h); + const T k = T(exp_k); + const T l = T(exp_l); + + const Eigen::Matrix e_pred_recip = Astar * h + Bstar * k + Cstar * l; // Ewald sphere centre is at -k_i = (0, 0, -inv_lambda) // Radial direction: outward normal at g_hkl @@ -99,35 +198,109 @@ struct PixelRefineResidual { const T eps_tangential_sq = dq_tang.squaredNorm(); // guaranteed ≥ 0 // ───────────────────────────────────────────────────────────── - return ceres::exp(- eps_radial * eps_radial / (R[0] * R[0]) - eps_tangential_sq / (R[1] * R[1])); - } + const T B_term = ceres::exp(- B[0] * e_pred_recip.squaredNorm() / 4.0); - template - bool operator()(const T *const G, - const T *const B, - const T *const R, - const T *const beam_corr, - const T *const p0, - T *residual) const { - if (R[0] < T(1e-10) || R[1] < T(1e-10)) - return false; + // Need to normalize by R[0] and R[1] + const T partiality = ceres::exp(- eps_radial * eps_radial / (R[0] * R[0]) - eps_tangential_sq / (R[1] * R[1])); - const T B_term = ceres::exp(B[0] * T(b_resolution_coeff)); + const T Ipred = partiality * Itrue * scale_factor[0] * B_term - Ibkg; + + // Need to weight by sigma + // I would like to use sigma based on Ipred and Ibkg_sigma - need to come up with a better approach + residual[0] = (Ipred - Iobs) / Ibkg_sigma; - const T partiality = CalcPartiality(R, beam_corr, p0); - residual[0] = (G[0] * partiality * B_term * T(lp) * T(Itrue) - - T(Iobs)) * T(weight); return true; } - const double integration_center_x, integration_center_y; + const double Itrue, Iobs, Ibkg, Ibkg_sigma; + const double obs_x, obs_y; const double inv_lambda; const double pixel_size; - const double det_dist_mm; - const double beam_x, beam_y; const double exp_h; const double exp_k; const double exp_l; - const Coord Astar, Bstar, Cstar; - const double c1,s1,c2,s2; -}; \ No newline at end of file + const double angle_rad; + gemmi::CrystalSystem symmetry; +}; + +PixelRefine::PixelRefine(const DiffractionExperiment &experiment, + const AzimuthalIntegrationMapping &mapping, + const std::vector &reference, + BraggPrediction &prediction) + : prediction(prediction), + mapping(mapping), + xpixel(experiment.GetXPixelsNum()), + ypixel(experiment.GetYPixelsNum()), + experiment(experiment), + hkl_key_generator(experiment.GetScalingSettings().GetMergeFriedel(), + experiment.GetSpaceGroupNumber().value_or(1)) { + for (const auto &ref: reference) + reference_data[hkl_key_generator(ref)] = ref.I; +} + +template +void PixelRefine::Run(const T *image, + const AzimuthalIntegrationProfile &profile, + PixelRefineData &data) { + + ceres::Problem problem; + + // We predict reflections based on initial geometry and default settings + // To be tuned later + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .max_hkl = 100, + .centering = data.centering + }; + prediction.Calc(experiment, data.latt, settings_prediction); + + auto azim_result = profile.GetResult(); + auto azim_std = profile.GetStd(); + + // For each reflection we select some area (3-5 pixels around it) + const int radius = 3; + + for (const auto &refl : prediction.GetReflections()) { + auto hkl = hkl_key_generator(refl); + // We only handle reflections that are present in the reference set + if (!reference_data.contains(hkl)) + continue; + const double I_true = reference_data[hkl]; + + int min_y = std::max(refl.predicted_y - radius, 0); + int max_y = std::min(refl.predicted_y + radius, ypixel - 1); + int min_x = std::max(refl.predicted_x - radius, 0); + int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + + for (int y = min_y; y <= max_y; ++y) { + for (int x = min_x; x <= max_x; ++x) { + const size_t npixel = xpixel * y + x; + int azim_bin = mapping.GetPixelToBin()[npixel]; + // If pixel is not mapped to azimuthal bin + // or pixel has special value (lowest/highest integer) + // it should be ignored for the purpose of this try + // We should check if pixel mask is needed, but for most workflows it is already applied + + if (azim_bin >= mapping.GetAzimuthalBinCount()) + continue; + if (image[npixel] == std::numeric_limits::max()) + continue; + if (std::is_signed_v() && (image[npixel] == std::numeric_limits::min())) + continue; + + // Get per-pixel polarization and solid angle correction for the pixel from the AzimuthalIntegrationMapping + // Warning! this is missing Lorentz correction, but we don't worry at the moment about it + // Important is -> this correction is also applied to background, so we must be consistent here + float correction = mapping.Corrections()[npixel]; + + // Get mean pixel value for background in the azimuthal bin + sigma + float bkg_value = azim_result[azim_bin]; + float bkg_sigma = azim_std[azim_bin]; + float pixel_value = image[npixel]; + + + } + } + } + +} \ No newline at end of file diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 1610ea9c..1ee0a9f1 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -6,6 +6,8 @@ #include "../bragg_prediction/BraggPrediction.h" #include "../common/DiffractionExperiment.h" #include "../common/AzimuthalIntegrationMapping.h" +#include "../common/AzimuthalIntegrationProfile.h" +#include "../scale_merge/HKLKey.h" struct PixelRefineData { DiffractionGeometry geom; @@ -25,12 +27,18 @@ class PixelRefine { BraggPrediction &prediction; const AzimuthalIntegrationMapping &mapping; const size_t xpixel, ypixel; + const DiffractionExperiment &experiment; + const HKLKeyGenerator hkl_key_generator; + std::map reference_data; public: PixelRefine(const DiffractionExperiment &experiment, - const AzimuthalIntegrationMapping &mapping; + const AzimuthalIntegrationMapping &mapping, + const std::vector &reference, BraggPrediction &prediction); - template - void Run(const T *image, PixelRefineData &data); + template + void Run(const T *image, + const AzimuthalIntegrationProfile &profile, + PixelRefineData &data); }; -- 2.52.0 From 66a48c426637ac4704ac95d3195a1e946e709aee Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 12:43:10 +0200 Subject: [PATCH 004/228] PixelRefine: Work in progress (Claude) --- .../pixel_refinement/CMakeLists.txt | 2 +- .../pixel_refinement/PixelRefine.cpp | 589 ++++++++++++++---- image_analysis/pixel_refinement/PixelRefine.h | 32 +- 3 files changed, 488 insertions(+), 135 deletions(-) diff --git a/image_analysis/pixel_refinement/CMakeLists.txt b/image_analysis/pixel_refinement/CMakeLists.txt index 5eb253dc..5e94b562 100644 --- a/image_analysis/pixel_refinement/CMakeLists.txt +++ b/image_analysis/pixel_refinement/CMakeLists.txt @@ -1,2 +1,2 @@ ADD_LIBRARY(JFJochPixelRefine PixelRefine.cpp PixelRefine.h) -TARGET_LINK_LIBRARIES(JFJochPixelRefine JFJochBraggPrediction JFJochCommon JFJochScaleMerge ceres_static) +TARGET_LINK_LIBRARIES(JFJochPixelRefine JFJochBraggPrediction JFJochCommon JFJochScaleMerge JFJochGeomRefinement ceres_static) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index fe1d2089..e734516c 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -3,79 +3,114 @@ #include "PixelRefine.h" +#include #include #include +#include "../geom_refinement/LatticeReduction.h" +namespace { + +// Per-pixel observation, in *corrected* intensity units (solid-angle and +// polarization correction already folded in, consistently for signal and +// background). Geometry-independent quantities are precomputed here so that the +// Ceres cost functor stays cheap. +struct PixelObs { + double x, y; // detector pixel coordinate + double Iobs; // corrected pixel value (signal + background) + double Ibkg; // corrected background estimate (azimuthal bin mean) + double weight; // 1 / sigma_pixel + double A_recip; // reciprocal-space area subtended by the pixel (Jacobian) + double angle_rad; // goniometer angle of this observation +}; + +// One reflection together with the pixels of its shoebox. +struct ReflGroup { + int h, k, l; + double d; + double Itrue; // reference intensity (held fixed) + double predicted_x, predicted_y; + std::vector pixels; +}; + +double SafeInv(double x, double fallback) { + if (!std::isfinite(x) || std::fabs(x) < 1e-30) + return fallback; + return 1.0 / x; +} + +} // namespace + +// --------------------------------------------------------------------------- +// Cost functor +// +// I_pred(pixel) = G * Itrue * B_term * P_radial * P_tangential + I_bkg +// +// B_term = exp(-B |q|^2 / 4) (Debye-Waller) +// P_radial = exp(-eps_r^2 / R0^2) (partiality: fraction of +// the mosaic blob on the +// Ewald sphere; NOT +// normalized, <= 1) +// P_tangential = A_recip/(pi R1^2) * exp(-eps_t^2/R1^2)(spatial profile on the +// detector, normalized so +// that sum over pixels ~ 1) +// +// The tangential factor is what makes this "profile fitting": summing +// I_pred - I_bkg over the shoebox reproduces G * Itrue * B_term * P_radial. +// The 1/(pi R1^2) normalization is the missing piece that decouples the profile +// width R1 from the overall scale G. +// --------------------------------------------------------------------------- struct PixelResidual { - // Assume that Itrue and Ibkg are already corrected with solid angle and polarization correction - PixelResidual(double x, double y, - double Itrue, double Iobs, - double Ibkg, double Ibkg_sigma, - double lambda, - double pixel_size, - double angle_rad, - double exp_h, double exp_k, - double exp_l, + PixelResidual(const PixelObs &obs, double Itrue, + double lambda, double pixel_size, + double exp_h, double exp_k, double exp_l, gemmi::CrystalSystem symmetry) - : Itrue(Itrue), Iobs(Iobs), - Ibkg(Ibkg), Ibkg_sigma(Ibkg_sigma), obs_x(x), - obs_y(y), - inv_lambda(1.0 / lambda), - pixel_size(pixel_size), - exp_h(exp_h), - exp_k(exp_k), - exp_l(exp_l), - angle_rad(angle_rad), - symmetry(symmetry) { + : Itrue(Itrue), Iobs(obs.Iobs), Ibkg(obs.Ibkg), weight(obs.weight), + A_recip(obs.A_recip), obs_x(obs.x), obs_y(obs.y), + inv_lambda(1.0 / lambda), pixel_size(pixel_size), + exp_h(exp_h), exp_k(exp_k), exp_l(exp_l), + angle_rad(obs.angle_rad), symmetry(symmetry) { if (std::fabs(lambda) < 1e-6) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Lambda cannot be close to zero"); } + // Maps a detector pixel through the current geometry + lattice into the + // reference reciprocal frame and returns: + // q_sq = |g_hkl|^2 (predicted node, for B-factor) + // eps_radial = deviation along Ewald normal (partiality direction) + // eps_tang_sq = squared deviation in the detector-tangential plane (profile) template - bool operator()(const T *const beam, - const T *const distance_mm, - const T *const detector_rot, - const T *const rotation_axis, - const T *const p0, - const T *const p1, - const T *const p2, - const T *const scale_factor, - const T *const B, - const T *const R, - T *residual) const { + bool GeometryTerms(const T *const beam, + const T *const distance_mm, + const T *const detector_rot, + const T *const rotation_axis, + const T *const p0, + const T *const p1, + const T *const p2, + T &q_sq, T &eps_radial, T &eps_tang_sq) const { // PyFAI convention (left-handed for rot1/rot2): - // poni_rot = Rz(-rot3) * Rx(-rot2) * Ry(+rot1) - // detector_rot[0] = rot1, detector_rot[1] = rot2 (rot3 = 0 assumed) - + // poni_rot = Rz(-rot3) * Rx(-rot2) * Ry(+rot1), rot3 = 0 assumed. const T rot1 = detector_rot[0]; const T rot2 = detector_rot[1]; - // Ry(+rot1): rotation around Y-axis const T c1 = ceres::cos(rot1); const T s1 = ceres::sin(rot1); - - // Rx(-rot2): rotation around X-axis with inverted sign (PyFAI left-handed) const T c2 = ceres::cos(rot2); const T s2 = ceres::sin(rot2); - // Detector coordinates in mm const T det_x = (T(obs_x) - beam[0]) * T(pixel_size); const T det_y = (T(obs_y) - beam[1]) * T(pixel_size); const T det_z = T(distance_mm[0]); - // Apply Ry(rot1) first: rotate around Y const T t1_x = c1 * det_x + s1 * det_z; const T t1_y = det_y; const T t1_z = -s1 * det_x + c1 * det_z; - // Then apply Rx(-rot2): rotate around X const T x = t1_x; const T y = c2 * t1_y + s2 * t1_z; const T z = -s2 * t1_y + c2 * t1_z; - // convert to recip space const T lab_norm = ceres::sqrt(x * x + y * y + z * z); const T inv_norm = T(1) / lab_norm; @@ -84,27 +119,25 @@ struct PixelResidual { recip_raw[1] = y * inv_norm * T(inv_lambda); recip_raw[2] = (z * inv_norm - T(1.0)) * T(inv_lambda); - // Apply goniometer "back-to-start" rotation: - // brings observed reciprocal from image orientation into reference crystal frame + // Goniometer "back-to-start" rotation: image frame -> reference frame. const T aa_back[3] = { T(angle_rad) * rotation_axis[0], T(angle_rad) * rotation_axis[1], T(angle_rad) * rotation_axis[2] }; - T recip_obs[3]; ceres::AngleAxisRotatePoint(aa_back, recip_raw, recip_obs); - const Eigen::Matrix e_obs_recip(recip_obs[0], recip_obs[1], recip_obs[2]); - // Build unit cell lengths and B (convention: columns are a, b, c prior to global rotation) + // Build cell lengths and the (unit) B matrix from the symmetry-specific + // parametrization (identical convention to XtalOptimizer::XtalResidual). Eigen::Matrix e_uc_len = Eigen::Matrix::Zero(); - Eigen::Matrix B = Eigen::Matrix::Identity(); + Eigen::Matrix Bmat = Eigen::Matrix::Identity(); if (symmetry == gemmi::CrystalSystem::Hexagonal) { e_uc_len << p1[0], p1[0], p1[2]; - B(0, 1) = T(-0.5); // cos(120) - B(1, 1) = T(sqrt(3.0) / 2.0); // sin(120) + Bmat(0, 1) = T(-0.5); + Bmat(1, 1) = T(sqrt(3.0) / 2.0); } else if (symmetry == gemmi::CrystalSystem::Orthorhombic) { e_uc_len << p1[0], p1[1], p1[2]; } else if (symmetry == gemmi::CrystalSystem::Tetragonal) { @@ -112,12 +145,10 @@ struct PixelResidual { } else if (symmetry == gemmi::CrystalSystem::Cubic) { e_uc_len << p1[0], p1[0], p1[0]; } else if (symmetry == gemmi::CrystalSystem::Monoclinic) { - // Unique axis b: alpha = gamma = 90°, beta free (angle between a and c) e_uc_len << p1[0], p1[1], p1[2]; - B(0, 2) = ceres::cos(p2[0]); - B(2, 2) = ceres::sin(p2[0]); + Bmat(0, 2) = ceres::cos(p2[0]); + Bmat(2, 2) = ceres::sin(p2[0]); } else { - // Triclinic: p1 = (a,b,c), p2 = (alpha, beta, gamma) in radians const T ca = ceres::cos(p2[0]); const T cb = ceres::cos(p2[1]); const T cg = ceres::cos(p2[2]); @@ -125,33 +156,26 @@ struct PixelResidual { e_uc_len << p1[0], p1[1], p1[2]; - B(0, 0) = T(1); - B(1, 0) = T(0); - B(2, 0) = T(0); - B(0, 1) = cg; - B(1, 1) = sg; - B(2, 1) = T(0); + Bmat(0, 1) = cg; + Bmat(1, 1) = sg; - // c vector components: const T cx = cb; const T cy = (ca - cb * cg) / sg; const T v = T(1) - cx * cx - cy * cy; const T cz = (v >= T(0)) ? ceres::sqrt(v) : T(0); - B(0, 2) = cx; - B(1, 2) = cy; - B(2, 2) = cz; + Bmat(0, 2) = cx; + Bmat(1, 2) = cy; + Bmat(2, 2) = cz; } - // Build unrotated direct lattice columns: (B * D), then rotate them by p0. - // This avoids AngleAxisToRotationMatrix + matrix multiplications. const T L0 = e_uc_len[0]; const T L1 = e_uc_len[1]; const T L2 = e_uc_len[2]; - T col0_unrot[3] = {B(0, 0) * L0, B(1, 0) * L0, B(2, 0) * L0}; - T col1_unrot[3] = {B(0, 1) * L1, B(1, 1) * L1, B(2, 1) * L1}; - T col2_unrot[3] = {B(0, 2) * L2, B(1, 2) * L2, B(2, 2) * L2}; + T col0_unrot[3] = {Bmat(0, 0) * L0, Bmat(1, 0) * L0, Bmat(2, 0) * L0}; + T col1_unrot[3] = {Bmat(0, 1) * L1, Bmat(1, 1) * L1, Bmat(2, 1) * L1}; + T col2_unrot[3] = {Bmat(0, 2) * L2, Bmat(1, 2) * L2, Bmat(2, 2) * L2}; T col0_rot[3], col1_rot[3], col2_rot[3]; ceres::AngleAxisRotatePoint(p0, col0_unrot, col0_rot); @@ -167,58 +191,92 @@ struct PixelResidual { const Eigen::Matrix AxB = A.cross(Bv); const T V = A.dot(BxC); + if (ceres::abs(V) < T(1e-12)) + return false; const T invV = T(1) / V; const Eigen::Matrix Astar = BxC * invV; const Eigen::Matrix Bstar = CxA * invV; const Eigen::Matrix Cstar = AxB * invV; - const T h = T(exp_h); - const T k = T(exp_k); - const T l = T(exp_l); + const Eigen::Matrix e_pred_recip = + Astar * T(exp_h) + Bstar * T(exp_k) + Cstar * T(exp_l); - const Eigen::Matrix e_pred_recip = Astar * h + Bstar * k + Cstar * l; + q_sq = e_pred_recip.squaredNorm(); - // Ewald sphere centre is at -k_i = (0, 0, -inv_lambda) - // Radial direction: outward normal at g_hkl + // Ewald sphere centre at -k_i = (0,0,-inv_lambda); radial normal at g_hkl. const Eigen::Matrix S_pred( e_pred_recip[0], e_pred_recip[1], - e_pred_recip[2] + T(inv_lambda) // g_hkl + k_i - ); + e_pred_recip[2] + T(inv_lambda)); const T S_pred_norm = S_pred.norm(); if (S_pred_norm < T(1e-10)) - return T(0); + return false; const Eigen::Matrix n_radial = S_pred / S_pred_norm; - const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - const T eps_radial = delta_q.dot(n_radial); + + eps_radial = delta_q.dot(n_radial); const Eigen::Matrix dq_tang = delta_q - eps_radial * n_radial; - const T eps_tangential_sq = dq_tang.squaredNorm(); // guaranteed ≥ 0 - // ───────────────────────────────────────────────────────────── - - const T B_term = ceres::exp(- B[0] * e_pred_recip.squaredNorm() / 4.0); - - // Need to normalize by R[0] and R[1] - const T partiality = ceres::exp(- eps_radial * eps_radial / (R[0] * R[0]) - eps_tangential_sq / (R[1] * R[1])); - - const T Ipred = partiality * Itrue * scale_factor[0] * B_term - Ibkg; - - // Need to weight by sigma - // I would like to use sigma based on Ipred and Ibkg_sigma - need to come up with a better approach - residual[0] = (Ipred - Iobs) / Ibkg_sigma; - + eps_tang_sq = dq_tang.squaredNorm(); return true; } - const double Itrue, Iobs, Ibkg, Ibkg_sigma; + // Assembles the full model intensity for the pixel from the geometry terms. + template + bool Model(const T *const beam, const T *const distance_mm, + const T *const detector_rot, const T *const rotation_axis, + const T *const p0, const T *const p1, const T *const p2, + const T *const scale_factor, const T *const B, const T *const R, + T &Ipred) const { + T q_sq, eps_radial, eps_tang_sq; + if (!GeometryTerms(beam, distance_mm, detector_rot, rotation_axis, + p0, p1, p2, q_sq, eps_radial, eps_tang_sq)) + return false; + + if (R[0] < T(1e-10) || R[1] < T(1e-10)) + return false; + + const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); + const T P_radial = ceres::exp(-eps_radial * eps_radial / (R[0] * R[0])); + + // Normalized 2D Gaussian profile in the detector-tangential plane, + // weighted by the reciprocal-space area of the pixel so that the sum of + // the profile over the shoebox is ~1 (intensity-conserving). + const T norm_tang = T(A_recip) / (T(M_PI) * R[1] * R[1]); + const T P_tang = norm_tang * ceres::exp(-eps_tang_sq / (R[1] * R[1])); + + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; + Ipred = signal + T(Ibkg); + return true; + } + + template + bool operator()(const T *const beam, + const T *const distance_mm, + const T *const detector_rot, + const T *const rotation_axis, + const T *const p0, + const T *const p1, + const T *const p2, + const T *const scale_factor, + const T *const B, + const T *const R, + T *residual) const { + T Ipred; + if (!Model(beam, distance_mm, detector_rot, rotation_axis, + p0, p1, p2, scale_factor, B, R, Ipred)) + return false; + + residual[0] = (Ipred - T(Iobs)) * T(weight); + return true; + } + + const double Itrue, Iobs, Ibkg, weight, A_recip; const double obs_x, obs_y; const double inv_lambda; const double pixel_size; - const double exp_h; - const double exp_k; - const double exp_l; + const double exp_h, exp_k, exp_l; const double angle_rad; gemmi::CrystalSystem symmetry; }; @@ -240,13 +298,23 @@ PixelRefine::PixelRefine(const DiffractionExperiment &experiment, template void PixelRefine::Run(const T *image, - const AzimuthalIntegrationProfile &profile, - PixelRefineData &data) { + const AzimuthalIntegrationProfile &profile, + PixelRefineData &data) { + data.solved = false; + data.reflections.clear(); - ceres::Problem problem; + const auto geom = data.geom; + const double lambda = geom.GetWavelength_A(); + const double pixel_size = geom.GetPixelSize_mm(); + const double distance_mm = geom.GetDetectorDistance_mm(); - // We predict reflections based on initial geometry and default settings - // To be tuned later + // Reciprocal-space area subtended by one pixel (small-angle approximation): + // |dq/dpixel| ~ pixel_size / (lambda * distance), so the area is its square. + // Used to normalize the tangential profile. Good enough for a prototype; a + // proper treatment would use the per-reflection 2-theta dependent Jacobian. + const double A_recip = std::pow(pixel_size / (lambda * distance_mm), 2.0); + + // Predict reflection positions with the current lattice/geometry. const BraggPredictionSettings settings_prediction{ .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), .max_hkl = 100, @@ -254,53 +322,316 @@ void PixelRefine::Run(const T *image, }; prediction.Calc(experiment, data.latt, settings_prediction); - auto azim_result = profile.GetResult(); - auto azim_std = profile.GetStd(); + const auto azim_result = profile.GetResult(); + const auto azim_std = profile.GetStd(); + const auto &pixel_to_bin = mapping.GetPixelToBin(); + const auto &corrections = mapping.Corrections(); + const int azim_bin_count = mapping.GetAzimuthalBinCount(); - // For each reflection we select some area (3-5 pixels around it) - const int radius = 3; + const double angle_rad = data.angle_deg * M_PI / 180.0; + const int radius = data.shoebox_radius; + // ---- Collect per-reflection shoebox pixels -------------------------------- + std::vector groups; for (const auto &refl : prediction.GetReflections()) { - auto hkl = hkl_key_generator(refl); - // We only handle reflections that are present in the reference set + const auto hkl = hkl_key_generator(refl); if (!reference_data.contains(hkl)) continue; - const double I_true = reference_data[hkl]; - int min_y = std::max(refl.predicted_y - radius, 0); - int max_y = std::min(refl.predicted_y + radius, ypixel - 1); - int min_x = std::max(refl.predicted_x - radius, 0); - int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + ReflGroup g; + g.h = refl.h; + g.k = refl.k; + g.l = refl.l; + g.d = refl.d; + g.Itrue = reference_data[hkl]; + g.predicted_x = refl.predicted_x; + g.predicted_y = refl.predicted_y; + + const int min_y = std::max(refl.predicted_y - radius, 0); + const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); + const int min_x = std::max(refl.predicted_x - radius, 0); + const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); for (int y = min_y; y <= max_y; ++y) { for (int x = min_x; x <= max_x; ++x) { const size_t npixel = xpixel * y + x; - int azim_bin = mapping.GetPixelToBin()[npixel]; - // If pixel is not mapped to azimuthal bin - // or pixel has special value (lowest/highest integer) - // it should be ignored for the purpose of this try - // We should check if pixel mask is needed, but for most workflows it is already applied + const int azim_bin = pixel_to_bin[npixel]; - if (azim_bin >= mapping.GetAzimuthalBinCount()) + // Skip pixels not mapped to an azimuthal bin or carrying a + // sentinel (masked / saturated) value. We assume the pixel mask + // is already applied upstream. + if (azim_bin >= azim_bin_count) continue; if (image[npixel] == std::numeric_limits::max()) continue; - if (std::is_signed_v() && (image[npixel] == std::numeric_limits::min())) + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) continue; - // Get per-pixel polarization and solid angle correction for the pixel from the AzimuthalIntegrationMapping - // Warning! this is missing Lorentz correction, but we don't worry at the moment about it - // Important is -> this correction is also applied to background, so we must be consistent here - float correction = mapping.Corrections()[npixel]; - - // Get mean pixel value for background in the azimuthal bin + sigma - float bkg_value = azim_result[azim_bin]; - float bkg_sigma = azim_std[azim_bin]; - float pixel_value = image[npixel]; + const double correction = corrections[npixel]; + const double Ibkg = azim_result[azim_bin] * 1.0; // already corrected units + const double Ibkg_sigma = azim_std[azim_bin]; + const double raw = static_cast(image[npixel]); + const double Iobs = raw * correction; + // Per-pixel variance: Poisson noise of the corrected counts + // (var(c*N) = c^2 * N = c * Iobs) plus the background spread. + double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; + if (!(var > 1.0)) + var = 1.0; + PixelObs obs{ + .x = static_cast(x), + .y = static_cast(y), + .Iobs = Iobs, + .Ibkg = Ibkg, + .weight = 1.0 / std::sqrt(var), + .A_recip = A_recip, + .angle_rad = angle_rad + }; + g.pixels.push_back(obs); } } + + if (!g.pixels.empty()) + groups.push_back(std::move(g)); + } + + if (groups.empty()) + return; + + // ---- Set up parameter blocks (mirrors XtalOptimizer for the geometry part) - + double beam[2] = {geom.GetBeamX_pxl(), geom.GetBeamY_pxl()}; + double dist_mm = distance_mm; + double detector_rot[2] = {geom.GetPoniRot1_rad(), geom.GetPoniRot2_rad()}; + double rot_vec[3] = {1.0, 0.0, 0.0}; + if (auto axis = geom.GetRotation()) { + rot_vec[0] = axis->GetAxis().x; + rot_vec[1] = axis->GetAxis().y; + rot_vec[2] = axis->GetAxis().z; + } + + double latt_vec0[3] = {0, 0, 0}; // orientation (Rodrigues) + double latt_vec1[3] = {0, 0, 0}; // lengths + double latt_vec2[3] = {0, 0, 0}; // angles (rad) + double beta = data.latt.GetUnitCell().beta; + + switch (data.crystal_system) { + case gemmi::CrystalSystem::Orthorhombic: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + break; + case gemmi::CrystalSystem::Tetragonal: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; + break; + case gemmi::CrystalSystem::Cubic: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; + break; + case gemmi::CrystalSystem::Hexagonal: + LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); + break; + case gemmi::CrystalSystem::Monoclinic: + LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); + latt_vec2[0] = beta; + break; + default: { + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + const auto uc = data.latt.GetUnitCell(); + latt_vec2[0] = uc.alpha * M_PI / 180.0; + latt_vec2[1] = uc.beta * M_PI / 180.0; + latt_vec2[2] = uc.gamma * M_PI / 180.0; + break; + } } -} \ No newline at end of file + ceres::Problem problem; + + for (const auto &g : groups) { + for (const auto &obs : g.pixels) { + auto *cost = new ceres::AutoDiffCostFunction< + PixelResidual, 1, 2, 1, 2, 3, 3, 3, 3, 1, 1, 2>( + new PixelResidual(obs, g.Itrue, lambda, pixel_size, + g.h, g.k, g.l, data.crystal_system)); + problem.AddResidualBlock(cost, new ceres::HuberLoss(3.0), + beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R); + } + } + + data.residual_count = problem.NumResidualBlocks(); + + // ---- Constrain / bound parameter blocks ----------------------------------- + if (!data.refine_orientation) + problem.SetParameterBlockConstant(latt_vec0); + + if (!data.refine_unit_cell) { + problem.SetParameterBlockConstant(latt_vec1); + problem.SetParameterBlockConstant(latt_vec2); + } else { + for (int i = 0; i < 3; ++i) { + problem.SetParameterLowerBound(latt_vec1, i, 5.0); + problem.SetParameterUpperBound(latt_vec1, i, 1000.0); + } + if (data.crystal_system != gemmi::CrystalSystem::Monoclinic && + data.crystal_system != gemmi::CrystalSystem::Triclinic) + problem.SetParameterBlockConstant(latt_vec2); + } + + if (!data.refine_beam_center) + problem.SetParameterBlockConstant(beam); + + if (!data.refine_distance) { + problem.SetParameterBlockConstant(&dist_mm); + } else { + problem.SetParameterLowerBound(&dist_mm, 0, dist_mm * 0.9); + problem.SetParameterUpperBound(&dist_mm, 0, dist_mm * 1.1); + } + + if (!data.refine_detector_angles) { + problem.SetParameterBlockConstant(detector_rot); + } else { + const double rng = 3.0 / 180.0 * M_PI; + for (int i = 0; i < 2; ++i) { + problem.SetParameterLowerBound(detector_rot, i, detector_rot[i] - rng); + problem.SetParameterUpperBound(detector_rot, i, detector_rot[i] + rng); + } + } + + if (!data.refine_rotation_axis) + problem.SetParameterBlockConstant(rot_vec); + + if (data.refine_scale) + problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); + else + problem.SetParameterBlockConstant(&data.scale_factor); + + if (!data.refine_B) + problem.SetParameterBlockConstant(&data.B_factor); + + if (data.refine_R) { + problem.SetParameterLowerBound(data.R, 0, 1e-5); + problem.SetParameterLowerBound(data.R, 1, 1e-5); + } else { + problem.SetParameterBlockConstant(data.R); + } + + // ---- Solve ---------------------------------------------------------------- + ceres::Solver::Options options; + options.linear_solver_type = ceres::DENSE_QR; + options.minimizer_progress_to_stdout = false; + options.logging_type = ceres::LoggingType::SILENT; + options.max_solver_time_in_seconds = data.max_time_s; + options.num_threads = 1; + + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + + data.final_cost = summary.final_cost; + data.solved = summary.IsSolutionUsable(); + + // ---- Write back refined geometry + lattice -------------------------------- + if (data.refine_beam_center) + data.geom.BeamX_pxl(beam[0]).BeamY_pxl(beam[1]); + if (data.refine_distance) + data.geom.DetectorDistance_mm(dist_mm); + if (data.refine_detector_angles) + data.geom.PoniRot1_rad(detector_rot[0]).PoniRot2_rad(detector_rot[1]); + + if (data.refine_orientation || data.refine_unit_cell) { + switch (data.crystal_system) { + case gemmi::CrystalSystem::Orthorhombic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Tetragonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Cubic: + latt_vec1[1] = latt_vec1[0]; + latt_vec1[2] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Hexagonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); + break; + case gemmi::CrystalSystem::Monoclinic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); + break; + default: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, + latt_vec2[0], latt_vec2[1], latt_vec2[2]); + break; + } + } + + // ---- Extract integrated reflections --------------------------------------- + // Profile-fitted intensity with the optimized profile P (shape only): + // I = sum_p [ P_p (Iobs_p - Ibkg_p) / v_p ] / sum_p [ P_p^2 / v_p ] + // var(I) = 1 / sum_p [ P_p^2 / v_p ] + // where P_p is the *shape* of the model per pixel (signal at G=Itrue=B=1). + data.reflections.reserve(groups.size()); + for (const auto &g : groups) { + double num = 0.0, den = 0.0, partiality_sum = 0.0, bkg_sum = 0.0; + size_t n = 0; + + for (const auto &obs : g.pixels) { + PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system); + double q_sq, eps_r, eps_t_sq; + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + continue; + if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) + continue; + + const double P_radial = std::exp(-eps_r * eps_r / (data.R[0] * data.R[0])); + const double norm_tang = obs.A_recip / (M_PI * data.R[1] * data.R[1]); + const double P_tang = norm_tang * std::exp(-eps_t_sq / (data.R[1] * data.R[1])); + const double P = P_radial * P_tang; // model shape per pixel (unit scale) + const double v = SafeInv(obs.weight * obs.weight, 1.0); // pixel variance + const double signal = obs.Iobs - obs.Ibkg; + + num += P * signal / v; + den += P * P / v; + partiality_sum += P_tang; // tangential profile sums to ~partiality + bkg_sum += obs.Ibkg; + ++n; + } + + Reflection r{}; + r.h = g.h; + r.k = g.k; + r.l = g.l; + r.d = static_cast(g.d); + r.predicted_x = static_cast(g.predicted_x); + r.predicted_y = static_cast(g.predicted_y); + r.observed_x = NAN; + r.observed_y = NAN; + r.rlp = 1.0f; + r.partiality = static_cast(partiality_sum); + + if (den > 0.0 && n > 0) { + const double I = num / den; + r.I = static_cast(I); + r.sigma = static_cast(std::sqrt(1.0 / den)); + r.bkg = static_cast(bkg_sum / static_cast(n)); + r.observed = true; + r.image_scale_corr = (r.partiality > 0.0f) ? r.rlp / r.partiality : NAN; + } else { + r.I = 0.0f; + r.sigma = NAN; + r.bkg = 0.0f; + r.observed = false; + } + data.reflections.push_back(r); + } +} + +// Explicit instantiations for the supported (uncompressed) image pixel types. +template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const int32_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const uint8_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const uint16_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const uint32_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 1ee0a9f1..5848cf15 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -10,17 +10,39 @@ #include "../scale_merge/HKLKey.h" struct PixelRefineData { + // --- model state (input as initial guess, output as refined result) --- DiffractionGeometry geom; CrystalLattice latt; gemmi::CrystalSystem crystal_system = gemmi::CrystalSystem::Triclinic; char centering = 'P'; - double B_factor = 0.0; - double scale_factor = 1.0; - double R[2] = {0.001, 0.001}; + double B_factor = 0.0; // Debye-Waller B (A^2) + double scale_factor = 1.0; // overall scale G + double R[2] = {0.005, 0.005}; // R[0] = radial (partiality) width, R[1] = tangential (profile) width (A^-1) - bool refine_beam_center = false; - bool refine_unit_cell = false; + // Goniometer: for a still image keep angle_deg = 0. For a wedge, pass the + // angle (deg) of the slice centre relative to the reference (phi=0) frame. + double angle_deg = 0.0; + + // --- what to refine --- + bool refine_orientation = true; // crystal orientation (p0) + bool refine_unit_cell = false; // cell lengths + angles + bool refine_beam_center = false; + bool refine_distance = false; + bool refine_detector_angles = false; + bool refine_rotation_axis = false; + bool refine_scale = true; + bool refine_B = false; + bool refine_R = true; + + double max_time_s = 5.0; + int shoebox_radius = 3; // half-size of the per-reflection pixel box + + // --- output --- + std::vector reflections; // profile-fitted integration result + bool solved = false; + double final_cost = NAN; + size_t residual_count = 0; }; class PixelRefine { -- 2.52.0 From 6f6098d5083bfd2d08c413f5c3068066b53e8212 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 13:44:39 +0200 Subject: [PATCH 005/228] PixelRefine: Work in progress (Claude) --- .../pixel_refinement/PixelRefine.cpp | 536 ++++++++++-------- image_analysis/pixel_refinement/PixelRefine.h | 67 +++ 2 files changed, 360 insertions(+), 243 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index e734516c..63971bf2 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -238,15 +238,27 @@ struct PixelResidual { return false; const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); - const T P_radial = ceres::exp(-eps_radial * eps_radial / (R[0] * R[0])); - // Normalized 2D Gaussian profile in the detector-tangential plane, - // weighted by the reciprocal-space area of the pixel so that the sum of - // the profile over the shoebox is ~1 (intensity-conserving). - const T norm_tang = T(A_recip) / (T(M_PI) * R[1] * R[1]); - const T P_tang = norm_tang * ceres::exp(-eps_tang_sq / (R[1] * R[1])); + // Full 3D reciprocal-space spot density modelled as a separable Gaussian, + // normalized so that its integral is 1 (intensity-conserving): + // radial: g_r(e) = exp(-e^2/R0^2) / (sqrt(pi) R0) [1/A^-1] + // tangential: g_t(e) = exp(-|e|^2/R1^2) / (pi R1^2) [1/A^-2] + // The detector pixel captures the fraction g_t * A_recip of the tangential + // profile (A_recip = reciprocal area the pixel subtends; sum over shoebox + // ~ 1). The radial factor is the still-image partiality: how far this + // reflection sits from the Ewald sphere. + // + // Caveat: a still samples the radial direction at a single offset, so the + // sqrt(pi) R0 normalization makes g_r a density (1/A^-1) rather than a + // dimensionless fraction. The leftover dimensional factor is absorbed by + // the free scale G; completing it physically needs an effective radial + // sampling width from the energy bandwidth + beam divergence (TODO). + const T g_radial = ceres::exp(-eps_radial * eps_radial / (R[0] * R[0])) + / (ceres::sqrt(T(M_PI)) * R[0]); + const T P_tang = T(A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) + / (T(M_PI) * R[1] * R[1]); - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; + const T signal = scale_factor[0] * T(Itrue) * B_term * g_radial * P_tang; Ipred = signal + T(Ibkg); return true; } @@ -303,24 +315,14 @@ void PixelRefine::Run(const T *image, data.solved = false; data.reflections.clear(); - const auto geom = data.geom; - const double lambda = geom.GetWavelength_A(); - const double pixel_size = geom.GetPixelSize_mm(); - const double distance_mm = geom.GetDetectorDistance_mm(); + const double lambda = data.geom.GetWavelength_A(); + const double pixel_size = data.geom.GetPixelSize_mm(); - // Reciprocal-space area subtended by one pixel (small-angle approximation): - // |dq/dpixel| ~ pixel_size / (lambda * distance), so the area is its square. - // Used to normalize the tangential profile. Good enough for a prototype; a - // proper treatment would use the per-reflection 2-theta dependent Jacobian. - const double A_recip = std::pow(pixel_size / (lambda * distance_mm), 2.0); - - // Predict reflection positions with the current lattice/geometry. const BraggPredictionSettings settings_prediction{ .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), .max_hkl = 100, .centering = data.centering }; - prediction.Calc(experiment, data.latt, settings_prediction); const auto azim_result = profile.GetResult(); const auto azim_std = profile.GetStd(); @@ -331,249 +333,295 @@ void PixelRefine::Run(const T *image, const double angle_rad = data.angle_deg * M_PI / 180.0; const int radius = data.shoebox_radius; - // ---- Collect per-reflection shoebox pixels -------------------------------- + // Exact reciprocal-space area a 1x1 pixel subtends, |dq/dx x dq/dy|, via + // finite differences of the detector->reciprocal map. This is the Jacobian + // between the curved Ewald-sphere sampling and flat reciprocal space, and it + // is exactly the geometric factor that plays the role of the Lorentz factor + // for stills: where the sphere grazes reciprocal space obliquely, a pixel + // covers more reciprocal volume and the captured fraction grows. It tracks + // the refined geometry because it reads the current data.geom each iteration. + auto recip_area = [&](double x, double y) -> double { + const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); + const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); + return (qx % qy).Length(); + }; + + // Mutable experiment whose geometry is re-synced from the refined data.geom + // before each prediction, so shoeboxes track the refined geometry/cell. + DiffractionExperiment exp_iter = experiment; + + // State retained after the loop for the final reflection extraction. std::vector groups; - for (const auto &refl : prediction.GetReflections()) { - const auto hkl = hkl_key_generator(refl); - if (!reference_data.contains(hkl)) - continue; - - ReflGroup g; - g.h = refl.h; - g.k = refl.k; - g.l = refl.l; - g.d = refl.d; - g.Itrue = reference_data[hkl]; - g.predicted_x = refl.predicted_x; - g.predicted_y = refl.predicted_y; - - const int min_y = std::max(refl.predicted_y - radius, 0); - const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); - const int min_x = std::max(refl.predicted_x - radius, 0); - const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); - - for (int y = min_y; y <= max_y; ++y) { - for (int x = min_x; x <= max_x; ++x) { - const size_t npixel = xpixel * y + x; - const int azim_bin = pixel_to_bin[npixel]; - - // Skip pixels not mapped to an azimuthal bin or carrying a - // sentinel (masked / saturated) value. We assume the pixel mask - // is already applied upstream. - if (azim_bin >= azim_bin_count) - continue; - if (image[npixel] == std::numeric_limits::max()) - continue; - if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) - continue; - - const double correction = corrections[npixel]; - const double Ibkg = azim_result[azim_bin] * 1.0; // already corrected units - const double Ibkg_sigma = azim_std[azim_bin]; - const double raw = static_cast(image[npixel]); - const double Iobs = raw * correction; - - // Per-pixel variance: Poisson noise of the corrected counts - // (var(c*N) = c^2 * N = c * Iobs) plus the background spread. - double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; - if (!(var > 1.0)) - var = 1.0; - - PixelObs obs{ - .x = static_cast(x), - .y = static_cast(y), - .Iobs = Iobs, - .Ibkg = Ibkg, - .weight = 1.0 / std::sqrt(var), - .A_recip = A_recip, - .angle_rad = angle_rad - }; - g.pixels.push_back(obs); - } - } - - if (!g.pixels.empty()) - groups.push_back(std::move(g)); - } - - if (groups.empty()) - return; - - // ---- Set up parameter blocks (mirrors XtalOptimizer for the geometry part) - - double beam[2] = {geom.GetBeamX_pxl(), geom.GetBeamY_pxl()}; - double dist_mm = distance_mm; - double detector_rot[2] = {geom.GetPoniRot1_rad(), geom.GetPoniRot2_rad()}; + double beam[2] = {0, 0}; + double dist_mm = data.geom.GetDetectorDistance_mm(); + double detector_rot[2] = {0, 0}; double rot_vec[3] = {1.0, 0.0, 0.0}; - if (auto axis = geom.GetRotation()) { - rot_vec[0] = axis->GetAxis().x; - rot_vec[1] = axis->GetAxis().y; - rot_vec[2] = axis->GetAxis().z; - } - double latt_vec0[3] = {0, 0, 0}; // orientation (Rodrigues) double latt_vec1[3] = {0, 0, 0}; // lengths double latt_vec2[3] = {0, 0, 0}; // angles (rad) - double beta = data.latt.GetUnitCell().beta; - switch (data.crystal_system) { - case gemmi::CrystalSystem::Orthorhombic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Tetragonal: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; - break; - case gemmi::CrystalSystem::Cubic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; - break; - case gemmi::CrystalSystem::Hexagonal: - LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Monoclinic: - LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); - latt_vec2[0] = beta; - break; - default: { - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - const auto uc = data.latt.GetUnitCell(); - latt_vec2[0] = uc.alpha * M_PI / 180.0; - latt_vec2[1] = uc.beta * M_PI / 180.0; - latt_vec2[2] = uc.gamma * M_PI / 180.0; - break; + const int n_iter = std::max(1, data.max_iterations); + for (int iter = 0; iter < n_iter; ++iter) { + // ---- 1. Re-sync prediction geometry from the (refined) model ---------- + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + + prediction.Calc(exp_iter, data.latt, settings_prediction); + + // ---- 2. Collect per-reflection shoebox pixels ------------------------- + groups.clear(); + for (const auto &refl : prediction.GetReflections()) { + const auto hkl = hkl_key_generator(refl); + if (!reference_data.contains(hkl)) + continue; + + ReflGroup g; + g.h = refl.h; + g.k = refl.k; + g.l = refl.l; + g.d = refl.d; + g.Itrue = reference_data[hkl]; + g.predicted_x = refl.predicted_x; + g.predicted_y = refl.predicted_y; + + const int min_y = std::max(refl.predicted_y - radius, 0); + const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); + const int min_x = std::max(refl.predicted_x - radius, 0); + const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + + for (int y = min_y; y <= max_y; ++y) { + for (int x = min_x; x <= max_x; ++x) { + const size_t npixel = xpixel * y + x; + const int azim_bin = pixel_to_bin[npixel]; + + // Skip pixels not mapped to an azimuthal bin or carrying a + // sentinel (masked / saturated) value. We assume the pixel + // mask is already applied upstream. + if (azim_bin >= azim_bin_count) + continue; + if (image[npixel] == std::numeric_limits::max()) + continue; + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) + continue; + + const double correction = corrections[npixel]; + const double Ibkg = azim_result[azim_bin]; // already in corrected units + const double Ibkg_sigma = azim_std[azim_bin]; + const double raw = static_cast(image[npixel]); + const double Iobs = raw * correction; + + // Per-pixel variance: Poisson noise of the corrected counts + // (var(c*N) = c^2 * N = c * Iobs) plus the background spread. + double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; + if (!(var > 1.0)) + var = 1.0; + + PixelObs obs{ + .x = static_cast(x), + .y = static_cast(y), + .Iobs = Iobs, + .Ibkg = Ibkg, + .weight = 1.0 / std::sqrt(var), + .A_recip = recip_area(x, y), + .angle_rad = angle_rad + }; + g.pixels.push_back(obs); + } + } + + if (!g.pixels.empty()) + groups.push_back(std::move(g)); } - } - ceres::Problem problem; + if (groups.empty()) + return; - for (const auto &g : groups) { - for (const auto &obs : g.pixels) { - auto *cost = new ceres::AutoDiffCostFunction< - PixelResidual, 1, 2, 1, 2, 3, 3, 3, 3, 1, 1, 2>( - new PixelResidual(obs, g.Itrue, lambda, pixel_size, - g.h, g.k, g.l, data.crystal_system)); - problem.AddResidualBlock(cost, new ceres::HuberLoss(3.0), - beam, &dist_mm, detector_rot, rot_vec, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R); + // ---- 3. Set up parameter blocks (geometry part mirrors XtalOptimizer) - + beam[0] = data.geom.GetBeamX_pxl(); + beam[1] = data.geom.GetBeamY_pxl(); + dist_mm = data.geom.GetDetectorDistance_mm(); + detector_rot[0] = data.geom.GetPoniRot1_rad(); + detector_rot[1] = data.geom.GetPoniRot2_rad(); + rot_vec[0] = 1.0; rot_vec[1] = 0.0; rot_vec[2] = 0.0; + if (auto axis = data.geom.GetRotation()) { + rot_vec[0] = axis->GetAxis().x; + rot_vec[1] = axis->GetAxis().y; + rot_vec[2] = axis->GetAxis().z; } - } - data.residual_count = problem.NumResidualBlocks(); - - // ---- Constrain / bound parameter blocks ----------------------------------- - if (!data.refine_orientation) - problem.SetParameterBlockConstant(latt_vec0); - - if (!data.refine_unit_cell) { - problem.SetParameterBlockConstant(latt_vec1); - problem.SetParameterBlockConstant(latt_vec2); - } else { - for (int i = 0; i < 3; ++i) { - problem.SetParameterLowerBound(latt_vec1, i, 5.0); - problem.SetParameterUpperBound(latt_vec1, i, 1000.0); - } - if (data.crystal_system != gemmi::CrystalSystem::Monoclinic && - data.crystal_system != gemmi::CrystalSystem::Triclinic) - problem.SetParameterBlockConstant(latt_vec2); - } - - if (!data.refine_beam_center) - problem.SetParameterBlockConstant(beam); - - if (!data.refine_distance) { - problem.SetParameterBlockConstant(&dist_mm); - } else { - problem.SetParameterLowerBound(&dist_mm, 0, dist_mm * 0.9); - problem.SetParameterUpperBound(&dist_mm, 0, dist_mm * 1.1); - } - - if (!data.refine_detector_angles) { - problem.SetParameterBlockConstant(detector_rot); - } else { - const double rng = 3.0 / 180.0 * M_PI; - for (int i = 0; i < 2; ++i) { - problem.SetParameterLowerBound(detector_rot, i, detector_rot[i] - rng); - problem.SetParameterUpperBound(detector_rot, i, detector_rot[i] + rng); - } - } - - if (!data.refine_rotation_axis) - problem.SetParameterBlockConstant(rot_vec); - - if (data.refine_scale) - problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); - else - problem.SetParameterBlockConstant(&data.scale_factor); - - if (!data.refine_B) - problem.SetParameterBlockConstant(&data.B_factor); - - if (data.refine_R) { - problem.SetParameterLowerBound(data.R, 0, 1e-5); - problem.SetParameterLowerBound(data.R, 1, 1e-5); - } else { - problem.SetParameterBlockConstant(data.R); - } - - // ---- Solve ---------------------------------------------------------------- - ceres::Solver::Options options; - options.linear_solver_type = ceres::DENSE_QR; - options.minimizer_progress_to_stdout = false; - options.logging_type = ceres::LoggingType::SILENT; - options.max_solver_time_in_seconds = data.max_time_s; - options.num_threads = 1; - - ceres::Solver::Summary summary; - ceres::Solve(options, &problem, &summary); - - data.final_cost = summary.final_cost; - data.solved = summary.IsSolutionUsable(); - - // ---- Write back refined geometry + lattice -------------------------------- - if (data.refine_beam_center) - data.geom.BeamX_pxl(beam[0]).BeamY_pxl(beam[1]); - if (data.refine_distance) - data.geom.DetectorDistance_mm(dist_mm); - if (data.refine_detector_angles) - data.geom.PoniRot1_rad(detector_rot[0]).PoniRot2_rad(detector_rot[1]); - - if (data.refine_orientation || data.refine_unit_cell) { + double beta = data.latt.GetUnitCell().beta; switch (data.crystal_system) { case gemmi::CrystalSystem::Orthorhombic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); break; case gemmi::CrystalSystem::Tetragonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; break; case gemmi::CrystalSystem::Cubic: - latt_vec1[1] = latt_vec1[0]; - latt_vec1[2] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; break; case gemmi::CrystalSystem::Hexagonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); + LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); break; case gemmi::CrystalSystem::Monoclinic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); + LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); + latt_vec2[0] = beta; break; - default: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, - latt_vec2[0], latt_vec2[1], latt_vec2[2]); + default: { + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + const auto uc = data.latt.GetUnitCell(); + latt_vec2[0] = uc.alpha * M_PI / 180.0; + latt_vec2[1] = uc.beta * M_PI / 180.0; + latt_vec2[2] = uc.gamma * M_PI / 180.0; break; + } } - } + + // ---- 4. Build the problem --------------------------------------------- + ceres::Problem problem; + for (const auto &g : groups) { + for (const auto &obs : g.pixels) { + auto *cost = new ceres::AutoDiffCostFunction< + PixelResidual, 1, 2, 1, 2, 3, 3, 3, 3, 1, 1, 2>( + new PixelResidual(obs, g.Itrue, lambda, pixel_size, + g.h, g.k, g.l, data.crystal_system)); + problem.AddResidualBlock(cost, new ceres::HuberLoss(3.0), + beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R); + } + } + data.residual_count = problem.NumResidualBlocks(); + + // ---- 5. Constrain / bound parameter blocks ---------------------------- + if (!data.refine_orientation) + problem.SetParameterBlockConstant(latt_vec0); + + if (!data.refine_unit_cell) { + problem.SetParameterBlockConstant(latt_vec1); + problem.SetParameterBlockConstant(latt_vec2); + } else { + for (int i = 0; i < 3; ++i) { + problem.SetParameterLowerBound(latt_vec1, i, 5.0); + problem.SetParameterUpperBound(latt_vec1, i, 1000.0); + } + if (data.crystal_system != gemmi::CrystalSystem::Monoclinic && + data.crystal_system != gemmi::CrystalSystem::Triclinic) + problem.SetParameterBlockConstant(latt_vec2); + } + + if (!data.refine_beam_center) + problem.SetParameterBlockConstant(beam); + + if (!data.refine_distance) { + problem.SetParameterBlockConstant(&dist_mm); + } else { + problem.SetParameterLowerBound(&dist_mm, 0, dist_mm * 0.9); + problem.SetParameterUpperBound(&dist_mm, 0, dist_mm * 1.1); + } + + if (!data.refine_detector_angles) { + problem.SetParameterBlockConstant(detector_rot); + } else { + const double rng = 3.0 / 180.0 * M_PI; + for (int i = 0; i < 2; ++i) { + problem.SetParameterLowerBound(detector_rot, i, detector_rot[i] - rng); + problem.SetParameterUpperBound(detector_rot, i, detector_rot[i] + rng); + } + } + + if (!data.refine_rotation_axis) + problem.SetParameterBlockConstant(rot_vec); + + if (data.refine_scale) + problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); + else + problem.SetParameterBlockConstant(&data.scale_factor); + + if (!data.refine_B) + problem.SetParameterBlockConstant(&data.B_factor); + + if (data.refine_R) { + problem.SetParameterLowerBound(data.R, 0, 1e-5); + problem.SetParameterLowerBound(data.R, 1, 1e-5); + } else { + problem.SetParameterBlockConstant(data.R); + } + + // ---- 6. Solve --------------------------------------------------------- + ceres::Solver::Options options; + options.linear_solver_type = ceres::DENSE_QR; + options.minimizer_progress_to_stdout = false; + options.logging_type = ceres::LoggingType::SILENT; + options.max_solver_time_in_seconds = data.max_time_s; + options.num_threads = 1; + + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + + data.final_cost = summary.final_cost; + data.solved = summary.IsSolutionUsable(); + + // ---- 7. Write refined geometry + lattice back into data --------------- + if (data.refine_beam_center) + data.geom.BeamX_pxl(beam[0]).BeamY_pxl(beam[1]); + if (data.refine_distance) + data.geom.DetectorDistance_mm(dist_mm); + if (data.refine_detector_angles) + data.geom.PoniRot1_rad(detector_rot[0]).PoniRot2_rad(detector_rot[1]); + + if (data.refine_orientation || data.refine_unit_cell) { + switch (data.crystal_system) { + case gemmi::CrystalSystem::Orthorhombic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Tetragonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Cubic: + latt_vec1[1] = latt_vec1[0]; + latt_vec1[2] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Hexagonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); + break; + case gemmi::CrystalSystem::Monoclinic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); + break; + default: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, + latt_vec2[0], latt_vec2[1], latt_vec2[2]); + break; + } + } + } // predict<->refine iterations // ---- Extract integrated reflections --------------------------------------- - // Profile-fitted intensity with the optimized profile P (shape only): - // I = sum_p [ P_p (Iobs_p - Ibkg_p) / v_p ] / sum_p [ P_p^2 / v_p ] - // var(I) = 1 / sum_p [ P_p^2 / v_p ] - // where P_p is the *shape* of the model per pixel (signal at G=Itrue=B=1). + // Two quantities are read back per reflection, using the *optimized* model: + // + // * Recorded (partial) intensity J by profile fitting against the normalized + // tangential profile P_t (sum over the shoebox ~ 1): + // J = sum_p [ P_t,p (Iobs_p - Ibkg_p) / v_p ] / sum_p [ P_t,p^2 / v_p ] + // var(J) = 1 / sum_p [ P_t,p^2 / v_p ] + // Because P_t is area-normalized, J estimates the integrated intensity + // actually recorded on this image (not the full-spot intensity). + // + // * Partiality: the profile-weighted mean of the *peak-normalized* radial + // factor exp(-eps_r^2/R0^2) in (0,1], i.e. how close to the Ewald sphere + // the reflection sits. Kept dimensionless for consistency with the rest of + // the pipeline (image_scale_corr = rlp / partiality, full I = J / partiality). data.reflections.reserve(groups.size()); for (const auto &g : groups) { - double num = 0.0, den = 0.0, partiality_sum = 0.0, bkg_sum = 0.0; + double num = 0.0, den = 0.0, bkg_sum = 0.0; + double radial_sum = 0.0, radial_w = 0.0; size_t n = 0; for (const auto &obs : g.pixels) { @@ -585,16 +633,19 @@ void PixelRefine::Run(const T *image, if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) continue; + // Normalized tangential profile (sum over shoebox ~ 1) -> fit weight. + const double P_t = obs.A_recip * std::exp(-eps_t_sq / (data.R[1] * data.R[1])) + / (M_PI * data.R[1] * data.R[1]); + // Peak-normalized radial factor (the partiality), in (0,1]. const double P_radial = std::exp(-eps_r * eps_r / (data.R[0] * data.R[0])); - const double norm_tang = obs.A_recip / (M_PI * data.R[1] * data.R[1]); - const double P_tang = norm_tang * std::exp(-eps_t_sq / (data.R[1] * data.R[1])); - const double P = P_radial * P_tang; // model shape per pixel (unit scale) + const double v = SafeInv(obs.weight * obs.weight, 1.0); // pixel variance const double signal = obs.Iobs - obs.Ibkg; - num += P * signal / v; - den += P * P / v; - partiality_sum += P_tang; // tangential profile sums to ~partiality + num += P_t * signal / v; + den += P_t * P_t / v; + radial_sum += P_radial * P_t; // weight partiality by the spot core + radial_w += P_t; bkg_sum += obs.Ibkg; ++n; } @@ -609,11 +660,10 @@ void PixelRefine::Run(const T *image, r.observed_x = NAN; r.observed_y = NAN; r.rlp = 1.0f; - r.partiality = static_cast(partiality_sum); + r.partiality = (radial_w > 0.0) ? static_cast(radial_sum / radial_w) : 1.0f; if (den > 0.0 && n > 0) { - const double I = num / den; - r.I = static_cast(I); + r.I = static_cast(num / den); r.sigma = static_cast(std::sqrt(1.0 / den)); r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 5848cf15..44ae21c8 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -9,6 +9,72 @@ #include "../common/AzimuthalIntegrationProfile.h" #include "../scale_merge/HKLKey.h" +// ============================================================================= +// PixelRefine — one optimization to rule geometry, integration and scaling +// ============================================================================= +// +// Intent +// ------ +// Classical crystallographic data processing is a one-way pipeline: +// +// spot finding -> indexing -> geometry refinement -> integration -> scaling -> merging +// +// Each stage consumes the previous stage's output and never talks back. The +// integrator trusts the refined geometry; the scaler trusts the integrated +// intensities; nothing downstream is ever allowed to correct an upstream +// parameter. Post-refinement and profile fitting were the field's partial +// answers to this: post-refinement lets merged intensities nudge per-image +// orientation/cell/mosaicity, and profile fitting lets a learned spot shape +// improve weak-reflection intensities. But both are narrow back-channels bolted +// onto a feed-forward pipeline — there is no end-to-end gradient that flows from +// the raw detector pixels all the way back to every parameter at once. +// +// PixelRefine is an experiment in doing the whole thing as a *single* least +// squares problem. We write down, for every pixel in a reflection's shoebox, the +// expected counts as an explicit forward model +// +// I_pred(pixel) = G * I_true * B_term * P_radial * P_tangential + I_bkg +// +// and let Ceres autodiff back-propagate the per-pixel residuals into ALL of: +// * detector geometry (beam centre, distance, tilt) +// * crystal orientation + unit cell +// * overall scale G and Debye-Waller B +// * the reciprocal-space spot widths R = (radial, tangential) +// simultaneously. Geometry refinement, profile-fitted integration and scaling +// then stop being separate stages: they are different parameters of one model, +// coupled through the same pixels, with full backpropagation between them. Once +// the model is differentiable end-to-end, things that used to need bespoke code +// — mosaicity refinement, profile fitting, partiality — fall out "for free" as +// extra parameters of the same forward model. +// +// How I_true enters +// ----------------- +// I_true is NOT refined here. It is a *fixed hypothesis* for the duration of a +// pass: the current best merged estimate of each reflection's full intensity. +// The intended outer loop is iterative, like EM / self-consistent field: +// +// repeat over the whole dataset: +// run PixelRefine on every image with the current I_true reference +// re-merge the resulting intensities -> new I_true +// until the reference stops changing +// +// So a single PixelRefine call answers "given this intensity hypothesis, what +// geometry/scale/profile best explains these pixels, and what intensities do I +// read back out?", and the dataset-level loop refines the hypothesis itself. +// +// Inner predict<->refine loop +// --------------------------- +// Within one image we also iterate (max_iterations): Bragg prediction places the +// shoeboxes, we refine, the refined geometry/cell feed the next prediction, etc. +// Initially the model geometry equals the experiment's DiffractionGeometry, but +// as refinement proceeds it diverges, so later predictions must use the *refined* +// data.geom rather than the static experiment geometry. +// +// Status: experimental prototype. The forward model (esp. the still-image +// Lorentz/partiality normalization) is deliberately simple and expected to +// evolve. See PixelRefine.cpp for the physics conventions and known caveats. +// ============================================================================= + struct PixelRefineData { // --- model state (input as initial guess, output as refined result) --- DiffractionGeometry geom; @@ -37,6 +103,7 @@ struct PixelRefineData { double max_time_s = 5.0; int shoebox_radius = 3; // half-size of the per-reflection pixel box + int max_iterations = 3; // inner predict<->refine cycles (re-predict with refined geom/latt) // --- output --- std::vector reflections; // profile-fitted integration result -- 2.52.0 From 6f311596078e99e92520aa667d9f66d62a02f121 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 13:59:01 +0200 Subject: [PATCH 006/228] PixelRefine: Add bandwidth contribution --- .../pixel_refinement/PixelRefine.cpp | 49 ++++++++++++++----- image_analysis/pixel_refinement/PixelRefine.h | 29 +++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 63971bf2..933edb40 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -29,6 +29,7 @@ struct ReflGroup { int h, k, l; double d; double Itrue; // reference intensity (held fixed) + double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) double predicted_x, predicted_y; std::vector pixels; }; @@ -47,10 +48,9 @@ double SafeInv(double x, double fallback) { // I_pred(pixel) = G * Itrue * B_term * P_radial * P_tangential + I_bkg // // B_term = exp(-B |q|^2 / 4) (Debye-Waller) -// P_radial = exp(-eps_r^2 / R0^2) (partiality: fraction of +// P_radial = exp(-eps_r^2 / R0_eff^2) (partiality: fraction of // the mosaic blob on the -// Ewald sphere; NOT -// normalized, <= 1) +// Ewald sphere; <= 1) // P_tangential = A_recip/(pi R1^2) * exp(-eps_t^2/R1^2)(spatial profile on the // detector, normalized so // that sum over pixels ~ 1) @@ -59,17 +59,28 @@ double SafeInv(double x, double fallback) { // I_pred - I_bkg over the shoebox reproduces G * Itrue * B_term * P_radial. // The 1/(pi R1^2) normalization is the missing piece that decouples the profile // width R1 from the overall scale G. +// +// X-ray bandwidth: a spread in lambda is a spread in the Ewald-sphere radius, +// i.e. a purely *radial* thickening of the shell. It adds (in quadrature) a +// resolution-dependent term to the radial width: +// R0_eff^2 = R0^2 + R_bw^2 , R_bw^2 = (b*lambda)^2 / (2 d^4) +// where b = relative bandwidth (sigma of dlambda/lambda). R_bw grows like 1/d^2, +// so bandwidth leaves low-resolution spots sharp and smears high-resolution ones +// radially - the pink-beam/DMM signature. R_bw_sq is a fixed per-reflection +// constant (b is known), so R0 keeps meaning "intrinsic" width (mosaic + +// divergence + beam). b = 0 makes R_bw = 0: a monochromatic no-op. // --------------------------------------------------------------------------- struct PixelResidual { PixelResidual(const PixelObs &obs, double Itrue, double lambda, double pixel_size, double exp_h, double exp_k, double exp_l, + double R_bw_sq, gemmi::CrystalSystem symmetry) : Itrue(Itrue), Iobs(obs.Iobs), Ibkg(obs.Ibkg), weight(obs.weight), A_recip(obs.A_recip), obs_x(obs.x), obs_y(obs.y), inv_lambda(1.0 / lambda), pixel_size(pixel_size), exp_h(exp_h), exp_k(exp_k), exp_l(exp_l), - angle_rad(obs.angle_rad), symmetry(symmetry) { + R_bw_sq(R_bw_sq), angle_rad(obs.angle_rad), symmetry(symmetry) { if (std::fabs(lambda) < 1e-6) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Lambda cannot be close to zero"); @@ -251,10 +262,11 @@ struct PixelResidual { // Caveat: a still samples the radial direction at a single offset, so the // sqrt(pi) R0 normalization makes g_r a density (1/A^-1) rather than a // dimensionless fraction. The leftover dimensional factor is absorbed by - // the free scale G; completing it physically needs an effective radial - // sampling width from the energy bandwidth + beam divergence (TODO). - const T g_radial = ceres::exp(-eps_radial * eps_radial / (R[0] * R[0])) - / (ceres::sqrt(T(M_PI)) * R[0]); + // the free scale G. The energy-bandwidth contribution to the radial width + // is folded in here via R_bw_sq (beam divergence is still TODO). + const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); + const T g_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq) + / (ceres::sqrt(T(M_PI)) * ceres::sqrt(R0_eff_sq)); const T P_tang = T(A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); @@ -289,6 +301,7 @@ struct PixelResidual { const double inv_lambda; const double pixel_size; const double exp_h, exp_k, exp_l; + const double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) const double angle_rad; gemmi::CrystalSystem symmetry; }; @@ -346,6 +359,17 @@ void PixelRefine::Run(const T *image, return (qx % qy).Length(); }; + // Bandwidth radial-width^2 (in the code's R = sqrt(2)*sigma convention): + // R_bw^2 = (b*lambda)^2 / (2 d^4), b = relative bandwidth (sigma). + // A fixed per-reflection constant; data.bandwidth == 0 -> monochromatic no-op. + const double bw = data.bandwidth; + auto bandwidth_radial_sq = [&](double d) -> double { + if (bw <= 0.0 || d <= 0.0) + return 0.0; + const double bl = bw * lambda; + return bl * bl / (2.0 * d * d * d * d); + }; + // Mutable experiment whose geometry is re-synced from the refined data.geom // before each prediction, so shoeboxes track the refined geometry/cell. DiffractionExperiment exp_iter = experiment; @@ -384,6 +408,7 @@ void PixelRefine::Run(const T *image, g.l = refl.l; g.d = refl.d; g.Itrue = reference_data[hkl]; + g.R_bw_sq = bandwidth_radial_sq(refl.d); g.predicted_x = refl.predicted_x; g.predicted_y = refl.predicted_y; @@ -489,7 +514,7 @@ void PixelRefine::Run(const T *image, auto *cost = new ceres::AutoDiffCostFunction< PixelResidual, 1, 2, 1, 2, 3, 3, 3, 3, 1, 1, 2>( new PixelResidual(obs, g.Itrue, lambda, pixel_size, - g.h, g.k, g.l, data.crystal_system)); + g.h, g.k, g.l, g.R_bw_sq, data.crystal_system)); problem.AddResidualBlock(cost, new ceres::HuberLoss(3.0), beam, &dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2, @@ -625,7 +650,7 @@ void PixelRefine::Run(const T *image, size_t n = 0; for (const auto &obs : g.pixels) { - PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system); + PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, g.R_bw_sq, data.crystal_system); double q_sq, eps_r, eps_t_sq; if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) @@ -637,7 +662,9 @@ void PixelRefine::Run(const T *image, const double P_t = obs.A_recip * std::exp(-eps_t_sq / (data.R[1] * data.R[1])) / (M_PI * data.R[1] * data.R[1]); // Peak-normalized radial factor (the partiality), in (0,1]. - const double P_radial = std::exp(-eps_r * eps_r / (data.R[0] * data.R[0])); + // Bandwidth-broadened radial width, matching the model in Model(). + const double R0_eff_sq = data.R[0] * data.R[0] + g.R_bw_sq; + const double P_radial = std::exp(-eps_r * eps_r / R0_eff_sq); const double v = SafeInv(obs.weight * obs.weight, 1.0); // pixel variance const double signal = obs.Iobs - obs.Ibkg; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 44ae21c8..07b280cf 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -70,6 +70,30 @@ // as refinement proceeds it diverges, so later predictions must use the *refined* // data.geom rather than the static experiment geometry. // +// On shoeboxes, tails, and gatekeeping +// ------------------------------------ +// We deliberately do NOT chase the full spot. A shoebox only needs to cover +// enough of a reflection for the fit to be meaningful; clipped tails are fine, +// because the partiality term already downweights whatever falls outside the +// well-modelled core. The premise - especially for serial crystallography - is +// that the problem is rarely *missing* information; it is *failing to gate out* +// information that is not meaningful. As long as the model knows a piece is +// missing (low partiality), it is safe to leave it missing. That flips the usual +// trade-off: rather than shrinking boxes to avoid contamination, we can grow them +// and let partiality decide, per pixel and per reflection, what actually carries +// signal. (For downstream integration this pixel-level gating is the point - keep +// only meaningful pixels, instead of a fixed geometric mask.) The bandwidth term +// below is part of the same idea: it tells the model where the radial tails are +// *expected* to be, so it can weight rather than blindly include them. +// +// X-ray bandwidth (optional) +// -------------------------- +// A finite bandwidth thickens the Ewald shell radially and smears spots along the +// radial direction, growing like 1/d^2 (the pink-beam/DMM signature). It enters +// as a fixed, resolution-dependent addition to the radial width R0 (see +// PixelRefineData::bandwidth and PixelRefine.cpp). It is OFF by default +// (bandwidth = 0, monochromatic); set it for DMM-type data, leave it for Si. +// // Status: experimental prototype. The forward model (esp. the still-image // Lorentz/partiality normalization) is deliberately simple and expected to // evolve. See PixelRefine.cpp for the physics conventions and known caveats. @@ -86,6 +110,11 @@ struct PixelRefineData { double scale_factor = 1.0; // overall scale G double R[2] = {0.005, 0.005}; // R[0] = radial (partiality) width, R[1] = tangential (profile) width (A^-1) + // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% + // FWHM DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening + // to R[0]. 0 = monochromatic (the term switches off entirely). + double bandwidth = 0.0; + // Goniometer: for a still image keep angle_deg = 0. For a wedge, pass the // angle (deg) of the slice centre relative to the reference (phi=0) frame. double angle_deg = 0.0; -- 2.52.0 From 155c53acd8d971a8d017d88437a3a363e25cf06c Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 15:37:27 +0200 Subject: [PATCH 007/228] jfjoch_process: First pixel refine integration --- common/DatasetSettings.cpp | 9 ++ common/DatasetSettings.h | 3 + common/DiffractionExperiment.cpp | 9 ++ common/DiffractionExperiment.h | 2 + common/IndexingSettings.h | 2 +- image_analysis/CMakeLists.txt | 2 +- image_analysis/IndexAndRefine.cpp | 115 +++++++++++++++--- image_analysis/IndexAndRefine.h | 26 +++- image_analysis/MXAnalysisWithoutFPGA.cpp | 2 +- .../pixel_refinement/PixelRefine.cpp | 26 ++-- image_analysis/pixel_refinement/PixelRefine.h | 8 +- tools/jfjoch_process.cpp | 34 +++++- 12 files changed, 203 insertions(+), 35 deletions(-) diff --git a/common/DatasetSettings.cpp b/common/DatasetSettings.cpp index 85bb81d5..2bcb144b 100644 --- a/common/DatasetSettings.cpp +++ b/common/DatasetSettings.cpp @@ -389,6 +389,15 @@ std::optional DatasetSettings::GetPolarizationFactor() const { return polarization_factor; } +DatasetSettings &DatasetSettings::BandwidthFWHM(const std::optional &input) { + bandwidth_fwhm = input; + return *this; +} + +std::optional DatasetSettings::GetBandwidthFWHM() const { + return bandwidth_fwhm; +} + float DatasetSettings::GetPoniRot3_rad() const { return poni_rot_3_rad; } diff --git a/common/DatasetSettings.h b/common/DatasetSettings.h index 9ec5672e..3d91c630 100644 --- a/common/DatasetSettings.h +++ b/common/DatasetSettings.h @@ -55,6 +55,7 @@ class DatasetSettings { bool write_nxmx_hdf5_master; std::optional polarization_factor; + std::optional bandwidth_fwhm; // relative X-ray bandwidth, FWHM of dlambda/lambda (e.g. 0.01 for 1%) float poni_rot_1_rad; float poni_rot_2_rad; float poni_rot_3_rad; @@ -99,6 +100,7 @@ public: DatasetSettings& PixelValueLowThreshold(const std::optional &input); DatasetSettings& PixelValueHighThreshold(const std::optional &input); DatasetSettings& PolarizationFactor(const std::optional &input); + DatasetSettings& BandwidthFWHM(const std::optional &input); DatasetSettings& PoniRot1_rad(float input); DatasetSettings& PoniRot2_rad(float input); DatasetSettings& PoniRot3_rad(float input); @@ -150,6 +152,7 @@ public: std::optional IsSaveCalibration() const; std::optional GetPolarizationFactor() const; + std::optional GetBandwidthFWHM() const; float GetPoniRot1_rad() const; float GetPoniRot2_rad() const; float GetPoniRot3_rad() const; diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index ffd5f4ad..6b1a3b66 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -1310,6 +1310,15 @@ std::optional DiffractionExperiment::GetPolarizationFactor() const { return dataset.GetPolarizationFactor(); } +DiffractionExperiment &DiffractionExperiment::BandwidthFWHM(const std::optional &input) { + dataset.BandwidthFWHM(input); + return *this; +} + +std::optional DiffractionExperiment::GetBandwidthFWHM() const { + return dataset.GetBandwidthFWHM(); +} + DiffractionExperiment &DiffractionExperiment::SaveCalibration(const std::optional &input) { dataset.SaveCalibration(input); return *this; diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index 131c3bff..a8bbb9a8 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -281,9 +281,11 @@ public: DiffractionExperiment& ApplySolidAngleCorr(bool input); DiffractionExperiment& PolarizationFactor(const std::optional &input); + DiffractionExperiment& BandwidthFWHM(const std::optional &input); bool GetApplySolidAngleCorr() const; std::optional GetPolarizationFactor() const; + std::optional GetBandwidthFWHM() const; int64_t GetUDPInterfaceCount() const; std::vector GetDetectorModuleConfig(const std::vector& net_config) const; diff --git a/common/IndexingSettings.h b/common/IndexingSettings.h index 6e900440..c428f02e 100644 --- a/common/IndexingSettings.h +++ b/common/IndexingSettings.h @@ -6,7 +6,7 @@ #include enum class IndexingAlgorithmEnum {FFBIDX, FFT, FFTW, Auto, None}; -enum class GeomRefinementAlgorithmEnum {None, OrientationOnly, BeamCenter}; +enum class GeomRefinementAlgorithmEnum {None, OrientationOnly, BeamCenter, PixelRefine}; class IndexingSettings { IndexingAlgorithmEnum algorithm; diff --git a/image_analysis/CMakeLists.txt b/image_analysis/CMakeLists.txt index 913870c5..1bf6e448 100644 --- a/image_analysis/CMakeLists.txt +++ b/image_analysis/CMakeLists.txt @@ -52,4 +52,4 @@ ADD_SUBDIRECTORY(image_preprocessing) ADD_SUBDIRECTORY(azint) ADD_SUBDIRECTORY(pixel_refinement) -TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge gemmi) +TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochPixelRefine gemmi) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index d436f2a7..5dc02d55 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -175,6 +175,10 @@ void IndexAndRefine::RefineGeometryIfNeeded(DataMessage &msg, IndexAndRefine::In XtalOptimizerRotationOnly(data, msg.spots, 0.05); break; case GeomRefinementAlgorithmEnum::BeamCenter: + case GeomRefinementAlgorithmEnum::PixelRefine: + // PixelRefine still benefits from the classical beam-center + cell + // refinement as a starting point; the pixel-level refinement runs + // later, during integration. if (XtalOptimizer(data, {msg.spots})) { outcome.experiment.BeamX_pxl(data.geom.GetBeamX_pxl()) .BeamY_pxl(data.geom.GetBeamY_pxl()); @@ -234,7 +238,9 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const IndexAndRefine::IndexingOutcome &outcome) { + const IndexAndRefine::IndexingOutcome &outcome, + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile) { if (!outcome.lattice_candidate) return; @@ -286,14 +292,31 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, .mosaicity_deg = std::fabs(mos_deg) }; - auto pred_start_time = std::chrono::steady_clock::now(); - auto nrefl = prediction.Calc(outcome.experiment, latt, settings_prediction); - auto pred_end_time = std::chrono::steady_clock::now(); - msg.bragg_prediction_time_s = std::chrono::duration(pred_end_time - pred_start_time).count(); + // Select the integration path: classical 2D integration, or the experimental + // PixelRefine joint geometry/profile/scale refinement (which also produces the + // integrated reflections that flow into the normal save/merge). + const bool use_pixel_refine = + experiment.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine + && !pixel_reference_.empty() && mapping && profile; - auto integration_start_time = std::chrono::steady_clock::now(); - i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), nrefl, msg.number); - msg.integrated_reflections = i_outcome.reflections.size(); + if (use_pixel_refine) { + auto integration_start_time = std::chrono::steady_clock::now(); + PixelRefineIntegrate(msg, image, prediction, outcome, *mapping, *profile, i_outcome); + msg.integrated_reflections = i_outcome.reflections.size(); + auto integration_end_time = std::chrono::steady_clock::now(); + msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); + } else { + auto pred_start_time = std::chrono::steady_clock::now(); + auto nrefl = prediction.Calc(outcome.experiment, latt, settings_prediction); + auto pred_end_time = std::chrono::steady_clock::now(); + msg.bragg_prediction_time_s = std::chrono::duration(pred_end_time - pred_start_time).count(); + + auto integration_start_time = std::chrono::steady_clock::now(); + i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), nrefl, msg.number); + msg.integrated_reflections = i_outcome.reflections.size(); + auto integration_end_time = std::chrono::steady_clock::now(); + msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); + } constexpr size_t kMaxReflections = 10000; if (i_outcome.reflections.size() > kMaxReflections) { @@ -314,10 +337,10 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, CalcISigma(msg, i_outcome.reflections); CalcWilsonBFactor(msg, i_outcome.reflections); - auto integration_end_time = std::chrono::steady_clock::now(); - msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); - - ScaleImage(msg, i_outcome); + // PixelRefine produces already-scaled reflections; only the classical path + // needs the separate ScaleOnTheFly step. + if (!use_pixel_refine) + ScaleImage(msg, i_outcome); // Copy reflections to outgoing message msg.reflections = i_outcome.reflections; @@ -331,7 +354,9 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, void IndexAndRefine::ProcessImage(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, - BraggPrediction &prediction) { + BraggPrediction &prediction, + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile) { if (!indexer_ || !spot_finding_settings.indexing) return; @@ -361,7 +386,7 @@ void IndexAndRefine::ProcessImage(DataMessage &msg, msg.lattice_type = outcome.symmetry; if (spot_finding_settings.quick_integration) - QuickPredictAndIntegrate(msg, spot_finding_settings, image, prediction, outcome); + QuickPredictAndIntegrate(msg, spot_finding_settings, image, prediction, outcome, mapping, profile); } std::optional IndexAndRefine::FinalizeRotationIndexing() { @@ -377,6 +402,7 @@ std::optional IndexAndRefine::FinalizeRotationIndexing() IndexAndRefine &IndexAndRefine::ReferenceIntensities(std::vector &reference) { scaling_engine = std::make_unique(experiment, reference); + pixel_reference_ = reference; // kept for the experimental PixelRefine path return *this; } @@ -396,6 +422,67 @@ void IndexAndRefine::ScaleImage(DataMessage &msg, IntegrationOutcome& outcome) { msg.image_scale_time_s = std::chrono::duration(scaling_end_time - scaling_start_time).count(); } +bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, + const CompressedImage &image, + BraggPrediction &prediction, + const IndexAndRefine::IndexingOutcome &outcome, + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile, + IntegrationOutcome &i_outcome) { + if (!outcome.lattice_candidate) + return false; + + // Build the engine once (lazy: needs the azimuthal mapping, known only here). + std::call_once(pixel_refine_once_, [&] { + pixel_refine_ = std::make_unique(experiment, mapping, pixel_reference_); + }); + if (!pixel_refine_) + return false; + + PixelRefineData prd; + prd.geom = outcome.experiment.GetDiffractionGeometry(); + prd.latt = *outcome.lattice_candidate; + prd.crystal_system = outcome.symmetry.crystal_system; + if (prd.crystal_system == gemmi::CrystalSystem::Trigonal) + prd.crystal_system = gemmi::CrystalSystem::Hexagonal; + prd.centering = outcome.symmetry.centering; + if (const auto bw = experiment.GetBandwidthFWHM()) + prd.bandwidth = bw.value() / 2.3548; // FWHM -> sigma + + std::vector buffer; + const uint8_t *ptr = image.GetUncompressedPtr(buffer); + switch (image.GetMode()) { + case CompressedImageMode::Int8: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + case CompressedImageMode::Int16: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + case CompressedImageMode::Int32: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + case CompressedImageMode::Uint8: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + case CompressedImageMode::Uint16: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + case CompressedImageMode::Uint32: + pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + default: + return false; + } + + // PixelRefine output flows into the normal save/merge path: the refined + // geometry/lattice and the already-scaled reflections become the outcome. + i_outcome.reflections = std::move(prd.reflections); + i_outcome.geom = prd.geom; + i_outcome.latt = prd.latt; + i_outcome.image_scale_g = static_cast(prd.scale_factor); + i_outcome.image_scale_b_factor_Ang2 = static_cast(prd.B_factor); + + msg.image_scale_factor = static_cast(prd.scale_factor); + if (prd.B_factor != 0.0) + msg.image_scale_b_factor = static_cast(prd.B_factor); + + return true; +} + ScalingResult IndexAndRefine::ScaleAllImages(const std::vector &reference, size_t nthreads) { ScaleOnTheFly scaling(experiment, reference); scaling.Scale(integration_outcome, nthreads); diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index 0f03ee97..7674392e 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -8,7 +8,10 @@ #include "../common/DiffractionSpot.h" #include "../common/DiffractionExperiment.h" +#include "../common/AzimuthalIntegrationMapping.h" +#include "../common/AzimuthalIntegrationProfile.h" #include "bragg_prediction/BraggPrediction.h" +#include "pixel_refinement/PixelRefine.h" #include "indexing/IndexerThreadPool.h" #include "lattice_search/LatticeSearch.h" #include "rotation_indexer/RotationIndexer.h" @@ -62,17 +65,36 @@ class IndexAndRefine { const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const IndexingOutcome &outcome); + const IndexingOutcome &outcome, + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile); std::unique_ptr scaling_engine; void ScaleImage(DataMessage &msg, IntegrationOutcome& outcome); + + // Experimental PixelRefine integration path (selected via + // GeomRefinementAlgorithmEnum::PixelRefine). Needs reference intensities; the + // engine is built lazily on first use (when the azimuthal mapping is known) + // and is safe to share across threads (prediction is supplied per call). + std::vector pixel_reference_; + std::unique_ptr pixel_refine_; + std::once_flag pixel_refine_once_; + bool PixelRefineIntegrate(DataMessage &msg, + const CompressedImage &image, + BraggPrediction &prediction, + const IndexingOutcome &outcome, + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile, + IntegrationOutcome &i_outcome); public: IndexAndRefine(const DiffractionExperiment &x, IndexerThreadPool *indexer); void AddImageToRotationIndexer(DataMessage &msg); void ForceRotationIndexerLattice(const CrystalLattice& lattice); - void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction); + void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction, + const AzimuthalIntegrationMapping *mapping = nullptr, + const AzimuthalIntegrationProfile *profile = nullptr); IndexAndRefine& ReferenceIntensities(std::vector &reference); ScalingResult ScaleAllImages(const std::vector &reference, size_t nthreads = 0); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 7bade492..6a638213 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -92,7 +92,7 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, if (spot_finding_settings.indexing) indexer.ProcessImage(output, spot_finding_settings, CompressedImage(preprocessor_buffer->getBuffer(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum()), - *prediction); + *prediction, &integration, &profile); } output.max_viable_pixel_value = ret.max_value; diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 933edb40..f2e4f93c 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -308,10 +308,8 @@ struct PixelResidual { PixelRefine::PixelRefine(const DiffractionExperiment &experiment, const AzimuthalIntegrationMapping &mapping, - const std::vector &reference, - BraggPrediction &prediction) - : prediction(prediction), - mapping(mapping), + const std::vector &reference) + : mapping(mapping), xpixel(experiment.GetXPixelsNum()), ypixel(experiment.GetYPixelsNum()), experiment(experiment), @@ -324,6 +322,7 @@ PixelRefine::PixelRefine(const DiffractionExperiment &experiment, template void PixelRefine::Run(const T *image, const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, PixelRefineData &data) { data.solved = false; data.reflections.clear(); @@ -694,7 +693,12 @@ void PixelRefine::Run(const T *image, r.sigma = static_cast(std::sqrt(1.0 / den)); r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; - r.image_scale_corr = (r.partiality > 0.0f) ? r.rlp / r.partiality : NAN; + // Put I onto a common (merge-ready) scale: I * image_scale_corr is the + // full-spot, scale/DW-corrected intensity. Fold in the refined G, the + // per-reflection B_term and the partiality (rlp = 1 here). + const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); + const double denom = static_cast(r.partiality) * data.scale_factor * B_term; + r.image_scale_corr = (denom > 0.0) ? static_cast(r.rlp / denom) : NAN; } else { r.I = 0.0f; r.sigma = NAN; @@ -706,9 +710,9 @@ void PixelRefine::Run(const T *image, } // Explicit instantiations for the supported (uncompressed) image pixel types. -template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); -template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); -template void PixelRefine::Run(const int32_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); -template void PixelRefine::Run(const uint8_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); -template void PixelRefine::Run(const uint16_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); -template void PixelRefine::Run(const uint32_t *, const AzimuthalIntegrationProfile &, PixelRefineData &); +template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const int32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 07b280cf..fec77c46 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -142,7 +142,6 @@ struct PixelRefineData { }; class PixelRefine { - BraggPrediction &prediction; const AzimuthalIntegrationMapping &mapping; const size_t xpixel, ypixel; const DiffractionExperiment &experiment; @@ -152,11 +151,14 @@ class PixelRefine { public: PixelRefine(const DiffractionExperiment &experiment, const AzimuthalIntegrationMapping &mapping, - const std::vector &reference, - BraggPrediction &prediction); + const std::vector &reference); + // The BraggPrediction is supplied per call (it is mutated): this keeps a + // single PixelRefine instance usable from several threads, each passing its + // own prediction buffer. Only `data` is written; PixelRefine state is const. template void Run(const T *image, const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, PixelRefineData &data); }; diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index f0f32aab..b01c05b1 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -58,7 +58,7 @@ void print_usage() { std::cout << " -X, --indexing-algorithm Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl; std::cout << " -S, --space-group Space group number - used for both indexing and scaling" << std::endl; std::cout << " -C, --unit-cell Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl; - std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice)" << std::endl; + std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|pixelrefine)" << std::endl; std::cout << std::endl; std::cout << " Scaling and merging" << std::endl; @@ -73,6 +73,10 @@ void print_usage() { std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; std::cout << " -z, --reference-mtz Reference MTZ file" << std::endl; + std::cout << std::endl; + + std::cout << " Pixel refinement (experimental, select via -r pixelrefine, needs --reference-mtz)" << std::endl; + std::cout << " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" << std::endl; } enum { @@ -87,7 +91,8 @@ enum { OPT_SCALING_OUTPUT, OPT_SINGLE_PASS_ROTATION, OPT_REDO_ROTATION_SPOTS, - OPT_FORCE_ROTATION_LATTICE + OPT_FORCE_ROTATION_LATTICE, + OPT_BANDWIDTH }; static option long_options[] = { @@ -123,6 +128,7 @@ static option long_options[] = { {"scaling-iterations", required_argument, nullptr, OPT_SCALING_ITERATIONS}, {"scaling-high-resolution", required_argument, nullptr, OPT_SCALING_HIGH_RESOLUTION}, {"scaling-output", required_argument, nullptr, OPT_SCALING_OUTPUT}, + {"bandwidth", required_argument, nullptr, OPT_BANDWIDTH}, {nullptr, 0, nullptr, 0} }; @@ -293,6 +299,8 @@ int main(int argc, char **argv) { int64_t scaling_iter = 3; std::optional forced_rotation_lattice; + std::optional bandwidth_fwhm; // relative FWHM of dlambda/lambda + IndexingAlgorithmEnum indexing_algorithm = IndexingAlgorithmEnum::Auto; GeomRefinementAlgorithmEnum refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; @@ -410,6 +418,8 @@ int main(int argc, char **argv) { refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; else if (alg == "orientation") refinement_algorithm = GeomRefinementAlgorithmEnum::OrientationOnly; + else if (alg == "pixelrefine") + refinement_algorithm = GeomRefinementAlgorithmEnum::PixelRefine; else { logger.Error("Invalid geom refinement algorithm: {}", alg); print_usage(); @@ -515,6 +525,13 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } break; + case OPT_BANDWIDTH: + bandwidth_fwhm = atof(optarg); + if (!(bandwidth_fwhm.value() >= 0.0f)) { + logger.Error("Invalid bandwidth: {}", optarg); + exit(EXIT_FAILURE); + } + break; default: print_usage(); @@ -616,6 +633,19 @@ int main(int argc, char **argv) { logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); } + // X-ray bandwidth: CLI overrides the value carried in the dataset; otherwise + // keep whatever the dataset provided (0 / none -> monochromatic). + if (bandwidth_fwhm) + experiment.BandwidthFWHM(bandwidth_fwhm); + if (experiment.GetBandwidthFWHM()) + logger.Info("X-ray bandwidth FWHM set to {:.4f}", experiment.GetBandwidthFWHM().value()); + + // PixelRefine integration needs reference intensities (the I_true hypothesis). + if (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine && reference_data.empty()) { + logger.Warning("-r pixelrefine needs --reference-mtz; falling back to beam_and_lattice"); + refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; + } + // Configure Indexing IndexingSettings indexing_settings; indexing_settings.Algorithm(indexing_algorithm); -- 2.52.0 From 96edee9b2d8e6fe08d305f54d5b3c5027adac7a3 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 15:47:12 +0200 Subject: [PATCH 008/228] jfjoch_writer: Add incident_wavelength_spread --- common/DiffractionExperiment.cpp | 5 +++++ common/JFJochMessages.h | 1 + docs/CBOR.md | 1 + frame_serialize/CBORStream2Deserializer.cpp | 2 ++ frame_serialize/CBORStream2Serializer.cpp | 1 + writer/HDF5NXmx.cpp | 2 ++ 6 files changed, 12 insertions(+) diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 6b1a3b66..3fac1a9c 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -641,6 +641,11 @@ void DiffractionExperiment::FillMessage(StartMessage &message) const { message.beam_center_y = GetBeamY_pxl(); message.detector_distance = GetDetectorDistance_mm() * 1e-3f; message.incident_wavelength = GetWavelength_A(); + // NXmx incident_wavelength_spread is the FWHM of the wavelength distribution, + // in wavelength units. We store bandwidth as a relative FWHM (dlambda/lambda), + // so convert to absolute: dlambda = (dlambda/lambda) * lambda. + if (const auto bw = GetBandwidthFWHM()) + message.incident_wavelength_spread = bw.value() * GetWavelength_A(); message.incident_energy = GetIncidentEnergy_keV() * 1e3f; message.image_size_x = GetXPixelsNum(); message.image_size_y = GetYPixelsNum(); diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index aa0b703f..067f938c 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -208,6 +208,7 @@ struct StartMessage { float incident_energy; float incident_wavelength; + std::optional incident_wavelength_spread; // NXmx incident_wavelength_spread: FWHM of dlambda (Angstrom) float frame_time; float count_time; diff --git a/docs/CBOR.md b/docs/CBOR.md index bdca03d4..58339cc6 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -27,6 +27,7 @@ There are minor differences at the moment: | image_size_y | uint64 | Image height \[pixels\] | X | | incident_energy | float | X-ray energy \[eV\] | X | | incident_wavelength | float | X-ray wavelength \[Angstrom\] | X | +| incident_wavelength_spread | float (optional) | FWHM of the X-ray wavelength distribution \[Angstrom\] (NXmx incident_wavelength_spread); omitted when the beam is monochromatic | | | frame_time | float | Frame time, if multiple frames per trigger \[s\] | X | | count_time | float | Exposure time \[s\] | X | | saturation_value | int64 | Maximum valid sample value | X | diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 0d2c70ae..2a598de6 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1182,6 +1182,8 @@ namespace { message.incident_energy = GetCBORFloat(value); else if (key == "incident_wavelength") message.incident_wavelength = GetCBORFloat(value); + else if (key == "incident_wavelength_spread") + message.incident_wavelength_spread = GetCBORFloat(value); else if (key == "frame_time") message.frame_time = GetCBORFloat(value); else if (key == "count_time") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 6debfc5c..a79370f1 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -632,6 +632,7 @@ void CBORStream2Serializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC(mapEncoder, "incident_energy", message.incident_energy); CBOR_ENC(mapEncoder, "incident_wavelength", message.incident_wavelength); + CBOR_ENC(mapEncoder, "incident_wavelength_spread", message.incident_wavelength_spread); CBOR_ENC(mapEncoder, "frame_time", message.frame_time); CBOR_ENC(mapEncoder, "count_time", message.count_time); diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index d2a3372d..2196dd56 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -480,6 +480,8 @@ void NXmx::Beam(const StartMessage &start) { HDF5Group group(*hdf5_file, "/entry/instrument/beam"); group.NXClass("NXbeam"); SaveScalar(group, "incident_wavelength", start.incident_wavelength)->Units("angstrom"); + if (start.incident_wavelength_spread) + SaveScalar(group, "incident_wavelength_spread", start.incident_wavelength_spread.value())->Units("angstrom"); if (start.total_flux) SaveScalar(group, "total_flux", start.total_flux.value())->Units("Hz"); } -- 2.52.0 From 32e91f7287790835f4ae74ec160d8549c5d5523e Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 15:56:20 +0200 Subject: [PATCH 009/228] Minor fixes --- image_analysis/UpdateReflectionResolution.h | 6 ++++-- image_analysis/indexing/FitProfileRadius.cpp | 3 --- image_analysis/scale_merge/Merge.cpp | 2 +- image_analysis/scale_merge/ScalingResult.cpp | 4 ++-- reader/JFJochHDF5Reader.cpp | 10 ++++++++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/image_analysis/UpdateReflectionResolution.h b/image_analysis/UpdateReflectionResolution.h index 46f909e1..695d2ba0 100644 --- a/image_analysis/UpdateReflectionResolution.h +++ b/image_analysis/UpdateReflectionResolution.h @@ -9,8 +9,10 @@ #include "IntegrationOutcome.h" struct ResolutionStats { - float d_low = std::numeric_limits::max(); - float d_high = 0.0f; + // d_high = highest resolution = smallest d (tracked via `d_high > d`); + // d_low = lowest resolution = largest d (tracked via `d_low < d`). + float d_low = 0.0f; + float d_high = std::numeric_limits::max(); int n_reflections = 0; int n_images = 0; }; diff --git a/image_analysis/indexing/FitProfileRadius.cpp b/image_analysis/indexing/FitProfileRadius.cpp index bef1f2a6..2c3ca8c2 100644 --- a/image_analysis/indexing/FitProfileRadius.cpp +++ b/image_analysis/indexing/FitProfileRadius.cpp @@ -1,9 +1,6 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - #include "FitProfileRadius.h" #include // std::nth_element #include // std::fabs diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index d455129e..8d69f811 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -406,7 +406,7 @@ std::ostream &operator<<(std::ostream &output, const MergeStatistics &in) { output << std::endl; output << fmt::format(" {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}", "d_min", "N_obs", "N_uniq", "N_possib", "Compl","", "CC1/2", "CCref") - << std::endl;; + << std::endl; output << fmt::format(" {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s}", "", "", "", "", "", "", "", "") << std::endl; for (const auto &sh: in.shells) { diff --git a/image_analysis/scale_merge/ScalingResult.cpp b/image_analysis/scale_merge/ScalingResult.cpp index 318948b9..c01b88e5 100644 --- a/image_analysis/scale_merge/ScalingResult.cpp +++ b/image_analysis/scale_merge/ScalingResult.cpp @@ -36,8 +36,8 @@ void ScalingResult::SaveToFile(const std::string &filename) { const std::string img_path = filename + "_image.dat"; std::ofstream img_file(img_path, std::ofstream::out | std::ofstream::trunc); if (!img_file) { - throw JFJochException(JFJochExceptionCategory::FileWriteError - , "Cannot open {} for writing"); + throw JFJochException(JFJochExceptionCategory::FileWriteError, + "Cannot open " + img_path + " for writing"); } for (size_t i = 0; i < image_scale_g.size(); ++i) { diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index ce997918..cefbba4e 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -498,8 +498,14 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) { det_distance = 0.1; // Set to 100 mm, if det distance is less than 1 mm dataset->experiment.DetectorDistance_mm(det_distance * 1000.0); - dataset->experiment.IncidentEnergy_keV( - WVL_1A_IN_KEV / master_file->GetFloat("/entry/instrument/beam/incident_wavelength")); + const float incident_wavelength_A = master_file->GetFloat("/entry/instrument/beam/incident_wavelength"); + dataset->experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / incident_wavelength_A); + + // NXmx incident_wavelength_spread is the absolute FWHM (Angstrom); store it + // as the relative bandwidth FWHM (dlambda/lambda) used internally. + if (const auto spread = master_file->GetOptFloat("/entry/instrument/beam/incident_wavelength_spread")) + if (incident_wavelength_A > 0.0f) + dataset->experiment.BandwidthFWHM(spread.value() / incident_wavelength_A); dataset->error_value = master_file->GetOptInt("/entry/instrument/detector/error_value"); -- 2.52.0 From 698a98be359b293aa9dfe220afb124309b578b28 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 20:06:53 +0200 Subject: [PATCH 010/228] PixelRefine: Claude fixed my bugs --- .../pixel_refinement/PixelRefine.cpp | 222 ++++++++++++++---- image_analysis/pixel_refinement/PixelRefine.h | 19 ++ 2 files changed, 193 insertions(+), 48 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index f2e4f93c..8a59a64c 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -319,6 +319,56 @@ PixelRefine::PixelRefine(const DiffractionExperiment &experiment, reference_data[hkl_key_generator(ref)] = ref.I; } +void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, + double beam[2], double &dist_mm, + double detector_rot[2], double rot_vec[3], + double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const { + beam[0] = data.geom.GetBeamX_pxl(); + beam[1] = data.geom.GetBeamY_pxl(); + dist_mm = data.geom.GetDetectorDistance_mm(); + detector_rot[0] = data.geom.GetPoniRot1_rad(); + detector_rot[1] = data.geom.GetPoniRot2_rad(); + rot_vec[0] = 1.0; rot_vec[1] = 0.0; rot_vec[2] = 0.0; + if (auto axis = data.geom.GetRotation()) { + rot_vec[0] = axis->GetAxis().x; + rot_vec[1] = axis->GetAxis().y; + rot_vec[2] = axis->GetAxis().z; + } + + for (int i = 0; i < 3; ++i) + latt_vec0[i] = latt_vec1[i] = latt_vec2[i] = 0.0; + + double beta = data.latt.GetUnitCell().beta; + switch (data.crystal_system) { + case gemmi::CrystalSystem::Orthorhombic: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + break; + case gemmi::CrystalSystem::Tetragonal: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; + break; + case gemmi::CrystalSystem::Cubic: + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; + break; + case gemmi::CrystalSystem::Hexagonal: + LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); + break; + case gemmi::CrystalSystem::Monoclinic: + LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); + latt_vec2[0] = beta; + break; + default: { + LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); + const auto uc = data.latt.GetUnitCell(); + latt_vec2[0] = uc.alpha * M_PI / 180.0; + latt_vec2[1] = uc.beta * M_PI / 180.0; + latt_vec2[2] = uc.gamma * M_PI / 180.0; + break; + } + } +} + template void PixelRefine::Run(const T *image, const AzimuthalIntegrationProfile &profile, @@ -340,7 +390,10 @@ void PixelRefine::Run(const T *image, const auto azim_std = profile.GetStd(); const auto &pixel_to_bin = mapping.GetPixelToBin(); const auto &corrections = mapping.Corrections(); - const int azim_bin_count = mapping.GetAzimuthalBinCount(); + // pixel_to_bin stores the *full* bin index (azimuthal_sector * q_bins + q_bin), + // so the valid range is the total number of bins, i.e. the profile size - NOT + // GetAzimuthalBinCount() (which is only the number of azimuthal sectors). + const int total_bin_count = static_cast(azim_result.size()); const double angle_rad = data.angle_deg * M_PI / 180.0; const int radius = data.shoebox_radius; @@ -392,11 +445,15 @@ void PixelRefine::Run(const T *image, .PoniRot1_rad(data.geom.GetPoniRot1_rad()) .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - prediction.Calc(exp_iter, data.latt, settings_prediction); + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); // ---- 2. Collect per-reflection shoebox pixels ------------------------- + // GetReflections() returns the full pre-sized buffer; only the first + // nrefl entries are valid for this image (the rest are stale/zeroed). groups.clear(); - for (const auto &refl : prediction.GetReflections()) { + const auto &predicted = prediction.GetReflections(); + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; const auto hkl = hkl_key_generator(refl); if (!reference_data.contains(hkl)) continue; @@ -421,10 +478,10 @@ void PixelRefine::Run(const T *image, const size_t npixel = xpixel * y + x; const int azim_bin = pixel_to_bin[npixel]; - // Skip pixels not mapped to an azimuthal bin or carrying a - // sentinel (masked / saturated) value. We assume the pixel - // mask is already applied upstream. - if (azim_bin >= azim_bin_count) + // Skip pixels not mapped to a bin or carrying a sentinel + // (masked / saturated) value. We assume the pixel mask is + // already applied upstream. + if (azim_bin >= total_bin_count) continue; if (image[npixel] == std::numeric_limits::max()) continue; @@ -464,47 +521,8 @@ void PixelRefine::Run(const T *image, return; // ---- 3. Set up parameter blocks (geometry part mirrors XtalOptimizer) - - beam[0] = data.geom.GetBeamX_pxl(); - beam[1] = data.geom.GetBeamY_pxl(); - dist_mm = data.geom.GetDetectorDistance_mm(); - detector_rot[0] = data.geom.GetPoniRot1_rad(); - detector_rot[1] = data.geom.GetPoniRot2_rad(); - rot_vec[0] = 1.0; rot_vec[1] = 0.0; rot_vec[2] = 0.0; - if (auto axis = data.geom.GetRotation()) { - rot_vec[0] = axis->GetAxis().x; - rot_vec[1] = axis->GetAxis().y; - rot_vec[2] = axis->GetAxis().z; - } - - double beta = data.latt.GetUnitCell().beta; - switch (data.crystal_system) { - case gemmi::CrystalSystem::Orthorhombic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Tetragonal: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; - break; - case gemmi::CrystalSystem::Cubic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; - break; - case gemmi::CrystalSystem::Hexagonal: - LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Monoclinic: - LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); - latt_vec2[0] = beta; - break; - default: { - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - const auto uc = data.latt.GetUnitCell(); - latt_vec2[0] = uc.alpha * M_PI / 180.0; - latt_vec2[1] = uc.beta * M_PI / 180.0; - latt_vec2[2] = uc.gamma * M_PI / 180.0; - break; - } - } + BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2); // ---- 4. Build the problem --------------------------------------------- ceres::Problem problem; @@ -709,6 +727,114 @@ void PixelRefine::Run(const T *image, } } +std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, + const PixelRefineData &data, + bool include_background) const { + std::vector img(xpixel * ypixel, 0.0f); + + const double lambda = data.geom.GetWavelength_A(); + const double pixel_size = data.geom.GetPixelSize_mm(); + const auto azim_result = profile.GetResult(); + const auto &pixel_to_bin = mapping.GetPixelToBin(); + const auto &corrections = mapping.Corrections(); + const int total_bin_count = static_cast(azim_result.size()); + const double angle_rad = data.angle_deg * M_PI / 180.0; + const int radius = data.shoebox_radius; + const double bw = data.bandwidth; + + auto recip_area = [&](double x, double y) -> double { + const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); + const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); + return (qx % qy).Length(); + }; + auto bandwidth_radial_sq = [&](double d) -> double { + if (bw <= 0.0 || d <= 0.0) + return 0.0; + const double bl = bw * lambda; + return bl * bl / (2.0 * d * d * d * d); + }; + + // The model works in solid-angle/polarization-corrected units (as in Run, + // where Iobs = raw * correction). Map back to raw detector units (/ correction) + // so the predicted image overlays directly on the original image. + auto to_raw = [&](size_t npixel, double corrected) -> float { + const double corr = corrections[npixel]; + return (corr > 0.0) ? static_cast(corrected / corr) : 0.0f; + }; + + // Background base layer (per-pixel azimuthal mean), full-frame pass. + if (include_background) { + for (size_t p = 0; p < img.size(); ++p) { + const int bin = pixel_to_bin[p]; + if (bin >= 0 && bin < total_bin_count) + img[p] = to_raw(p, azim_result[bin]); + } + } + + double beam[2], dist_mm, detector_rot[2], rot_vec[3]; + double latt_vec0[3], latt_vec1[3], latt_vec2[3]; + BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2); + + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .max_hkl = 100, + .centering = data.centering + }; + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + const auto &predicted = prediction.GetReflections(); + + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; + const auto it = reference_data.find(hkl_key_generator(refl)); + if (it == reference_data.end()) + continue; + + const double Itrue = it->second; + const double R_bw_sq = bandwidth_radial_sq(refl.d); + + const int min_y = std::max(refl.predicted_y - radius, 0); + const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); + const int min_x = std::max(refl.predicted_x - radius, 0); + const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + + for (int y = min_y; y <= max_y; ++y) { + for (int x = min_x; x <= max_x; ++x) { + const size_t npixel = xpixel * y + x; + + // Pure Bragg signal: Ibkg = 0 so Model() returns signal only; the + // background is already laid down above. Same code path as Run. + PixelObs obs{ + .x = static_cast(x), + .y = static_cast(y), + .Iobs = 0.0, + .Ibkg = 0.0, + .weight = 1.0, + .A_recip = recip_area(x, y), + .angle_rad = angle_rad + }; + PixelResidual pr(obs, Itrue, lambda, pixel_size, + refl.h, refl.k, refl.l, R_bw_sq, data.crystal_system); + + double signal = 0.0; + if (pr.Model(beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R, signal)) + img[npixel] += to_raw(npixel, signal); + } + } + } + + return img; +} + // Explicit instantiations for the supported (uncompressed) image pixel types. template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index fec77c46..ae805f4e 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -148,6 +148,14 @@ class PixelRefine { const HKLKeyGenerator hkl_key_generator; std::map reference_data; + + // Fills the Ceres parameter blocks (geometry + symmetry-aware lattice + // parametrization) from the current model state. Shared by Run and + // PredictImage so both walk identical geometry/lattice code. + void BuildParameterBlocks(const PixelRefineData &data, + double beam[2], double &dist_mm, + double detector_rot[2], double rot_vec[3], + double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; public: PixelRefine(const DiffractionExperiment &experiment, const AzimuthalIntegrationMapping &mapping, @@ -161,4 +169,15 @@ public: const AzimuthalIntegrationProfile &profile, BraggPrediction &prediction, PixelRefineData &data); + + // Render the forward model as a full detector image (raw detector units, so + // it overlays directly on the original image). Uses the *same* per-pixel + // model path (PixelResidual::Model) as the optimizer, evaluated in double + // precision - slow but exact. For each reference reflection it adds the Bragg + // signal over its shoebox; with include_background it also lays down the + // azimuthal background. Diagnostic tool, not on the hot path. + std::vector PredictImage(const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, + const PixelRefineData &data, + bool include_background = true) const; }; -- 2.52.0 From 05711a107702faf387243c38848fead091235565 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 21:22:19 +0200 Subject: [PATCH 011/228] jfjoch_viewer: Add pixel refinw and magnifier windows (to be tested) --- .../pixel_refinement/PixelRefine.cpp | 33 ++-- viewer/CMakeLists.txt | 5 + viewer/JFJochImageReadingWorker.cpp | 172 +++++++++++++++- viewer/JFJochImageReadingWorker.h | 26 +++ viewer/JFJochViewerWindow.cpp | 37 ++++ viewer/image_viewer/JFJochImage.cpp | 21 ++ viewer/image_viewer/JFJochImage.h | 10 + viewer/windows/JFJochMagnifierWindow.cpp | 35 ++++ viewer/windows/JFJochMagnifierWindow.h | 28 +++ viewer/windows/JFJochPixelRefineWindow.cpp | 186 ++++++++++++++++++ viewer/windows/JFJochPixelRefineWindow.h | 73 +++++++ viewer/windows/PixelRefineParams.h | 33 ++++ 12 files changed, 646 insertions(+), 13 deletions(-) create mode 100644 viewer/windows/JFJochMagnifierWindow.cpp create mode 100644 viewer/windows/JFJochMagnifierWindow.h create mode 100644 viewer/windows/JFJochPixelRefineWindow.cpp create mode 100644 viewer/windows/JFJochPixelRefineWindow.h create mode 100644 viewer/windows/PixelRefineParams.h diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 8a59a64c..cf64c993 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -436,6 +436,7 @@ void PixelRefine::Run(const T *image, double latt_vec1[3] = {0, 0, 0}; // lengths double latt_vec2[3] = {0, 0, 0}; // angles (rad) + const bool eval_only = (data.max_iterations <= 0); const int n_iter = std::max(1, data.max_iterations); for (int iter = 0; iter < n_iter; ++iter) { // ---- 1. Re-sync prediction geometry from the (refined) model ---------- @@ -595,19 +596,29 @@ void PixelRefine::Run(const T *image, problem.SetParameterBlockConstant(data.R); } - // ---- 6. Solve --------------------------------------------------------- - ceres::Solver::Options options; - options.linear_solver_type = ceres::DENSE_QR; - options.minimizer_progress_to_stdout = false; - options.logging_type = ceres::LoggingType::SILENT; - options.max_solver_time_in_seconds = data.max_time_s; - options.num_threads = 1; + // ---- 6. Solve (or, for max_iterations<=0, just evaluate the cost) ----- + // Evaluate-only is the live-residual path: it reports the current cost + // without moving any parameter, so a UI can show how good the present + // R0/R1/bandwidth/geometry are as the user drags sliders. + if (eval_only) { + double cost = 0.0; + problem.Evaluate(ceres::Problem::EvaluateOptions(), &cost, nullptr, nullptr, nullptr); + data.final_cost = cost; + data.solved = true; + } else { + ceres::Solver::Options options; + options.linear_solver_type = ceres::DENSE_QR; + options.minimizer_progress_to_stdout = false; + options.logging_type = ceres::LoggingType::SILENT; + options.max_solver_time_in_seconds = data.max_time_s; + options.num_threads = 1; - ceres::Solver::Summary summary; - ceres::Solve(options, &problem, &summary); + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); - data.final_cost = summary.final_cost; - data.solved = summary.IsSolutionUsable(); + data.final_cost = summary.final_cost; + data.solved = summary.IsSolutionUsable(); + } // ---- 7. Write refined geometry + lattice back into data --------------- if (data.refine_beam_center) diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 0db42e76..70981081 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -86,6 +86,11 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView ${APP_RESOURCES} windows/JFJochViewerReciprocalSpaceWindow.cpp windows/JFJochViewerReciprocalSpaceWindow.h + windows/JFJochPixelRefineWindow.cpp + windows/JFJochPixelRefineWindow.h + windows/JFJochMagnifierWindow.cpp + windows/JFJochMagnifierWindow.h + windows/PixelRefineParams.h ) TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::DBus Qt6::Concurrent diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 6d7cf8c7..1ad60bd0 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -8,6 +8,8 @@ #include #include "JFJochImageReadingWorker.h" +#include "../image_analysis/LoadFCalcFromMtz.h" +#include "../image_analysis/bragg_prediction/BraggPredictionFactory.h" #include "../image_analysis/geom_refinement/AssignSpotsToRings.h" #include "../image_analysis/spot_finding/StrongPixelSet.h" #include "../image_analysis/spot_finding/SpotUtils.h" @@ -66,6 +68,7 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se : QObject(parent), indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { + qRegisterMetaType("PixelRefineParams"); spot_finding_settings = settings;; indexing = std::make_unique(indexing_settings); @@ -300,6 +303,14 @@ void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) index_and_refine = std::make_unique(curr_experiment, indexing.get()); image_analysis = std::make_unique(curr_experiment, *azint_mapping, dataset->pixel_mask, *index_and_refine.get()); + + // PixelRefine state is tied to the experiment/mapping; rebuild lazily. + pixel_refine_.reset(); + pixel_pred_.reset(); + last_profile_.reset(); + // Keep scale-on-the-fly alive across dataset reloads. + if (!pixel_reference_.empty()) + index_and_refine->ReferenceIntensities(pixel_reference_); } } @@ -444,8 +455,10 @@ void JFJochImageReadingWorker::ReanalyzeImage_i() { new_image_dataset->azimuthal_bins = azint_mapping->GetAzimuthalBinCount(); new_image_dataset->q_bins = azint_mapping->GetQBinCount(); - AzimuthalIntegrationProfile azint_profile(*azint_mapping); - image_analysis->Analyze(new_image->ImageData(),azint_profile, spot_finding_settings); + // Retain the profile (PixelRefine needs it). AzimuthalIntegrationProfile holds + // a mutex (non-copyable), so keep it via unique_ptr re-created each analysis. + last_profile_ = std::make_unique(*azint_mapping); + image_analysis->Analyze(new_image->ImageData(), *last_profile_, spot_finding_settings); current_image_ptr = new_image; auto end_time = std::chrono::high_resolution_clock::now(); @@ -705,3 +718,158 @@ void JFJochImageReadingWorker::LoadSpots(int64_t start_image, int64_t end_image, result = file_reader.ReadAllSpots(start_image, end_image, stride); emit spotsLoaded(result); } + +// --------------------------------------------------------------------------- +// Experimental PixelRefine +// --------------------------------------------------------------------------- +void JFJochImageReadingWorker::EnsurePixelRefine_i() { + if (!pixel_refine_ && azint_mapping && !pixel_reference_.empty()) + pixel_refine_ = std::make_unique(curr_experiment, *azint_mapping, pixel_reference_); + if (!pixel_pred_) + pixel_pred_ = CreateBraggPrediction(curr_experiment.IsRotationIndexing()); +} + +bool JFJochImageReadingWorker::BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p) const { + if (!current_image_ptr || !index_and_refine || !current_image.has_value()) + return false; + + const auto &outcomes = index_and_refine->GetIntegrationOutcome(); + const int64_t n = current_image.value(); + if (n < 0 || n >= static_cast(outcomes.size())) + return false; + + const auto &io = outcomes[n]; + if (io.reflections.empty()) // image not indexed/integrated + return false; + + d.geom = io.geom; + d.latt = io.latt; + + const auto < = current_image_ptr->ImageData().lattice_type; + if (lt) { + d.crystal_system = lt->crystal_system; + d.centering = lt->centering; + } + if (d.crystal_system == gemmi::CrystalSystem::Trigonal) + d.crystal_system = gemmi::CrystalSystem::Hexagonal; + + d.R[0] = p.R0; + d.R[1] = p.R1; + d.bandwidth = p.bandwidth_fwhm / 2.3548; // FWHM -> sigma + d.scale_factor = p.scale_factor; + d.B_factor = p.B_factor; + if (std::isfinite(p.beam_x) && std::isfinite(p.beam_y)) + d.geom.BeamX_pxl(static_cast(p.beam_x)).BeamY_pxl(static_cast(p.beam_y)); + + d.refine_orientation = p.refine_orientation; + d.refine_unit_cell = p.refine_unit_cell; + d.refine_beam_center = p.refine_beam_center; + d.refine_scale = p.refine_scale; + d.refine_B = p.refine_B; + d.refine_R = p.refine_R; + d.max_iterations = p.max_iterations; + return true; +} + +std::shared_ptr JFJochImageReadingWorker::WrapFloatImage_i(const std::vector &img) const { + auto si = std::make_shared(); + si->image = CompressedImage(img, curr_experiment.GetXPixelsNum(), curr_experiment.GetYPixelsNum()); + return si; +} + +void JFJochImageReadingWorker::LoadReference(QString path) { + QMutexLocker ul(&m); + try { + pixel_reference_ = LoadFCalcFromMtz(path.toStdString()); + if (index_and_refine) + index_and_refine->ReferenceIntensities(pixel_reference_); // enables scale-on-the-fly too + pixel_refine_.reset(); // rebuild with new reference + emit pixelRefineStatus(QString("Loaded %1 reference reflections").arg(pixel_reference_.size())); + } catch (const std::exception &e) { + emit pixelRefineStatus(QString("Failed to load reference: %1").arg(e.what())); + } +} + +void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { + QMutexLocker ul(&m); + if (!last_profile_) { + emit pixelRefineStatus("Analyze an image first"); + return; + } + EnsurePixelRefine_i(); + if (!pixel_refine_) { + emit pixelRefineStatus("Load reference data first"); + return; + } + + PixelRefineData d; + if (!BuildPixelSeed_i(d, params)) { + emit pixelRefineStatus("Current image is not indexed"); + return; + } + + // Preview = evaluate only: do not move any parameter. + d.refine_orientation = d.refine_unit_cell = d.refine_beam_center = false; + d.refine_scale = d.refine_B = d.refine_R = false; + d.max_iterations = 0; + + try { + const auto &img32 = current_image_ptr->Image(); + pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); + emit pixelRefineResidual(d.final_cost, static_cast(d.reflections.size())); + + auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); + emit predictedImageReady(WrapFloatImage_i(pred)); + } catch (const std::exception &e) { + emit pixelRefineStatus(QString("PixelRefine preview failed: %1").arg(e.what())); + } +} + +void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { + QMutexLocker ul(&m); + if (!last_profile_) { + emit pixelRefineStatus("Analyze an image first"); + return; + } + EnsurePixelRefine_i(); + if (!pixel_refine_) { + emit pixelRefineStatus("Load reference data first"); + return; + } + + PixelRefineData d; + if (!BuildPixelSeed_i(d, params)) { + emit pixelRefineStatus("Current image is not indexed"); + return; + } + if (d.max_iterations <= 0) + d.max_iterations = 3; + + try { + const auto &img32 = current_image_ptr->Image(); + pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); + + // Push refined values back so the sliders follow the optimizer. + PixelRefineParams out = params; + out.R0 = d.R[0]; + out.R1 = d.R[1]; + out.bandwidth_fwhm = d.bandwidth * 2.3548; // sigma -> FWHM + out.scale_factor = d.scale_factor; + out.B_factor = d.B_factor; + out.beam_x = d.geom.GetBeamX_pxl(); + out.beam_y = d.geom.GetBeamY_pxl(); + emit pixelRefineParamsRefined(out); + emit pixelRefineResidual(d.final_cost, static_cast(d.reflections.size())); + + auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); + emit predictedImageReady(WrapFloatImage_i(pred)); + + // Show the refined predictions on the main image too. + auto new_image = std::make_shared(*current_image_ptr); + new_image->ImageData().reflections = d.reflections; + current_image_ptr = new_image; + emit imageLoaded(current_image_ptr); + } catch (const std::exception &e) { + emit pixelRefineStatus(QString("PixelRefine failed: %1").arg(e.what())); + } +} diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index 4d419f9d..6ab9e429 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -15,7 +15,10 @@ #include "../common/Logger.h" #include "../reader/JFJochHttpReader.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" +#include "../image_analysis/pixel_refinement/PixelRefine.h" +#include "../image_analysis/bragg_prediction/BraggPrediction.h" #include "SimpleImage.h" +#include "windows/PixelRefineParams.h" #include "../common/MovingAverage.h" Q_DECLARE_METATYPE(std::shared_ptr) @@ -57,6 +60,18 @@ private: std::unique_ptr image_analysis; std::unique_ptr index_and_refine; + // Experimental PixelRefine support. last_profile_ keeps the azimuthal profile + // of the most recently analyzed image (PixelRefine needs it); the engine and a + // dedicated prediction buffer are built lazily once a reference is loaded. + std::unique_ptr last_profile_; + std::vector pixel_reference_; + std::unique_ptr pixel_refine_; + std::unique_ptr pixel_pred_; + + void EnsurePixelRefine_i(); + bool BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p) const; + std::shared_ptr WrapFloatImage_i(const std::vector &img) const; + std::unique_ptr roi; SpotFindingSettings spot_finding_settings; @@ -117,6 +132,12 @@ signals: void fileLoadError(QString title, QString message); void fileLoadRetryStatus(bool active, QString message); + // PixelRefine (experimental) + void predictedImageReady(std::shared_ptr image); + void pixelRefineResidual(double cost, int64_t n_reflections); + void pixelRefineParamsRefined(PixelRefineParams params); + void pixelRefineStatus(QString message); + public: JFJochImageReadingWorker(const SpotFindingSettings &settings, const DiffractionExperiment& experiment, QObject *parent = nullptr); ~JFJochImageReadingWorker() override = default; @@ -153,4 +174,9 @@ public slots: void LoadCalibration(QString dataset); void setAutoLoadMode(AutoloadMode mode); void setAutoLoadJump(int64_t val); + + // PixelRefine (experimental) + void LoadReference(QString path); + void PixelRefinePreview(PixelRefineParams params); + void PixelRefineRun(PixelRefineParams params); }; diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index e43fcc76..de383008 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -25,6 +25,9 @@ #include "toolbar/JFJochViewerToolbarImage.h" #include "windows/JFJoch2DAzintImageWindow.h" #include "windows/JFJochAzIntWindow.h" +#include "windows/JFJochPixelRefineWindow.h" +#include "windows/JFJochMagnifierWindow.h" +#include "image_viewer/JFJochImage.h" #include JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString &file) : QMainWindow(parent) { @@ -103,6 +106,8 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString auto azintWindow = new JFJochAzIntWindow(experiment.GetAzimuthalIntegrationSettings(), this); auto azintImageWindow = new JFJoch2DAzintImageWindow(this); + auto pixelRefineWindow = new JFJochPixelRefineWindow(this); + auto magnifierWindow = new JFJochMagnifierWindow(this); menuBar->AddWindowEntry(tableWindow, "Image list"); menuBar->AddWindowEntry(spotWindow, "Spot list"); @@ -113,6 +118,8 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString menuBar->AddWindowEntry(reciprocalWindow, "Reciprocal space viewer"); menuBar->AddWindowEntry(azintWindow, "Azimuthal integration settings"); menuBar->AddWindowEntry(azintImageWindow, "Azimuthal integration 2D image"); + menuBar->AddWindowEntry(pixelRefineWindow, "PixelRefine (experimental)"); + menuBar->AddWindowEntry(magnifierWindow, "Magnifier"); if (dbus) { // Create adaptor attached to this window @@ -333,6 +340,36 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString connect(azintImageWindow, &JFJoch2DAzintImageWindow::zoomOnBin, viewer, &JFJochDiffractionImage::centerOnSpot); + // --- PixelRefine (experimental) --- + connect(reading_worker, &JFJochImageReadingWorker::imageLoaded, + pixelRefineWindow, &JFJochHelperWindow::imageLoaded); + connect(pixelRefineWindow, &JFJochPixelRefineWindow::paramsChanged, + reading_worker, &JFJochImageReadingWorker::PixelRefinePreview); + connect(pixelRefineWindow, &JFJochPixelRefineWindow::refineRequested, + reading_worker, &JFJochImageReadingWorker::PixelRefineRun); + connect(pixelRefineWindow, &JFJochPixelRefineWindow::loadReferenceRequested, + reading_worker, &JFJochImageReadingWorker::LoadReference); + connect(reading_worker, &JFJochImageReadingWorker::predictedImageReady, + pixelRefineWindow, &JFJochPixelRefineWindow::setPredictedImage); + connect(reading_worker, &JFJochImageReadingWorker::pixelRefineResidual, + pixelRefineWindow, &JFJochPixelRefineWindow::setResidual); + connect(reading_worker, &JFJochImageReadingWorker::pixelRefineParamsRefined, + pixelRefineWindow, &JFJochPixelRefineWindow::setRefinedParams); + connect(reading_worker, &JFJochImageReadingWorker::pixelRefineStatus, + pixelRefineWindow, &JFJochPixelRefineWindow::setStatus); + + // Lock the predicted-image viewport to the original image (both directions). + connect(viewer, &JFJochImage::viewportChanged, + pixelRefineWindow->imageView(), &JFJochImage::applyViewport); + connect(pixelRefineWindow->imageView(), &JFJochImage::viewportChanged, + viewer, &JFJochImage::applyViewport); + + // --- Magnifier --- + connect(reading_worker, &JFJochImageReadingWorker::imageLoaded, + magnifierWindow, &JFJochHelperWindow::imageLoaded); + connect(viewer, &JFJochImage::hoverScenePos, + magnifierWindow, &JFJochMagnifierWindow::centerAt); + // Ensure worker is deleted in its own thread when the thread stops connect(reading_thread, &QThread::finished, reading_worker, &QObject::deleteLater); diff --git a/viewer/image_viewer/JFJochImage.cpp b/viewer/image_viewer/JFJochImage.cpp index 7700dc4f..ed030ef0 100644 --- a/viewer/image_viewer/JFJochImage.cpp +++ b/viewer/image_viewer/JFJochImage.cpp @@ -119,6 +119,7 @@ void JFJochImage::wheelEvent(QWheelEvent *event) { translate(delta.x(), delta.y()); // Shift the view updateOverlay(); + emitViewportChanged(); } } @@ -188,6 +189,7 @@ void JFJochImage::mouseMoveEvent(QMouseEvent *event) { const QPointF scenePos = mapToScene(event->pos()); mouseHover(event); + emit hoverScenePos(scenePos); QPointF delta; switch (mouse_event_type) { @@ -199,6 +201,7 @@ void JFJochImage::mouseMoveEvent(QMouseEvent *event) { verticalScrollBar()->setValue(verticalScrollBar()->value() - viewDelta.y()); updateOverlay(); + emitViewportChanged(); break; } case MouseEventType::DrawingROI: @@ -679,6 +682,24 @@ void JFJochImage::centerOnSpot(QPointF point) { // If W or H = 0, then conditions are never satisfied if (point.x() >= 0 && point.x() < W && point.y() >= 0 && point.y() < H) centerOn(point); + emitViewportChanged(); +} + +void JFJochImage::emitViewportChanged() { + if (m_applyingViewport || !scene()) + return; + emit viewportChanged(transform(), mapToScene(viewport()->rect().center())); +} + +void JFJochImage::applyViewport(QTransform transform, QPointF center) { + if (m_applyingViewport || !scene()) + return; + m_applyingViewport = true; + setTransform(transform); + scale_factor = transform.m11(); + centerOn(center); + updateOverlay(); + m_applyingViewport = false; } void JFJochImage::writePixelLabels() { diff --git a/viewer/image_viewer/JFJochImage.h b/viewer/image_viewer/JFJochImage.h index 4e8b1ad8..593cda04 100644 --- a/viewer/image_viewer/JFJochImage.h +++ b/viewer/image_viewer/JFJochImage.h @@ -5,6 +5,8 @@ #include #include +#include +#include #include "../../common/ColorScale.h" #include "../../common/JFJochMessages.h" @@ -15,6 +17,11 @@ class JFJochImage : public QGraphicsView { bool m_adjustForegroundWithWheel = false; + // Viewport-lock support: guard prevents the emit<->apply ping-pong between + // two linked views, and the helper broadcasts the current transform+center. + bool m_applyingViewport = false; + void emitViewportChanged(); + void DrawROI(); virtual void addCustomOverlay(); void updateROI(); @@ -106,6 +113,8 @@ signals: void roiBoxUpdated(QRect box); void roiCircleUpdated(double x, double y, double radius); void roiCalculated(ROIMessage &output); + void viewportChanged(QTransform transform, QPointF center); + void hoverScenePos(QPointF scenePos); private slots: void onScroll(int value); public slots: @@ -118,6 +127,7 @@ public slots: void SetROICircle(double x, double y, double radius); void centerOnSpot(QPointF point); + void applyViewport(QTransform transform, QPointF center); void fitToView(); void adjustForeground(bool input); diff --git a/viewer/windows/JFJochMagnifierWindow.cpp b/viewer/windows/JFJochMagnifierWindow.cpp new file mode 100644 index 00000000..266a5056 --- /dev/null +++ b/viewer/windows/JFJochMagnifierWindow.cpp @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochMagnifierWindow.h" +#include "../image_viewer/JFJochSimpleImage.h" +#include "../SimpleImage.h" + +#include + +JFJochMagnifierWindow::JFJochMagnifierWindow(QWidget *parent) + : JFJochHelperWindow(parent) { + setWindowTitle("Magnifier"); + m_image = new JFJochSimpleImage(this); + setCentralWidget(m_image); + resize(320, 320); +} + +void JFJochMagnifierWindow::imageLoaded(std::shared_ptr image) { + if (!image) { + m_have_image = false; + m_image->setImage(nullptr); + return; + } + const auto &exp = image->Dataset().experiment; + auto si = std::make_shared(); + si->image = CompressedImage(image->Image(), exp.GetXPixelsNum(), exp.GetYPixelsNum()); + m_image->setImage(si); + m_have_image = true; +} + +void JFJochMagnifierWindow::centerAt(QPointF scenePos) { + if (!m_have_image || !isVisible()) + return; + m_image->applyViewport(QTransform::fromScale(m_magnification, m_magnification), scenePos); +} diff --git a/viewer/windows/JFJochMagnifierWindow.h b/viewer/windows/JFJochMagnifierWindow.h new file mode 100644 index 00000000..91ec2621 --- /dev/null +++ b/viewer/windows/JFJochMagnifierWindow.h @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include "JFJochHelperWindow.h" +#include + +class JFJochSimpleImage; + +// ADXV-style magnifier: a small window showing a high-zoom close-up of the main +// image that follows the cursor. Fed the original image (converted to a +// SimpleImage) and re-centered on each hover position. +class JFJochMagnifierWindow : public JFJochHelperWindow { + Q_OBJECT + + JFJochSimpleImage *m_image; + double m_magnification = 12.0; + bool m_have_image = false; + +public: + explicit JFJochMagnifierWindow(QWidget *parent = nullptr); + + void imageLoaded(std::shared_ptr image) override; + +public slots: + void centerAt(QPointF scenePos); +}; diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp new file mode 100644 index 00000000..364cf5df --- /dev/null +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochPixelRefineWindow.h" +#include "../image_viewer/JFJochSimpleImage.h" + +#include +#include +#include +#include +#include +#include +#include + +JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) + : JFJochHelperWindow(parent) { + setWindowTitle("PixelRefine (experimental)"); + + auto central = new QWidget(this); + setCentralWidget(central); + auto layout = new QHBoxLayout(central); + + // --- predicted image (left, expanding) --------------------------------- + m_image = new JFJochSimpleImage(this); + m_image->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding); + layout->addWidget(m_image, 1); + + // --- control panel (right) --------------------------------------------- + auto controls = new QWidget(this); + controls->setMinimumWidth(320); + auto controlsLayout = new QVBoxLayout(controls); + layout->addWidget(controls, 0); + + auto paramBox = new QGroupBox(tr("Model parameters"), this); + auto form = new QFormLayout(paramBox); + + m_R0 = new SliderPlusBox(1e-4, 0.05, 1e-4, 4, this); m_R0->setValue(0.005); + m_R1 = new SliderPlusBox(1e-4, 0.05, 1e-4, 4, this); m_R1->setValue(0.005); + m_bw = new SliderPlusBox(0.0, 0.05, 1e-4, 4, this); m_bw->setValue(0.0); + m_scale = new SliderPlusBox(1e-3, 1e4, 1e-3, 3, this, SliderPlusBox::Logarithmic); m_scale->setValue(1.0); + m_B = new SliderPlusBox(0.0, 200.0, 0.1, 1, this); m_B->setValue(0.0); + m_beamx = new SliderPlusBox(0.0, 5000.0, 0.5, 1, this); m_beamx->setValue(0.0); + m_beamy = new SliderPlusBox(0.0, 5000.0, 0.5, 1, this); m_beamy->setValue(0.0); + + form->addRow(tr("R0 radial [Å⁻¹]:"), m_R0); + form->addRow(tr("R1 tangential [Å⁻¹]:"), m_R1); + form->addRow(tr("Bandwidth FWHM (Δλ/λ):"), m_bw); + form->addRow(tr("Scale G:"), m_scale); + form->addRow(tr("B-factor [Ų]:"), m_B); + + m_overrideBeam = new QCheckBox(tr("Override beam centre"), this); + form->addRow(QString(), m_overrideBeam); + form->addRow(tr("Beam X [px]:"), m_beamx); + form->addRow(tr("Beam Y [px]:"), m_beamy); + m_beamx->setEnabled(false); + m_beamy->setEnabled(false); + + controlsLayout->addWidget(paramBox); + + // --- what "Refine" is allowed to move ---------------------------------- + auto refBox = new QGroupBox(tr("Refine (Ceres)"), this); + auto refLayout = new QVBoxLayout(refBox); + m_refOrientation = new QCheckBox(tr("Orientation"), this); m_refOrientation->setChecked(true); + m_refCell = new QCheckBox(tr("Unit cell"), this); + m_refBeam = new QCheckBox(tr("Beam centre"), this); + m_refScale = new QCheckBox(tr("Scale G"), this); m_refScale->setChecked(true); + m_refB = new QCheckBox(tr("B-factor"), this); + m_refR = new QCheckBox(tr("Widths R0/R1"), this); m_refR->setChecked(true); + for (auto *cb : {m_refOrientation, m_refCell, m_refBeam, m_refScale, m_refB, m_refR}) + refLayout->addWidget(cb); + controlsLayout->addWidget(refBox); + + // --- buttons + readouts ------------------------------------------------- + m_loadRef = new QPushButton(tr("Load reference MTZ…"), this); + m_refine = new QPushButton(tr("Refine"), this); + controlsLayout->addWidget(m_loadRef); + controlsLayout->addWidget(m_refine); + + m_residual = new QLabel(tr("Residual: —"), this); + m_status = new QLabel(QString(), this); + m_status->setWordWrap(true); + m_status->setStyleSheet("color: rgb(80, 80, 80);"); + controlsLayout->addWidget(m_residual); + controlsLayout->addWidget(m_status); + controlsLayout->addStretch(1); + + // --- debounce timer for live preview ----------------------------------- + m_debounce = new QTimer(this); + m_debounce->setSingleShot(true); + m_debounce->setInterval(150); + connect(m_debounce, &QTimer::timeout, this, [this] { + emit paramsChanged(currentParams()); + }); + + for (auto *s : {m_R0, m_R1, m_bw, m_scale, m_B, m_beamx, m_beamy}) + connect(s, &SliderPlusBox::valueChanged, this, [this](double) { onControlChanged(); }); + + connect(m_overrideBeam, &QCheckBox::toggled, this, [this](bool on) { + m_beamx->setEnabled(on); + m_beamy->setEnabled(on); + onControlChanged(); + }); + + connect(m_loadRef, &QPushButton::clicked, this, [this] { + const QString path = QFileDialog::getOpenFileName( + this, tr("Load reference MTZ"), QString(), tr("MTZ files (*.mtz);;All files (*)")); + if (!path.isEmpty()) + emit loadReferenceRequested(path); + }); + + connect(m_refine, &QPushButton::clicked, this, [this] { + PixelRefineParams p = currentParams(); + p.max_iterations = 5; + emit refineRequested(p); + }); +} + +PixelRefineParams JFJochPixelRefineWindow::currentParams() const { + PixelRefineParams p; + p.R0 = m_R0->value(); + p.R1 = m_R1->value(); + p.bandwidth_fwhm = m_bw->value(); + p.scale_factor = m_scale->value(); + p.B_factor = m_B->value(); + if (m_overrideBeam->isChecked()) { + p.beam_x = m_beamx->value(); + p.beam_y = m_beamy->value(); + } else { + p.beam_x = NAN; + p.beam_y = NAN; + } + p.refine_orientation = m_refOrientation->isChecked(); + p.refine_unit_cell = m_refCell->isChecked(); + p.refine_beam_center = m_refBeam->isChecked(); + p.refine_scale = m_refScale->isChecked(); + p.refine_B = m_refB->isChecked(); + p.refine_R = m_refR->isChecked(); + return p; +} + +void JFJochPixelRefineWindow::onControlChanged() { + if (m_suppress) + return; + m_debounce->start(); +} + +void JFJochPixelRefineWindow::imageLoaded(std::shared_ptr image) { + if (m_beamInit || !image) + return; + const auto geom = image->Dataset().experiment.GetDiffractionGeometry(); + m_beamx->setMax(static_cast(image->Dataset().experiment.GetXPixelsNum())); + m_beamy->setMax(static_cast(image->Dataset().experiment.GetYPixelsNum())); + m_suppress = true; + m_beamx->setValue(geom.GetBeamX_pxl()); + m_beamy->setValue(geom.GetBeamY_pxl()); + m_suppress = false; + m_beamInit = true; +} + +void JFJochPixelRefineWindow::setPredictedImage(std::shared_ptr image) { + m_image->setImage(std::move(image)); +} + +void JFJochPixelRefineWindow::setResidual(double cost, int64_t n_reflections) { + m_residual->setText(tr("Residual: %1 (%2 reflections)") + .arg(cost, 0, 'g', 6) + .arg(n_reflections)); +} + +void JFJochPixelRefineWindow::setRefinedParams(PixelRefineParams params) { + m_suppress = true; + m_R0->setValue(params.R0); + m_R1->setValue(params.R1); + m_bw->setValue(params.bandwidth_fwhm); + m_scale->setValue(params.scale_factor); + m_B->setValue(params.B_factor); + if (std::isfinite(params.beam_x) && std::isfinite(params.beam_y)) { + m_beamx->setValue(params.beam_x); + m_beamy->setValue(params.beam_y); + } + m_suppress = false; +} + +void JFJochPixelRefineWindow::setStatus(QString message) { + m_status->setText(message); +} diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h new file mode 100644 index 00000000..7e7f167c --- /dev/null +++ b/viewer/windows/JFJochPixelRefineWindow.h @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include "JFJochHelperWindow.h" +#include "PixelRefineParams.h" +#include "../SimpleImage.h" +#include "../widgets/SliderPlusBox.h" + +#include +#include +#include +#include +#include + +class JFJochSimpleImage; + +// Experimental PixelRefine control window: sliders for the forward-model +// parameters, a live (debounced) predicted-image preview, a residual readout, +// "Load reference" and "Refine" buttons. The predicted-image view is exposed so +// the main window can lock its viewport to the original image. +class JFJochPixelRefineWindow : public JFJochHelperWindow { + Q_OBJECT + + JFJochSimpleImage *m_image; + + SliderPlusBox *m_R0; + SliderPlusBox *m_R1; + SliderPlusBox *m_bw; + SliderPlusBox *m_scale; + SliderPlusBox *m_B; + SliderPlusBox *m_beamx; + SliderPlusBox *m_beamy; + + QCheckBox *m_overrideBeam; + QCheckBox *m_refOrientation; + QCheckBox *m_refCell; + QCheckBox *m_refBeam; + QCheckBox *m_refScale; + QCheckBox *m_refB; + QCheckBox *m_refR; + + QLabel *m_residual; + QLabel *m_status; + QPushButton *m_loadRef; + QPushButton *m_refine; + + QTimer *m_debounce; + bool m_suppress = false; // guard while pushing refined params into sliders + bool m_beamInit = false; // beam-centre sliders initialised from geometry + + PixelRefineParams currentParams() const; + void onControlChanged(); + +public: + explicit JFJochPixelRefineWindow(QWidget *parent = nullptr); + + JFJochSimpleImage *imageView() const { return m_image; } + + void imageLoaded(std::shared_ptr image) override; + +signals: + void paramsChanged(PixelRefineParams params); // debounced live preview + void refineRequested(PixelRefineParams params); // "Refine" button + void loadReferenceRequested(QString path); // "Load reference" button + +public slots: + void setPredictedImage(std::shared_ptr image); + void setResidual(double cost, int64_t n_reflections); + void setRefinedParams(PixelRefineParams params); + void setStatus(QString message); +}; diff --git a/viewer/windows/PixelRefineParams.h b/viewer/windows/PixelRefineParams.h new file mode 100644 index 00000000..4b266e62 --- /dev/null +++ b/viewer/windows/PixelRefineParams.h @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +// Parameters exchanged between the PixelRefine window (sliders/buttons) and the +// reading worker. bandwidth here is the *FWHM* of dlambda/lambda (user-facing); +// the worker converts it to the sigma that PixelRefineData expects. beam_x/beam_y +// are NaN to mean "keep the current refined geometry". +struct PixelRefineParams { + double R0 = 0.005; // radial / partiality width (A^-1) + double R1 = 0.005; // tangential / profile width (A^-1) + double bandwidth_fwhm = 0.0; // relative bandwidth FWHM (dlambda/lambda) + double scale_factor = 1.0; // overall scale G + double B_factor = 0.0; // Debye-Waller B (A^2) + double beam_x = NAN; // detector beam centre X (px); NaN = keep current + double beam_y = NAN; // detector beam centre Y (px); NaN = keep current + + // What the "Refine" button is allowed to move (ignored by the preview path). + bool refine_orientation = true; + bool refine_unit_cell = false; + bool refine_beam_center = false; + bool refine_scale = true; + bool refine_B = false; + bool refine_R = true; + + int max_iterations = 3; // <=0 means evaluate-only (preview / residual) +}; + +Q_DECLARE_METATYPE(PixelRefineParams) -- 2.52.0 From e8a9b1840d2a01a8aea55a5f09a5d35436643d22 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 8 Jun 2026 22:45:22 +0200 Subject: [PATCH 012/228] PixelRefine: Make it faster by doing one cell calculation per shoe-box --- .../pixel_refinement/PixelRefine.cpp | 412 +++++++++++------- 1 file changed, 260 insertions(+), 152 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index cf64c993..3b6ccdfe 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -40,6 +40,139 @@ double SafeInv(double x, double fallback) { return 1.0 / x; } +// Per-pixel: map a detector pixel through the current geometry into the +// reference reciprocal frame. Cheap (a few trig + one rotation); depends on the +// pixel and the detector geometry, not on the lattice. +template +void ObservedRecip(const T *beam, const T *distance_mm, const T *detector_rot, + const T *rotation_axis, double obs_x, double obs_y, + double pixel_size, double inv_lambda, double angle_rad, + Eigen::Matrix &e_obs_recip) { + // PyFAI convention (left-handed for rot1/rot2): rot3 = 0 assumed. + const T c1 = ceres::cos(detector_rot[0]); + const T s1 = ceres::sin(detector_rot[0]); + const T c2 = ceres::cos(detector_rot[1]); + const T s2 = ceres::sin(detector_rot[1]); + + const T det_x = (T(obs_x) - beam[0]) * T(pixel_size); + const T det_y = (T(obs_y) - beam[1]) * T(pixel_size); + const T det_z = T(distance_mm[0]); + + const T t1_x = c1 * det_x + s1 * det_z; + const T t1_y = det_y; + const T t1_z = -s1 * det_x + c1 * det_z; + + const T x = t1_x; + const T y = c2 * t1_y + s2 * t1_z; + const T z = -s2 * t1_y + c2 * t1_z; + + const T inv_norm = T(1) / ceres::sqrt(x * x + y * y + z * z); + + T recip_raw[3] = { + x * inv_norm * T(inv_lambda), + y * inv_norm * T(inv_lambda), + (z * inv_norm - T(1.0)) * T(inv_lambda) + }; + const T aa_back[3] = { + T(angle_rad) * rotation_axis[0], + T(angle_rad) * rotation_axis[1], + T(angle_rad) * rotation_axis[2] + }; + T recip_obs[3]; + ceres::AngleAxisRotatePoint(aa_back, recip_raw, recip_obs); + e_obs_recip = Eigen::Matrix(recip_obs[0], recip_obs[1], recip_obs[2]); +} + +// Per-reflection: predicted node g_hkl, |g_hkl|^2, and the Ewald-sphere normal. +// This is the expensive part (symmetry-aware B matrix, three rotations, cross +// products) - it depends only on the lattice (p0,p1,p2) and hkl, so for a whole +// shoebox it can be computed once. Convention identical to XtalOptimizer. +template +bool PredictedNode(const T *p0, const T *p1, const T *p2, + double exp_h, double exp_k, double exp_l, + gemmi::CrystalSystem symmetry, double inv_lambda, + Eigen::Matrix &e_pred_recip, + Eigen::Matrix &n_radial, T &q_sq) { + Eigen::Matrix e_uc_len = Eigen::Matrix::Zero(); + Eigen::Matrix Bmat = Eigen::Matrix::Identity(); + + if (symmetry == gemmi::CrystalSystem::Hexagonal) { + e_uc_len << p1[0], p1[0], p1[2]; + Bmat(0, 1) = T(-0.5); + Bmat(1, 1) = T(sqrt(3.0) / 2.0); + } else if (symmetry == gemmi::CrystalSystem::Orthorhombic) { + e_uc_len << p1[0], p1[1], p1[2]; + } else if (symmetry == gemmi::CrystalSystem::Tetragonal) { + e_uc_len << p1[0], p1[0], p1[2]; + } else if (symmetry == gemmi::CrystalSystem::Cubic) { + e_uc_len << p1[0], p1[0], p1[0]; + } else if (symmetry == gemmi::CrystalSystem::Monoclinic) { + e_uc_len << p1[0], p1[1], p1[2]; + Bmat(0, 2) = ceres::cos(p2[0]); + Bmat(2, 2) = ceres::sin(p2[0]); + } else { + const T ca = ceres::cos(p2[0]); + const T cb = ceres::cos(p2[1]); + const T cg = ceres::cos(p2[2]); + const T sg = ceres::sin(p2[2]); + + e_uc_len << p1[0], p1[1], p1[2]; + + Bmat(0, 1) = cg; + Bmat(1, 1) = sg; + + const T cx = cb; + const T cy = (ca - cb * cg) / sg; + const T v = T(1) - cx * cx - cy * cy; + const T cz = (v >= T(0)) ? ceres::sqrt(v) : T(0); + + Bmat(0, 2) = cx; + Bmat(1, 2) = cy; + Bmat(2, 2) = cz; + } + + const T L0 = e_uc_len[0]; + const T L1 = e_uc_len[1]; + const T L2 = e_uc_len[2]; + + T col0_unrot[3] = {Bmat(0, 0) * L0, Bmat(1, 0) * L0, Bmat(2, 0) * L0}; + T col1_unrot[3] = {Bmat(0, 1) * L1, Bmat(1, 1) * L1, Bmat(2, 1) * L1}; + T col2_unrot[3] = {Bmat(0, 2) * L2, Bmat(1, 2) * L2, Bmat(2, 2) * L2}; + + T col0_rot[3], col1_rot[3], col2_rot[3]; + ceres::AngleAxisRotatePoint(p0, col0_unrot, col0_rot); + ceres::AngleAxisRotatePoint(p0, col1_unrot, col1_rot); + ceres::AngleAxisRotatePoint(p0, col2_unrot, col2_rot); + + const Eigen::Matrix A(col0_rot[0], col0_rot[1], col0_rot[2]); + const Eigen::Matrix Bv(col1_rot[0], col1_rot[1], col1_rot[2]); + const Eigen::Matrix C(col2_rot[0], col2_rot[1], col2_rot[2]); + + const Eigen::Matrix BxC = Bv.cross(C); + const Eigen::Matrix CxA = C.cross(A); + const Eigen::Matrix AxB = A.cross(Bv); + + const T Vol = A.dot(BxC); + if (ceres::abs(Vol) < T(1e-12)) + return false; + const T invV = T(1) / Vol; + + e_pred_recip = (BxC * T(exp_h) + CxA * T(exp_k) + AxB * T(exp_l)) * invV; + q_sq = e_pred_recip.squaredNorm(); + + // Ewald sphere centre at -k_i = (0,0,-inv_lambda); radial normal at g_hkl. + const Eigen::Matrix S_pred( + e_pred_recip[0], + e_pred_recip[1], + e_pred_recip[2] + T(inv_lambda)); + const T S_pred_norm = S_pred.norm(); + if (S_pred_norm < T(1e-10)) + return false; + + n_radial = S_pred / S_pred_norm; + return true; +} + } // namespace // --------------------------------------------------------------------------- @@ -100,136 +233,18 @@ struct PixelResidual { const T *const p1, const T *const p2, T &q_sq, T &eps_radial, T &eps_tang_sq) const { - // PyFAI convention (left-handed for rot1/rot2): - // poni_rot = Rz(-rot3) * Rx(-rot2) * Ry(+rot1), rot3 = 0 assumed. - const T rot1 = detector_rot[0]; - const T rot2 = detector_rot[1]; + Eigen::Matrix e_obs_recip; + ObservedRecip(beam, distance_mm, detector_rot, rotation_axis, + obs_x, obs_y, pixel_size, inv_lambda, angle_rad, e_obs_recip); - const T c1 = ceres::cos(rot1); - const T s1 = ceres::sin(rot1); - const T c2 = ceres::cos(rot2); - const T s2 = ceres::sin(rot2); - - const T det_x = (T(obs_x) - beam[0]) * T(pixel_size); - const T det_y = (T(obs_y) - beam[1]) * T(pixel_size); - const T det_z = T(distance_mm[0]); - - const T t1_x = c1 * det_x + s1 * det_z; - const T t1_y = det_y; - const T t1_z = -s1 * det_x + c1 * det_z; - - const T x = t1_x; - const T y = c2 * t1_y + s2 * t1_z; - const T z = -s2 * t1_y + c2 * t1_z; - - const T lab_norm = ceres::sqrt(x * x + y * y + z * z); - const T inv_norm = T(1) / lab_norm; - - T recip_raw[3]; - recip_raw[0] = x * inv_norm * T(inv_lambda); - recip_raw[1] = y * inv_norm * T(inv_lambda); - recip_raw[2] = (z * inv_norm - T(1.0)) * T(inv_lambda); - - // Goniometer "back-to-start" rotation: image frame -> reference frame. - const T aa_back[3] = { - T(angle_rad) * rotation_axis[0], - T(angle_rad) * rotation_axis[1], - T(angle_rad) * rotation_axis[2] - }; - T recip_obs[3]; - ceres::AngleAxisRotatePoint(aa_back, recip_raw, recip_obs); - const Eigen::Matrix e_obs_recip(recip_obs[0], recip_obs[1], recip_obs[2]); - - // Build cell lengths and the (unit) B matrix from the symmetry-specific - // parametrization (identical convention to XtalOptimizer::XtalResidual). - Eigen::Matrix e_uc_len = Eigen::Matrix::Zero(); - Eigen::Matrix Bmat = Eigen::Matrix::Identity(); - - if (symmetry == gemmi::CrystalSystem::Hexagonal) { - e_uc_len << p1[0], p1[0], p1[2]; - Bmat(0, 1) = T(-0.5); - Bmat(1, 1) = T(sqrt(3.0) / 2.0); - } else if (symmetry == gemmi::CrystalSystem::Orthorhombic) { - e_uc_len << p1[0], p1[1], p1[2]; - } else if (symmetry == gemmi::CrystalSystem::Tetragonal) { - e_uc_len << p1[0], p1[0], p1[2]; - } else if (symmetry == gemmi::CrystalSystem::Cubic) { - e_uc_len << p1[0], p1[0], p1[0]; - } else if (symmetry == gemmi::CrystalSystem::Monoclinic) { - e_uc_len << p1[0], p1[1], p1[2]; - Bmat(0, 2) = ceres::cos(p2[0]); - Bmat(2, 2) = ceres::sin(p2[0]); - } else { - const T ca = ceres::cos(p2[0]); - const T cb = ceres::cos(p2[1]); - const T cg = ceres::cos(p2[2]); - const T sg = ceres::sin(p2[2]); - - e_uc_len << p1[0], p1[1], p1[2]; - - Bmat(0, 1) = cg; - Bmat(1, 1) = sg; - - const T cx = cb; - const T cy = (ca - cb * cg) / sg; - const T v = T(1) - cx * cx - cy * cy; - const T cz = (v >= T(0)) ? ceres::sqrt(v) : T(0); - - Bmat(0, 2) = cx; - Bmat(1, 2) = cy; - Bmat(2, 2) = cz; - } - - const T L0 = e_uc_len[0]; - const T L1 = e_uc_len[1]; - const T L2 = e_uc_len[2]; - - T col0_unrot[3] = {Bmat(0, 0) * L0, Bmat(1, 0) * L0, Bmat(2, 0) * L0}; - T col1_unrot[3] = {Bmat(0, 1) * L1, Bmat(1, 1) * L1, Bmat(2, 1) * L1}; - T col2_unrot[3] = {Bmat(0, 2) * L2, Bmat(1, 2) * L2, Bmat(2, 2) * L2}; - - T col0_rot[3], col1_rot[3], col2_rot[3]; - ceres::AngleAxisRotatePoint(p0, col0_unrot, col0_rot); - ceres::AngleAxisRotatePoint(p0, col1_unrot, col1_rot); - ceres::AngleAxisRotatePoint(p0, col2_unrot, col2_rot); - - const Eigen::Matrix A(col0_rot[0], col0_rot[1], col0_rot[2]); - const Eigen::Matrix Bv(col1_rot[0], col1_rot[1], col1_rot[2]); - const Eigen::Matrix C(col2_rot[0], col2_rot[1], col2_rot[2]); - - const Eigen::Matrix BxC = Bv.cross(C); - const Eigen::Matrix CxA = C.cross(A); - const Eigen::Matrix AxB = A.cross(Bv); - - const T V = A.dot(BxC); - if (ceres::abs(V) < T(1e-12)) - return false; - const T invV = T(1) / V; - - const Eigen::Matrix Astar = BxC * invV; - const Eigen::Matrix Bstar = CxA * invV; - const Eigen::Matrix Cstar = AxB * invV; - - const Eigen::Matrix e_pred_recip = - Astar * T(exp_h) + Bstar * T(exp_k) + Cstar * T(exp_l); - - q_sq = e_pred_recip.squaredNorm(); - - // Ewald sphere centre at -k_i = (0,0,-inv_lambda); radial normal at g_hkl. - const Eigen::Matrix S_pred( - e_pred_recip[0], - e_pred_recip[1], - e_pred_recip[2] + T(inv_lambda)); - const T S_pred_norm = S_pred.norm(); - if (S_pred_norm < T(1e-10)) + Eigen::Matrix e_pred_recip, n_radial; + if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, + e_pred_recip, n_radial, q_sq)) return false; - const Eigen::Matrix n_radial = S_pred / S_pred_norm; const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - eps_radial = delta_q.dot(n_radial); - const Eigen::Matrix dq_tang = delta_q - eps_radial * n_radial; - eps_tang_sq = dq_tang.squaredNorm(); + eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); return true; } @@ -250,27 +265,25 @@ struct PixelResidual { const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); - // Full 3D reciprocal-space spot density modelled as a separable Gaussian, - // normalized so that its integral is 1 (intensity-conserving): - // radial: g_r(e) = exp(-e^2/R0^2) / (sqrt(pi) R0) [1/A^-1] - // tangential: g_t(e) = exp(-|e|^2/R1^2) / (pi R1^2) [1/A^-2] - // The detector pixel captures the fraction g_t * A_recip of the tangential - // profile (A_recip = reciprocal area the pixel subtends; sum over shoebox - // ~ 1). The radial factor is the still-image partiality: how far this - // reflection sits from the Ewald sphere. + // Separable Gaussian spot model: + // radial P_r(e) = exp(-e^2/R0_eff^2) (peak-normalized, in (0,1]) + // tangent g_t(e) = exp(-|e|^2/R1^2) / (pi R1^2) [1/A^-2] + // The pixel captures the fraction g_t * A_recip of the tangential profile + // (A_recip = reciprocal area the pixel subtends; sum over shoebox ~ 1). + // The radial factor is the still-image partiality (how far the reflection + // sits from the Ewald sphere); the overall scale is carried by the free G. // - // Caveat: a still samples the radial direction at a single offset, so the - // sqrt(pi) R0 normalization makes g_r a density (1/A^-1) rather than a - // dimensionless fraction. The leftover dimensional factor is absorbed by - // the free scale G. The energy-bandwidth contribution to the radial width - // is folded in here via R_bw_sq (beam divergence is still TODO). + // IMPORTANT: the radial factor MUST use the same convention here as the + // extraction's `partiality` (peak-normalized), otherwise image_scale_corr + // = 1/(partiality*G*B) does not invert the model and a leftover, R0_eff- + // dependent (hence resolution-dependent) factor biases the intensities. + // R0_eff folds in the energy-bandwidth broadening via R_bw_sq. const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); - const T g_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq) - / (ceres::sqrt(T(M_PI)) * ceres::sqrt(R0_eff_sq)); + const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); const T P_tang = T(A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); - const T signal = scale_factor[0] * T(Itrue) * B_term * g_radial * P_tang; + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; Ipred = signal + T(Ibkg); return true; } @@ -306,6 +319,85 @@ struct PixelResidual { gemmi::CrystalSystem symmetry; }; +// --------------------------------------------------------------------------- +// Per-shoebox cost functor +// +// One residual block per reflection emitting N residuals (one per shoebox pixel). +// The expensive per-reflection geometry (PredictedNode: symmetry-aware B matrix, +// three rotations, cross products) is computed ONCE; only the cheap per-pixel +// ObservedRecip + Gaussian profile run in the pixel loop. This is identical in +// value to the old one-block-per-pixel formulation but ~(pixels-per-shoebox)x +// fewer evaluations of the costly node computation. Uses the same shared helpers +// (and hence the same conventions) as PixelResidual. +// --------------------------------------------------------------------------- +struct ShoeboxResidual { + ShoeboxResidual(const ReflGroup &g, double lambda, double pixel_size, + gemmi::CrystalSystem symmetry) + : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), + exp_h(g.h), exp_k(g.k), exp_l(g.l), + inv_lambda(1.0 / lambda), pixel_size(pixel_size), + angle_rad(g.pixels.empty() ? 0.0 : g.pixels.front().angle_rad), + symmetry(symmetry) {} + + template + bool operator()(const T *const *params, T *residual) const { + // Parameter blocks (order matches AddParameterBlock in Run): + // 0 beam[2] 1 distance[1] 2 detector_rot[2] 3 rotation_axis[3] + // 4 p0[3] 5 p1[3] 6 p2[3] 7 scale[1] 8 B[1] 9 R[2] + const T *beam = params[0]; + const T *distance_mm = params[1]; + const T *detector_rot = params[2]; + const T *rotation_axis = params[3]; + const T *p0 = params[4]; + const T *p1 = params[5]; + const T *p2 = params[6]; + const T *scale_factor = params[7]; + const T *B = params[8]; + const T *R = params[9]; + + if (R[0] < T(1e-10) || R[1] < T(1e-10)) + return false; + + // --- per-reflection: computed once --------------------------------- + Eigen::Matrix e_pred_recip, n_radial; + T q_sq; + if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, + e_pred_recip, n_radial, q_sq)) + return false; + + const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); + const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); + + // --- per-pixel loop ------------------------------------------------- + for (size_t i = 0; i < pixels.size(); ++i) { + const PixelObs &obs = pixels[i]; + + Eigen::Matrix e_obs_recip; + ObservedRecip(beam, distance_mm, detector_rot, rotation_axis, + obs.x, obs.y, pixel_size, inv_lambda, angle_rad, e_obs_recip); + + const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; + const T eps_radial = delta_q.dot(n_radial); + const T eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); + + const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); + const T P_tang = T(obs.A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) + / (T(M_PI) * R[1] * R[1]); + + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; + const T Ipred = signal + T(obs.Ibkg); + residual[i] = (Ipred - T(obs.Iobs)) * T(obs.weight); + } + return true; + } + + std::vector pixels; + const double Itrue, R_bw_sq; + const double exp_h, exp_k, exp_l; + const double inv_lambda, pixel_size, angle_rad; + gemmi::CrystalSystem symmetry; +}; + PixelRefine::PixelRefine(const DiffractionExperiment &experiment, const AzimuthalIntegrationMapping &mapping, const std::vector &reference) @@ -526,20 +618,36 @@ void PixelRefine::Run(const T *image, latt_vec0, latt_vec1, latt_vec2); // ---- 4. Build the problem --------------------------------------------- + // One residual block per shoebox (N residuals), so the expensive + // per-reflection node geometry is evaluated once per reflection instead + // of once per pixel. ceres::Problem problem; + size_t residual_pixels = 0; for (const auto &g : groups) { - for (const auto &obs : g.pixels) { - auto *cost = new ceres::AutoDiffCostFunction< - PixelResidual, 1, 2, 1, 2, 3, 3, 3, 3, 1, 1, 2>( - new PixelResidual(obs, g.Itrue, lambda, pixel_size, - g.h, g.k, g.l, g.R_bw_sq, data.crystal_system)); - problem.AddResidualBlock(cost, new ceres::HuberLoss(3.0), - beam, &dist_mm, detector_rot, rot_vec, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R); - } + auto *cost = new ceres::DynamicAutoDiffCostFunction( + new ShoeboxResidual(g, lambda, pixel_size, data.crystal_system)); + cost->AddParameterBlock(2); // beam + cost->AddParameterBlock(1); // distance + cost->AddParameterBlock(2); // detector_rot + cost->AddParameterBlock(3); // rotation_axis + cost->AddParameterBlock(3); // p0 (orientation) + cost->AddParameterBlock(3); // p1 (lengths) + cost->AddParameterBlock(3); // p2 (angles) + cost->AddParameterBlock(1); // scale G + cost->AddParameterBlock(1); // B + cost->AddParameterBlock(2); // R + cost->SetNumResiduals(static_cast(g.pixels.size())); + // No robust loss here: a per-block (whole-shoebox) Huber would act on + // the sum of ~N squared residuals and mis-scale, unlike the previous + // per-pixel Huber. Per-pixel sigma weighting is retained; per-pixel + // outlier rejection (zingers) is a TODO if needed. + problem.AddResidualBlock(cost, nullptr, + beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R); + residual_pixels += g.pixels.size(); } - data.residual_count = problem.NumResidualBlocks(); + data.residual_count = residual_pixels; // ---- 5. Constrain / bound parameter blocks ---------------------------- if (!data.refine_orientation) -- 2.52.0 From 254462b9f202a7c61f7e06ce102416a6c0aa4d13 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 07:28:13 +0200 Subject: [PATCH 013/228] jfjoch_process: Clarify what happens in scaling when reference data provided --- tools/jfjoch_process.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index b01c05b1..47cd0736 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -932,10 +932,17 @@ int main(int argc, char **argv) { if (end_msg.indexing_rate.has_value() && end_msg.indexing_rate > 0 && (run_scaling || !reference_data.empty())) { - logger.Info("Running scaling (mosaicity refinement) ..."); - if (reference_data.empty()) { - // If reference data are given, there is live scaling (no need to go again) + const bool pixel_refine_path = + (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine); + + // Scaling is only the classical, no-reference ScaleOnTheFly post-pass. + // - With a reference (classical path): per-image live scaling already ran. + // - With PixelRefine: each image was scaled by PixelRefine itself. + // In both of those cases we must NOT run ScaleOnTheFly again - we go + // straight to merging on the per-image image_scale_corr. + if (reference_data.empty() && !pixel_refine_path) { + logger.Info("Running scaling ..."); ScalingResult scale_result(0); auto scale_start = std::chrono::steady_clock::now(); @@ -957,6 +964,10 @@ int main(int argc, char **argv) { auto scale_end = std::chrono::steady_clock::now(); double scale_time = std::chrono::duration(scale_end - scale_start).count(); logger.Info("Scaling completed in {:.2f} s", scale_time); + } else if (pixel_refine_path) { + logger.Info("PixelRefine scaled each image during processing; merging directly (no ScaleOnTheFly)"); + } else { + logger.Info("Reference provided: per-image live scaling already applied; merging directly"); } auto merge_start = std::chrono::steady_clock::now(); -- 2.52.0 From e4b66f9cd9a39ebbba1214cd2f65332d820faecc Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 08:04:27 +0200 Subject: [PATCH 014/228] PIxelRefine: Another iteration --- image_analysis/IndexAndRefine.cpp | 5 ++ .../pixel_refinement/PixelRefine.cpp | 77 ++++++++++++++----- image_analysis/pixel_refinement/PixelRefine.h | 2 + viewer/JFJochImageReadingWorker.cpp | 4 +- viewer/JFJochImageReadingWorker.h | 2 +- viewer/windows/JFJochPixelRefineWindow.cpp | 7 +- viewer/windows/JFJochPixelRefineWindow.h | 2 +- 7 files changed, 73 insertions(+), 26 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 5dc02d55..c297de04 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -475,8 +475,13 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, i_outcome.latt = prd.latt; i_outcome.image_scale_g = static_cast(prd.scale_factor); i_outcome.image_scale_b_factor_Ang2 = static_cast(prd.B_factor); + if (std::isfinite(prd.cc)) { + i_outcome.image_scale_cc = static_cast(prd.cc); + i_outcome.image_scale_cc_n = prd.cc_n; + } msg.image_scale_factor = static_cast(prd.scale_factor); + msg.image_scale_cc = i_outcome.image_scale_cc; if (prd.B_factor != 0.0) msg.image_scale_b_factor = static_cast(prd.B_factor); diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 3b6ccdfe..b2d2578f 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -766,19 +766,20 @@ void PixelRefine::Run(const T *image, } // predict<->refine iterations // ---- Extract integrated reflections --------------------------------------- - // Two quantities are read back per reflection, using the *optimized* model: + // Profile fitting gives the recorded amplitude (against the normalized + // tangential profile P_t): + // J = sum_p[ P_t,p (Iobs_p - Ibkg_p)/v_p ] / sum_p[ P_t,p^2 / v_p ] + // ~ G * Itrue * B_term * partiality (recorded intensity) + // var(J) = 1 / sum_p[ P_t,p^2 / v_p ] // - // * Recorded (partial) intensity J by profile fitting against the normalized - // tangential profile P_t (sum over the shoebox ~ 1): - // J = sum_p [ P_t,p (Iobs_p - Ibkg_p) / v_p ] / sum_p [ P_t,p^2 / v_p ] - // var(J) = 1 / sum_p [ P_t,p^2 / v_p ] - // Because P_t is area-normalized, J estimates the integrated intensity - // actually recorded on this image (not the full-spot intensity). - // - // * Partiality: the profile-weighted mean of the *peak-normalized* radial - // factor exp(-eps_r^2/R0^2) in (0,1], i.e. how close to the Ewald sphere - // the reflection sits. Kept dimensionless for consistency with the rest of - // the pipeline (image_scale_corr = rlp / partiality, full I = J / partiality). + // Output split (Merge multiplies r.I * image_scale_corr and weights by + // 1/(sigma*image_scale_corr)^2 - see Merge.cpp): + // r.I = J / (B_term * partiality) = G * Itrue (B/partiality corrected) + // r.sigma = sqrt(var(J)) / (B_term * partiality) + // r.partiality = profile-weighted peak radial factor in (0,1] (Merge filter only) + // r.image_scale_corr = 1/G (per-image scale ONLY) + // so r.I * image_scale_corr = Itrue. B and partiality live on the intensity, + // G lives on image_scale_corr - one clean meaning per field. data.reflections.reserve(groups.size()); for (const auto &g : groups) { double num = 0.0, den = 0.0, bkg_sum = 0.0; @@ -826,16 +827,22 @@ void PixelRefine::Run(const T *image, r.partiality = (radial_w > 0.0) ? static_cast(radial_sum / radial_w) : 1.0f; if (den > 0.0 && n > 0) { - r.I = static_cast(num / den); - r.sigma = static_cast(std::sqrt(1.0 / den)); + const double I_amp = num / den; // ~ G*Itrue*B_term*partiality + const double sigma_amp = std::sqrt(1.0 / den); + const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); + const double corr = static_cast(r.partiality) * B_term; // B & partiality r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; - // Put I onto a common (merge-ready) scale: I * image_scale_corr is the - // full-spot, scale/DW-corrected intensity. Fold in the refined G, the - // per-reflection B_term and the partiality (rlp = 1 here). - const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); - const double denom = static_cast(r.partiality) * data.scale_factor * B_term; - r.image_scale_corr = (denom > 0.0) ? static_cast(r.rlp / denom) : NAN; + + if (corr > 0.0 && data.scale_factor > 0.0) { + r.I = static_cast(I_amp / corr); // = G*Itrue + r.sigma = static_cast(sigma_amp / corr); + r.image_scale_corr = static_cast(1.0 / data.scale_factor); // G only + } else { + r.I = static_cast(I_amp); + r.sigma = static_cast(sigma_amp); + r.image_scale_corr = NAN; + } } else { r.I = 0.0f; r.sigma = NAN; @@ -844,6 +851,36 @@ void PixelRefine::Run(const T *image, } data.reflections.push_back(r); } + + // ---- Per-image CC vs reference (the half/ref correlation diagnostic) ------- + // Pearson CC of the scaled estimate (r.I * image_scale_corr = Itrue_est) + // against the reference intensities, over the matched reflections. + { + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; + size_t cn = 0; + for (const auto &r : data.reflections) { + if (!r.observed || !std::isfinite(r.I) || !std::isfinite(r.image_scale_corr)) + continue; + const auto it = reference_data.find(hkl_key_generator(r)); + if (it == reference_data.end()) + continue; + const double x = static_cast(r.I) * static_cast(r.image_scale_corr); + const double y = it->second; + if (!std::isfinite(x) || !std::isfinite(y)) + continue; + sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y; ++cn; + } + data.cc = NAN; + data.cc_n = static_cast(cn); + if (cn >= 3) { + const double nd = static_cast(cn); + const double cov = sxy - sx * sy / nd; + const double vx = sxx - sx * sx / nd; + const double vy = syy - sy * sy / nd; + if (vx > 0.0 && vy > 0.0) + data.cc = cov / std::sqrt(vx * vy); + } + } } std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile &profile, diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index ae805f4e..9522fc91 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -139,6 +139,8 @@ struct PixelRefineData { bool solved = false; double final_cost = NAN; size_t residual_count = 0; + double cc = NAN; // per-image CC of scaled intensities vs reference + int64_t cc_n = 0; // number of reflections in the CC }; class PixelRefine { diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 1ad60bd0..08eb7ad7 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -816,7 +816,7 @@ void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { try { const auto &img32 = current_image_ptr->Image(); pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); - emit pixelRefineResidual(d.final_cost, static_cast(d.reflections.size())); + emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); emit predictedImageReady(WrapFloatImage_i(pred)); @@ -859,7 +859,7 @@ void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { out.beam_x = d.geom.GetBeamX_pxl(); out.beam_y = d.geom.GetBeamY_pxl(); emit pixelRefineParamsRefined(out); - emit pixelRefineResidual(d.final_cost, static_cast(d.reflections.size())); + emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); emit predictedImageReady(WrapFloatImage_i(pred)); diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index 6ab9e429..a5b219df 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -134,7 +134,7 @@ signals: // PixelRefine (experimental) void predictedImageReady(std::shared_ptr image); - void pixelRefineResidual(double cost, int64_t n_reflections); + void pixelRefineResidual(double cost, double cc, int64_t n_reflections); void pixelRefineParamsRefined(PixelRefineParams params); void pixelRefineStatus(QString message); diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index 364cf5df..6cce6a99 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -161,9 +161,12 @@ void JFJochPixelRefineWindow::setPredictedImage(std::shared_ptrsetImage(std::move(image)); } -void JFJochPixelRefineWindow::setResidual(double cost, int64_t n_reflections) { - m_residual->setText(tr("Residual: %1 (%2 reflections)") +void JFJochPixelRefineWindow::setResidual(double cost, double cc, int64_t n_reflections) { + const QString cc_str = std::isfinite(cc) ? QString::number(cc * 100.0, 'f', 1) + "%" + : QStringLiteral("—"); + m_residual->setText(tr("Residual: %1 CC: %2 (%3 reflections)") .arg(cost, 0, 'g', 6) + .arg(cc_str) .arg(n_reflections)); } diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h index 7e7f167c..b508497e 100644 --- a/viewer/windows/JFJochPixelRefineWindow.h +++ b/viewer/windows/JFJochPixelRefineWindow.h @@ -67,7 +67,7 @@ signals: public slots: void setPredictedImage(std::shared_ptr image); - void setResidual(double cost, int64_t n_reflections); + void setResidual(double cost, double cc, int64_t n_reflections); void setRefinedParams(PixelRefineParams params); void setStatus(QString message); }; -- 2.52.0 From 5735302691dff3055f65dcf3b2a994ce9b9c6d94 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 11:06:16 +0200 Subject: [PATCH 015/228] Merge: CC1/2 limit adjustment --- image_analysis/scale_merge/Merge.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 8d69f811..4f4fd8ee 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -22,7 +22,14 @@ MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x) scaling_settings(x.GetScalingSettings()), indexing_settings(x.GetIndexingSettings()), high_resolution_limit(scaling_settings.GetHighResolutionLimit_A()), - image_cc_limit(scaling_settings.GetMinCCForImage()), + // A min-image-CC of 0 (the default) means "no limit": leave the optional + // empty so the per-image CC cut is inactive. Otherwise a 0.0 threshold + // would silently drop every image with a non-positive per-image CC (which + // also wrongly zeroed N_obs in MergeStats, since it masks with cc_mask=true + // while the merge keeps all images). + image_cc_limit(scaling_settings.GetMinCCForImage() > 0.0 + ? std::optional(scaling_settings.GetMinCCForImage()) + : std::nullopt), min_partiality(scaling_settings.GetMinPartiality()), generator(scaling_settings.GetMergeFriedel(), space_group_number) { } -- 2.52.0 From 30dcc98f896cebbcfa58fb64304a29522eaff729 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 9 Jun 2026 12:03:45 +0200 Subject: [PATCH 016/228] JFJochMagnifierWindow: Zoom is saved ... it is not optimal (when image is first loaded, than it starts with weird zoom), but can be fixed later --- viewer/image_viewer/JFJochImage.cpp | 13 +++++++++++++ viewer/image_viewer/JFJochImage.h | 3 ++- viewer/windows/JFJochMagnifierWindow.cpp | 5 ++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/viewer/image_viewer/JFJochImage.cpp b/viewer/image_viewer/JFJochImage.cpp index ed030ef0..a11db19e 100644 --- a/viewer/image_viewer/JFJochImage.cpp +++ b/viewer/image_viewer/JFJochImage.cpp @@ -948,4 +948,17 @@ void JFJochImage::adjustForeground(bool input) { void JFJochImage::beforeOverlayCleared() {} +double JFJochImage::GetScaleFactor() const { + return scale_factor; +} +void JFJochImage::setZoom(double input) { + if (std::isfinite(input) && input > 0) { + scale_factor = input; + if (!scene()) + return; + scale(input, input); + updateOverlay(); + emitViewportChanged(); + } +} diff --git a/viewer/image_viewer/JFJochImage.h b/viewer/image_viewer/JFJochImage.h index 593cda04..f16ec3ec 100644 --- a/viewer/image_viewer/JFJochImage.h +++ b/viewer/image_viewer/JFJochImage.h @@ -131,7 +131,8 @@ public slots: void fitToView(); void adjustForeground(bool input); - + void setZoom(double input); public: explicit JFJochImage(QWidget *parent = nullptr); + double GetScaleFactor() const; }; \ No newline at end of file diff --git a/viewer/windows/JFJochMagnifierWindow.cpp b/viewer/windows/JFJochMagnifierWindow.cpp index 266a5056..8027652e 100644 --- a/viewer/windows/JFJochMagnifierWindow.cpp +++ b/viewer/windows/JFJochMagnifierWindow.cpp @@ -11,6 +11,7 @@ JFJochMagnifierWindow::JFJochMagnifierWindow(QWidget *parent) : JFJochHelperWindow(parent) { setWindowTitle("Magnifier"); m_image = new JFJochSimpleImage(this); + m_image->setZoom(m_magnification); setCentralWidget(m_image); resize(320, 320); } @@ -25,11 +26,13 @@ void JFJochMagnifierWindow::imageLoaded(std::shared_ptr auto si = std::make_shared(); si->image = CompressedImage(image->Image(), exp.GetXPixelsNum(), exp.GetYPixelsNum()); m_image->setImage(si); + m_image->setZoom(m_magnification); m_have_image = true; } void JFJochMagnifierWindow::centerAt(QPointF scenePos) { if (!m_have_image || !isVisible()) return; - m_image->applyViewport(QTransform::fromScale(m_magnification, m_magnification), scenePos); + double scale = m_image->GetScaleFactor(); + m_image->applyViewport(QTransform::fromScale(scale, scale), scenePos); } -- 2.52.0 From 8a582b8a90604ffc1c6201af22967d64d6be631e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 9 Jun 2026 12:09:40 +0200 Subject: [PATCH 017/228] JFJochMagnifierWindow: Fixed --- viewer/image_viewer/JFJochImage.cpp | 2 +- viewer/windows/JFJochMagnifierWindow.cpp | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/viewer/image_viewer/JFJochImage.cpp b/viewer/image_viewer/JFJochImage.cpp index a11db19e..0e278524 100644 --- a/viewer/image_viewer/JFJochImage.cpp +++ b/viewer/image_viewer/JFJochImage.cpp @@ -957,7 +957,7 @@ void JFJochImage::setZoom(double input) { scale_factor = input; if (!scene()) return; - scale(input, input); + setTransform(QTransform::fromScale(input, input)); updateOverlay(); emitViewportChanged(); } diff --git a/viewer/windows/JFJochMagnifierWindow.cpp b/viewer/windows/JFJochMagnifierWindow.cpp index 8027652e..06f1f109 100644 --- a/viewer/windows/JFJochMagnifierWindow.cpp +++ b/viewer/windows/JFJochMagnifierWindow.cpp @@ -22,11 +22,18 @@ void JFJochMagnifierWindow::imageLoaded(std::shared_ptr m_image->setImage(nullptr); return; } + + const double scale = m_have_image ? m_image->GetScaleFactor() : m_magnification; + const QPointF center = m_have_image + ? m_image->mapToScene(m_image->viewport()->rect().center()) + : QPointF(image->Dataset().experiment.GetXPixelsNum() * 0.5, + image->Dataset().experiment.GetYPixelsNum() * 0.5); + const auto &exp = image->Dataset().experiment; auto si = std::make_shared(); si->image = CompressedImage(image->Image(), exp.GetXPixelsNum(), exp.GetYPixelsNum()); m_image->setImage(si); - m_image->setZoom(m_magnification); + m_image->applyViewport(QTransform::fromScale(scale, scale), center); m_have_image = true; } -- 2.52.0 From feca63f4b9be1516cbdbaadb85b30f86af812a9d Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 12:35:06 +0200 Subject: [PATCH 018/228] jfjoch_viewer: fixes to pixel refine --- viewer/JFJochImageReadingWorker.cpp | 41 +++++++++++++++++----- viewer/JFJochImageReadingWorker.h | 2 +- viewer/windows/JFJochPixelRefineWindow.cpp | 28 ++++++++++----- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 08eb7ad7..6582700c 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -729,18 +729,31 @@ void JFJochImageReadingWorker::EnsurePixelRefine_i() { pixel_pred_ = CreateBraggPrediction(curr_experiment.IsRotationIndexing()); } -bool JFJochImageReadingWorker::BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p) const { - if (!current_image_ptr || !index_and_refine || !current_image.has_value()) +bool JFJochImageReadingWorker::BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p, QString &reason) const { + if (!current_image_ptr || !index_and_refine || !current_image.has_value()) { + reason = "No image loaded"; return false; + } const auto &outcomes = index_and_refine->GetIntegrationOutcome(); const int64_t n = current_image.value(); - if (n < 0 || n >= static_cast(outcomes.size())) + if (n < 0 || n >= static_cast(outcomes.size())) { + reason = "No analysis result for current image"; return false; + } const auto &io = outcomes[n]; - if (io.reflections.empty()) // image not indexed/integrated + if (io.reflections.empty()) { + // The "indexed" badge in the viewer comes from indexing (lattice_type); + // PixelRefine instead seeds from the *integration* outcome, which is only + // populated when Quick integration is enabled and succeeds for this image. + // Distinguish the two so the user knows what to turn on. + if (current_image_ptr->ImageData().lattice_type.has_value()) + reason = "Image is indexed but not integrated - enable Quick integration"; + else + reason = "Current image is not indexed"; return false; + } d.geom = io.geom; d.latt = io.latt; @@ -773,7 +786,15 @@ bool JFJochImageReadingWorker::BuildPixelSeed_i(PixelRefineData &d, const PixelR std::shared_ptr JFJochImageReadingWorker::WrapFloatImage_i(const std::vector &img) const { auto si = std::make_shared(); - si->image = CompressedImage(img, curr_experiment.GetXPixelsNum(), curr_experiment.GetYPixelsNum()); + // CompressedImage is a non-owning view over its data pointer. predictedImageReady + // is a queued cross-thread connection, so the source float vector must outlive the + // emit: copy it into SimpleImage::buffer (which the shared_ptr keeps alive) instead + // of aliasing the caller's temporary, otherwise loadImageInternal() reads freed + // memory in the GUI thread (SIGSEGV). + si->buffer.resize(img.size() * sizeof(float)); + std::memcpy(si->buffer.data(), img.data(), si->buffer.size()); + si->image = CompressedImage(si->buffer, curr_experiment.GetXPixelsNum(), curr_experiment.GetYPixelsNum(), + CompressedImageMode::Float32); return si; } @@ -803,8 +824,9 @@ void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { } PixelRefineData d; - if (!BuildPixelSeed_i(d, params)) { - emit pixelRefineStatus("Current image is not indexed"); + QString seed_reason; + if (!BuildPixelSeed_i(d, params, seed_reason)) { + emit pixelRefineStatus(seed_reason); return; } @@ -838,8 +860,9 @@ void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { } PixelRefineData d; - if (!BuildPixelSeed_i(d, params)) { - emit pixelRefineStatus("Current image is not indexed"); + QString seed_reason; + if (!BuildPixelSeed_i(d, params, seed_reason)) { + emit pixelRefineStatus(seed_reason); return; } if (d.max_iterations <= 0) diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index a5b219df..93d4240b 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -69,7 +69,7 @@ private: std::unique_ptr pixel_pred_; void EnsurePixelRefine_i(); - bool BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p) const; + bool BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p, QString &reason) const; std::shared_ptr WrapFloatImage_i(const std::vector &img) const; std::unique_ptr roi; diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index 6cce6a99..bd4e1927 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -145,16 +145,26 @@ void JFJochPixelRefineWindow::onControlChanged() { } void JFJochPixelRefineWindow::imageLoaded(std::shared_ptr image) { - if (m_beamInit || !image) + if (!image) return; - const auto geom = image->Dataset().experiment.GetDiffractionGeometry(); - m_beamx->setMax(static_cast(image->Dataset().experiment.GetXPixelsNum())); - m_beamy->setMax(static_cast(image->Dataset().experiment.GetYPixelsNum())); - m_suppress = true; - m_beamx->setValue(geom.GetBeamX_pxl()); - m_beamy->setValue(geom.GetBeamY_pxl()); - m_suppress = false; - m_beamInit = true; + + // Initialise the beam-centre sliders from the geometry once. + if (!m_beamInit) { + const auto geom = image->Dataset().experiment.GetDiffractionGeometry(); + m_beamx->setMax(static_cast(image->Dataset().experiment.GetXPixelsNum())); + m_beamy->setMax(static_cast(image->Dataset().experiment.GetYPixelsNum())); + m_suppress = true; + m_beamx->setValue(geom.GetBeamX_pxl()); + m_beamy->setValue(geom.GetBeamY_pxl()); + m_suppress = false; + m_beamInit = true; + } + + // Request a predicted-image preview for the (re)loaded image. Without this the + // preview only refreshed on a slider change, so opening/reanalyzing an image + // left the predicted view empty. The worker no-ops if there is no reference or + // the image is not integrated yet. + onControlChanged(); } void JFJochPixelRefineWindow::setPredictedImage(std::shared_ptr image) { -- 2.52.0 From 2d202f1d44ed1a7afb6ab12b2cd38af1c39de4e5 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 13:20:46 +0200 Subject: [PATCH 019/228] jfjoch_viewer: fixes to pixel refine window --- viewer/image_viewer/JFJochImage.cpp | 4 ++-- viewer/image_viewer/JFJochImage.h | 6 ++++++ viewer/image_viewer/JFJochSimpleImage.cpp | 3 +++ viewer/windows/JFJochPixelRefineWindow.cpp | 4 ++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/viewer/image_viewer/JFJochImage.cpp b/viewer/image_viewer/JFJochImage.cpp index 0e278524..7f151a92 100644 --- a/viewer/image_viewer/JFJochImage.cpp +++ b/viewer/image_viewer/JFJochImage.cpp @@ -752,9 +752,9 @@ void JFJochImage::writePixelLabels() { if (std::abs(val - nearest) < 1e-6) numBuf = QString::number(static_cast(val)); else if (absVal < 1e4) - numBuf = QString::number(val, 'f', 3); + numBuf = QString::number(val, 'f', label_decimals_); else - numBuf = QString::number(val, 'f', 2); + numBuf = QString::number(val, 'f', std::min(label_decimals_, 2)); pText = &numBuf; } else { numBuf = QString::number(val, 'e', 1); diff --git a/viewer/image_viewer/JFJochImage.h b/viewer/image_viewer/JFJochImage.h index f16ec3ec..5b3816ea 100644 --- a/viewer/image_viewer/JFJochImage.h +++ b/viewer/image_viewer/JFJochImage.h @@ -54,6 +54,12 @@ protected: bool initial_fit_done_ = false; QColor feature_color = Qt::magenta; + + // Decimal places for non-integer per-pixel value labels. Float images (e.g. the + // PixelRefine prediction) are unreadable with many decimals, so subclasses can + // lower this. + int label_decimals_ = 3; + float foreground = 10.0; float background = 0.0; ColorScale color_scale; diff --git a/viewer/image_viewer/JFJochSimpleImage.cpp b/viewer/image_viewer/JFJochSimpleImage.cpp index 700537f8..24bdcad6 100644 --- a/viewer/image_viewer/JFJochSimpleImage.cpp +++ b/viewer/image_viewer/JFJochSimpleImage.cpp @@ -18,6 +18,9 @@ JFJochSimpleImage::JFJochSimpleImage(QWidget *parent) // Keep overlays in pixel units independent of zoom (for labels font sizing) setViewportUpdateMode(QGraphicsView::FullViewportUpdate); + + // The predicted/float image is unreadable with 3-decimal per-pixel labels. + label_decimals_ = 1; } void JFJochSimpleImage::setImage(std::shared_ptr img) { diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index bd4e1927..5ff21ea6 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -109,6 +109,10 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) }); connect(m_refine, &QPushButton::clicked, this, [this] { + // Cancel any pending live-preview: otherwise a debounce armed by a slider + // move just before this click fires after the refine and overwrites the + // refined residual/preview with the stale pre-refine slider values. + m_debounce->stop(); PixelRefineParams p = currentParams(); p.max_iterations = 5; emit refineRequested(p); -- 2.52.0 From 003fea1b1e6acc9e8c03bb162985a6ec1f52b2b0 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 13:28:35 +0200 Subject: [PATCH 020/228] jfjoch_viewer: fix sorting by indexing status --- viewer/windows/JFJochViewerImageListWindow.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/viewer/windows/JFJochViewerImageListWindow.cpp b/viewer/windows/JFJochViewerImageListWindow.cpp index bf5bf698..abd2dc5c 100644 --- a/viewer/windows/JFJochViewerImageListWindow.cpp +++ b/viewer/windows/JFJochViewerImageListWindow.cpp @@ -84,6 +84,12 @@ void JFJochViewerImageListWindow::addDataRow(int imageNumber, double backgroundE rowItems.append(bgItem); QStandardItem *indexingItem = new QStandardItem(indexingResult); + if (indexingResult == "Yes") + indexingItem->setData(1, ScaleSortRole); + else if (indexingResult == "No") + indexingItem->setData(0, ScaleSortRole); + else + indexingItem->setData(-1, ScaleSortRole); rowItems.append(indexingItem); QStandardItem *spotItem = new QStandardItem(); -- 2.52.0 From 6af22b6a0ccd88744c45298d8978dafaa5a3b8b0 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 13:28:56 +0200 Subject: [PATCH 021/228] jfjoch_viewer: show image CC based on standard pipeline --- viewer/windows/JFJochPixelRefineWindow.cpp | 12 ++++++++++++ viewer/windows/JFJochPixelRefineWindow.h | 1 + 2 files changed, 13 insertions(+) diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index 5ff21ea6..230d4974 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -77,10 +77,12 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) controlsLayout->addWidget(m_refine); m_residual = new QLabel(tr("Residual: —"), this); + m_pipelineCC = new QLabel(tr("Pipeline CC (ref): —"), this); m_status = new QLabel(QString(), this); m_status->setWordWrap(true); m_status->setStyleSheet("color: rgb(80, 80, 80);"); controlsLayout->addWidget(m_residual); + controlsLayout->addWidget(m_pipelineCC); controlsLayout->addWidget(m_status); controlsLayout->addStretch(1); @@ -164,6 +166,16 @@ void JFJochPixelRefineWindow::imageLoaded(std::shared_ptrImageData().image_scale_cc; + if (pipeline_cc.has_value() && std::isfinite(pipeline_cc.value())) + m_pipelineCC->setText(tr("Pipeline CC (ref): %1%") + .arg(pipeline_cc.value() * 100.0, 0, 'f', 1)); + else + m_pipelineCC->setText(tr("Pipeline CC (ref): —")); + // Request a predicted-image preview for the (re)loaded image. Without this the // preview only refreshed on a slider change, so opening/reanalyzing an image // left the predicted view empty. The worker no-ops if there is no reference or diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h index b508497e..7940ce16 100644 --- a/viewer/windows/JFJochPixelRefineWindow.h +++ b/viewer/windows/JFJochPixelRefineWindow.h @@ -42,6 +42,7 @@ class JFJochPixelRefineWindow : public JFJochHelperWindow { QCheckBox *m_refR; QLabel *m_residual; + QLabel *m_pipelineCC; // first-image CC vs reference from the standard ScaleOnTheFly pipeline QLabel *m_status; QPushButton *m_loadRef; QPushButton *m_refine; -- 2.52.0 From 6c85aaba2b6bbcbdf3d73ce5718875e9b811aa02 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 15:01:44 +0200 Subject: [PATCH 022/228] BraggPrediction: Include X-ray bandwidth --- image_analysis/IndexAndRefine.cpp | 4 +++- .../bragg_prediction/BraggPrediction.cpp | 20 ++++++++++++++++++- .../bragg_prediction/BraggPrediction.h | 6 ++++++ .../bragg_prediction/BraggPredictionGPU.cu | 20 ++++++++++++++++--- .../bragg_prediction/BraggPredictionGPU.h | 1 + .../pixel_refinement/PixelRefine.cpp | 6 ++++-- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index c297de04..98b1aed6 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -289,7 +289,9 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, .max_hkl = 100, .centering = outcome.symmetry.centering, .wedge_deg = std::fabs(wedge_deg), - .mosaicity_deg = std::fabs(mos_deg) + .mosaicity_deg = std::fabs(mos_deg), + // FWHM -> sigma; 0 when monochromatic, leaving the prediction unchanged. + .bandwidth_sigma = experiment.GetBandwidthFWHM().value_or(0.0f) / 2.3548f }; // Select the integration path: classical 2D integration, or the experimental diff --git a/image_analysis/bragg_prediction/BraggPrediction.cpp b/image_analysis/bragg_prediction/BraggPrediction.cpp index 962ae45d..4c134e38 100644 --- a/image_analysis/bragg_prediction/BraggPrediction.cpp +++ b/image_analysis/bragg_prediction/BraggPrediction.cpp @@ -4,6 +4,13 @@ #include "BraggPrediction.h" #include "../bragg_integration/SystematicAbsence.h" +namespace { + // Number of bandwidth sigmas included in the (radially thickened) Ewald-shell + // acceptance window. 3σ captures essentially the whole pink-beam smear; matches + // the conservative end of the mosaicity cutoff used by callers. + constexpr float kBandwidthCutoffSigmas = 3.0f; +} + BraggPrediction::BraggPrediction(int max_reflections) : max_reflections(max_reflections), reflections(max_reflections) {} @@ -76,7 +83,18 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal float S_len = sqrtf(S_x * S_x + S_y * S_y + S_z * S_z); float dist_ewald_sphere = std::fabs(S_len - one_over_wavelength); - if (dist_ewald_sphere <= settings.ewald_dist_cutoff ) { + // Energy bandwidth thickens the Ewald shell radially: at the + // diffraction condition |S|-1/λ shifts by recip_z·(Δλ/λ), i.e. + // σ_bw = |recip_z|·bandwidth_sigma (= bλ/2d², identical to + // PixelRefine's R_bw). Broaden the acceptance window in quadrature so + // high-resolution shells (smeared most, ∝1/d²) are not clipped. + float radial_cutoff = settings.ewald_dist_cutoff; + if (settings.bandwidth_sigma > 0.0f) { + const float bw_tol = kBandwidthCutoffSigmas * settings.bandwidth_sigma * std::fabs(recip_z); + radial_cutoff = std::sqrt(radial_cutoff * radial_cutoff + bw_tol * bw_tol); + } + + if (dist_ewald_sphere <= radial_cutoff ) { const float s0_p0 = S0.x * recip_x + S0.y * recip_y + S0.z * recip_z; const float val = s0_sq * recip_sq - s0_p0 * s0_p0; diff --git a/image_analysis/bragg_prediction/BraggPrediction.h b/image_analysis/bragg_prediction/BraggPrediction.h index 6aaf4fc5..13d1c7ed 100644 --- a/image_analysis/bragg_prediction/BraggPrediction.h +++ b/image_analysis/bragg_prediction/BraggPrediction.h @@ -18,6 +18,12 @@ struct BraggPredictionSettings { float mosaicity_deg = 0.2f; float min_zeta = 0.05; float mosaicity_multiplier = 4.0; + // Relative X-ray bandwidth Δλ/λ expressed as a Gaussian sigma (0 = monochromatic). + // When > 0 the Ewald-shell acceptance is thickened radially per reflection by + // σ_bw = |recip_z|·bandwidth_sigma (= bλ/2d²), so the 1/d² pink-beam smear no + // longer clips high-resolution reflections. Must stay the last member so existing + // designated initializers remain valid. + float bandwidth_sigma = 0.0f; }; class BraggPrediction { diff --git a/image_analysis/bragg_prediction/BraggPredictionGPU.cu b/image_analysis/bragg_prediction/BraggPredictionGPU.cu index b15c34bc..cf8f8cc6 100644 --- a/image_analysis/bragg_prediction/BraggPredictionGPU.cu +++ b/image_analysis/bragg_prediction/BraggPredictionGPU.cu @@ -8,6 +8,10 @@ #include namespace { + // Number of bandwidth sigmas included in the (radially thickened) Ewald-shell + // acceptance window. Mirrors the CPU BraggPrediction path. + constexpr float kBandwidthCutoffSigmas = 3.0f; + __device__ inline bool is_odd(int v) { return (v & 1) != 0; } __device__ inline float angle_from_ewald_sphere_deg(const Coord &S0, float recip_x, float recip_y, float recip_z, float recip_sq) { @@ -95,7 +99,14 @@ namespace { float Sz = recip_z + C.S0.z; float S_len = sqrtf(Sx * Sx + Sy * Sy + Sz * Sz); float dist_ewald = fabsf(S_len - C.one_over_wavelength); - if (dist_ewald > C.ewald_cutoff) return false; + // Energy bandwidth thickens the Ewald shell radially: σ_bw = |recip_z|·(Δλ/λ) + // (= bλ/2d²). Broaden the acceptance window in quadrature (see CPU path). + float radial_cutoff = C.ewald_cutoff; + if (C.bandwidth_sigma > 0.0f) { + const float bw_tol = kBandwidthCutoffSigmas * C.bandwidth_sigma * fabsf(recip_z); + radial_cutoff = sqrtf(radial_cutoff * radial_cutoff + bw_tol * bw_tol); + } + if (dist_ewald > radial_cutoff) return false; float Srx = C.rot[0] * Sx + C.rot[1] * Sy + C.rot[2] * Sz; float Sry = C.rot[3] * Sx + C.rot[4] * Sy + C.rot[5] * Sz; float Srz = C.rot[6] * Sx + C.rot[7] * Sy + C.rot[8] * Sz; @@ -145,7 +156,8 @@ namespace { const CrystalLattice &lattice, float high_res_A, float ewald_dist_cutoff, - char centering) { + char centering, + float bandwidth_sigma) { KernelConsts kc{}; auto geom = experiment.GetDiffractionGeometry(); kc.det_width_pxl = static_cast(experiment.GetXPixelsNum()); @@ -157,6 +169,7 @@ namespace { kc.one_over_dmax_sq = one_over_dmax * one_over_dmax; kc.one_over_wavelength = 1.0f / geom.GetWavelength_A(); kc.ewald_cutoff = ewald_dist_cutoff; + kc.bandwidth_sigma = bandwidth_sigma; kc.Astar = lattice.Astar(); kc.Bstar = lattice.Bstar(); kc.Cstar = lattice.Cstar(); @@ -178,7 +191,8 @@ int BraggPredictionGPU::Calc(const DiffractionExperiment &experiment, const CrystalLattice &lattice, const BraggPredictionSettings &settings) { // Build constants on host - KernelConsts hK = BuildKernelConsts(experiment, lattice, settings.high_res_A, settings.ewald_dist_cutoff, settings.centering); + KernelConsts hK = BuildKernelConsts(experiment, lattice, settings.high_res_A, settings.ewald_dist_cutoff, + settings.centering, settings.bandwidth_sigma); cudaMemcpyAsync(dK, &hK, sizeof(KernelConsts), cudaMemcpyHostToDevice, stream); cudaMemsetAsync(d_count, 0, sizeof(int), stream); diff --git a/image_analysis/bragg_prediction/BraggPredictionGPU.h b/image_analysis/bragg_prediction/BraggPredictionGPU.h index 2bf7a27b..8c6a8486 100644 --- a/image_analysis/bragg_prediction/BraggPredictionGPU.h +++ b/image_analysis/bragg_prediction/BraggPredictionGPU.h @@ -18,6 +18,7 @@ struct KernelConsts { float one_over_wavelength; float one_over_dmax_sq; float ewald_cutoff; + float bandwidth_sigma; // relative Δλ/λ (sigma); 0 = monochromatic Coord Astar, Bstar, Cstar, S0; float rot[9]; char centering; diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index b2d2578f..8f78e16c 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -475,7 +475,8 @@ void PixelRefine::Run(const T *image, const BraggPredictionSettings settings_prediction{ .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), .max_hkl = 100, - .centering = data.centering + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma }; const auto azim_result = profile.GetResult(); @@ -942,7 +943,8 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & const BraggPredictionSettings settings_prediction{ .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), .max_hkl = 100, - .centering = data.centering + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma }; const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); -- 2.52.0 From efe882f4b689f38c7bcbc01eba68d58960557377 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 16:28:17 +0200 Subject: [PATCH 023/228] jfjoch_viewer: Better display (to be tested) of pixel refine --- .../pixel_refinement/PixelRefine.cpp | 123 ++++++++++++++++++ image_analysis/pixel_refinement/PixelRefine.h | 12 ++ viewer/JFJochImageReadingWorker.cpp | 80 +++++++++++- viewer/JFJochImageReadingWorker.h | 12 ++ viewer/JFJochViewerWindow.cpp | 3 + viewer/image_viewer/JFJochSimpleImage.cpp | 25 ++++ viewer/image_viewer/JFJochSimpleImage.h | 8 ++ viewer/windows/JFJochPixelRefineWindow.cpp | 12 ++ viewer/windows/JFJochPixelRefineWindow.h | 3 + viewer/windows/PixelRefineParams.h | 8 ++ 10 files changed, 282 insertions(+), 4 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 8f78e16c..225778f7 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -993,6 +993,122 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & return img; } +template +std::vector PixelRefine::ChiSquaredImage(const T *image, + const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, + const PixelRefineData &data) const { + std::vector img(xpixel * ypixel, 0.0f); + + const double lambda = data.geom.GetWavelength_A(); + const double pixel_size = data.geom.GetPixelSize_mm(); + const auto azim_result = profile.GetResult(); + const auto azim_std = profile.GetStd(); + const auto &pixel_to_bin = mapping.GetPixelToBin(); + const auto &corrections = mapping.Corrections(); + const int total_bin_count = static_cast(azim_result.size()); + const double angle_rad = data.angle_deg * M_PI / 180.0; + const int radius = data.shoebox_radius; + const double bw = data.bandwidth; + + auto recip_area = [&](double x, double y) -> double { + const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); + const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); + return (qx % qy).Length(); + }; + auto bandwidth_radial_sq = [&](double d) -> double { + if (bw <= 0.0 || d <= 0.0) + return 0.0; + const double bl = bw * lambda; + return bl * bl / (2.0 * d * d * d * d); + }; + + double beam[2], dist_mm, detector_rot[2], rot_vec[3]; + double latt_vec0[3], latt_vec1[3], latt_vec2[3]; + BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2); + + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) + }; + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + const auto &predicted = prediction.GetReflections(); + + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; + const auto it = reference_data.find(hkl_key_generator(refl)); + if (it == reference_data.end()) + continue; + + const double Itrue = it->second; + const double R_bw_sq = bandwidth_radial_sq(refl.d); + + const int min_y = std::max(refl.predicted_y - radius, 0); + const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); + const int min_x = std::max(refl.predicted_x - radius, 0); + const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + + for (int y = min_y; y <= max_y; ++y) { + for (int x = min_x; x <= max_x; ++x) { + const size_t npixel = xpixel * y + x; + const int azim_bin = pixel_to_bin[npixel]; + + // Same gating as Run(): only pixels that actually enter the fit. + if (azim_bin >= total_bin_count) + continue; + if (image[npixel] == std::numeric_limits::max()) + continue; + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) + continue; + + const double correction = corrections[npixel]; + const double Ibkg = azim_result[azim_bin]; + const double Ibkg_sigma = azim_std[azim_bin]; + const double raw = static_cast(image[npixel]); + const double Iobs = raw * correction; + + double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; + if (!(var > 1.0)) + var = 1.0; + const double weight = 1.0 / std::sqrt(var); + + PixelObs obs{ + .x = static_cast(x), + .y = static_cast(y), + .Iobs = Iobs, + .Ibkg = Ibkg, + .weight = weight, + .A_recip = recip_area(x, y), + .angle_rad = angle_rad + }; + PixelResidual pr(obs, Itrue, lambda, pixel_size, + refl.h, refl.k, refl.l, R_bw_sq, data.crystal_system); + + double Ipred = 0.0; + if (pr.Model(beam, &dist_mm, detector_rot, rot_vec, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R, Ipred)) { + // residual_i = (I_pred - I_obs) * weight (== Ceres residual); + // its square is this pixel's contribution to the cost. + const double rw = (Ipred - Iobs) * weight; + img[npixel] += static_cast(rw * rw); + } + } + } + } + + return img; +} + // Explicit instantiations for the supported (uncompressed) image pixel types. template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); @@ -1000,3 +1116,10 @@ template void PixelRefine::Run(const int32_t *, const AzimuthalIntegrat template void PixelRefine::Run(const uint8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const uint16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const uint32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); + +template std::vector PixelRefine::ChiSquaredImage(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const int32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 9522fc91..73a63094 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -182,4 +182,16 @@ public: BraggPrediction &prediction, const PixelRefineData &data, bool include_background = true) const; + + // Render the per-pixel chi-square (cost density) that the optimizer actually + // minimizes: for every shoebox pixel that enters the fit it stores the squared + // weighted residual ((I_pred - I_obs)/sigma)^2 in *corrected* units - identical + // to the Ceres residual_i^2 - accumulating where shoeboxes overlap. Pixels that + // are not part of any shoebox stay 0; masked/saturated pixels (skipped by the + // fit) also stay 0. Summing the image gives ~2*final_cost. Diagnostic tool. + template + std::vector ChiSquaredImage(const T *image, + const AzimuthalIntegrationProfile &profile, + BraggPrediction &prediction, + const PixelRefineData &data) const; }; diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 6582700c..8f90cb11 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -8,6 +8,7 @@ #include #include "JFJochImageReadingWorker.h" +#include "../reader/JFJochReaderImage.h" // JFJochReaderImage + GAP/ERROR/SATURATED sentinels #include "../image_analysis/LoadFCalcFromMtz.h" #include "../image_analysis/bragg_prediction/BraggPredictionFactory.h" #include "../image_analysis/geom_refinement/AssignSpotsToRings.h" @@ -69,6 +70,7 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { qRegisterMetaType("PixelRefineParams"); + qRegisterMetaType>("QVector"); spot_finding_settings = settings;; indexing = std::make_unique(indexing_settings); @@ -798,6 +800,74 @@ std::shared_ptr JFJochImageReadingWorker::WrapFloatImage_i(const st return si; } +void JFJochImageReadingWorker::SquaredResidualWithImage_i(std::vector &pred) const { + // PredictImage() returns raw detector units (same as the measured counts), so + // pred - measured is the per-pixel residual the model fails to explain. We plot + // |pred - measured|^2: sign-free, so it needs no diverging colour scale and just + // highlights where the model disagrees most. Masked / saturated pixels carry + // sentinels rather than counts, so no comparison is possible -> NaN (gap). + if (!current_image_ptr) + return; + const auto &img = current_image_ptr->Image(); + const size_t n = std::min(pred.size(), img.size()); + for (size_t i = 0; i < n; ++i) { + const int32_t v = img[i]; + if (v == GAP_PXL_VALUE || v == ERROR_PXL_VALUE || v == SATURATED_PXL_VALUE) { + pred[i] = NAN; + } else { + const float diff = pred[i] - static_cast(v); + pred[i] = diff * diff; + } + } +} + +void JFJochImageReadingWorker::MaskMeasuredSentinels_i(std::vector &img) const { + // The chi^2 image is 0 outside shoeboxes; show masked/saturated pixels as a gap + // (NaN) instead, so they read as "not comparable" rather than "zero cost". + if (!current_image_ptr) + return; + const auto &measured = current_image_ptr->Image(); + const size_t n = std::min(img.size(), measured.size()); + for (size_t i = 0; i < n; ++i) { + const int32_t v = measured[i]; + if (v == GAP_PXL_VALUE || v == ERROR_PXL_VALUE || v == SATURATED_PXL_VALUE) + img[i] = NAN; + } +} + +QVector JFJochImageReadingWorker::BuildShoeboxes_i(const PixelRefineData &data) const { + // One rectangle per fitted reflection: the shoebox the optimizer summed over, + // centred on the predicted position with half-size data.shoebox_radius. + QVector boxes; + boxes.reserve(static_cast(data.reflections.size())); + const int r = data.shoebox_radius; + const int side = 2 * r + 1; + for (const auto &refl : data.reflections) { + if (!std::isfinite(refl.predicted_x) || !std::isfinite(refl.predicted_y)) + continue; + const int cx = static_cast(std::lround(refl.predicted_x)); + const int cy = static_cast(std::lround(refl.predicted_y)); + boxes.push_back(QRect(cx - r, cy - r, side, side)); + } + return boxes; +} + +std::vector JFJochImageReadingWorker::BuildDisplayImage_i(const PixelRefineData &data, + int display_mode) const { + if (display_mode == PixelRefineParams::ChiSquared) { + // The cost density the optimizer actually minimizes (weighted residual^2). + const auto &img32 = current_image_ptr->Image(); + auto chi2 = pixel_refine_->ChiSquaredImage(img32.data(), *last_profile_, *pixel_pred_, data); + MaskMeasuredSentinels_i(chi2); + return chi2; + } + + auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, data, true); + if (display_mode == PixelRefineParams::SquaredDifference) + SquaredResidualWithImage_i(pred); + return pred; +} + void JFJochImageReadingWorker::LoadReference(QString path) { QMutexLocker ul(&m); try { @@ -840,8 +910,9 @@ void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); - auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); - emit predictedImageReady(WrapFloatImage_i(pred)); + auto display = BuildDisplayImage_i(d, params.display_mode); + emit predictedImageReady(WrapFloatImage_i(display)); + emit predictedShoeboxes(BuildShoeboxes_i(d)); } catch (const std::exception &e) { emit pixelRefineStatus(QString("PixelRefine preview failed: %1").arg(e.what())); } @@ -884,8 +955,9 @@ void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { emit pixelRefineParamsRefined(out); emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); - auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, d, true); - emit predictedImageReady(WrapFloatImage_i(pred)); + auto display = BuildDisplayImage_i(d, params.display_mode); + emit predictedImageReady(WrapFloatImage_i(display)); + emit predictedShoeboxes(BuildShoeboxes_i(d)); // Show the refined predictions on the main image too. auto new_image = std::make_shared(*current_image_ptr); diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index 93d4240b..501b557d 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -71,6 +71,17 @@ private: void EnsurePixelRefine_i(); bool BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p, QString &reason) const; std::shared_ptr WrapFloatImage_i(const std::vector &img) const; + // Turn a predicted image into the squared residual |predicted - measured|^2 in + // place. Masked/saturated pixels become NaN (rendered as a gap: no comparison + // possible), not 0. + void SquaredResidualWithImage_i(std::vector &pred) const; + // Mark masked/saturated pixels of the current image as NaN (gap) in a float + // image, leaving the rest untouched (used for the chi^2 view). + void MaskMeasuredSentinels_i(std::vector &img) const; + // Build the per-reflection shoebox rectangles for the last refine/preview. + QVector BuildShoeboxes_i(const PixelRefineData &data) const; + // Build the float image to display for the given PixelRefineParams::DisplayMode. + std::vector BuildDisplayImage_i(const PixelRefineData &data, int display_mode) const; std::unique_ptr roi; @@ -134,6 +145,7 @@ signals: // PixelRefine (experimental) void predictedImageReady(std::shared_ptr image); + void predictedShoeboxes(QVector boxes); // per-reflection optimization windows void pixelRefineResidual(double cost, double cc, int64_t n_reflections); void pixelRefineParamsRefined(PixelRefineParams params); void pixelRefineStatus(QString message); diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index de383008..9fa6a302 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -28,6 +28,7 @@ #include "windows/JFJochPixelRefineWindow.h" #include "windows/JFJochMagnifierWindow.h" #include "image_viewer/JFJochImage.h" +#include "image_viewer/JFJochSimpleImage.h" #include JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString &file) : QMainWindow(parent) { @@ -351,6 +352,8 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString reading_worker, &JFJochImageReadingWorker::LoadReference); connect(reading_worker, &JFJochImageReadingWorker::predictedImageReady, pixelRefineWindow, &JFJochPixelRefineWindow::setPredictedImage); + connect(reading_worker, &JFJochImageReadingWorker::predictedShoeboxes, + pixelRefineWindow->imageView(), &JFJochSimpleImage::setShoeboxes); connect(reading_worker, &JFJochImageReadingWorker::pixelRefineResidual, pixelRefineWindow, &JFJochPixelRefineWindow::setResidual); connect(reading_worker, &JFJochImageReadingWorker::pixelRefineParamsRefined, diff --git a/viewer/image_viewer/JFJochSimpleImage.cpp b/viewer/image_viewer/JFJochSimpleImage.cpp index 24bdcad6..d3494a8f 100644 --- a/viewer/image_viewer/JFJochSimpleImage.cpp +++ b/viewer/image_viewer/JFJochSimpleImage.cpp @@ -40,6 +40,31 @@ void JFJochSimpleImage::setImage(std::shared_ptr img) { } } +void JFJochSimpleImage::setShoeboxes(QVector boxes) { + shoeboxes_ = std::move(boxes); + // Redraw overlays on the current image (no-op if no image yet). + updateOverlay(); +} + +void JFJochSimpleImage::addCustomOverlay() { + if (shoeboxes_.isEmpty() || !scene()) + return; + + // Cosmetic 1-px outline so the box edges stay thin at any zoom; only draw the + // ones currently in view (there can be hundreds of reflections). + const QRectF visibleRect = mapToScene(viewport()->geometry()).boundingRect(); + QPen pen(QColor(0, 220, 255), 0); // cyan, distinct from the prediction colours + pen.setCosmetic(true); + + for (const QRect &b : shoeboxes_) { + const QRectF r(b.x(), b.y(), b.width(), b.height()); + if (!visibleRect.intersects(r)) + continue; + auto *item = scene()->addRect(r, pen); + addOverlayItem(item); + } +} + void JFJochSimpleImage::mouseHover(QMouseEvent *event) { if (image_) { const QPointF scenePos = mapToScene(event->pos()); diff --git a/viewer/image_viewer/JFJochSimpleImage.h b/viewer/image_viewer/JFJochSimpleImage.h index 8d7251e4..40abac06 100644 --- a/viewer/image_viewer/JFJochSimpleImage.h +++ b/viewer/image_viewer/JFJochSimpleImage.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -18,14 +19,21 @@ class JFJochSimpleImage : public JFJochImage { Q_OBJECT std::shared_ptr image_; + + // Per-reflection shoebox rectangles (pixel coordinates) to overlay: the pixels + // PixelRefine actually summed over. Empty = nothing drawn. + QVector shoeboxes_; + // Prepare image template void loadImageInternal(const uint8_t *input); void loadImageInternal(); void mouseHover(QMouseEvent *event) override; + void addCustomOverlay() override; public: explicit JFJochSimpleImage(QWidget *parent = nullptr); public slots: void setImage(std::shared_ptr img); + void setShoeboxes(QVector boxes); }; diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index 230d4974..7f7a64ec 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -31,6 +31,15 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) auto controlsLayout = new QVBoxLayout(controls); layout->addWidget(controls, 0); + // --- what the left image shows ------------------------------------------ + m_displayMode = new QComboBox(this); + m_displayMode->addItem(tr("Prediction")); + m_displayMode->addItem(tr("Squared difference |pred - image|²")); + m_displayMode->addItem(tr("χ² (weighted residual = LSQ cost)")); + auto displayForm = new QFormLayout(); + displayForm->addRow(tr("Display:"), m_displayMode); + controlsLayout->addLayout(displayForm); + auto paramBox = new QGroupBox(tr("Model parameters"), this); auto form = new QFormLayout(paramBox); @@ -97,6 +106,8 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) for (auto *s : {m_R0, m_R1, m_bw, m_scale, m_B, m_beamx, m_beamy}) connect(s, &SliderPlusBox::valueChanged, this, [this](double) { onControlChanged(); }); + connect(m_displayMode, &QComboBox::currentIndexChanged, this, [this](int) { onControlChanged(); }); + connect(m_overrideBeam, &QCheckBox::toggled, this, [this](bool on) { m_beamx->setEnabled(on); m_beamy->setEnabled(on); @@ -141,6 +152,7 @@ PixelRefineParams JFJochPixelRefineWindow::currentParams() const { p.refine_scale = m_refScale->isChecked(); p.refine_B = m_refB->isChecked(); p.refine_R = m_refR->isChecked(); + p.display_mode = m_displayMode->currentIndex(); return p; } diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h index 7940ce16..99c628c2 100644 --- a/viewer/windows/JFJochPixelRefineWindow.h +++ b/viewer/windows/JFJochPixelRefineWindow.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -33,6 +34,8 @@ class JFJochPixelRefineWindow : public JFJochHelperWindow { SliderPlusBox *m_beamx; SliderPlusBox *m_beamy; + QComboBox *m_displayMode; // Prediction vs. Difference (prediction - image) + QCheckBox *m_overrideBeam; QCheckBox *m_refOrientation; QCheckBox *m_refCell; diff --git a/viewer/windows/PixelRefineParams.h b/viewer/windows/PixelRefineParams.h index 4b266e62..64f6e27b 100644 --- a/viewer/windows/PixelRefineParams.h +++ b/viewer/windows/PixelRefineParams.h @@ -28,6 +28,14 @@ struct PixelRefineParams { bool refine_R = true; int max_iterations = 3; // <=0 means evaluate-only (preview / residual) + + // Display only (no effect on the fit): what the preview/refine image shows. + enum DisplayMode : int { + Prediction = 0, // forward-model image + SquaredDifference = 1, // |prediction - measured|^2 (raw, unweighted) + ChiSquared = 2 // ((prediction - measured)/sigma)^2 = the LSQ cost density + }; + int display_mode = Prediction; }; Q_DECLARE_METATYPE(PixelRefineParams) -- 2.52.0 From e051eed033cd2e36ad85d725ea98bda4f9511648 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 9 Jun 2026 19:43:34 +0200 Subject: [PATCH 024/228] jfjoch_process: Remove postrefinement --- common/ScalingSettings.h | 2 +- image_analysis/scale_merge/ScaleOnTheFly.cpp | 170 +------------------ tools/jfjoch_process.cpp | 2 - 3 files changed, 2 insertions(+), 172 deletions(-) diff --git a/common/ScalingSettings.h b/common/ScalingSettings.h index a20bcd21..f7c724f8 100644 --- a/common/ScalingSettings.h +++ b/common/ScalingSettings.h @@ -6,7 +6,7 @@ #include #include "JFJochException.h" -enum class PartialityModel { Fixed, Rotation, Unity, Postrefinement }; +enum class PartialityModel { Fixed, Rotation, Unity }; enum class IntensityFormat { Text, mmCIF, MTZ}; class ScalingSettings { diff --git a/image_analysis/scale_merge/ScaleOnTheFly.cpp b/image_analysis/scale_merge/ScaleOnTheFly.cpp index f5765374..46fc6cea 100644 --- a/image_analysis/scale_merge/ScaleOnTheFly.cpp +++ b/image_analysis/scale_merge/ScaleOnTheFly.cpp @@ -105,145 +105,6 @@ namespace { double partiality; }; -struct ScalingPostRefResidual : public ScalingResidual { - // Postrefinement for still images - // - // In this algorithm at the point of post-refinement we don't anymore care for where maximum of - // the reflection was located and if it fits the observed position. - // This reflections was already integrated and we cannot integrate it better at this point. - // But, we could adjust partiality to indicate that this reflection was wrongly predicted. - // I.e., integrated position was far away from true reflection, so partiality must be low. - // This is an empiric model and need to see if this will work in practice at all. - // I hope it will allow the model to find that reflections were misindexed. - // We assume at this point that initial indexing was done properly and integration was generally OK => most low resolution reflections fit correctly - // Yet we know, that small errors in indexing are inducing misalignment at high resolution - sometimes it is visible that high-resolution reflections - // are away from shoe-boxes observed in the image, if we can catch this at post-refinement/scaling step, this would be great. - // Next logical step is to do this pixel-wise - for each pixel refine partiality and merge pixels - // This should work for per-image scaling, or even, maybe, for full rotation datasets (3600 images) - // Then we could properly take into account misalignment of shoe-box center vs. partiality and also remove most pixels - // in the shoe-box that don't really contribute to the reflection. - // But...it could also drift to downweighting partiality for all high resolution reflections to make loss function "fake happy". - - // We assume rot3 == 0. Rot3 is not really helping much in crystallography (other than fixing polarization correction) - ScalingPostRefResidual(const Reflection &r, double Itrue, double sigma, - const DiffractionGeometry &geometry, - const CrystalLattice &lattice) - : ScalingResidual(r, Itrue, sigma), - integration_center_x(r.predicted_x), - integration_center_y(r.predicted_y), - inv_lambda(SafeInv(geometry.GetWavelength_A(), 0.0)), - pixel_size(geometry.GetPixelSize_mm()), - det_dist_mm(geometry.GetDetectorDistance_mm()), - beam_x(geometry.GetBeamX_pxl()), - beam_y(geometry.GetBeamY_pxl()), - exp_h(r.h), - exp_k(r.k), - exp_l(r.l), - Astar(lattice.Astar()), - Bstar(lattice.Bstar()), - Cstar(lattice.Cstar()), - c1(std::cos(geometry.GetPoniRot1_rad())), - s1(std::sin(geometry.GetPoniRot1_rad())), - c2(std::cos(geometry.GetPoniRot2_rad())), - s2(std::sin(geometry.GetPoniRot2_rad())) { - } - - template - T CalcPartiality(const T *const R, - const T *const beam_corr, - const T *const p0) const { - // Detector coordinates in mm - const T det_x = (T(integration_center_x) - beam_x - beam_corr[0]) * T(pixel_size); - const T det_y = (T(integration_center_y) - beam_y - beam_corr[1]) * T(pixel_size); - const T det_z = T(det_dist_mm); - - // Apply Ry(rot1) first: rotate around Y - const T t1_x = T(c1) * det_x + T(s1) * det_z; - const T t1_y = det_y; - const T t1_z = T(-s1) * det_x + T(c1) * det_z; - - // Then apply Rx(-rot2): rotate around X - const T x = t1_x; - const T y = T(c2) * t1_y + T(s2) * t1_z; - const T z = - T(s2) * t1_y + T(c2) * t1_z; - - // convert to recip space - const T lab_norm = ceres::sqrt(x * x + y * y + z * z); - const T inv_norm = T(1) / lab_norm; - - T recip_obs[3]; - recip_obs[0] = x * inv_norm * inv_lambda; - recip_obs[1] = y * inv_norm * inv_lambda; - recip_obs[2] = (z * inv_norm - T(1.0)) * inv_lambda; - - const Eigen::Matrix e_obs_recip(recip_obs[0], recip_obs[1], recip_obs[2]); - - const T astar_unrot[3] = {T(Astar.x), T(Astar.y), T(Astar.z)}; - const T bstar_unrot[3] = {T(Bstar.x), T(Bstar.y), T(Bstar.z)}; - const T cstar_unrot[3] = {T(Cstar.x), T(Cstar.y), T(Cstar.z)}; - - T astar_rot[3], bstar_rot[3], cstar_rot[3]; - - ceres::AngleAxisRotatePoint(p0, astar_unrot, astar_rot); - ceres::AngleAxisRotatePoint(p0, bstar_unrot, bstar_rot); - ceres::AngleAxisRotatePoint(p0, cstar_unrot, cstar_rot); - - const Eigen::Matrix e_pred_recip(T(exp_h) * astar_rot[0] + T(exp_k) * bstar_rot[0] + T(exp_l) * cstar_rot[0], - T(exp_h) * astar_rot[1] + T(exp_k) * bstar_rot[1] + T(exp_l) * cstar_rot[1], - T(exp_h) * astar_rot[2] + T(exp_k) * bstar_rot[2] + T(exp_l) * cstar_rot[2] - ); - - // Ewald sphere centre is at -k_i = (0, 0, -inv_lambda) - // Radial direction: outward normal at g_hkl - const Eigen::Matrix S_pred( - e_pred_recip[0], - e_pred_recip[1], - e_pred_recip[2] + T(inv_lambda) // g_hkl + k_i - ); - const T S_pred_norm = S_pred.norm(); - if (S_pred_norm < T(1e-10)) - return T(0); - - const Eigen::Matrix n_radial = S_pred / S_pred_norm; - - const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - const T eps_radial = delta_q.dot(n_radial); - const Eigen::Matrix dq_tang = delta_q - eps_radial * n_radial; - const T eps_tangential_sq = dq_tang.squaredNorm(); // guaranteed ≥ 0 - // ───────────────────────────────────────────────────────────── - - return ceres::exp(- eps_radial * eps_radial / (R[0] * R[0]) - eps_tangential_sq / (R[1] * R[1])); - } - - template - bool operator()(const T *const G, - const T *const B, - const T *const R, - const T *const beam_corr, - const T *const p0, - T *residual) const { - if (R[0] < T(1e-10) || R[1] < T(1e-10)) - return false; - - const T B_term = ceres::exp(B[0] * T(b_resolution_coeff)); - - const T partiality = CalcPartiality(R, beam_corr, p0); - residual[0] = (G[0] * partiality * B_term * T(lp) * T(Itrue) - - T(Iobs)) * T(weight); - return true; - } - - const double integration_center_x, integration_center_y; - const double inv_lambda; - const double pixel_size; - const double det_dist_mm; - const double beam_x, beam_y; - const double exp_h; - const double exp_k; - const double exp_l; - const Coord Astar, Bstar, Cstar; - const double c1,s1,c2,s2; -}; struct RotationNormRegularizer { explicit RotationNormRegularizer(double weight) : weight(weight) {} @@ -282,7 +143,6 @@ bool ScaleOnTheFly::Accept(const Reflection &r) const { return std::isfinite(r.zeta) && r.zeta > 0.0f; case PartialityModel::Fixed: case PartialityModel::Unity: - case PartialityModel::Postrefinement: return true; } @@ -395,12 +255,6 @@ void ScaleOnTheFly::Scale(IntegrationOutcome &integration_outcome) const { problem.AddResidualBlock(cost, nullptr, &result.G, &result.B); } break; - case PartialityModel::Postrefinement: { - auto *cost = new ceres::AutoDiffCostFunction( - new ScalingPostRefResidual(r, Itrue, sigma, integration_outcome.geom, integration_outcome.latt)); - problem.AddResidualBlock(cost, nullptr, &result.G, &result.B, result.R, result.beam_corr, result.p0); - } - break; case PartialityModel::Rotation: { auto *cost = new ceres::AutoDiffCostFunction( new ScalingRotationResidual(r, Itrue, sigma)); @@ -460,21 +314,6 @@ void ScaleOnTheFly::Scale(IntegrationOutcome &integration_outcome) const { problem.SetParameterUpperBound(&result.mos, 0, s.GetMaxMosaicity()); } - if (model == PartialityModel::Postrefinement) { - problem.SetParameterLowerBound(result.R, 0, 1e-6); - problem.SetParameterLowerBound(result.R, 1, 1e-6); - - // Beam center can be off by max 1 pixel - problem.SetParameterLowerBound(result.beam_corr, 0, -1); - problem.SetParameterUpperBound(result.beam_corr, 0, 1); - problem.SetParameterLowerBound(result.beam_corr, 1, -1); - problem.SetParameterUpperBound(result.beam_corr, 1, 1); - - auto *angle_reg_cost = new ceres::AutoDiffCostFunction( - new RotationNormRegularizer(0.05)); - problem.AddResidualBlock(angle_reg_cost, nullptr, result.p0); - } - ceres::Solver::Options options; options.linear_solver_type = ceres::DENSE_QR; options.minimizer_progress_to_stdout = false; @@ -495,19 +334,12 @@ void ScaleOnTheFly::Scale(IntegrationOutcome &integration_outcome) const { r.partiality = RotationPartiality(r.delta_phi_deg, r.zeta, result.mos, result.wedge); break; } - case PartialityModel::Postrefinement: { - ScalingPostRefResidual residual(r, 0, 0, integration_outcome.geom, integration_outcome.latt); - r.partiality = static_cast(residual.CalcPartiality(result.R, result.beam_corr, result.p0)); - } - break; default: // For fixed partiality there is no need to change anything break; } const double denom = B_term * r.partiality * result.G; - if (std::isfinite(r.rlp) && - std::isfinite(denom) && - denom > 0.0) { + if (std::isfinite(r.rlp) && std::isfinite(denom) && denom > 0.0) { r.image_scale_corr = static_cast(r.rlp / denom); } else { r.image_scale_corr = NAN; diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 47cd0736..5876418b 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -469,8 +469,6 @@ int main(int argc, char **argv) { partiality_model = PartialityModel::Fixed; else if (strcmp(optarg, "rot") == 0) partiality_model = PartialityModel::Rotation; - else if (strcmp(optarg, "postref") == 0) - partiality_model = PartialityModel::Postrefinement; else { logger.Error("Invalid partiality mode: {}", optarg); print_usage(); -- 2.52.0 From e4230bc14ee9a0b5a127d24d1cb71ca486820de4 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 14:41:36 +0200 Subject: [PATCH 025/228] Bragg integration: option to use azimuthal integration profile --- common/BraggIntegrationSettings.cpp | 9 + common/BraggIntegrationSettings.h | 5 +- image_analysis/IndexAndRefine.cpp | 14 +- image_analysis/IndexAndRefine.h | 8 +- image_analysis/MXAnalysisAfterFPGA.cpp | 11 +- image_analysis/MXAnalysisAfterFPGA.h | 5 +- image_analysis/MXAnalysisWithoutFPGA.cpp | 2 +- .../bragg_integration/BraggIntegrate2D.cpp | 219 +++++++++++++----- .../bragg_integration/BraggIntegrate2D.h | 10 +- receiver/JFJochReceiver.cpp | 25 +- receiver/JFJochReceiver.h | 2 +- receiver/JFJochReceiverFPGA.cpp | 11 +- receiver/JFJochReceiverLite.cpp | 4 +- tests/BraggIntegrate2DTest.cpp | 6 +- tools/jfjoch_scale.cpp | 2 - 15 files changed, 218 insertions(+), 115 deletions(-) diff --git a/common/BraggIntegrationSettings.cpp b/common/BraggIntegrationSettings.cpp index 3fe0a861..0528084e 100644 --- a/common/BraggIntegrationSettings.cpp +++ b/common/BraggIntegrationSettings.cpp @@ -87,3 +87,12 @@ float BraggIntegrationSettings::GetDMinLimit_A() const { float BraggIntegrationSettings::GetMinimumSigmaInRegardsToI() const { return minimum_sigma_in_regards_to_i; } + +bool BraggIntegrationSettings::IsUseAzimProfile() const { + return use_azim_profile; +} + +BraggIntegrationSettings &BraggIntegrationSettings::UseAzimProfile(bool input) { + use_azim_profile = input; + return *this; +} diff --git a/common/BraggIntegrationSettings.h b/common/BraggIntegrationSettings.h index 60ab4059..2c15921d 100644 --- a/common/BraggIntegrationSettings.h +++ b/common/BraggIntegrationSettings.h @@ -12,14 +12,14 @@ class BraggIntegrationSettings { float d_min_limit_A = 1.0; std::optional fixed_profile_radius; float minimum_sigma_in_regards_to_i = 0.02; - + bool use_azim_profile = false; public: BraggIntegrationSettings& R1(float input); BraggIntegrationSettings& R2(float input); BraggIntegrationSettings& R3(float input); BraggIntegrationSettings& DMinLimit_A(float input); BraggIntegrationSettings& FixedProfileRadius_recipA(std::optional input); - + BraggIntegrationSettings& UseAzimProfile(bool input); [[nodiscard]] float GetR1() const; [[nodiscard]] float GetR2() const; @@ -28,4 +28,5 @@ public: [[nodiscard]] float GetDMinLimit_A() const; [[nodiscard]] float GetMinimumSigmaInRegardsToI() const; + [[nodiscard]] bool IsUseAzimProfile() const; }; diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 98b1aed6..459af31e 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -239,8 +239,8 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, const CompressedImage &image, BraggPrediction &prediction, const IndexAndRefine::IndexingOutcome &outcome, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile) { + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile) { if (!outcome.lattice_candidate) return; @@ -299,11 +299,11 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, // integrated reflections that flow into the normal save/merge). const bool use_pixel_refine = experiment.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine - && !pixel_reference_.empty() && mapping && profile; + && !pixel_reference_.empty(); if (use_pixel_refine) { auto integration_start_time = std::chrono::steady_clock::now(); - PixelRefineIntegrate(msg, image, prediction, outcome, *mapping, *profile, i_outcome); + PixelRefineIntegrate(msg, image, prediction, outcome, mapping, profile, i_outcome); msg.integrated_reflections = i_outcome.reflections.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); @@ -314,7 +314,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, msg.bragg_prediction_time_s = std::chrono::duration(pred_end_time - pred_start_time).count(); auto integration_start_time = std::chrono::steady_clock::now(); - i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), nrefl, msg.number); + i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), mapping, profile, nrefl, msg.number); msg.integrated_reflections = i_outcome.reflections.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); @@ -357,8 +357,8 @@ void IndexAndRefine::ProcessImage(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile) { + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile) { if (!indexer_ || !spot_finding_settings.indexing) return; diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index 7674392e..d73fc2f1 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -66,8 +66,8 @@ class IndexAndRefine { const CompressedImage &image, BraggPrediction &prediction, const IndexingOutcome &outcome, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile); + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile); std::unique_ptr scaling_engine; void ScaleImage(DataMessage &msg, IntegrationOutcome& outcome); @@ -93,8 +93,8 @@ public: void ForceRotationIndexerLattice(const CrystalLattice& lattice); void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction, - const AzimuthalIntegrationMapping *mapping = nullptr, - const AzimuthalIntegrationProfile *profile = nullptr); + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile); IndexAndRefine& ReferenceIntensities(std::vector &reference); ScalingResult ScaleAllImages(const std::vector &reference, size_t nthreads = 0); diff --git a/image_analysis/MXAnalysisAfterFPGA.cpp b/image_analysis/MXAnalysisAfterFPGA.cpp index a5347236..96d8430d 100644 --- a/image_analysis/MXAnalysisAfterFPGA.cpp +++ b/image_analysis/MXAnalysisAfterFPGA.cpp @@ -25,10 +25,12 @@ double stddev(const std::vector &v) { } -MXAnalysisAfterFPGA::MXAnalysisAfterFPGA(const DiffractionExperiment &in_experiment, IndexAndRefine &indexer) +MXAnalysisAfterFPGA::MXAnalysisAfterFPGA(const DiffractionExperiment &in_experiment, IndexAndRefine &indexer, + AzimuthalIntegrationMapping &mapping) : experiment(in_experiment), indexer(indexer), - prediction(CreateBraggPrediction(experiment.IsRotationIndexing())) { + prediction(CreateBraggPrediction(experiment.IsRotationIndexing())), + mapping(mapping) { if (experiment.IsSpotFindingEnabled()) find_spots = true; } @@ -103,7 +105,8 @@ void MXAnalysisAfterFPGA::ReadFromCPU(DeviceOutput *output, const SpotFindingSet } } -void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSettings& spot_finding_settings) { +void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSettings &spot_finding_settings, + const AzimuthalIntegrationProfile &profile) { if (find_spots && (state == State::Enabled)) { const auto t0 = std::chrono::steady_clock::now(); SpotAnalyze(experiment, spot_finding_settings, spots, message); @@ -111,7 +114,7 @@ void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSetting spot_finding_time_total += (t1 - t0); if (spot_finding_settings.indexing) - indexer.ProcessImage(message, spot_finding_settings, message.image, *prediction); + indexer.ProcessImage(message, spot_finding_settings, message.image, *prediction, mapping, profile); } if (spot_finding_timing_active) { diff --git a/image_analysis/MXAnalysisAfterFPGA.h b/image_analysis/MXAnalysisAfterFPGA.h index ccddf87a..e3b708ac 100644 --- a/image_analysis/MXAnalysisAfterFPGA.h +++ b/image_analysis/MXAnalysisAfterFPGA.h @@ -28,9 +28,10 @@ class MXAnalysisAfterFPGA { std::chrono::duration spot_finding_time_total{0.0}; bool spot_finding_timing_active = false; + const AzimuthalIntegrationMapping &mapping; public: - MXAnalysisAfterFPGA(const DiffractionExperiment& experiment, IndexAndRefine &indexer); + MXAnalysisAfterFPGA(const DiffractionExperiment& experiment, IndexAndRefine &indexer, AzimuthalIntegrationMapping &mapping); void ReadFromFPGA(const DeviceOutput* output, const SpotFindingSettings& settings, @@ -40,5 +41,5 @@ public: const SpotFindingSettings &settings, size_t module_number); - void Process(DataMessage &message, const SpotFindingSettings& settings); + void Process(DataMessage &message, const SpotFindingSettings& settings, const AzimuthalIntegrationProfile &profile); }; diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 6a638213..62b6379f 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -92,7 +92,7 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, if (spot_finding_settings.indexing) indexer.ProcessImage(output, spot_finding_settings, CompressedImage(preprocessor_buffer->getBuffer(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum()), - *prediction, &integration, &profile); + *prediction, integration, profile); } output.max_viable_pixel_value = ret.max_value; diff --git a/image_analysis/bragg_integration/BraggIntegrate2D.cpp b/image_analysis/bragg_integration/BraggIntegrate2D.cpp index b616ecc1..35ec4849 100644 --- a/image_analysis/bragg_integration/BraggIntegrate2D.cpp +++ b/image_analysis/bragg_integration/BraggIntegrate2D.cpp @@ -8,65 +8,62 @@ #include namespace { + template + float Median(std::vector &values) { + if (values.empty()) + return 0.0f; -template -float Median(std::vector &values) { - if (values.empty()) - return 0.0f; + const size_t middle = values.size() / 2; + std::nth_element(values.begin(), values.begin() + middle, values.end()); - const size_t middle = values.size() / 2; - std::nth_element(values.begin(), values.begin() + middle, values.end()); + if (values.size() % 2 == 1) + return static_cast(values[middle]); - if (values.size() % 2 == 1) - return static_cast(values[middle]); + const T upper = values[middle]; + std::nth_element(values.begin(), values.begin() + middle - 1, values.begin() + middle); + const T lower = values[middle - 1]; - const T upper = values[middle]; - std::nth_element(values.begin(), values.begin() + middle - 1, values.begin() + middle); - const T lower = values[middle - 1]; + return 0.5f * static_cast(lower + upper); + } - return 0.5f * static_cast(lower + upper); -} + void MarkReflectionMask(std::vector &mask, + size_t xpixel, size_t ypixel, + const Reflection &r, float r_2, float r_2_sq) { + int64_t x0 = std::floor(r.predicted_x - r_2 - 1.0f); + int64_t x1 = std::ceil(r.predicted_x + r_2 + 1.0f); + int64_t y0 = std::floor(r.predicted_y - r_2 - 1.0f); + int64_t y1 = std::ceil(r.predicted_y + r_2 + 1.0f); -void MarkReflectionMask(std::vector &mask, - size_t xpixel, size_t ypixel, - const Reflection &r, float r_2, float r_2_sq) { + if (x0 < 0) + x0 = 0; + if (y0 < 0) + y0 = 0; + if (x1 >= static_cast(xpixel)) + x1 = static_cast(xpixel) - 1; + if (y1 >= static_cast(ypixel)) + y1 = static_cast(ypixel) - 1; - int64_t x0 = std::floor(r.predicted_x - r_2 - 1.0f); - int64_t x1 = std::ceil(r.predicted_x + r_2 + 1.0f); - int64_t y0 = std::floor(r.predicted_y - r_2 - 1.0f); - int64_t y1 = std::ceil(r.predicted_y + r_2 + 1.0f); - - if (x0 < 0) - x0 = 0; - if (y0 < 0) - y0 = 0; - if (x1 >= static_cast(xpixel)) - x1 = static_cast(xpixel) - 1; - if (y1 >= static_cast(ypixel)) - y1 = static_cast(ypixel) - 1; - - for (int64_t y = y0; y <= y1; y++) { - for (int64_t x = x0; x <= x1; x++) { - const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) - + (y - r.predicted_y) * (y - r.predicted_y); - if (dist_sq < r_2_sq) - mask[y * xpixel + x] = 1; + for (int64_t y = y0; y <= y1; y++) { + for (int64_t x = x0; x <= x1; x++) { + const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) + + (y - r.predicted_y) * (y - r.predicted_y); + if (dist_sq < r_2_sq) + mask[y * xpixel + x] = 1; + } } } -} -std::vector BuildReflectionMask(const std::vector &predicted, - size_t npredicted, - size_t xpixel, size_t ypixel, - float r_2, float r_2_sq) { - std::vector mask(xpixel * ypixel, 0); + std::vector BuildReflectionMask(const std::vector &predicted, + size_t npredicted, + size_t xpixel, size_t ypixel, + float r_2, float r_2_sq) { + std::vector mask(xpixel * ypixel, 0); - for (size_t i = 0; i < npredicted; i++) - MarkReflectionMask(mask, xpixel, ypixel, predicted.at(i), r_2, r_2_sq); - - return mask; -} + for (size_t i = 0; i < npredicted; i++) + MarkReflectionMask(mask, xpixel, ypixel, predicted.at(i), r_2, r_2_sq); + return mask; + } } // namespace template @@ -75,7 +72,6 @@ void IntegrateReflection(Reflection &r, const T *image, const std::vector +void IntegrateReflectionAzim(Reflection &r, const T *image, + size_t xpixel, size_t ypixel, + int64_t special_value, int64_t saturation, + float r_1, float r_1_sq, + const std::vector &pixel_to_bin, + const std::vector &bkg_mean, + float minimum_sigma_in_regards_to_i) { + int64_t x0 = std::floor(r.predicted_x - r_1 - 1.0); + int64_t x1 = std::ceil(r.predicted_x + r_1 + 1.0); + int64_t y0 = std::floor(r.predicted_y - r_1 - 1.0); + int64_t y1 = std::ceil(r.predicted_y + r_1 + 1.0); + x0 = std::max(0L, x0); + x1 = std::min(static_cast(xpixel - 1), x1); + y0 = std::max(0L, y0); + y1 = std::min(static_cast(ypixel - 1), y1); + + int64_t I_sum = 0; + int64_t I_npixel_inner = 0; + int64_t I_npixel_integrated = 0; + int64_t I_sum_x = 0; + int64_t I_sum_y = 0; + + std::vector bkg_values; + bkg_values.reserve(static_cast((x1 - x0 + 1) * (y1 - y0 + 1))); + + float bkg = 0.0; + + for (int64_t y = y0; y <= y1; y++) { + for (int64_t x = x0; x <= x1; x++) { + const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) + + (y - r.predicted_y) * (y - r.predicted_y); + const auto pixel = image[y * xpixel + x]; + auto azim_bint = pixel_to_bin[y * xpixel + x]; + + if (dist_sq < r_1_sq) + I_npixel_inner++; + + if (azim_bint >= bkg_mean.size()) + continue; + + if (pixel == special_value || pixel == saturation) + continue; + + if (dist_sq < r_1_sq) { + bkg += bkg_values.at(azim_bint); + I_sum += pixel; + I_sum_x += x * pixel; + I_sum_y += y * pixel; + I_npixel_integrated++; + } + } + } + + if ((I_npixel_integrated == I_npixel_inner) && (bkg_values.size() > 5)) { + r.bkg = bkg / static_cast(I_npixel_integrated); + r.I = static_cast(I_sum) - bkg; + if (I_sum > 0) { + r.observed_x = static_cast(I_sum_x) / static_cast(I_sum); + r.observed_y = static_cast(I_sum_y) / static_cast(I_sum); + } + + // sigma is max of the: + // - 1 (for zero photons) + // - Poisson noise (sqrt(I_sum)) (for in between) + // - minimum_sigma_in_regards_to_i of Intensity (for very large numbers) + r.sigma = std::max(1.0f, r.I * minimum_sigma_in_regards_to_i); + if (I_sum > 0) + r.sigma = std::max(r.sigma, std::sqrt(static_cast(I_sum))); + r.observed = true; + } else { + r.I = 0; + r.bkg = 0; + r.sigma = NAN; + r.observed = false; + } +} + template std::vector IntegrateInternal(const DiffractionExperiment &experiment, const CompressedImage &image, + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile, const std::vector &predicted, size_t npredicted, int64_t special_value, int64_t saturation, int64_t image_number) { - std::vector ret; ret.reserve(npredicted); @@ -164,24 +239,36 @@ std::vector IntegrateInternal(const DiffractionExperiment &experimen std::vector buffer; auto ptr = reinterpret_cast(image.GetUncompressedPtr(buffer)); - const float r_3 = settings.GetR3(); - const float r_1_sq = settings.GetR1() * settings.GetR1(); + const float r_1 = settings.GetR1(); const float r_2 = settings.GetR2(); - const float r_2_sq = settings.GetR2() * settings.GetR2(); - const float r_3_sq = settings.GetR3() * settings.GetR3(); + const float r_3 = settings.GetR3(); + + const float r_1_sq = r_1 * r_1; + const float r_2_sq = r_2 * r_2; + const float r_3_sq = r_3 * r_3; const float minimum_sigma_in_regards_to_i = settings.GetMinimumSigmaInRegardsToI(); const auto reflection_mask = BuildReflectionMask(predicted, npredicted, image.GetWidth(), image.GetHeight(), r_2, r_2_sq); + const auto &pixel_to_bin = mapping.GetPixelToBin(); + const auto p = profile.GetResult(); + for (int i = 0; i < npredicted; i++) { auto r = predicted.at(i); - IntegrateReflection(r, ptr, reflection_mask, image.GetWidth(), image.GetHeight(), special_value, saturation, - r_3, r_1_sq, r_2_sq, r_3_sq, minimum_sigma_in_regards_to_i); + if (settings.IsUseAzimProfile()) { + IntegrateReflectionAzim(r, ptr, image.GetWidth(), image.GetHeight(), + special_value, saturation, + r_1, r_1_sq, pixel_to_bin, p, minimum_sigma_in_regards_to_i); + } else { + IntegrateReflection(r, ptr, reflection_mask, image.GetWidth(), image.GetHeight(), special_value, saturation, + r_3, r_1_sq, r_2_sq, r_3_sq, minimum_sigma_in_regards_to_i); + } if (r.observed) { if (experiment.GetPolarizationFactor()) - r.rlp /= geom.CalcAzIntPolarizationCorr(r.predicted_x, r.predicted_y, experiment.GetPolarizationFactor().value()); + r.rlp /= geom.CalcAzIntPolarizationCorr(r.predicted_x, r.predicted_y, + experiment.GetPolarizationFactor().value()); r.image_scale_corr = r.rlp / r.partiality; r.image_number = static_cast(image_number); ret.emplace_back(r); @@ -193,6 +280,8 @@ std::vector IntegrateInternal(const DiffractionExperiment &experimen std::vector BraggIntegrate2D(const DiffractionExperiment &experiment, const CompressedImage &image, const std::vector &predicted, + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile, size_t npredicted, int64_t image_number) { if (image.GetCompressedSize() == 0 || predicted.empty()) @@ -200,17 +289,23 @@ std::vector BraggIntegrate2D(const DiffractionExperiment &experiment switch (image.GetMode()) { case CompressedImageMode::Int8: - return IntegrateInternal(experiment, image, predicted, npredicted, INT8_MIN, INT8_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + INT8_MIN, INT8_MAX, image_number); case CompressedImageMode::Int16: - return IntegrateInternal(experiment, image, predicted, npredicted, INT16_MIN, INT16_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + INT16_MIN, INT16_MAX, image_number); case CompressedImageMode::Int32: - return IntegrateInternal(experiment, image, predicted, npredicted, INT32_MIN, INT32_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + INT32_MIN, INT32_MAX, image_number); case CompressedImageMode::Uint8: - return IntegrateInternal(experiment, image, predicted, npredicted, UINT8_MAX, UINT8_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + UINT8_MAX, UINT8_MAX, image_number); case CompressedImageMode::Uint16: - return IntegrateInternal(experiment, image, predicted, npredicted, UINT16_MAX, UINT16_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + UINT16_MAX, UINT16_MAX, image_number); case CompressedImageMode::Uint32: - return IntegrateInternal(experiment, image, predicted, npredicted, UINT32_MAX, UINT32_MAX, image_number); + return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, + UINT32_MAX, UINT32_MAX, image_number); default: throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image mode not supported"); diff --git a/image_analysis/bragg_integration/BraggIntegrate2D.h b/image_analysis/bragg_integration/BraggIntegrate2D.h index 353757b5..3045ff7d 100644 --- a/image_analysis/bragg_integration/BraggIntegrate2D.h +++ b/image_analysis/bragg_integration/BraggIntegrate2D.h @@ -1,17 +1,17 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -#ifndef JFJOCH_BRAGGINTEGRATE2D_H -#define JFJOCH_BRAGGINTEGRATE2D_H +#pragma once #include #include "../../common/DiffractionExperiment.h" #include "../../common/Reflection.h" +#include "../../common/AzimuthalIntegrationProfile.h" std::vector BraggIntegrate2D(const DiffractionExperiment &experiment, const CompressedImage &image, const std::vector &predicted, + const AzimuthalIntegrationMapping &mapping, + const AzimuthalIntegrationProfile &profile, size_t npredicted, - int64_t image_number); - -#endif //JFJOCH_BRAGGINTEGRATE2D_H + int64_t image_number); \ No newline at end of file diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 64e82e33..1aa8ce1e 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -31,7 +31,8 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, serialmx_filter(in_experiment), numa_policy(in_numa_policy), pixel_mask(in_pixel_mask), - indexer(experiment, indexing_thread_pool) { + indexer(experiment, indexing_thread_pool), + az_int_mapping(experiment, pixel_mask) { logger.Info("Initializing receiver"); // Ensure there is nothing running for now if (!image_buffer.Finalize(std::chrono::seconds(1))) @@ -43,13 +44,7 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, current_status.SetEfficiency({}); current_status.SetStatus(JFJochReceiverStatus{}); // GetStatus() is virtual function and cannot be called yet! - auto start_time_point = std::chrono::steady_clock::now(); - az_int_mapping = std::make_unique(experiment, pixel_mask); - auto end_time_point = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration(end_time_point - start_time_point); - logger.Info("Azimuthal integration mapping done in {:.5f} s with {} threads", duration.count(), az_int_mapping->GetNThreads()); - - plots.Setup(experiment, *az_int_mapping); + plots.Setup(experiment, az_int_mapping); push_images_to_writer = (experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty()); } @@ -111,13 +106,13 @@ void JFJochReceiver::SendStartMessage() { StartMessage message{}; experiment.FillMessage(message); message.arm_date = time_UTC(std::chrono::system_clock::now()); - message.az_int_q_bin_count = az_int_mapping->GetQBinCount(); - message.az_int_bin_to_q = az_int_mapping->GetBinToQ(); - message.az_int_bin_to_two_theta = az_int_mapping->GetBinToTwoTheta(); - message.az_int_phi_bin_count = az_int_mapping->GetAzimuthalBinCount(); - if (az_int_mapping->GetAzimuthalBinCount() > 1) { - message.az_int_bin_to_phi = az_int_mapping->GetBinToPhi(); - message.az_int_map = az_int_mapping->GetPixelToBin(); + message.az_int_q_bin_count = az_int_mapping.GetQBinCount(); + message.az_int_bin_to_q = az_int_mapping.GetBinToQ(); + message.az_int_bin_to_two_theta = az_int_mapping.GetBinToTwoTheta(); + message.az_int_phi_bin_count = az_int_mapping.GetAzimuthalBinCount(); + if (az_int_mapping.GetAzimuthalBinCount() > 1) { + message.az_int_bin_to_phi = az_int_mapping.GetBinToPhi(); + message.az_int_map = az_int_mapping.GetPixelToBin(); } message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress(); message.rois = experiment.ROI().ExportMetadata(); diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 83918987..79be2802 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -80,7 +80,7 @@ protected: std::vector> adu_histogram_module; PixelMask pixel_mask; - std::unique_ptr az_int_mapping; + AzimuthalIntegrationMapping az_int_mapping; std::optional max_delay; std::mutex max_delay_mutex; diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 802b7faf..a3ac6384 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -297,7 +297,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { try { numa_policy.Bind(threadid); - analyzer = std::make_unique(experiment, indexer); + analyzer = std::make_unique(experiment, indexer, az_int_mapping); } catch (const JFJochException &e) { frame_transformation_ready.count_down(); logger.Error("Thread setup error {}", e.what()); @@ -309,9 +309,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { frame_transformation_ready.count_down(); - uint16_t az_int_min_bin = std::floor(az_int_mapping->QToBin(experiment.GetLowQForBkgEstimate_recipA())); - uint16_t az_int_max_bin = std::ceil(az_int_mapping->QToBin(experiment.GetHighQForBkgEstimate_recipA())); - int64_t image_number; while (images_to_go.Get(image_number) != 0) { try { @@ -338,7 +335,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { ImageMetadata metadata(experiment); - AzimuthalIntegrationProfile az_int_profile_image(*az_int_mapping); + AzimuthalIntegrationProfile az_int_profile_image(az_int_mapping); auto local_spot_finding_settings = GetSpotFindingSettings(); @@ -428,7 +425,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { experiment.GetYPixelsNum(), experiment.GetImageMode(), CompressionAlgorithm::NO_COMPRESSION); - analyzer->Process(message, local_spot_finding_settings); + analyzer->Process(message, local_spot_finding_settings, az_int_profile_image); auto status = image_buffer.GetStatus(); message.receiver_buf_available = status.available_slots; @@ -681,5 +678,5 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) { acquisition_device[data_stream].InitializeROIMap(experiment, roi_map); // Initialize data processing - acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping); + acquisition_device[data_stream].InitializeDataProcessing(experiment, az_int_mapping); } diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index e986956e..8c04b1cf 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -242,7 +242,7 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { measurement_started.wait(); try { - analysis = std::make_unique(experiment, *az_int_mapping, pixel_mask, indexer); + analysis = std::make_unique(experiment, az_int_mapping, pixel_mask, indexer); } catch (const JFJochException &e) { Cancel(e); return; @@ -279,7 +279,7 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { auto image_start_time = std::chrono::high_resolution_clock::now(); - AzimuthalIntegrationProfile profile(*az_int_mapping); + AzimuthalIntegrationProfile profile(az_int_mapping); analysis->Analyze(data_msg, profile, GetSpotFindingSettings()); auto image_end_time = std::chrono::high_resolution_clock::now(); diff --git a/tests/BraggIntegrate2DTest.cpp b/tests/BraggIntegrate2DTest.cpp index 75ba2a42..149515c0 100644 --- a/tests/BraggIntegrate2DTest.cpp +++ b/tests/BraggIntegrate2DTest.cpp @@ -59,6 +59,10 @@ TEST_CASE("BraggIntegrate2D_RejectsReflectionsFromBackgroundUsingR2MaskAndMedian Reflection r1 = MakeReflection(10.0f, 10.0f); Reflection r2 = MakeReflection(16.0f, 10.0f); + PixelMask mask(experiment); + AzimuthalIntegrationMapping mapping(experiment, mask); + AzimuthalIntegrationProfile profile(mapping); + for (int y = 0; y < static_cast(height); y++) { for (int x = 0; x < static_cast(width); x++) { auto &pixel = image_data[y * width + x]; @@ -74,7 +78,7 @@ TEST_CASE("BraggIntegrate2D_RejectsReflectionsFromBackgroundUsingR2MaskAndMedian CompressedImage image(image_data, width, height); std::vector predicted = {r1, r2}; - auto integrated = BraggIntegrate2D(experiment, image, predicted, predicted.size(), 17); + auto integrated = BraggIntegrate2D(experiment, image, predicted, mapping, profile, predicted.size(), 17); REQUIRE(integrated.size() == 2); diff --git a/tools/jfjoch_scale.cpp b/tools/jfjoch_scale.cpp index 7f169940..e6da83f2 100644 --- a/tools/jfjoch_scale.cpp +++ b/tools/jfjoch_scale.cpp @@ -148,8 +148,6 @@ int main(int argc, char **argv) { partiality_model = PartialityModel::Fixed; else if (strcmp(optarg, "rot") == 0) partiality_model = PartialityModel::Rotation; - else if (strcmp(optarg, "postref") == 0) - partiality_model = PartialityModel::Postrefinement; else { logger.Error("Invalid partiality mode: {}", optarg); print_usage(); -- 2.52.0 From b22d5929a1b3e9b565338cfccf91a184a04e02ff Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 14:52:58 +0200 Subject: [PATCH 026/228] jfjoch_process: Add option to use azimuthal integration as background for Bragg integration --- tools/jfjoch_process.cpp | 575 ++++++++++++++++++++++----------------- 1 file changed, 321 insertions(+), 254 deletions(-) diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 5876418b..4c8d070c 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -45,39 +45,75 @@ void print_usage() { std::cout << " Spot finding" << std::endl; std::cout << " --spot-sigma Noise sigma level for spot finding (default: 3.0)" << std::endl; - std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << std::endl; - std::cout << " --spot-high-resolution High resolution limit for spot finding (default: 1.5)" << std::endl; + std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << + std::endl; + std::cout << " --spot-high-resolution High resolution limit for spot finding (default: 1.5)" << + std::endl; std::cout << " --max-spots Max spot count (default: 250)" << std::endl; std::cout << std::endl; + std::cout << " Azimuthal integration" << std::endl; + std::cout << " -Q, --azim-q-spacing Q spacing for azimuthal integration (default: 0.01)" << + std::endl; + std::cout << " --azim-q-min Minimum Q value for azimuthal integration (default: 0.0)" << + std::endl; + std::cout << " --azim-q-max Maximum Q value for azimuthal integration (default: 5.0)" << + std::endl; + std::cout << std::endl; + std::cout << " Indexing" << std::endl; - std::cout << " -R, --two-pass-rotation[=num] Two-pass offline rotation indexing (optional: number of images, default: 30)" << std::endl; - std::cout << " --single-pass-rotation[=num] Use online-like single-pass rotation indexing (optional: min angular range deg)" << std::endl; + std::cout << + " -R, --two-pass-rotation[=num] Two-pass offline rotation indexing (optional: number of images, default: 30)" + << std::endl; + std::cout << + " --single-pass-rotation[=num] Use online-like single-pass rotation indexing (optional: min angular range deg)" + << std::endl; std::cout << " --redo-rotation-spots Redo spot finding for two-pass rotation indexing" << std::endl; - std::cout << " --force-rotation-lattice Force rotation indexer with external lattice (in Angstrom) : \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, skips first pass)" << std::endl; + std::cout << + " --force-rotation-lattice Force rotation indexer with external lattice (in Angstrom) : \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, skips first pass)" + << std::endl; std::cout << " -X, --indexing-algorithm Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl; - std::cout << " -S, --space-group Space group number - used for both indexing and scaling" << std::endl; + std::cout << " -S, --space-group Space group number - used for both indexing and scaling" << + std::endl; std::cout << " -C, --unit-cell Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl; - std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|pixelrefine)" << std::endl; + std::cout << + " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|pixelrefine)" + << std::endl; + std::cout << std::endl; + + std::cout << " Integration" << std::endl; + std::cout << + " --integration-use-azim Background from Bragg peak integration is based on azimuthal integration results" + << std::endl; std::cout << std::endl; std::cout << " Scaling and merging" << std::endl; - std::cout << " -M, --scale-merge Scale and merge (refine mosaicity) and write scaled.hkl + image.dat" << std::endl; - std::cout << " -P, --partiality Partiality refinement fixed|rot|unity (default: fixed)" << std::endl; + std::cout << + " -M, --scale-merge Scale and merge (refine mosaicity) and write scaled.hkl + image.dat" << + std::endl; + std::cout << " -P, --partiality Partiality refinement fixed|rot|unity (default: fixed)" << + std::endl; std::cout << " -A, --anomalous Anomalous mode (don't merge Friedel pairs)" << std::endl; std::cout << " -B, --refine-bfactor Refine per image B-factor" << std::endl; - std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << std::endl; - std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << std::endl; - std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; + std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << + std::endl; + std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << + std::endl; + std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << + std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; - std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; - std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; + std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" + << std::endl; + std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" + << std::endl; std::cout << " -z, --reference-mtz Reference MTZ file" << std::endl; std::cout << std::endl; std::cout << " Pixel refinement (experimental, select via -r pixelrefine, needs --reference-mtz)" << std::endl; - std::cout << " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" << std::endl; - } + std::cout << + " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" + << std::endl; +} enum { OPT_SPOT_SIGMA = 1000, @@ -92,7 +128,10 @@ enum { OPT_SINGLE_PASS_ROTATION, OPT_REDO_ROTATION_SPOTS, OPT_FORCE_ROTATION_LATTICE, - OPT_BANDWIDTH + OPT_BANDWIDTH, + OPT_INTEGRATION_USE_AZIM, + OPT_AZIM_Q_MIN, + OPT_AZIM_Q_MAX }; static option long_options[] = { @@ -112,13 +151,16 @@ static option long_options[] = { {"wedge", optional_argument, nullptr, 'w'}, {"scale-merge", no_argument, nullptr, 'M'}, {"refine", required_argument, nullptr, 'r'}, + {"azim-q-spacing", required_argument, nullptr, 'Q'}, + {"azim-q-min", required_argument, nullptr, OPT_AZIM_Q_MIN}, + {"azim-q-max", required_argument, nullptr, OPT_AZIM_Q_MAX}, {"two-pass-rotation", optional_argument, nullptr, 'R'}, {"single-pass-rotation", optional_argument, nullptr, OPT_SINGLE_PASS_ROTATION}, {"redo-rotation-spots", no_argument, nullptr, OPT_REDO_ROTATION_SPOTS}, {"force-rotation-lattice", required_argument, nullptr, OPT_FORCE_ROTATION_LATTICE}, - + {"integration-use-azim", no_argument, nullptr, OPT_INTEGRATION_USE_AZIM}, {"spot-sigma", required_argument, nullptr, OPT_SPOT_SIGMA}, {"spot-threshold", required_argument, nullptr, OPT_SPOT_THRESHOLD}, {"spot-high-resolution", required_argument, nullptr, OPT_SPOT_RESOLUTION}, @@ -181,7 +223,6 @@ std::optional parse_unit_cell_arg(const char *arg) { return std::nullopt; - UnitCell uc{}; if (!parse_float_strict(parts[0], uc.a)) return std::nullopt; if (!parse_float_strict(parts[1], uc.b)) return std::nullopt; @@ -297,6 +338,11 @@ int main(int argc, char **argv) { double min_partiality = 0.02; double min_image_cc = 0.0; int64_t scaling_iter = 3; + std::optional azim_q_spacing; + std::optional azim_q_min; + std::optional azim_q_max; + bool use_azim_for_integration = false; + std::optional forced_rotation_lattice; std::optional bandwidth_fwhm; // relative FWHM of dlambda/lambda @@ -317,225 +363,237 @@ int main(int argc, char **argv) { int opt; int option_index = 0; - const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABw::S:MP:r:"; + const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABw::S:MP:r:Q:"; - while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { - switch (opt) { - case 'o': - output_prefix = optarg; - break; - case 'v': - verbose = true; - break; - case 'N': - nthreads = atoi(optarg); - break; - case 's': - start_image = atoi(optarg); - break; - case 'e': - end_image = atoi(optarg); - break; - case 't': - image_stride = atoi(optarg); - break; - case 'R': - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); + while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { + switch (opt) { + case 'o': + output_prefix = optarg; + break; + case 'v': + verbose = true; + break; + case 'N': + nthreads = atoi(optarg); + break; + case 's': + start_image = atoi(optarg); + break; + case 'e': + end_image = atoi(optarg); + break; + case 't': + image_stride = atoi(optarg); + break; + case 'R': + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); + } + rotation_indexing = true; + two_pass_rotation = true; + if (optarg) + rotation_indexing_image_count = atoi(optarg); + + break; + case 'Q': + azim_q_spacing = atof(optarg); + break; + case OPT_AZIM_Q_MIN: + azim_q_min = atof(optarg); + break; + case OPT_AZIM_Q_MAX: + azim_q_max = atof(optarg); + break; + case OPT_INTEGRATION_USE_AZIM: + use_azim_for_integration = true; + break; + case OPT_SINGLE_PASS_ROTATION: + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); + } + rotation_indexing = true; + two_pass_rotation = false; + + if (optarg) + rotation_indexing_range = atof(optarg); + break; + case OPT_REDO_ROTATION_SPOTS: + reuse_rotation_spots = false; + break; + case OPT_FORCE_ROTATION_LATTICE: { + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); + } + rotation_indexing = true; + + auto latt = parse_lattice_arg(optarg); + if (!latt.has_value()) { + logger.Error( + "Invalid rotation lattice. Expected: \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, comma-separated). Got: {}", + optarg ? optarg : ""); + print_usage(); + exit(EXIT_FAILURE); + } + forced_rotation_lattice = latt; + auto uc = latt->GetUnitCell(); + logger.Info( + "Forced rotation lattice set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", + uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); + break; } - rotation_indexing = true; - two_pass_rotation = true; - if (optarg) - rotation_indexing_image_count = atoi(optarg); + case 'X': { + std::string alg = optarg ? optarg : ""; + std::transform(alg.begin(), alg.end(), alg.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); - break; - case OPT_SINGLE_PASS_ROTATION: - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); + if (alg == "ffbidx") + indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; + else if (alg == "fft") + indexing_algorithm = IndexingAlgorithmEnum::FFT; + else if (alg == "fftw") + indexing_algorithm = IndexingAlgorithmEnum::FFTW; + else if (alg == "auto") + indexing_algorithm = IndexingAlgorithmEnum::Auto; + else if (alg == "none") + indexing_algorithm = IndexingAlgorithmEnum::None; + else { + logger.Error("Invalid indexing algorithm: {}", alg); + print_usage(); + exit(EXIT_FAILURE); + } + break; } - rotation_indexing = true; - two_pass_rotation = false; - - if (optarg) - rotation_indexing_range = atof(optarg); - break; - case OPT_REDO_ROTATION_SPOTS: - reuse_rotation_spots = false; - break; - case OPT_FORCE_ROTATION_LATTICE: { - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); + case 'r': { + std::string alg = optarg ? optarg : ""; + std::transform(alg.begin(), alg.end(), alg.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (alg == "none") + refinement_algorithm = GeomRefinementAlgorithmEnum::None; + else if (alg == "beam_and_lattice") + refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; + else if (alg == "orientation") + refinement_algorithm = GeomRefinementAlgorithmEnum::OrientationOnly; + else if (alg == "pixelrefine") + refinement_algorithm = GeomRefinementAlgorithmEnum::PixelRefine; + else { + logger.Error("Invalid geom refinement algorithm: {}", alg); + print_usage(); + exit(EXIT_FAILURE); + } + break; } - rotation_indexing = true; - - auto latt = parse_lattice_arg(optarg); - if (!latt.has_value()) { - logger.Error( - "Invalid rotation lattice. Expected: \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, comma-separated). Got: {}", - optarg ? optarg : ""); - print_usage(); - exit(EXIT_FAILURE); + case 'C': { + auto uc = parse_unit_cell_arg(optarg); + if (!uc.has_value()) { + logger.Error( + "Invalid unit cell. Expected: \"a,b,c,alpha,beta,gamma\" (6 floats, comma-separated, no spaces). Got: {}", + optarg ? optarg : ""); + print_usage(); + exit(EXIT_FAILURE); + } + fixed_reference_unit_cell = uc; + logger.Info( + "Fixed reference unit cell set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", + uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma); + break; } - forced_rotation_lattice = latt; - auto uc = latt->GetUnitCell(); - logger.Info( - "Forced rotation lattice set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", - uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); - break; - } - case 'X': { - std::string alg = optarg ? optarg : ""; - std::transform(alg.begin(), alg.end(), alg.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - - if (alg == "ffbidx") - indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; - else if (alg == "fft") + case 'z': + ref_mtz = optarg; + break; + case 'F': indexing_algorithm = IndexingAlgorithmEnum::FFT; - else if (alg == "fftw") - indexing_algorithm = IndexingAlgorithmEnum::FFTW; - else if (alg == "auto") - indexing_algorithm = IndexingAlgorithmEnum::Auto; - else if (alg == "none") - indexing_algorithm = IndexingAlgorithmEnum::None; - else { - logger.Error("Invalid indexing algorithm: {}", alg); - print_usage(); - exit(EXIT_FAILURE); - } - break; - } - case 'r': { - std::string alg = optarg ? optarg : ""; - std::transform(alg.begin(), alg.end(), alg.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (alg == "none") - refinement_algorithm = GeomRefinementAlgorithmEnum::None; - else if (alg == "beam_and_lattice") - refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; - else if (alg == "orientation") - refinement_algorithm = GeomRefinementAlgorithmEnum::OrientationOnly; - else if (alg == "pixelrefine") - refinement_algorithm = GeomRefinementAlgorithmEnum::PixelRefine; - else { - logger.Error("Invalid geom refinement algorithm: {}", alg); - print_usage(); - exit(EXIT_FAILURE); - } - break; - } - case 'C': { - auto uc = parse_unit_cell_arg(optarg); - if (!uc.has_value()) { - logger.Error( - "Invalid unit cell. Expected: \"a,b,c,alpha,beta,gamma\" (6 floats, comma-separated, no spaces). Got: {}", - optarg ? optarg : ""); - print_usage(); - exit(EXIT_FAILURE); - } - fixed_reference_unit_cell = uc; - logger.Info( - "Fixed reference unit cell set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", - uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma); - break; - } - case 'z': - ref_mtz = optarg; - break; - case 'F': - indexing_algorithm = IndexingAlgorithmEnum::FFT; - break; - case 'A': - anomalous_mode = true; - break; - case 'B': - refine_bfactor = true; - break; - case 'w': - refine_wedge = true; - if (optarg) - wedge_for_scaling = std::stod(optarg); - break; - case 'S': - space_group_number = atoi(optarg); - break; - case 'P': - if (strcmp(optarg, "unity") == 0) - partiality_model = PartialityModel::Unity; - else if (strcmp(optarg, "fixed") == 0) - partiality_model = PartialityModel::Fixed; - else if (strcmp(optarg, "rot") == 0) - partiality_model = PartialityModel::Rotation; - else { - logger.Error("Invalid partiality mode: {}", optarg); - print_usage(); - exit(EXIT_FAILURE); - } - break; - case OPT_SPOT_SIGMA: - sigma_spot_finding = atof(optarg); - logger.Info("Noise threshold level for spot finding set to {:.2f} sigma", sigma_spot_finding); - break; - case OPT_SPOT_THRESHOLD: - photon_count_threshold_spot_finding = atoi(optarg); - logger.Info("Photon-count threshold level for spot finding set to {:d}", - photon_count_threshold_spot_finding); - break; - case OPT_SPOT_RESOLUTION: - d_min_spot_finding = atof(optarg); - logger.Info("High resolution limit for spot finding set to {:.2f} A", d_min_spot_finding); - break; - case OPT_MAX_SPOTS: - max_spot_count_override = atoll(optarg); - logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); - break; - case 'M': - run_scaling = true; - break; - case OPT_MIN_PARTIALITY: - min_partiality = std::stod(optarg); - break; - case OPT_MIN_IMAGE_CC: - min_image_cc = std::stod(optarg); - break; - case OPT_SCALING_HIGH_RESOLUTION: - d_min_scale_merge = atof(optarg); - break; - case OPT_SCALING_OUTPUT: - if (strcmp(optarg, "mtz") == 0) { - intensity_format = IntensityFormat::MTZ; - } else if (strcmp(optarg, "cif") == 0) { - intensity_format = IntensityFormat::mmCIF; - } else if (strcmp(optarg, "txt") == 0) { - intensity_format = IntensityFormat::Text; - } else { - logger.Error("Invalid intensity format: {}", optarg); - exit(EXIT_FAILURE); - } - break; - case OPT_SCALING_ITERATIONS: - scaling_iter = atoi(optarg); - if (scaling_iter <= 0) { - logger.Error("Invalid scaling iteration count: {}", scaling_iter); - exit(EXIT_FAILURE); - } - break; - case OPT_BANDWIDTH: - bandwidth_fwhm = atof(optarg); - if (!(bandwidth_fwhm.value() >= 0.0f)) { - logger.Error("Invalid bandwidth: {}", optarg); - exit(EXIT_FAILURE); - } - break; + break; + case 'A': + anomalous_mode = true; + break; + case 'B': + refine_bfactor = true; + break; + case 'w': + refine_wedge = true; + if (optarg) + wedge_for_scaling = std::stod(optarg); + break; + case 'S': + space_group_number = atoi(optarg); + break; + case 'P': + if (strcmp(optarg, "unity") == 0) + partiality_model = PartialityModel::Unity; + else if (strcmp(optarg, "fixed") == 0) + partiality_model = PartialityModel::Fixed; + else if (strcmp(optarg, "rot") == 0) + partiality_model = PartialityModel::Rotation; + else { + logger.Error("Invalid partiality mode: {}", optarg); + print_usage(); + exit(EXIT_FAILURE); + } + break; + case OPT_SPOT_SIGMA: + sigma_spot_finding = atof(optarg); + logger.Info("Noise threshold level for spot finding set to {:.2f} sigma", sigma_spot_finding); + break; + case OPT_SPOT_THRESHOLD: + photon_count_threshold_spot_finding = atoi(optarg); + logger.Info("Photon-count threshold level for spot finding set to {:d}", + photon_count_threshold_spot_finding); + break; + case OPT_SPOT_RESOLUTION: + d_min_spot_finding = atof(optarg); + logger.Info("High resolution limit for spot finding set to {:.2f} A", d_min_spot_finding); + break; + case OPT_MAX_SPOTS: + max_spot_count_override = atoll(optarg); + logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); + break; + case 'M': + run_scaling = true; + break; + case OPT_MIN_PARTIALITY: + min_partiality = std::stod(optarg); + break; + case OPT_MIN_IMAGE_CC: + min_image_cc = std::stod(optarg); + break; + case OPT_SCALING_HIGH_RESOLUTION: + d_min_scale_merge = atof(optarg); + break; + case OPT_SCALING_OUTPUT: + if (strcmp(optarg, "mtz") == 0) { + intensity_format = IntensityFormat::MTZ; + } else if (strcmp(optarg, "cif") == 0) { + intensity_format = IntensityFormat::mmCIF; + } else if (strcmp(optarg, "txt") == 0) { + intensity_format = IntensityFormat::Text; + } else { + logger.Error("Invalid intensity format: {}", optarg); + exit(EXIT_FAILURE); + } + break; + case OPT_SCALING_ITERATIONS: + scaling_iter = atoi(optarg); + if (scaling_iter <= 0) { + logger.Error("Invalid scaling iteration count: {}", scaling_iter); + exit(EXIT_FAILURE); + } + break; + case OPT_BANDWIDTH: + bandwidth_fwhm = atof(optarg); + if (!(bandwidth_fwhm.value() >= 0.0f)) { + logger.Error("Invalid bandwidth: {}", optarg); + exit(EXIT_FAILURE); + } + break; - default: - print_usage(); - exit(EXIT_FAILURE); + default: + print_usage(); + exit(EXIT_FAILURE); + } } -} if (optind != argc - 1) { logger.Error("Input file not specified"); @@ -603,7 +661,7 @@ int main(int argc, char **argv) { if (images_to_process <= 0) { logger.Warning("No images to process (Start: {}, End: {} Stride: {}, Total: {})", start_image, end_image, - image_stride, total_images_in_file); + image_stride, total_images_in_file); return 0; } @@ -668,6 +726,17 @@ int main(int argc, char **argv) { experiment.ImportScalingSettings(scaling_settings); + BraggIntegrationSettings bragg_integration_settings; + bragg_integration_settings.UseAzimProfile(use_azim_for_integration); + experiment.ImportBraggIntegrationSettings(bragg_integration_settings); + + AzimuthalIntegrationSettings azimuthal_integration_settings; + if (azim_q_spacing) + azimuthal_integration_settings.QSpacing_recipA(azim_q_spacing.value()); + azimuthal_integration_settings.QRange_recipA(azim_q_min.value_or(azimuthal_integration_settings.GetLowQ_recipA()), + azim_q_max.value_or(azimuthal_integration_settings.GetHighQ_recipA())); + experiment.ImportAzimuthalIntegrationSettings(azimuthal_integration_settings); + SpotFindingSettings spot_settings; spot_settings.enable = true; spot_settings.indexing = true; @@ -680,9 +749,6 @@ int main(int argc, char **argv) { // Initialize Analysis Components PixelMask pixel_mask = dataset->pixel_mask; - // If dataset has a mask you wish to use, you might need to load it into pixel_mask here - // e.g. pixel_mask.LoadUserMask(dataset->pixel_mask, ...); - AzimuthalIntegrationMapping mapping(experiment, pixel_mask); IndexerThreadPool indexer_pool(experiment.GetIndexingSettings()); @@ -768,7 +834,7 @@ int main(int argc, char **argv) { analysis.Analyze(msg, profile, first_pass_spot_settings); } - indexer.AddImageToRotationIndexer(msg); + indexer.AddImageToRotationIndexer(msg); } catch (const std::exception &e) { logger.Warning("Failed to add image {} to first-pass rotation indexing: {}", image_idx, e.what()); } @@ -814,7 +880,8 @@ int main(int argc, char **argv) { msg.image = img->image; msg.number = image_ordinal; msg.original_number = image_idx; - if (dataset->efficiency.size() > image_idx) msg.image_collection_efficiency = dataset->efficiency[image_idx]; + if (dataset->efficiency.size() > image_idx) + msg.image_collection_efficiency = dataset->efficiency[image_idx]; total_uncompressed_bytes += msg.image.GetUncompressedSize(); @@ -921,8 +988,8 @@ int main(int argc, char **argv) { if (consensus_cell) { logger.Info("Consensus unit cell found in {:.2f} ms", consensus_duration * 1e3); logger.Info("UC: a={:.2f} b={:.2f} c={:.2f} alpha={:.2f} beta={:.2f} gamma={:.2f}", - consensus_cell->a, consensus_cell->b, consensus_cell->c, - consensus_cell->alpha, consensus_cell->beta, consensus_cell->gamma); + consensus_cell->a, consensus_cell->b, consensus_cell->c, + consensus_cell->alpha, consensus_cell->beta, consensus_cell->gamma); } else logger.Info("Consensus unit cell not found - calculation tool {:.2f} ms", consensus_duration * 1e3); end_msg.unit_cell = consensus_cell; @@ -930,9 +997,8 @@ int main(int argc, char **argv) { if (end_msg.indexing_rate.has_value() && end_msg.indexing_rate > 0 && (run_scaling || !reference_data.empty())) { - const bool pixel_refine_path = - (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine); + (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine); // Scaling is only the classical, no-reference ScaleOnTheFly post-pass. // - With a reference (classical path): per-image live scaling already ran. @@ -973,11 +1039,12 @@ int main(int argc, char **argv) { MergeOnTheFly merge_engine(experiment); if (consensus_cell.has_value()) merge_engine.ReferenceCell(*consensus_cell); - for (auto &i : indexer.GetIntegrationOutcome()) + for (auto &i: indexer.GetIntegrationOutcome()) merge_engine.AddImage(i); auto merged_reflections = merge_engine.ExportReflections(); - auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), reference_data); + auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), + reference_data); auto merge_end = std::chrono::steady_clock::now(); double merge_time = std::chrono::duration(merge_end - merge_start).count(); @@ -1031,17 +1098,17 @@ int main(int argc, char **argv) { auto image_mean_time = plots.GetMeanProcessingTime(); std::cout << fmt::format( - "Per-image time: (mean; milliseconds): decompress {:.2f} preprocess {:.2f} azint {:.2f} spot finding {:.2f} indexing {:.2f} refinement {:.2f} indexing analysis {:.2f} prediction {:.2f} integration {:.2f} scaling {:.2f} total {:.2f}", - image_mean_time.compression * 1e3, - image_mean_time.preprocessing * 1e3, - image_mean_time.azint * 1e3, - image_mean_time.spot_finding * 1e3, - image_mean_time.indexing * 1e3, - image_mean_time.refinement * 1e3, - image_mean_time.indexing_analysis * 1e3, - image_mean_time.bragg_prediction * 1e3, - image_mean_time.integration * 1e3, - image_mean_time.image_scale * 1e3, - image_mean_time.processing * 1e3) - << std::endl; + "Per-image time: (mean; milliseconds): decompress {:.2f} preprocess {:.2f} azint {:.2f} spot finding {:.2f} indexing {:.2f} refinement {:.2f} indexing analysis {:.2f} prediction {:.2f} integration {:.2f} scaling {:.2f} total {:.2f}", + image_mean_time.compression * 1e3, + image_mean_time.preprocessing * 1e3, + image_mean_time.azint * 1e3, + image_mean_time.spot_finding * 1e3, + image_mean_time.indexing * 1e3, + image_mean_time.refinement * 1e3, + image_mean_time.indexing_analysis * 1e3, + image_mean_time.bragg_prediction * 1e3, + image_mean_time.integration * 1e3, + image_mean_time.image_scale * 1e3, + image_mean_time.processing * 1e3) + << std::endl; } -- 2.52.0 From 579d36fe71f336f104a9e836bdadeb3fffe35e48 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 16:36:37 +0200 Subject: [PATCH 027/228] Revert "jfjoch_process: Add option to use azimuthal integration as background for Bragg integration" This reverts commit b22d5929a1b3e9b565338cfccf91a184a04e02ff. --- tools/jfjoch_process.cpp | 575 +++++++++++++++++---------------------- 1 file changed, 254 insertions(+), 321 deletions(-) diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 4c8d070c..5876418b 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -45,75 +45,39 @@ void print_usage() { std::cout << " Spot finding" << std::endl; std::cout << " --spot-sigma Noise sigma level for spot finding (default: 3.0)" << std::endl; - std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << - std::endl; - std::cout << " --spot-high-resolution High resolution limit for spot finding (default: 1.5)" << - std::endl; + std::cout << " --spot-threshold Photon count threshold for spot finding (default: 10)" << std::endl; + std::cout << " --spot-high-resolution High resolution limit for spot finding (default: 1.5)" << std::endl; std::cout << " --max-spots Max spot count (default: 250)" << std::endl; std::cout << std::endl; - std::cout << " Azimuthal integration" << std::endl; - std::cout << " -Q, --azim-q-spacing Q spacing for azimuthal integration (default: 0.01)" << - std::endl; - std::cout << " --azim-q-min Minimum Q value for azimuthal integration (default: 0.0)" << - std::endl; - std::cout << " --azim-q-max Maximum Q value for azimuthal integration (default: 5.0)" << - std::endl; - std::cout << std::endl; - std::cout << " Indexing" << std::endl; - std::cout << - " -R, --two-pass-rotation[=num] Two-pass offline rotation indexing (optional: number of images, default: 30)" - << std::endl; - std::cout << - " --single-pass-rotation[=num] Use online-like single-pass rotation indexing (optional: min angular range deg)" - << std::endl; + std::cout << " -R, --two-pass-rotation[=num] Two-pass offline rotation indexing (optional: number of images, default: 30)" << std::endl; + std::cout << " --single-pass-rotation[=num] Use online-like single-pass rotation indexing (optional: min angular range deg)" << std::endl; std::cout << " --redo-rotation-spots Redo spot finding for two-pass rotation indexing" << std::endl; - std::cout << - " --force-rotation-lattice Force rotation indexer with external lattice (in Angstrom) : \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, skips first pass)" - << std::endl; + std::cout << " --force-rotation-lattice Force rotation indexer with external lattice (in Angstrom) : \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, skips first pass)" << std::endl; std::cout << " -X, --indexing-algorithm Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl; - std::cout << " -S, --space-group Space group number - used for both indexing and scaling" << - std::endl; + std::cout << " -S, --space-group Space group number - used for both indexing and scaling" << std::endl; std::cout << " -C, --unit-cell Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl; - std::cout << - " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|pixelrefine)" - << std::endl; - std::cout << std::endl; - - std::cout << " Integration" << std::endl; - std::cout << - " --integration-use-azim Background from Bragg peak integration is based on azimuthal integration results" - << std::endl; + std::cout << " -r, --refine Geometry refinement algorithm (none|orientation|beam_and_lattice|pixelrefine)" << std::endl; std::cout << std::endl; std::cout << " Scaling and merging" << std::endl; - std::cout << - " -M, --scale-merge Scale and merge (refine mosaicity) and write scaled.hkl + image.dat" << - std::endl; - std::cout << " -P, --partiality Partiality refinement fixed|rot|unity (default: fixed)" << - std::endl; + std::cout << " -M, --scale-merge Scale and merge (refine mosaicity) and write scaled.hkl + image.dat" << std::endl; + std::cout << " -P, --partiality Partiality refinement fixed|rot|unity (default: fixed)" << std::endl; std::cout << " -A, --anomalous Anomalous mode (don't merge Friedel pairs)" << std::endl; std::cout << " -B, --refine-bfactor Refine per image B-factor" << std::endl; - std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << - std::endl; - std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << - std::endl; - std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << - std::endl; + std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << std::endl; + std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << std::endl; + std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; - std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" - << std::endl; - std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" - << std::endl; + std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; + std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; std::cout << " -z, --reference-mtz Reference MTZ file" << std::endl; std::cout << std::endl; std::cout << " Pixel refinement (experimental, select via -r pixelrefine, needs --reference-mtz)" << std::endl; - std::cout << - " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" - << std::endl; -} + std::cout << " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" << std::endl; + } enum { OPT_SPOT_SIGMA = 1000, @@ -128,10 +92,7 @@ enum { OPT_SINGLE_PASS_ROTATION, OPT_REDO_ROTATION_SPOTS, OPT_FORCE_ROTATION_LATTICE, - OPT_BANDWIDTH, - OPT_INTEGRATION_USE_AZIM, - OPT_AZIM_Q_MIN, - OPT_AZIM_Q_MAX + OPT_BANDWIDTH }; static option long_options[] = { @@ -151,16 +112,13 @@ static option long_options[] = { {"wedge", optional_argument, nullptr, 'w'}, {"scale-merge", no_argument, nullptr, 'M'}, {"refine", required_argument, nullptr, 'r'}, - {"azim-q-spacing", required_argument, nullptr, 'Q'}, - {"azim-q-min", required_argument, nullptr, OPT_AZIM_Q_MIN}, - {"azim-q-max", required_argument, nullptr, OPT_AZIM_Q_MAX}, {"two-pass-rotation", optional_argument, nullptr, 'R'}, {"single-pass-rotation", optional_argument, nullptr, OPT_SINGLE_PASS_ROTATION}, {"redo-rotation-spots", no_argument, nullptr, OPT_REDO_ROTATION_SPOTS}, {"force-rotation-lattice", required_argument, nullptr, OPT_FORCE_ROTATION_LATTICE}, - {"integration-use-azim", no_argument, nullptr, OPT_INTEGRATION_USE_AZIM}, + {"spot-sigma", required_argument, nullptr, OPT_SPOT_SIGMA}, {"spot-threshold", required_argument, nullptr, OPT_SPOT_THRESHOLD}, {"spot-high-resolution", required_argument, nullptr, OPT_SPOT_RESOLUTION}, @@ -223,6 +181,7 @@ std::optional parse_unit_cell_arg(const char *arg) { return std::nullopt; + UnitCell uc{}; if (!parse_float_strict(parts[0], uc.a)) return std::nullopt; if (!parse_float_strict(parts[1], uc.b)) return std::nullopt; @@ -338,11 +297,6 @@ int main(int argc, char **argv) { double min_partiality = 0.02; double min_image_cc = 0.0; int64_t scaling_iter = 3; - std::optional azim_q_spacing; - std::optional azim_q_min; - std::optional azim_q_max; - bool use_azim_for_integration = false; - std::optional forced_rotation_lattice; std::optional bandwidth_fwhm; // relative FWHM of dlambda/lambda @@ -363,237 +317,225 @@ int main(int argc, char **argv) { int opt; int option_index = 0; - const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABw::S:MP:r:Q:"; + const char *short_opts = "vo:N:s:e:t:R::X:C:z:FABw::S:MP:r:"; - while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { - switch (opt) { - case 'o': - output_prefix = optarg; - break; - case 'v': - verbose = true; - break; - case 'N': - nthreads = atoi(optarg); - break; - case 's': - start_image = atoi(optarg); - break; - case 'e': - end_image = atoi(optarg); - break; - case 't': - image_stride = atoi(optarg); - break; - case 'R': - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); - } - rotation_indexing = true; - two_pass_rotation = true; - if (optarg) - rotation_indexing_image_count = atoi(optarg); - - break; - case 'Q': - azim_q_spacing = atof(optarg); - break; - case OPT_AZIM_Q_MIN: - azim_q_min = atof(optarg); - break; - case OPT_AZIM_Q_MAX: - azim_q_max = atof(optarg); - break; - case OPT_INTEGRATION_USE_AZIM: - use_azim_for_integration = true; - break; - case OPT_SINGLE_PASS_ROTATION: - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); - } - rotation_indexing = true; - two_pass_rotation = false; - - if (optarg) - rotation_indexing_range = atof(optarg); - break; - case OPT_REDO_ROTATION_SPOTS: - reuse_rotation_spots = false; - break; - case OPT_FORCE_ROTATION_LATTICE: { - if (rotation_indexing) { - logger.Error("Rotation indexing already enabled"); - exit(EXIT_FAILURE); - } - rotation_indexing = true; - - auto latt = parse_lattice_arg(optarg); - if (!latt.has_value()) { - logger.Error( - "Invalid rotation lattice. Expected: \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, comma-separated). Got: {}", - optarg ? optarg : ""); - print_usage(); - exit(EXIT_FAILURE); - } - forced_rotation_lattice = latt; - auto uc = latt->GetUnitCell(); - logger.Info( - "Forced rotation lattice set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", - uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); - break; + while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { + switch (opt) { + case 'o': + output_prefix = optarg; + break; + case 'v': + verbose = true; + break; + case 'N': + nthreads = atoi(optarg); + break; + case 's': + start_image = atoi(optarg); + break; + case 'e': + end_image = atoi(optarg); + break; + case 't': + image_stride = atoi(optarg); + break; + case 'R': + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); } - case 'X': { - std::string alg = optarg ? optarg : ""; - std::transform(alg.begin(), alg.end(), alg.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); + rotation_indexing = true; + two_pass_rotation = true; + if (optarg) + rotation_indexing_image_count = atoi(optarg); - if (alg == "ffbidx") - indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; - else if (alg == "fft") - indexing_algorithm = IndexingAlgorithmEnum::FFT; - else if (alg == "fftw") - indexing_algorithm = IndexingAlgorithmEnum::FFTW; - else if (alg == "auto") - indexing_algorithm = IndexingAlgorithmEnum::Auto; - else if (alg == "none") - indexing_algorithm = IndexingAlgorithmEnum::None; - else { - logger.Error("Invalid indexing algorithm: {}", alg); - print_usage(); - exit(EXIT_FAILURE); - } - break; + break; + case OPT_SINGLE_PASS_ROTATION: + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); } - case 'r': { - std::string alg = optarg ? optarg : ""; - std::transform(alg.begin(), alg.end(), alg.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (alg == "none") - refinement_algorithm = GeomRefinementAlgorithmEnum::None; - else if (alg == "beam_and_lattice") - refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; - else if (alg == "orientation") - refinement_algorithm = GeomRefinementAlgorithmEnum::OrientationOnly; - else if (alg == "pixelrefine") - refinement_algorithm = GeomRefinementAlgorithmEnum::PixelRefine; - else { - logger.Error("Invalid geom refinement algorithm: {}", alg); - print_usage(); - exit(EXIT_FAILURE); - } - break; - } - case 'C': { - auto uc = parse_unit_cell_arg(optarg); - if (!uc.has_value()) { - logger.Error( - "Invalid unit cell. Expected: \"a,b,c,alpha,beta,gamma\" (6 floats, comma-separated, no spaces). Got: {}", - optarg ? optarg : ""); - print_usage(); - exit(EXIT_FAILURE); - } - fixed_reference_unit_cell = uc; - logger.Info( - "Fixed reference unit cell set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", - uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma); - break; - } - case 'z': - ref_mtz = optarg; - break; - case 'F': - indexing_algorithm = IndexingAlgorithmEnum::FFT; - break; - case 'A': - anomalous_mode = true; - break; - case 'B': - refine_bfactor = true; - break; - case 'w': - refine_wedge = true; - if (optarg) - wedge_for_scaling = std::stod(optarg); - break; - case 'S': - space_group_number = atoi(optarg); - break; - case 'P': - if (strcmp(optarg, "unity") == 0) - partiality_model = PartialityModel::Unity; - else if (strcmp(optarg, "fixed") == 0) - partiality_model = PartialityModel::Fixed; - else if (strcmp(optarg, "rot") == 0) - partiality_model = PartialityModel::Rotation; - else { - logger.Error("Invalid partiality mode: {}", optarg); - print_usage(); - exit(EXIT_FAILURE); - } - break; - case OPT_SPOT_SIGMA: - sigma_spot_finding = atof(optarg); - logger.Info("Noise threshold level for spot finding set to {:.2f} sigma", sigma_spot_finding); - break; - case OPT_SPOT_THRESHOLD: - photon_count_threshold_spot_finding = atoi(optarg); - logger.Info("Photon-count threshold level for spot finding set to {:d}", - photon_count_threshold_spot_finding); - break; - case OPT_SPOT_RESOLUTION: - d_min_spot_finding = atof(optarg); - logger.Info("High resolution limit for spot finding set to {:.2f} A", d_min_spot_finding); - break; - case OPT_MAX_SPOTS: - max_spot_count_override = atoll(optarg); - logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); - break; - case 'M': - run_scaling = true; - break; - case OPT_MIN_PARTIALITY: - min_partiality = std::stod(optarg); - break; - case OPT_MIN_IMAGE_CC: - min_image_cc = std::stod(optarg); - break; - case OPT_SCALING_HIGH_RESOLUTION: - d_min_scale_merge = atof(optarg); - break; - case OPT_SCALING_OUTPUT: - if (strcmp(optarg, "mtz") == 0) { - intensity_format = IntensityFormat::MTZ; - } else if (strcmp(optarg, "cif") == 0) { - intensity_format = IntensityFormat::mmCIF; - } else if (strcmp(optarg, "txt") == 0) { - intensity_format = IntensityFormat::Text; - } else { - logger.Error("Invalid intensity format: {}", optarg); - exit(EXIT_FAILURE); - } - break; - case OPT_SCALING_ITERATIONS: - scaling_iter = atoi(optarg); - if (scaling_iter <= 0) { - logger.Error("Invalid scaling iteration count: {}", scaling_iter); - exit(EXIT_FAILURE); - } - break; - case OPT_BANDWIDTH: - bandwidth_fwhm = atof(optarg); - if (!(bandwidth_fwhm.value() >= 0.0f)) { - logger.Error("Invalid bandwidth: {}", optarg); - exit(EXIT_FAILURE); - } - break; + rotation_indexing = true; + two_pass_rotation = false; - default: + if (optarg) + rotation_indexing_range = atof(optarg); + break; + case OPT_REDO_ROTATION_SPOTS: + reuse_rotation_spots = false; + break; + case OPT_FORCE_ROTATION_LATTICE: { + if (rotation_indexing) { + logger.Error("Rotation indexing already enabled"); + exit(EXIT_FAILURE); + } + rotation_indexing = true; + + auto latt = parse_lattice_arg(optarg); + if (!latt.has_value()) { + logger.Error( + "Invalid rotation lattice. Expected: \"a0x,a0y,a0z,a1x,a1y,a1z,a2x,a2y,a2z\" (9 floats, comma-separated). Got: {}", + optarg ? optarg : ""); print_usage(); exit(EXIT_FAILURE); + } + forced_rotation_lattice = latt; + auto uc = latt->GetUnitCell(); + logger.Info( + "Forced rotation lattice set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", + uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); + break; } + case 'X': { + std::string alg = optarg ? optarg : ""; + std::transform(alg.begin(), alg.end(), alg.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + if (alg == "ffbidx") + indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; + else if (alg == "fft") + indexing_algorithm = IndexingAlgorithmEnum::FFT; + else if (alg == "fftw") + indexing_algorithm = IndexingAlgorithmEnum::FFTW; + else if (alg == "auto") + indexing_algorithm = IndexingAlgorithmEnum::Auto; + else if (alg == "none") + indexing_algorithm = IndexingAlgorithmEnum::None; + else { + logger.Error("Invalid indexing algorithm: {}", alg); + print_usage(); + exit(EXIT_FAILURE); + } + break; + } + case 'r': { + std::string alg = optarg ? optarg : ""; + std::transform(alg.begin(), alg.end(), alg.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (alg == "none") + refinement_algorithm = GeomRefinementAlgorithmEnum::None; + else if (alg == "beam_and_lattice") + refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; + else if (alg == "orientation") + refinement_algorithm = GeomRefinementAlgorithmEnum::OrientationOnly; + else if (alg == "pixelrefine") + refinement_algorithm = GeomRefinementAlgorithmEnum::PixelRefine; + else { + logger.Error("Invalid geom refinement algorithm: {}", alg); + print_usage(); + exit(EXIT_FAILURE); + } + break; + } + case 'C': { + auto uc = parse_unit_cell_arg(optarg); + if (!uc.has_value()) { + logger.Error( + "Invalid unit cell. Expected: \"a,b,c,alpha,beta,gamma\" (6 floats, comma-separated, no spaces). Got: {}", + optarg ? optarg : ""); + print_usage(); + exit(EXIT_FAILURE); + } + fixed_reference_unit_cell = uc; + logger.Info( + "Fixed reference unit cell set: a={:.3f} b={:.3f} c={:.3f} alpha={:.3f} beta={:.3f} gamma={:.3f}", + uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma); + break; + } + case 'z': + ref_mtz = optarg; + break; + case 'F': + indexing_algorithm = IndexingAlgorithmEnum::FFT; + break; + case 'A': + anomalous_mode = true; + break; + case 'B': + refine_bfactor = true; + break; + case 'w': + refine_wedge = true; + if (optarg) + wedge_for_scaling = std::stod(optarg); + break; + case 'S': + space_group_number = atoi(optarg); + break; + case 'P': + if (strcmp(optarg, "unity") == 0) + partiality_model = PartialityModel::Unity; + else if (strcmp(optarg, "fixed") == 0) + partiality_model = PartialityModel::Fixed; + else if (strcmp(optarg, "rot") == 0) + partiality_model = PartialityModel::Rotation; + else { + logger.Error("Invalid partiality mode: {}", optarg); + print_usage(); + exit(EXIT_FAILURE); + } + break; + case OPT_SPOT_SIGMA: + sigma_spot_finding = atof(optarg); + logger.Info("Noise threshold level for spot finding set to {:.2f} sigma", sigma_spot_finding); + break; + case OPT_SPOT_THRESHOLD: + photon_count_threshold_spot_finding = atoi(optarg); + logger.Info("Photon-count threshold level for spot finding set to {:d}", + photon_count_threshold_spot_finding); + break; + case OPT_SPOT_RESOLUTION: + d_min_spot_finding = atof(optarg); + logger.Info("High resolution limit for spot finding set to {:.2f} A", d_min_spot_finding); + break; + case OPT_MAX_SPOTS: + max_spot_count_override = atoll(optarg); + logger.Info("Max spot count overridden to {}", max_spot_count_override.value()); + break; + case 'M': + run_scaling = true; + break; + case OPT_MIN_PARTIALITY: + min_partiality = std::stod(optarg); + break; + case OPT_MIN_IMAGE_CC: + min_image_cc = std::stod(optarg); + break; + case OPT_SCALING_HIGH_RESOLUTION: + d_min_scale_merge = atof(optarg); + break; + case OPT_SCALING_OUTPUT: + if (strcmp(optarg, "mtz") == 0) { + intensity_format = IntensityFormat::MTZ; + } else if (strcmp(optarg, "cif") == 0) { + intensity_format = IntensityFormat::mmCIF; + } else if (strcmp(optarg, "txt") == 0) { + intensity_format = IntensityFormat::Text; + } else { + logger.Error("Invalid intensity format: {}", optarg); + exit(EXIT_FAILURE); + } + break; + case OPT_SCALING_ITERATIONS: + scaling_iter = atoi(optarg); + if (scaling_iter <= 0) { + logger.Error("Invalid scaling iteration count: {}", scaling_iter); + exit(EXIT_FAILURE); + } + break; + case OPT_BANDWIDTH: + bandwidth_fwhm = atof(optarg); + if (!(bandwidth_fwhm.value() >= 0.0f)) { + logger.Error("Invalid bandwidth: {}", optarg); + exit(EXIT_FAILURE); + } + break; + + default: + print_usage(); + exit(EXIT_FAILURE); } +} if (optind != argc - 1) { logger.Error("Input file not specified"); @@ -661,7 +603,7 @@ int main(int argc, char **argv) { if (images_to_process <= 0) { logger.Warning("No images to process (Start: {}, End: {} Stride: {}, Total: {})", start_image, end_image, - image_stride, total_images_in_file); + image_stride, total_images_in_file); return 0; } @@ -726,17 +668,6 @@ int main(int argc, char **argv) { experiment.ImportScalingSettings(scaling_settings); - BraggIntegrationSettings bragg_integration_settings; - bragg_integration_settings.UseAzimProfile(use_azim_for_integration); - experiment.ImportBraggIntegrationSettings(bragg_integration_settings); - - AzimuthalIntegrationSettings azimuthal_integration_settings; - if (azim_q_spacing) - azimuthal_integration_settings.QSpacing_recipA(azim_q_spacing.value()); - azimuthal_integration_settings.QRange_recipA(azim_q_min.value_or(azimuthal_integration_settings.GetLowQ_recipA()), - azim_q_max.value_or(azimuthal_integration_settings.GetHighQ_recipA())); - experiment.ImportAzimuthalIntegrationSettings(azimuthal_integration_settings); - SpotFindingSettings spot_settings; spot_settings.enable = true; spot_settings.indexing = true; @@ -749,6 +680,9 @@ int main(int argc, char **argv) { // Initialize Analysis Components PixelMask pixel_mask = dataset->pixel_mask; + // If dataset has a mask you wish to use, you might need to load it into pixel_mask here + // e.g. pixel_mask.LoadUserMask(dataset->pixel_mask, ...); + AzimuthalIntegrationMapping mapping(experiment, pixel_mask); IndexerThreadPool indexer_pool(experiment.GetIndexingSettings()); @@ -834,7 +768,7 @@ int main(int argc, char **argv) { analysis.Analyze(msg, profile, first_pass_spot_settings); } - indexer.AddImageToRotationIndexer(msg); + indexer.AddImageToRotationIndexer(msg); } catch (const std::exception &e) { logger.Warning("Failed to add image {} to first-pass rotation indexing: {}", image_idx, e.what()); } @@ -880,8 +814,7 @@ int main(int argc, char **argv) { msg.image = img->image; msg.number = image_ordinal; msg.original_number = image_idx; - if (dataset->efficiency.size() > image_idx) - msg.image_collection_efficiency = dataset->efficiency[image_idx]; + if (dataset->efficiency.size() > image_idx) msg.image_collection_efficiency = dataset->efficiency[image_idx]; total_uncompressed_bytes += msg.image.GetUncompressedSize(); @@ -988,8 +921,8 @@ int main(int argc, char **argv) { if (consensus_cell) { logger.Info("Consensus unit cell found in {:.2f} ms", consensus_duration * 1e3); logger.Info("UC: a={:.2f} b={:.2f} c={:.2f} alpha={:.2f} beta={:.2f} gamma={:.2f}", - consensus_cell->a, consensus_cell->b, consensus_cell->c, - consensus_cell->alpha, consensus_cell->beta, consensus_cell->gamma); + consensus_cell->a, consensus_cell->b, consensus_cell->c, + consensus_cell->alpha, consensus_cell->beta, consensus_cell->gamma); } else logger.Info("Consensus unit cell not found - calculation tool {:.2f} ms", consensus_duration * 1e3); end_msg.unit_cell = consensus_cell; @@ -997,8 +930,9 @@ int main(int argc, char **argv) { if (end_msg.indexing_rate.has_value() && end_msg.indexing_rate > 0 && (run_scaling || !reference_data.empty())) { + const bool pixel_refine_path = - (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine); + (refinement_algorithm == GeomRefinementAlgorithmEnum::PixelRefine); // Scaling is only the classical, no-reference ScaleOnTheFly post-pass. // - With a reference (classical path): per-image live scaling already ran. @@ -1039,12 +973,11 @@ int main(int argc, char **argv) { MergeOnTheFly merge_engine(experiment); if (consensus_cell.has_value()) merge_engine.ReferenceCell(*consensus_cell); - for (auto &i: indexer.GetIntegrationOutcome()) + for (auto &i : indexer.GetIntegrationOutcome()) merge_engine.AddImage(i); auto merged_reflections = merge_engine.ExportReflections(); - auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), - reference_data); + auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), reference_data); auto merge_end = std::chrono::steady_clock::now(); double merge_time = std::chrono::duration(merge_end - merge_start).count(); @@ -1098,17 +1031,17 @@ int main(int argc, char **argv) { auto image_mean_time = plots.GetMeanProcessingTime(); std::cout << fmt::format( - "Per-image time: (mean; milliseconds): decompress {:.2f} preprocess {:.2f} azint {:.2f} spot finding {:.2f} indexing {:.2f} refinement {:.2f} indexing analysis {:.2f} prediction {:.2f} integration {:.2f} scaling {:.2f} total {:.2f}", - image_mean_time.compression * 1e3, - image_mean_time.preprocessing * 1e3, - image_mean_time.azint * 1e3, - image_mean_time.spot_finding * 1e3, - image_mean_time.indexing * 1e3, - image_mean_time.refinement * 1e3, - image_mean_time.indexing_analysis * 1e3, - image_mean_time.bragg_prediction * 1e3, - image_mean_time.integration * 1e3, - image_mean_time.image_scale * 1e3, - image_mean_time.processing * 1e3) - << std::endl; + "Per-image time: (mean; milliseconds): decompress {:.2f} preprocess {:.2f} azint {:.2f} spot finding {:.2f} indexing {:.2f} refinement {:.2f} indexing analysis {:.2f} prediction {:.2f} integration {:.2f} scaling {:.2f} total {:.2f}", + image_mean_time.compression * 1e3, + image_mean_time.preprocessing * 1e3, + image_mean_time.azint * 1e3, + image_mean_time.spot_finding * 1e3, + image_mean_time.indexing * 1e3, + image_mean_time.refinement * 1e3, + image_mean_time.indexing_analysis * 1e3, + image_mean_time.bragg_prediction * 1e3, + image_mean_time.integration * 1e3, + image_mean_time.image_scale * 1e3, + image_mean_time.processing * 1e3) + << std::endl; } -- 2.52.0 From 59281f6330fbed591052960b16e34dbabe05a398 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 16:36:37 +0200 Subject: [PATCH 028/228] Revert "Bragg integration: option to use azimuthal integration profile" This reverts commit e4230bc14ee9a0b5a127d24d1cb71ca486820de4. --- common/BraggIntegrationSettings.cpp | 9 - common/BraggIntegrationSettings.h | 5 +- image_analysis/IndexAndRefine.cpp | 14 +- image_analysis/IndexAndRefine.h | 8 +- image_analysis/MXAnalysisAfterFPGA.cpp | 11 +- image_analysis/MXAnalysisAfterFPGA.h | 5 +- image_analysis/MXAnalysisWithoutFPGA.cpp | 2 +- .../bragg_integration/BraggIntegrate2D.cpp | 219 +++++------------- .../bragg_integration/BraggIntegrate2D.h | 10 +- receiver/JFJochReceiver.cpp | 25 +- receiver/JFJochReceiver.h | 2 +- receiver/JFJochReceiverFPGA.cpp | 11 +- receiver/JFJochReceiverLite.cpp | 4 +- tests/BraggIntegrate2DTest.cpp | 6 +- tools/jfjoch_scale.cpp | 2 + 15 files changed, 115 insertions(+), 218 deletions(-) diff --git a/common/BraggIntegrationSettings.cpp b/common/BraggIntegrationSettings.cpp index 0528084e..3fe0a861 100644 --- a/common/BraggIntegrationSettings.cpp +++ b/common/BraggIntegrationSettings.cpp @@ -87,12 +87,3 @@ float BraggIntegrationSettings::GetDMinLimit_A() const { float BraggIntegrationSettings::GetMinimumSigmaInRegardsToI() const { return minimum_sigma_in_regards_to_i; } - -bool BraggIntegrationSettings::IsUseAzimProfile() const { - return use_azim_profile; -} - -BraggIntegrationSettings &BraggIntegrationSettings::UseAzimProfile(bool input) { - use_azim_profile = input; - return *this; -} diff --git a/common/BraggIntegrationSettings.h b/common/BraggIntegrationSettings.h index 2c15921d..60ab4059 100644 --- a/common/BraggIntegrationSettings.h +++ b/common/BraggIntegrationSettings.h @@ -12,14 +12,14 @@ class BraggIntegrationSettings { float d_min_limit_A = 1.0; std::optional fixed_profile_radius; float minimum_sigma_in_regards_to_i = 0.02; - bool use_azim_profile = false; + public: BraggIntegrationSettings& R1(float input); BraggIntegrationSettings& R2(float input); BraggIntegrationSettings& R3(float input); BraggIntegrationSettings& DMinLimit_A(float input); BraggIntegrationSettings& FixedProfileRadius_recipA(std::optional input); - BraggIntegrationSettings& UseAzimProfile(bool input); + [[nodiscard]] float GetR1() const; [[nodiscard]] float GetR2() const; @@ -28,5 +28,4 @@ public: [[nodiscard]] float GetDMinLimit_A() const; [[nodiscard]] float GetMinimumSigmaInRegardsToI() const; - [[nodiscard]] bool IsUseAzimProfile() const; }; diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 459af31e..98b1aed6 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -239,8 +239,8 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, const CompressedImage &image, BraggPrediction &prediction, const IndexAndRefine::IndexingOutcome &outcome, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile) { + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile) { if (!outcome.lattice_candidate) return; @@ -299,11 +299,11 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, // integrated reflections that flow into the normal save/merge). const bool use_pixel_refine = experiment.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine - && !pixel_reference_.empty(); + && !pixel_reference_.empty() && mapping && profile; if (use_pixel_refine) { auto integration_start_time = std::chrono::steady_clock::now(); - PixelRefineIntegrate(msg, image, prediction, outcome, mapping, profile, i_outcome); + PixelRefineIntegrate(msg, image, prediction, outcome, *mapping, *profile, i_outcome); msg.integrated_reflections = i_outcome.reflections.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); @@ -314,7 +314,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, msg.bragg_prediction_time_s = std::chrono::duration(pred_end_time - pred_start_time).count(); auto integration_start_time = std::chrono::steady_clock::now(); - i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), mapping, profile, nrefl, msg.number); + i_outcome.reflections = BraggIntegrate2D(outcome.experiment, image, prediction.GetReflections(), nrefl, msg.number); msg.integrated_reflections = i_outcome.reflections.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); @@ -357,8 +357,8 @@ void IndexAndRefine::ProcessImage(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile) { + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile) { if (!indexer_ || !spot_finding_settings.indexing) return; diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index d73fc2f1..7674392e 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -66,8 +66,8 @@ class IndexAndRefine { const CompressedImage &image, BraggPrediction &prediction, const IndexingOutcome &outcome, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile); + const AzimuthalIntegrationMapping *mapping, + const AzimuthalIntegrationProfile *profile); std::unique_ptr scaling_engine; void ScaleImage(DataMessage &msg, IntegrationOutcome& outcome); @@ -93,8 +93,8 @@ public: void ForceRotationIndexerLattice(const CrystalLattice& lattice); void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile); + const AzimuthalIntegrationMapping *mapping = nullptr, + const AzimuthalIntegrationProfile *profile = nullptr); IndexAndRefine& ReferenceIntensities(std::vector &reference); ScalingResult ScaleAllImages(const std::vector &reference, size_t nthreads = 0); diff --git a/image_analysis/MXAnalysisAfterFPGA.cpp b/image_analysis/MXAnalysisAfterFPGA.cpp index 96d8430d..a5347236 100644 --- a/image_analysis/MXAnalysisAfterFPGA.cpp +++ b/image_analysis/MXAnalysisAfterFPGA.cpp @@ -25,12 +25,10 @@ double stddev(const std::vector &v) { } -MXAnalysisAfterFPGA::MXAnalysisAfterFPGA(const DiffractionExperiment &in_experiment, IndexAndRefine &indexer, - AzimuthalIntegrationMapping &mapping) +MXAnalysisAfterFPGA::MXAnalysisAfterFPGA(const DiffractionExperiment &in_experiment, IndexAndRefine &indexer) : experiment(in_experiment), indexer(indexer), - prediction(CreateBraggPrediction(experiment.IsRotationIndexing())), - mapping(mapping) { + prediction(CreateBraggPrediction(experiment.IsRotationIndexing())) { if (experiment.IsSpotFindingEnabled()) find_spots = true; } @@ -105,8 +103,7 @@ void MXAnalysisAfterFPGA::ReadFromCPU(DeviceOutput *output, const SpotFindingSet } } -void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSettings &spot_finding_settings, - const AzimuthalIntegrationProfile &profile) { +void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSettings& spot_finding_settings) { if (find_spots && (state == State::Enabled)) { const auto t0 = std::chrono::steady_clock::now(); SpotAnalyze(experiment, spot_finding_settings, spots, message); @@ -114,7 +111,7 @@ void MXAnalysisAfterFPGA::Process(DataMessage &message, const SpotFindingSetting spot_finding_time_total += (t1 - t0); if (spot_finding_settings.indexing) - indexer.ProcessImage(message, spot_finding_settings, message.image, *prediction, mapping, profile); + indexer.ProcessImage(message, spot_finding_settings, message.image, *prediction); } if (spot_finding_timing_active) { diff --git a/image_analysis/MXAnalysisAfterFPGA.h b/image_analysis/MXAnalysisAfterFPGA.h index e3b708ac..ccddf87a 100644 --- a/image_analysis/MXAnalysisAfterFPGA.h +++ b/image_analysis/MXAnalysisAfterFPGA.h @@ -28,10 +28,9 @@ class MXAnalysisAfterFPGA { std::chrono::duration spot_finding_time_total{0.0}; bool spot_finding_timing_active = false; - const AzimuthalIntegrationMapping &mapping; public: - MXAnalysisAfterFPGA(const DiffractionExperiment& experiment, IndexAndRefine &indexer, AzimuthalIntegrationMapping &mapping); + MXAnalysisAfterFPGA(const DiffractionExperiment& experiment, IndexAndRefine &indexer); void ReadFromFPGA(const DeviceOutput* output, const SpotFindingSettings& settings, @@ -41,5 +40,5 @@ public: const SpotFindingSettings &settings, size_t module_number); - void Process(DataMessage &message, const SpotFindingSettings& settings, const AzimuthalIntegrationProfile &profile); + void Process(DataMessage &message, const SpotFindingSettings& settings); }; diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 62b6379f..6a638213 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -92,7 +92,7 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, if (spot_finding_settings.indexing) indexer.ProcessImage(output, spot_finding_settings, CompressedImage(preprocessor_buffer->getBuffer(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum()), - *prediction, integration, profile); + *prediction, &integration, &profile); } output.max_viable_pixel_value = ret.max_value; diff --git a/image_analysis/bragg_integration/BraggIntegrate2D.cpp b/image_analysis/bragg_integration/BraggIntegrate2D.cpp index 35ec4849..b616ecc1 100644 --- a/image_analysis/bragg_integration/BraggIntegrate2D.cpp +++ b/image_analysis/bragg_integration/BraggIntegrate2D.cpp @@ -8,62 +8,65 @@ #include namespace { - template - float Median(std::vector &values) { - if (values.empty()) - return 0.0f; - const size_t middle = values.size() / 2; - std::nth_element(values.begin(), values.begin() + middle, values.end()); +template +float Median(std::vector &values) { + if (values.empty()) + return 0.0f; - if (values.size() % 2 == 1) - return static_cast(values[middle]); + const size_t middle = values.size() / 2; + std::nth_element(values.begin(), values.begin() + middle, values.end()); - const T upper = values[middle]; - std::nth_element(values.begin(), values.begin() + middle - 1, values.begin() + middle); - const T lower = values[middle - 1]; + if (values.size() % 2 == 1) + return static_cast(values[middle]); - return 0.5f * static_cast(lower + upper); - } + const T upper = values[middle]; + std::nth_element(values.begin(), values.begin() + middle - 1, values.begin() + middle); + const T lower = values[middle - 1]; - void MarkReflectionMask(std::vector &mask, - size_t xpixel, size_t ypixel, - const Reflection &r, float r_2, float r_2_sq) { - int64_t x0 = std::floor(r.predicted_x - r_2 - 1.0f); - int64_t x1 = std::ceil(r.predicted_x + r_2 + 1.0f); - int64_t y0 = std::floor(r.predicted_y - r_2 - 1.0f); - int64_t y1 = std::ceil(r.predicted_y + r_2 + 1.0f); + return 0.5f * static_cast(lower + upper); +} - if (x0 < 0) - x0 = 0; - if (y0 < 0) - y0 = 0; - if (x1 >= static_cast(xpixel)) - x1 = static_cast(xpixel) - 1; - if (y1 >= static_cast(ypixel)) - y1 = static_cast(ypixel) - 1; +void MarkReflectionMask(std::vector &mask, + size_t xpixel, size_t ypixel, + const Reflection &r, float r_2, float r_2_sq) { - for (int64_t y = y0; y <= y1; y++) { - for (int64_t x = x0; x <= x1; x++) { - const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) - + (y - r.predicted_y) * (y - r.predicted_y); - if (dist_sq < r_2_sq) - mask[y * xpixel + x] = 1; - } + int64_t x0 = std::floor(r.predicted_x - r_2 - 1.0f); + int64_t x1 = std::ceil(r.predicted_x + r_2 + 1.0f); + int64_t y0 = std::floor(r.predicted_y - r_2 - 1.0f); + int64_t y1 = std::ceil(r.predicted_y + r_2 + 1.0f); + + if (x0 < 0) + x0 = 0; + if (y0 < 0) + y0 = 0; + if (x1 >= static_cast(xpixel)) + x1 = static_cast(xpixel) - 1; + if (y1 >= static_cast(ypixel)) + y1 = static_cast(ypixel) - 1; + + for (int64_t y = y0; y <= y1; y++) { + for (int64_t x = x0; x <= x1; x++) { + const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) + + (y - r.predicted_y) * (y - r.predicted_y); + if (dist_sq < r_2_sq) + mask[y * xpixel + x] = 1; } } +} - std::vector BuildReflectionMask(const std::vector &predicted, - size_t npredicted, - size_t xpixel, size_t ypixel, - float r_2, float r_2_sq) { - std::vector mask(xpixel * ypixel, 0); +std::vector BuildReflectionMask(const std::vector &predicted, + size_t npredicted, + size_t xpixel, size_t ypixel, + float r_2, float r_2_sq) { + std::vector mask(xpixel * ypixel, 0); - for (size_t i = 0; i < npredicted; i++) - MarkReflectionMask(mask, xpixel, ypixel, predicted.at(i), r_2, r_2_sq); + for (size_t i = 0; i < npredicted; i++) + MarkReflectionMask(mask, xpixel, ypixel, predicted.at(i), r_2, r_2_sq); + + return mask; +} - return mask; - } } // namespace template @@ -72,6 +75,7 @@ void IntegrateReflection(Reflection &r, const T *image, const std::vector -void IntegrateReflectionAzim(Reflection &r, const T *image, - size_t xpixel, size_t ypixel, - int64_t special_value, int64_t saturation, - float r_1, float r_1_sq, - const std::vector &pixel_to_bin, - const std::vector &bkg_mean, - float minimum_sigma_in_regards_to_i) { - int64_t x0 = std::floor(r.predicted_x - r_1 - 1.0); - int64_t x1 = std::ceil(r.predicted_x + r_1 + 1.0); - int64_t y0 = std::floor(r.predicted_y - r_1 - 1.0); - int64_t y1 = std::ceil(r.predicted_y + r_1 + 1.0); - x0 = std::max(0L, x0); - x1 = std::min(static_cast(xpixel - 1), x1); - y0 = std::max(0L, y0); - y1 = std::min(static_cast(ypixel - 1), y1); - - int64_t I_sum = 0; - int64_t I_npixel_inner = 0; - int64_t I_npixel_integrated = 0; - int64_t I_sum_x = 0; - int64_t I_sum_y = 0; - - std::vector bkg_values; - bkg_values.reserve(static_cast((x1 - x0 + 1) * (y1 - y0 + 1))); - - float bkg = 0.0; - - for (int64_t y = y0; y <= y1; y++) { - for (int64_t x = x0; x <= x1; x++) { - const float dist_sq = (x - r.predicted_x) * (x - r.predicted_x) - + (y - r.predicted_y) * (y - r.predicted_y); - const auto pixel = image[y * xpixel + x]; - auto azim_bint = pixel_to_bin[y * xpixel + x]; - - if (dist_sq < r_1_sq) - I_npixel_inner++; - - if (azim_bint >= bkg_mean.size()) - continue; - - if (pixel == special_value || pixel == saturation) - continue; - - if (dist_sq < r_1_sq) { - bkg += bkg_values.at(azim_bint); - I_sum += pixel; - I_sum_x += x * pixel; - I_sum_y += y * pixel; - I_npixel_integrated++; - } - } - } - - if ((I_npixel_integrated == I_npixel_inner) && (bkg_values.size() > 5)) { - r.bkg = bkg / static_cast(I_npixel_integrated); - r.I = static_cast(I_sum) - bkg; - if (I_sum > 0) { - r.observed_x = static_cast(I_sum_x) / static_cast(I_sum); - r.observed_y = static_cast(I_sum_y) / static_cast(I_sum); - } - - // sigma is max of the: - // - 1 (for zero photons) - // - Poisson noise (sqrt(I_sum)) (for in between) - // - minimum_sigma_in_regards_to_i of Intensity (for very large numbers) - r.sigma = std::max(1.0f, r.I * minimum_sigma_in_regards_to_i); - if (I_sum > 0) - r.sigma = std::max(r.sigma, std::sqrt(static_cast(I_sum))); - r.observed = true; - } else { - r.I = 0; - r.bkg = 0; - r.sigma = NAN; - r.observed = false; - } -} - template std::vector IntegrateInternal(const DiffractionExperiment &experiment, const CompressedImage &image, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile, const std::vector &predicted, size_t npredicted, int64_t special_value, int64_t saturation, int64_t image_number) { + std::vector ret; ret.reserve(npredicted); @@ -239,36 +164,24 @@ std::vector IntegrateInternal(const DiffractionExperiment &experimen std::vector buffer; auto ptr = reinterpret_cast(image.GetUncompressedPtr(buffer)); - const float r_1 = settings.GetR1(); - const float r_2 = settings.GetR2(); const float r_3 = settings.GetR3(); - - const float r_1_sq = r_1 * r_1; - const float r_2_sq = r_2 * r_2; - const float r_3_sq = r_3 * r_3; + const float r_1_sq = settings.GetR1() * settings.GetR1(); + const float r_2 = settings.GetR2(); + const float r_2_sq = settings.GetR2() * settings.GetR2(); + const float r_3_sq = settings.GetR3() * settings.GetR3(); const float minimum_sigma_in_regards_to_i = settings.GetMinimumSigmaInRegardsToI(); const auto reflection_mask = BuildReflectionMask(predicted, npredicted, image.GetWidth(), image.GetHeight(), r_2, r_2_sq); - const auto &pixel_to_bin = mapping.GetPixelToBin(); - const auto p = profile.GetResult(); - for (int i = 0; i < npredicted; i++) { auto r = predicted.at(i); - if (settings.IsUseAzimProfile()) { - IntegrateReflectionAzim(r, ptr, image.GetWidth(), image.GetHeight(), - special_value, saturation, - r_1, r_1_sq, pixel_to_bin, p, minimum_sigma_in_regards_to_i); - } else { - IntegrateReflection(r, ptr, reflection_mask, image.GetWidth(), image.GetHeight(), special_value, saturation, - r_3, r_1_sq, r_2_sq, r_3_sq, minimum_sigma_in_regards_to_i); - } + IntegrateReflection(r, ptr, reflection_mask, image.GetWidth(), image.GetHeight(), special_value, saturation, + r_3, r_1_sq, r_2_sq, r_3_sq, minimum_sigma_in_regards_to_i); if (r.observed) { if (experiment.GetPolarizationFactor()) - r.rlp /= geom.CalcAzIntPolarizationCorr(r.predicted_x, r.predicted_y, - experiment.GetPolarizationFactor().value()); + r.rlp /= geom.CalcAzIntPolarizationCorr(r.predicted_x, r.predicted_y, experiment.GetPolarizationFactor().value()); r.image_scale_corr = r.rlp / r.partiality; r.image_number = static_cast(image_number); ret.emplace_back(r); @@ -280,8 +193,6 @@ std::vector IntegrateInternal(const DiffractionExperiment &experimen std::vector BraggIntegrate2D(const DiffractionExperiment &experiment, const CompressedImage &image, const std::vector &predicted, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile, size_t npredicted, int64_t image_number) { if (image.GetCompressedSize() == 0 || predicted.empty()) @@ -289,23 +200,17 @@ std::vector BraggIntegrate2D(const DiffractionExperiment &experiment switch (image.GetMode()) { case CompressedImageMode::Int8: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - INT8_MIN, INT8_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, INT8_MIN, INT8_MAX, image_number); case CompressedImageMode::Int16: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - INT16_MIN, INT16_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, INT16_MIN, INT16_MAX, image_number); case CompressedImageMode::Int32: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - INT32_MIN, INT32_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, INT32_MIN, INT32_MAX, image_number); case CompressedImageMode::Uint8: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - UINT8_MAX, UINT8_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, UINT8_MAX, UINT8_MAX, image_number); case CompressedImageMode::Uint16: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - UINT16_MAX, UINT16_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, UINT16_MAX, UINT16_MAX, image_number); case CompressedImageMode::Uint32: - return IntegrateInternal(experiment, image, mapping, profile, predicted, npredicted, - UINT32_MAX, UINT32_MAX, image_number); + return IntegrateInternal(experiment, image, predicted, npredicted, UINT32_MAX, UINT32_MAX, image_number); default: throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image mode not supported"); diff --git a/image_analysis/bragg_integration/BraggIntegrate2D.h b/image_analysis/bragg_integration/BraggIntegrate2D.h index 3045ff7d..353757b5 100644 --- a/image_analysis/bragg_integration/BraggIntegrate2D.h +++ b/image_analysis/bragg_integration/BraggIntegrate2D.h @@ -1,17 +1,17 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -#pragma once +#ifndef JFJOCH_BRAGGINTEGRATE2D_H +#define JFJOCH_BRAGGINTEGRATE2D_H #include #include "../../common/DiffractionExperiment.h" #include "../../common/Reflection.h" -#include "../../common/AzimuthalIntegrationProfile.h" std::vector BraggIntegrate2D(const DiffractionExperiment &experiment, const CompressedImage &image, const std::vector &predicted, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile, size_t npredicted, - int64_t image_number); \ No newline at end of file + int64_t image_number); + +#endif //JFJOCH_BRAGGINTEGRATE2D_H diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 1aa8ce1e..64e82e33 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -31,8 +31,7 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, serialmx_filter(in_experiment), numa_policy(in_numa_policy), pixel_mask(in_pixel_mask), - indexer(experiment, indexing_thread_pool), - az_int_mapping(experiment, pixel_mask) { + indexer(experiment, indexing_thread_pool) { logger.Info("Initializing receiver"); // Ensure there is nothing running for now if (!image_buffer.Finalize(std::chrono::seconds(1))) @@ -44,7 +43,13 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, current_status.SetEfficiency({}); current_status.SetStatus(JFJochReceiverStatus{}); // GetStatus() is virtual function and cannot be called yet! - plots.Setup(experiment, az_int_mapping); + auto start_time_point = std::chrono::steady_clock::now(); + az_int_mapping = std::make_unique(experiment, pixel_mask); + auto end_time_point = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration(end_time_point - start_time_point); + logger.Info("Azimuthal integration mapping done in {:.5f} s with {} threads", duration.count(), az_int_mapping->GetNThreads()); + + plots.Setup(experiment, *az_int_mapping); push_images_to_writer = (experiment.GetImageNum() > 0) && (!experiment.GetFilePrefix().empty()); } @@ -106,13 +111,13 @@ void JFJochReceiver::SendStartMessage() { StartMessage message{}; experiment.FillMessage(message); message.arm_date = time_UTC(std::chrono::system_clock::now()); - message.az_int_q_bin_count = az_int_mapping.GetQBinCount(); - message.az_int_bin_to_q = az_int_mapping.GetBinToQ(); - message.az_int_bin_to_two_theta = az_int_mapping.GetBinToTwoTheta(); - message.az_int_phi_bin_count = az_int_mapping.GetAzimuthalBinCount(); - if (az_int_mapping.GetAzimuthalBinCount() > 1) { - message.az_int_bin_to_phi = az_int_mapping.GetBinToPhi(); - message.az_int_map = az_int_mapping.GetPixelToBin(); + message.az_int_q_bin_count = az_int_mapping->GetQBinCount(); + message.az_int_bin_to_q = az_int_mapping->GetBinToQ(); + message.az_int_bin_to_two_theta = az_int_mapping->GetBinToTwoTheta(); + message.az_int_phi_bin_count = az_int_mapping->GetAzimuthalBinCount(); + if (az_int_mapping->GetAzimuthalBinCount() > 1) { + message.az_int_bin_to_phi = az_int_mapping->GetBinToPhi(); + message.az_int_map = az_int_mapping->GetPixelToBin(); } message.writer_notification_zmq_addr = image_pusher.GetWriterNotificationSocketAddress(); message.rois = experiment.ROI().ExportMetadata(); diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 79be2802..83918987 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -80,7 +80,7 @@ protected: std::vector> adu_histogram_module; PixelMask pixel_mask; - AzimuthalIntegrationMapping az_int_mapping; + std::unique_ptr az_int_mapping; std::optional max_delay; std::mutex max_delay_mutex; diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index a3ac6384..802b7faf 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -297,7 +297,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { try { numa_policy.Bind(threadid); - analyzer = std::make_unique(experiment, indexer, az_int_mapping); + analyzer = std::make_unique(experiment, indexer); } catch (const JFJochException &e) { frame_transformation_ready.count_down(); logger.Error("Thread setup error {}", e.what()); @@ -309,6 +309,9 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { frame_transformation_ready.count_down(); + uint16_t az_int_min_bin = std::floor(az_int_mapping->QToBin(experiment.GetLowQForBkgEstimate_recipA())); + uint16_t az_int_max_bin = std::ceil(az_int_mapping->QToBin(experiment.GetHighQForBkgEstimate_recipA())); + int64_t image_number; while (images_to_go.Get(image_number) != 0) { try { @@ -335,7 +338,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { ImageMetadata metadata(experiment); - AzimuthalIntegrationProfile az_int_profile_image(az_int_mapping); + AzimuthalIntegrationProfile az_int_profile_image(*az_int_mapping); auto local_spot_finding_settings = GetSpotFindingSettings(); @@ -425,7 +428,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { experiment.GetYPixelsNum(), experiment.GetImageMode(), CompressionAlgorithm::NO_COMPRESSION); - analyzer->Process(message, local_spot_finding_settings, az_int_profile_image); + analyzer->Process(message, local_spot_finding_settings); auto status = image_buffer.GetStatus(); message.receiver_buf_available = status.available_slots; @@ -678,5 +681,5 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) { acquisition_device[data_stream].InitializeROIMap(experiment, roi_map); // Initialize data processing - acquisition_device[data_stream].InitializeDataProcessing(experiment, az_int_mapping); + acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping); } diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index 8c04b1cf..e986956e 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -242,7 +242,7 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { measurement_started.wait(); try { - analysis = std::make_unique(experiment, az_int_mapping, pixel_mask, indexer); + analysis = std::make_unique(experiment, *az_int_mapping, pixel_mask, indexer); } catch (const JFJochException &e) { Cancel(e); return; @@ -279,7 +279,7 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { auto image_start_time = std::chrono::high_resolution_clock::now(); - AzimuthalIntegrationProfile profile(az_int_mapping); + AzimuthalIntegrationProfile profile(*az_int_mapping); analysis->Analyze(data_msg, profile, GetSpotFindingSettings()); auto image_end_time = std::chrono::high_resolution_clock::now(); diff --git a/tests/BraggIntegrate2DTest.cpp b/tests/BraggIntegrate2DTest.cpp index 149515c0..75ba2a42 100644 --- a/tests/BraggIntegrate2DTest.cpp +++ b/tests/BraggIntegrate2DTest.cpp @@ -59,10 +59,6 @@ TEST_CASE("BraggIntegrate2D_RejectsReflectionsFromBackgroundUsingR2MaskAndMedian Reflection r1 = MakeReflection(10.0f, 10.0f); Reflection r2 = MakeReflection(16.0f, 10.0f); - PixelMask mask(experiment); - AzimuthalIntegrationMapping mapping(experiment, mask); - AzimuthalIntegrationProfile profile(mapping); - for (int y = 0; y < static_cast(height); y++) { for (int x = 0; x < static_cast(width); x++) { auto &pixel = image_data[y * width + x]; @@ -78,7 +74,7 @@ TEST_CASE("BraggIntegrate2D_RejectsReflectionsFromBackgroundUsingR2MaskAndMedian CompressedImage image(image_data, width, height); std::vector predicted = {r1, r2}; - auto integrated = BraggIntegrate2D(experiment, image, predicted, mapping, profile, predicted.size(), 17); + auto integrated = BraggIntegrate2D(experiment, image, predicted, predicted.size(), 17); REQUIRE(integrated.size() == 2); diff --git a/tools/jfjoch_scale.cpp b/tools/jfjoch_scale.cpp index e6da83f2..7f169940 100644 --- a/tools/jfjoch_scale.cpp +++ b/tools/jfjoch_scale.cpp @@ -148,6 +148,8 @@ int main(int argc, char **argv) { partiality_model = PartialityModel::Fixed; else if (strcmp(optarg, "rot") == 0) partiality_model = PartialityModel::Rotation; + else if (strcmp(optarg, "postref") == 0) + partiality_model = PartialityModel::Postrefinement; else { logger.Error("Invalid partiality mode: {}", optarg); print_usage(); -- 2.52.0 From 7478c0390f6271988060bbecd4b2dc56cdf57e31 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 16:37:15 +0200 Subject: [PATCH 029/228] jfjoch_scale: Remove postrefinement model option --- tools/jfjoch_scale.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/jfjoch_scale.cpp b/tools/jfjoch_scale.cpp index 7f169940..e6da83f2 100644 --- a/tools/jfjoch_scale.cpp +++ b/tools/jfjoch_scale.cpp @@ -148,8 +148,6 @@ int main(int argc, char **argv) { partiality_model = PartialityModel::Fixed; else if (strcmp(optarg, "rot") == 0) partiality_model = PartialityModel::Rotation; - else if (strcmp(optarg, "postref") == 0) - partiality_model = PartialityModel::Postrefinement; else { logger.Error("Invalid partiality mode: {}", optarg); print_usage(); -- 2.52.0 From bd5fef7f61c3da61cb34b51f9a86f332f3e81408 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 10 Jun 2026 18:36:12 +0200 Subject: [PATCH 030/228] PixelRefine: Simplify (remove Lorentz correction, remove background from azimuthal integration) --- image_analysis/IndexAndRefine.cpp | 32 +- image_analysis/IndexAndRefine.h | 10 +- image_analysis/MXAnalysisWithoutFPGA.cpp | 2 +- .../pixel_refinement/PixelRefine.cpp | 474 ++++++++++-------- image_analysis/pixel_refinement/PixelRefine.h | 52 +- viewer/JFJochImageReadingWorker.cpp | 18 +- 6 files changed, 316 insertions(+), 272 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 98b1aed6..4f3865c3 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -238,9 +238,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const IndexAndRefine::IndexingOutcome &outcome, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile) { + const IndexAndRefine::IndexingOutcome &outcome) { if (!outcome.lattice_candidate) return; @@ -299,11 +297,11 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, // integrated reflections that flow into the normal save/merge). const bool use_pixel_refine = experiment.GetIndexingSettings().GetGeomRefinementAlgorithm() == GeomRefinementAlgorithmEnum::PixelRefine - && !pixel_reference_.empty() && mapping && profile; + && !pixel_reference_.empty(); if (use_pixel_refine) { auto integration_start_time = std::chrono::steady_clock::now(); - PixelRefineIntegrate(msg, image, prediction, outcome, *mapping, *profile, i_outcome); + PixelRefineIntegrate(msg, image, prediction, outcome, i_outcome); msg.integrated_reflections = i_outcome.reflections.size(); auto integration_end_time = std::chrono::steady_clock::now(); msg.integration_time_s = std::chrono::duration(integration_end_time - integration_start_time).count(); @@ -356,9 +354,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg, void IndexAndRefine::ProcessImage(DataMessage &msg, const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, - BraggPrediction &prediction, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile) { + BraggPrediction &prediction) { if (!indexer_ || !spot_finding_settings.indexing) return; @@ -388,7 +384,7 @@ void IndexAndRefine::ProcessImage(DataMessage &msg, msg.lattice_type = outcome.symmetry; if (spot_finding_settings.quick_integration) - QuickPredictAndIntegrate(msg, spot_finding_settings, image, prediction, outcome, mapping, profile); + QuickPredictAndIntegrate(msg, spot_finding_settings, image, prediction, outcome); } std::optional IndexAndRefine::FinalizeRotationIndexing() { @@ -428,15 +424,13 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, const CompressedImage &image, BraggPrediction &prediction, const IndexAndRefine::IndexingOutcome &outcome, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile, IntegrationOutcome &i_outcome) { if (!outcome.lattice_candidate) return false; - // Build the engine once (lazy: needs the azimuthal mapping, known only here). + // Build the engine once (lazy). std::call_once(pixel_refine_once_, [&] { - pixel_refine_ = std::make_unique(experiment, mapping, pixel_reference_); + pixel_refine_ = std::make_unique(experiment, pixel_reference_); }); if (!pixel_refine_) return false; @@ -455,17 +449,17 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, const uint8_t *ptr = image.GetUncompressedPtr(buffer); switch (image.GetMode()) { case CompressedImageMode::Int8: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; case CompressedImageMode::Int16: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; case CompressedImageMode::Int32: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; case CompressedImageMode::Uint8: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; case CompressedImageMode::Uint16: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; case CompressedImageMode::Uint32: - pixel_refine_->Run(reinterpret_cast(ptr), profile, prediction, prd); break; + pixel_refine_->Run(reinterpret_cast(ptr), prediction, prd); break; default: return false; } diff --git a/image_analysis/IndexAndRefine.h b/image_analysis/IndexAndRefine.h index 7674392e..b8b3e9f2 100644 --- a/image_analysis/IndexAndRefine.h +++ b/image_analysis/IndexAndRefine.h @@ -65,9 +65,7 @@ class IndexAndRefine { const SpotFindingSettings &spot_finding_settings, const CompressedImage &image, BraggPrediction &prediction, - const IndexingOutcome &outcome, - const AzimuthalIntegrationMapping *mapping, - const AzimuthalIntegrationProfile *profile); + const IndexingOutcome &outcome); std::unique_ptr scaling_engine; void ScaleImage(DataMessage &msg, IntegrationOutcome& outcome); @@ -83,8 +81,6 @@ class IndexAndRefine { const CompressedImage &image, BraggPrediction &prediction, const IndexingOutcome &outcome, - const AzimuthalIntegrationMapping &mapping, - const AzimuthalIntegrationProfile &profile, IntegrationOutcome &i_outcome); public: IndexAndRefine(const DiffractionExperiment &x, IndexerThreadPool *indexer); @@ -92,9 +88,7 @@ public: void AddImageToRotationIndexer(DataMessage &msg); void ForceRotationIndexerLattice(const CrystalLattice& lattice); - void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction, - const AzimuthalIntegrationMapping *mapping = nullptr, - const AzimuthalIntegrationProfile *profile = nullptr); + void ProcessImage(DataMessage &msg, const SpotFindingSettings &settings, const CompressedImage &image, BraggPrediction &prediction); IndexAndRefine& ReferenceIntensities(std::vector &reference); ScalingResult ScaleAllImages(const std::vector &reference, size_t nthreads = 0); diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 6a638213..7bade492 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -92,7 +92,7 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, if (spot_finding_settings.indexing) indexer.ProcessImage(output, spot_finding_settings, CompressedImage(preprocessor_buffer->getBuffer(), experiment.GetXPixelsNum(), experiment.GetYPixelsNum()), - *prediction, &integration, &profile); + *prediction); } output.max_viable_pixel_value = ret.max_value; diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 225778f7..f41925b3 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -3,6 +3,11 @@ #include "PixelRefine.h" +#include +#include +#include +#include + #include #include #include @@ -11,17 +16,14 @@ namespace { -// Per-pixel observation, in *corrected* intensity units (solid-angle and -// polarization correction already folded in, consistently for signal and -// background). Geometry-independent quantities are precomputed here so that the -// Ceres cost functor stays cheap. +// Per-pixel observation, in *raw* detector counts (no per-pixel solid-angle or +// polarization correction - same units the "normal" integrator works in; the +// per-reflection polarization correction is applied via ReflGroup::pol). struct PixelObs { double x, y; // detector pixel coordinate - double Iobs; // corrected pixel value (signal + background) - double Ibkg; // corrected background estimate (azimuthal bin mean) + double Iobs; // raw pixel value (signal + background) + double Ibkg; // local background estimate (per-shoebox level, raw counts) double weight; // 1 / sigma_pixel - double A_recip; // reciprocal-space area subtended by the pixel (Jacobian) - double angle_rad; // goniometer angle of this observation }; // One reflection together with the pixels of its shoebox. @@ -30,6 +32,7 @@ struct ReflGroup { double d; double Itrue; // reference intensity (held fixed) double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) + double pol; // per-reflection polarization correction (raw = true * pol) double predicted_x, predicted_y; std::vector pixels; }; @@ -40,13 +43,99 @@ double SafeInv(double x, double fallback) { return 1.0 / x; } +// Median of a vector (in place, partially reorders it). +double MedianInPlace(std::vector &v) { + if (v.empty()) + return 0.0; + const size_t mid = v.size() / 2; + std::nth_element(v.begin(), v.begin() + mid, v.end()); + if (v.size() % 2 == 1) + return v[mid]; + const double hi = v[mid]; + std::nth_element(v.begin(), v.begin() + mid - 1, v.begin() + mid); + return 0.5 * (v[mid - 1] + hi); +} + +// Mask marking the *core* (radius `radius`) of every predicted spot, so that the +// local-background sampling of one reflection never picks up a neighbouring +// reflection's signal. Same idea as BraggIntegrate2D::BuildReflectionMask. +std::vector BuildSpotMask(const std::vector &predicted, int nrefl, + size_t xpixel, size_t ypixel, int radius) { + std::vector mask(xpixel * ypixel, 0); + const double r_sq = static_cast(radius) * radius; + for (int i = 0; i < nrefl; ++i) { + const auto &r = predicted[i]; + const int cx = static_cast(std::lround(r.predicted_x)); + const int cy = static_cast(std::lround(r.predicted_y)); + const int x0 = std::max(0, cx - radius); + const int x1 = std::min(static_cast(xpixel) - 1, cx + radius); + const int y0 = std::max(0, cy - radius); + const int y1 = std::min(static_cast(ypixel) - 1, cy + radius); + for (int y = y0; y <= y1; ++y) { + for (int x = x0; x <= x1; ++x) { + const double dx = x - r.predicted_x; + const double dy = y - r.predicted_y; + if (dx * dx + dy * dy <= r_sq) + mask[static_cast(xpixel) * y + x] = 1; + } + } + } + return mask; +} + +// Local flat background around one shoebox, in raw detector counts. Samples the +// square ring shoebox_radius < max(|dx|,|dy|) <= bkg_outer_radius centred on the +// spot, dropping pixels that belong to any spot core (spot_mask) or carry a +// masked/saturated sentinel, and returns the median (robust to residual spot +// tails / zingers). Mirrors the local-background of BraggIntegrate2D, replacing +// the azimuthal-bin mean that proved a poor proxy for reflection background. +template +bool EstimateLocalBackground(const T *image, + const std::vector &spot_mask, + size_t xpixel, size_t ypixel, + double cx, double cy, + int shoebox_radius, int bkg_outer_radius, + double &bkg_mean) { + const int icx = static_cast(std::lround(cx)); + const int icy = static_cast(std::lround(cy)); + const int x0 = std::max(0, icx - bkg_outer_radius); + const int x1 = std::min(static_cast(xpixel) - 1, icx + bkg_outer_radius); + const int y0 = std::max(0, icy - bkg_outer_radius); + const int y1 = std::min(static_cast(ypixel) - 1, icy + bkg_outer_radius); + + std::vector vals; + vals.reserve(static_cast((x1 - x0 + 1) * (y1 - y0 + 1))); + for (int y = y0; y <= y1; ++y) { + for (int x = x0; x <= x1; ++x) { + // Skip the square shoebox core: that is signal, not background. + if (std::abs(x - icx) <= shoebox_radius && std::abs(y - icy) <= shoebox_radius) + continue; + const size_t np = static_cast(xpixel) * y + x; + if (spot_mask[np]) + continue; + const T raw = image[np]; + if (raw == std::numeric_limits::max()) + continue; + if (std::is_signed_v && raw == std::numeric_limits::min()) + continue; + vals.push_back(static_cast(raw)); + } + } + + if (vals.size() < 5) + return false; + + bkg_mean = MedianInPlace(vals); + return true; +} + // Per-pixel: map a detector pixel through the current geometry into the // reference reciprocal frame. Cheap (a few trig + one rotation); depends on the // pixel and the detector geometry, not on the lattice. template void ObservedRecip(const T *beam, const T *distance_mm, const T *detector_rot, - const T *rotation_axis, double obs_x, double obs_y, - double pixel_size, double inv_lambda, double angle_rad, + double obs_x, double obs_y, + double pixel_size, double inv_lambda, Eigen::Matrix &e_obs_recip) { // PyFAI convention (left-handed for rot1/rot2): rot3 = 0 assumed. const T c1 = ceres::cos(detector_rot[0]); @@ -73,14 +162,7 @@ void ObservedRecip(const T *beam, const T *distance_mm, const T *detector_rot, y * inv_norm * T(inv_lambda), (z * inv_norm - T(1.0)) * T(inv_lambda) }; - const T aa_back[3] = { - T(angle_rad) * rotation_axis[0], - T(angle_rad) * rotation_axis[1], - T(angle_rad) * rotation_axis[2] - }; - T recip_obs[3]; - ceres::AngleAxisRotatePoint(aa_back, recip_raw, recip_obs); - e_obs_recip = Eigen::Matrix(recip_obs[0], recip_obs[1], recip_obs[2]); + e_obs_recip = Eigen::Matrix(recip_raw[0], recip_raw[1], recip_raw[2]); } // Per-reflection: predicted node g_hkl, |g_hkl|^2, and the Ewald-sphere normal. @@ -178,20 +260,23 @@ bool PredictedNode(const T *p0, const T *p1, const T *p2, // --------------------------------------------------------------------------- // Cost functor // -// I_pred(pixel) = G * Itrue * B_term * P_radial * P_tangential + I_bkg +// I_pred(pixel) = G * Itrue * B_term * P_radial * P_tangential * pol + I_bkg // // B_term = exp(-B |q|^2 / 4) (Debye-Waller) // P_radial = exp(-eps_r^2 / R0_eff^2) (partiality: fraction of // the mosaic blob on the // Ewald sphere; <= 1) -// P_tangential = A_recip/(pi R1^2) * exp(-eps_t^2/R1^2)(spatial profile on the -// detector, normalized so -// that sum over pixels ~ 1) +// P_tangential = exp(-eps_t^2/R1^2) / (pi R1^2) (Gaussian spatial profile +// in the Ewald tangent plane) +// pol = per-reflection polarization correction (raw = true * pol), +// evaluated once at the predicted spot position (as in +// BraggIntegrate2D). 1 if polarization is disabled. // -// The tangential factor is what makes this "profile fitting": summing -// I_pred - I_bkg over the shoebox reproduces G * Itrue * B_term * P_radial. -// The 1/(pi R1^2) normalization is the missing piece that decouples the profile -// width R1 from the overall scale G. +// Everything is in *raw* detector counts: there is no per-pixel solid-angle or +// area (Lorentz/Jacobian) weighting - each pixel counts equally, like the normal +// integrator. The tangential factor is what makes this "profile fitting"; the +// 1/(pi R1^2) normalization keeps the profile width R1 from soaking up the +// overall scale G. // // X-ray bandwidth: a spread in lambda is a spread in the Ewald-sphere radius, // i.e. a purely *radial* thickening of the shell. It adds (in quadrature) a @@ -207,13 +292,13 @@ struct PixelResidual { PixelResidual(const PixelObs &obs, double Itrue, double lambda, double pixel_size, double exp_h, double exp_k, double exp_l, - double R_bw_sq, + double R_bw_sq, double pol, gemmi::CrystalSystem symmetry) : Itrue(Itrue), Iobs(obs.Iobs), Ibkg(obs.Ibkg), weight(obs.weight), - A_recip(obs.A_recip), obs_x(obs.x), obs_y(obs.y), + obs_x(obs.x), obs_y(obs.y), inv_lambda(1.0 / lambda), pixel_size(pixel_size), exp_h(exp_h), exp_k(exp_k), exp_l(exp_l), - R_bw_sq(R_bw_sq), angle_rad(obs.angle_rad), symmetry(symmetry) { + R_bw_sq(R_bw_sq), pol(pol), symmetry(symmetry) { if (std::fabs(lambda) < 1e-6) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Lambda cannot be close to zero"); @@ -228,14 +313,14 @@ struct PixelResidual { bool GeometryTerms(const T *const beam, const T *const distance_mm, const T *const detector_rot, - const T *const rotation_axis, const T *const p0, const T *const p1, const T *const p2, T &q_sq, T &eps_radial, T &eps_tang_sq) const { Eigen::Matrix e_obs_recip; - ObservedRecip(beam, distance_mm, detector_rot, rotation_axis, - obs_x, obs_y, pixel_size, inv_lambda, angle_rad, e_obs_recip); + ObservedRecip(beam, distance_mm, detector_rot, + obs_x, obs_y, pixel_size, inv_lambda, + e_obs_recip); Eigen::Matrix e_pred_recip, n_radial; if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, @@ -251,12 +336,12 @@ struct PixelResidual { // Assembles the full model intensity for the pixel from the geometry terms. template bool Model(const T *const beam, const T *const distance_mm, - const T *const detector_rot, const T *const rotation_axis, + const T *const detector_rot, const T *const p0, const T *const p1, const T *const p2, const T *const scale_factor, const T *const B, const T *const R, T &Ipred) const { T q_sq, eps_radial, eps_tang_sq; - if (!GeometryTerms(beam, distance_mm, detector_rot, rotation_axis, + if (!GeometryTerms(beam, distance_mm, detector_rot, p0, p1, p2, q_sq, eps_radial, eps_tang_sq)) return false; @@ -268,10 +353,9 @@ struct PixelResidual { // Separable Gaussian spot model: // radial P_r(e) = exp(-e^2/R0_eff^2) (peak-normalized, in (0,1]) // tangent g_t(e) = exp(-|e|^2/R1^2) / (pi R1^2) [1/A^-2] - // The pixel captures the fraction g_t * A_recip of the tangential profile - // (A_recip = reciprocal area the pixel subtends; sum over shoebox ~ 1). - // The radial factor is the still-image partiality (how far the reflection - // sits from the Ewald sphere); the overall scale is carried by the free G. + // Every pixel counts equally (no area/Lorentz weighting); the radial factor + // is the still-image partiality (how far the reflection sits from the Ewald + // sphere); the overall scale is carried by the free G. // // IMPORTANT: the radial factor MUST use the same convention here as the // extraction's `partiality` (peak-normalized), otherwise image_scale_corr @@ -280,10 +364,10 @@ struct PixelResidual { // R0_eff folds in the energy-bandwidth broadening via R_bw_sq. const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); - const T P_tang = T(A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) + const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); Ipred = signal + T(Ibkg); return true; } @@ -292,7 +376,6 @@ struct PixelResidual { bool operator()(const T *const beam, const T *const distance_mm, const T *const detector_rot, - const T *const rotation_axis, const T *const p0, const T *const p1, const T *const p2, @@ -301,21 +384,20 @@ struct PixelResidual { const T *const R, T *residual) const { T Ipred; - if (!Model(beam, distance_mm, detector_rot, rotation_axis, - p0, p1, p2, scale_factor, B, R, Ipred)) + if (!Model(beam, distance_mm, detector_rot, p0, p1, p2, scale_factor, B, R, Ipred)) return false; residual[0] = (Ipred - T(Iobs)) * T(weight); return true; } - const double Itrue, Iobs, Ibkg, weight, A_recip; + const double Itrue, Iobs, Ibkg, weight; const double obs_x, obs_y; const double inv_lambda; const double pixel_size; const double exp_h, exp_k, exp_l; const double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) - const double angle_rad; + const double pol; // per-reflection polarization correction gemmi::CrystalSystem symmetry; }; @@ -333,27 +415,25 @@ struct PixelResidual { struct ShoeboxResidual { ShoeboxResidual(const ReflGroup &g, double lambda, double pixel_size, gemmi::CrystalSystem symmetry) - : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), + : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), pol(g.pol), exp_h(g.h), exp_k(g.k), exp_l(g.l), inv_lambda(1.0 / lambda), pixel_size(pixel_size), - angle_rad(g.pixels.empty() ? 0.0 : g.pixels.front().angle_rad), symmetry(symmetry) {} template bool operator()(const T *const *params, T *residual) const { // Parameter blocks (order matches AddParameterBlock in Run): - // 0 beam[2] 1 distance[1] 2 detector_rot[2] 3 rotation_axis[3] - // 4 p0[3] 5 p1[3] 6 p2[3] 7 scale[1] 8 B[1] 9 R[2] + // 0 beam[2] 1 distance[1] 2 detector_rot[2] + // 3 p0[3] 4 p1[3] 5 p2[3] 6 scale[1] 7 B[1] 8 R[2] const T *beam = params[0]; const T *distance_mm = params[1]; const T *detector_rot = params[2]; - const T *rotation_axis = params[3]; - const T *p0 = params[4]; - const T *p1 = params[5]; - const T *p2 = params[6]; - const T *scale_factor = params[7]; - const T *B = params[8]; - const T *R = params[9]; + const T *p0 = params[3]; + const T *p1 = params[4]; + const T *p2 = params[5]; + const T *scale_factor = params[6]; + const T *B = params[7]; + const T *R = params[8]; if (R[0] < T(1e-10) || R[1] < T(1e-10)) return false; @@ -373,18 +453,18 @@ struct ShoeboxResidual { const PixelObs &obs = pixels[i]; Eigen::Matrix e_obs_recip; - ObservedRecip(beam, distance_mm, detector_rot, rotation_axis, - obs.x, obs.y, pixel_size, inv_lambda, angle_rad, e_obs_recip); + ObservedRecip(beam, distance_mm, detector_rot, + obs.x, obs.y, pixel_size, inv_lambda, e_obs_recip); const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; const T eps_radial = delta_q.dot(n_radial); const T eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); - const T P_tang = T(obs.A_recip) * ceres::exp(-eps_tang_sq / (R[1] * R[1])) + const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang; + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); const T Ipred = signal + T(obs.Ibkg); residual[i] = (Ipred - T(obs.Iobs)) * T(obs.weight); } @@ -392,17 +472,15 @@ struct ShoeboxResidual { } std::vector pixels; - const double Itrue, R_bw_sq; + const double Itrue, R_bw_sq, pol; const double exp_h, exp_k, exp_l; - const double inv_lambda, pixel_size, angle_rad; + const double inv_lambda, pixel_size; gemmi::CrystalSystem symmetry; }; PixelRefine::PixelRefine(const DiffractionExperiment &experiment, - const AzimuthalIntegrationMapping &mapping, const std::vector &reference) - : mapping(mapping), - xpixel(experiment.GetXPixelsNum()), + : xpixel(experiment.GetXPixelsNum()), ypixel(experiment.GetYPixelsNum()), experiment(experiment), hkl_key_generator(experiment.GetScalingSettings().GetMergeFriedel(), @@ -413,19 +491,13 @@ PixelRefine::PixelRefine(const DiffractionExperiment &experiment, void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, double beam[2], double &dist_mm, - double detector_rot[2], double rot_vec[3], + double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const { beam[0] = data.geom.GetBeamX_pxl(); beam[1] = data.geom.GetBeamY_pxl(); dist_mm = data.geom.GetDetectorDistance_mm(); detector_rot[0] = data.geom.GetPoniRot1_rad(); detector_rot[1] = data.geom.GetPoniRot2_rad(); - rot_vec[0] = 1.0; rot_vec[1] = 0.0; rot_vec[2] = 0.0; - if (auto axis = data.geom.GetRotation()) { - rot_vec[0] = axis->GetAxis().x; - rot_vec[1] = axis->GetAxis().y; - rot_vec[2] = axis->GetAxis().z; - } for (int i = 0; i < 3; ++i) latt_vec0[i] = latt_vec1[i] = latt_vec2[i] = 0.0; @@ -463,7 +535,6 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, template void PixelRefine::Run(const T *image, - const AzimuthalIntegrationProfile &profile, BraggPrediction &prediction, PixelRefineData &data) { data.solved = false; @@ -479,29 +550,17 @@ void PixelRefine::Run(const T *image, .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma }; - const auto azim_result = profile.GetResult(); - const auto azim_std = profile.GetStd(); - const auto &pixel_to_bin = mapping.GetPixelToBin(); - const auto &corrections = mapping.Corrections(); - // pixel_to_bin stores the *full* bin index (azimuthal_sector * q_bins + q_bin), - // so the valid range is the total number of bins, i.e. the profile size - NOT - // GetAzimuthalBinCount() (which is only the number of azimuthal sectors). - const int total_bin_count = static_cast(azim_result.size()); - - const double angle_rad = data.angle_deg * M_PI / 180.0; const int radius = data.shoebox_radius; + const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); - // Exact reciprocal-space area a 1x1 pixel subtends, |dq/dx x dq/dy|, via - // finite differences of the detector->reciprocal map. This is the Jacobian - // between the curved Ewald-sphere sampling and flat reciprocal space, and it - // is exactly the geometric factor that plays the role of the Lorentz factor - // for stills: where the sphere grazes reciprocal space obliquely, a pixel - // covers more reciprocal volume and the captured fraction grows. It tracks - // the refined geometry because it reads the current data.geom each iteration. - auto recip_area = [&](double x, double y) -> double { - const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); - const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); - return (qx % qy).Length(); + // Per-reflection polarization correction (raw = true * pol), evaluated once at + // the predicted spot - same handling as BraggIntegrate2D. Identity if disabled. + const auto pol_factor = experiment.GetPolarizationFactor(); + auto polarization = [&](double x, double y) -> double { + if (!pol_factor) + return 1.0; + return data.geom.CalcAzIntPolarizationCorr(static_cast(x), static_cast(y), + pol_factor.value()); }; // Bandwidth radial-width^2 (in the code's R = sqrt(2)*sigma convention): @@ -524,7 +583,6 @@ void PixelRefine::Run(const T *image, double beam[2] = {0, 0}; double dist_mm = data.geom.GetDetectorDistance_mm(); double detector_rot[2] = {0, 0}; - double rot_vec[3] = {1.0, 0.0, 0.0}; double latt_vec0[3] = {0, 0, 0}; // orientation (Rodrigues) double latt_vec1[3] = {0, 0, 0}; // lengths double latt_vec2[3] = {0, 0, 0}; // angles (rad) @@ -546,12 +604,27 @@ void PixelRefine::Run(const T *image, // nrefl entries are valid for this image (the rest are stale/zeroed). groups.clear(); const auto &predicted = prediction.GetReflections(); + + // Spot-core mask over ALL predicted reflections, so each reflection's + // local background ignores pixels that belong to a neighbouring spot. + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); + for (int ri = 0; ri < nrefl; ++ri) { const auto &refl = predicted[ri]; const auto hkl = hkl_key_generator(refl); if (!reference_data.contains(hkl)) continue; + // Local flat background from the ring around the shoebox (raw counts). + // No azimuthal fallback: if we cannot estimate a clean local background + // the reflection is dropped, exactly as BraggIntegrate2D marks it + // unobserved when fewer than a handful of background pixels survive. + double Ibkg = 0.0; + if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg)) + continue; + ReflGroup g; g.h = refl.h; g.k = refl.k; @@ -559,6 +632,7 @@ void PixelRefine::Run(const T *image, g.d = refl.d; g.Itrue = reference_data[hkl]; g.R_bw_sq = bandwidth_radial_sq(refl.d); + g.pol = polarization(refl.predicted_x, refl.predicted_y); g.predicted_x = refl.predicted_x; g.predicted_y = refl.predicted_y; @@ -570,27 +644,18 @@ void PixelRefine::Run(const T *image, for (int y = min_y; y <= max_y; ++y) { for (int x = min_x; x <= max_x; ++x) { const size_t npixel = xpixel * y + x; - const int azim_bin = pixel_to_bin[npixel]; - // Skip pixels not mapped to a bin or carrying a sentinel - // (masked / saturated) value. We assume the pixel mask is - // already applied upstream. - if (azim_bin >= total_bin_count) - continue; + // Skip sentinel (masked / saturated) pixels. We assume the pixel + // mask is already applied upstream (encoded as the sentinel). if (image[npixel] == std::numeric_limits::max()) continue; if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) continue; - const double correction = corrections[npixel]; - const double Ibkg = azim_result[azim_bin]; // already in corrected units - const double Ibkg_sigma = azim_std[azim_bin]; - const double raw = static_cast(image[npixel]); - const double Iobs = raw * correction; + const double Iobs = static_cast(image[npixel]); // raw counts - // Per-pixel variance: Poisson noise of the corrected counts - // (var(c*N) = c^2 * N = c * Iobs) plus the background spread. - double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; + // Per-pixel variance: Poisson noise of the raw counts. + double var = std::max(Iobs, 0.0); if (!(var > 1.0)) var = 1.0; @@ -599,9 +664,7 @@ void PixelRefine::Run(const T *image, .y = static_cast(y), .Iobs = Iobs, .Ibkg = Ibkg, - .weight = 1.0 / std::sqrt(var), - .A_recip = recip_area(x, y), - .angle_rad = angle_rad + .weight = 1.0 / std::sqrt(var) }; g.pixels.push_back(obs); } @@ -615,7 +678,7 @@ void PixelRefine::Run(const T *image, return; // ---- 3. Set up parameter blocks (geometry part mirrors XtalOptimizer) - - BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, + BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); // ---- 4. Build the problem --------------------------------------------- @@ -630,7 +693,6 @@ void PixelRefine::Run(const T *image, cost->AddParameterBlock(2); // beam cost->AddParameterBlock(1); // distance cost->AddParameterBlock(2); // detector_rot - cost->AddParameterBlock(3); // rotation_axis cost->AddParameterBlock(3); // p0 (orientation) cost->AddParameterBlock(3); // p1 (lengths) cost->AddParameterBlock(3); // p2 (angles) @@ -643,7 +705,7 @@ void PixelRefine::Run(const T *image, // per-pixel Huber. Per-pixel sigma weighting is retained; per-pixel // outlier rejection (zingers) is a TODO if needed. problem.AddResidualBlock(cost, nullptr, - beam, &dist_mm, detector_rot, rot_vec, + beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, &data.scale_factor, &data.B_factor, data.R); residual_pixels += g.pixels.size(); @@ -687,9 +749,6 @@ void PixelRefine::Run(const T *image, } } - if (!data.refine_rotation_axis) - problem.SetParameterBlockConstant(rot_vec); - if (data.refine_scale) problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); else @@ -767,20 +826,20 @@ void PixelRefine::Run(const T *image, } // predict<->refine iterations // ---- Extract integrated reflections --------------------------------------- - // Profile fitting gives the recorded amplitude (against the normalized - // tangential profile P_t): + // Profile fitting gives the recorded amplitude (against the tangential profile + // P_t): // J = sum_p[ P_t,p (Iobs_p - Ibkg_p)/v_p ] / sum_p[ P_t,p^2 / v_p ] - // ~ G * Itrue * B_term * partiality (recorded intensity) + // ~ G * Itrue * B_term * partiality * pol (recorded raw counts) // var(J) = 1 / sum_p[ P_t,p^2 / v_p ] // // Output split (Merge multiplies r.I * image_scale_corr and weights by // 1/(sigma*image_scale_corr)^2 - see Merge.cpp): - // r.I = J / (B_term * partiality) = G * Itrue (B/partiality corrected) - // r.sigma = sqrt(var(J)) / (B_term * partiality) + // r.I = J / (B_term * partiality * pol) = G * Itrue + // r.sigma = sqrt(var(J)) / (B_term * partiality * pol) // r.partiality = profile-weighted peak radial factor in (0,1] (Merge filter only) // r.image_scale_corr = 1/G (per-image scale ONLY) - // so r.I * image_scale_corr = Itrue. B and partiality live on the intensity, - // G lives on image_scale_corr - one clean meaning per field. + // so r.I * image_scale_corr = Itrue. B, partiality and polarization live on the + // intensity, G lives on image_scale_corr - one clean meaning per field. data.reflections.reserve(groups.size()); for (const auto &g : groups) { double num = 0.0, den = 0.0, bkg_sum = 0.0; @@ -788,16 +847,16 @@ void PixelRefine::Run(const T *image, size_t n = 0; for (const auto &obs : g.pixels) { - PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, g.R_bw_sq, data.crystal_system); + PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, g.R_bw_sq, g.pol, data.crystal_system); double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, rot_vec, - latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, + eps_r, eps_t_sq)) continue; if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) continue; - // Normalized tangential profile (sum over shoebox ~ 1) -> fit weight. - const double P_t = obs.A_recip * std::exp(-eps_t_sq / (data.R[1] * data.R[1])) + // Tangential profile shape -> fit weight (every pixel counts equally). + const double P_t = std::exp(-eps_t_sq / (data.R[1] * data.R[1])) / (M_PI * data.R[1] * data.R[1]); // Peak-normalized radial factor (the partiality), in (0,1]. // Bandwidth-broadened radial width, matching the model in Model(). @@ -828,10 +887,10 @@ void PixelRefine::Run(const T *image, r.partiality = (radial_w > 0.0) ? static_cast(radial_sum / radial_w) : 1.0f; if (den > 0.0 && n > 0) { - const double I_amp = num / den; // ~ G*Itrue*B_term*partiality + const double I_amp = num / den; // ~ G*Itrue*B_term*partiality*pol const double sigma_amp = std::sqrt(1.0 / den); const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); - const double corr = static_cast(r.partiality) * B_term; // B & partiality + const double corr = static_cast(r.partiality) * B_term * g.pol; // B, partiality & pol r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; @@ -884,7 +943,8 @@ void PixelRefine::Run(const T *image, } } -std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile &profile, +template +std::vector PixelRefine::PredictImage(const T *image, BraggPrediction &prediction, const PixelRefineData &data, bool include_background) const { @@ -892,18 +952,16 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); - const auto azim_result = profile.GetResult(); - const auto &pixel_to_bin = mapping.GetPixelToBin(); - const auto &corrections = mapping.Corrections(); - const int total_bin_count = static_cast(azim_result.size()); - const double angle_rad = data.angle_deg * M_PI / 180.0; const int radius = data.shoebox_radius; + const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); const double bw = data.bandwidth; - auto recip_area = [&](double x, double y) -> double { - const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); - const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); - return (qx % qy).Length(); + const auto pol_factor = experiment.GetPolarizationFactor(); + auto polarization = [&](double x, double y) -> double { + if (!pol_factor) + return 1.0; + return data.geom.CalcAzIntPolarizationCorr(static_cast(x), static_cast(y), + pol_factor.value()); }; auto bandwidth_radial_sq = [&](double d) -> double { if (bw <= 0.0 || d <= 0.0) @@ -912,26 +970,9 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & return bl * bl / (2.0 * d * d * d * d); }; - // The model works in solid-angle/polarization-corrected units (as in Run, - // where Iobs = raw * correction). Map back to raw detector units (/ correction) - // so the predicted image overlays directly on the original image. - auto to_raw = [&](size_t npixel, double corrected) -> float { - const double corr = corrections[npixel]; - return (corr > 0.0) ? static_cast(corrected / corr) : 0.0f; - }; - - // Background base layer (per-pixel azimuthal mean), full-frame pass. - if (include_background) { - for (size_t p = 0; p < img.size(); ++p) { - const int bin = pixel_to_bin[p]; - if (bin >= 0 && bin < total_bin_count) - img[p] = to_raw(p, azim_result[bin]); - } - } - - double beam[2], dist_mm, detector_rot[2], rot_vec[3]; + double beam[2], dist_mm, detector_rot[2]; double latt_vec0[3], latt_vec1[3], latt_vec2[3]; - BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2); + BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); DiffractionExperiment exp_iter = experiment; exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) @@ -948,6 +989,7 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & }; const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); for (int ri = 0; ri < nrefl; ++ri) { const auto &refl = predicted[ri]; @@ -957,6 +999,16 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & const double Itrue = it->second; const double R_bw_sq = bandwidth_radial_sq(refl.d); + const double pol = polarization(refl.predicted_x, refl.predicted_y); + + // Local background straight from the actual image (flat per shoebox), laid + // into the box so the prediction overlays the real frame - the same model + // path Run() fits, now reproduced faithfully because we have the image. + double Ibkg = 0.0; + const bool have_bkg = include_background && + EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg); const int min_y = std::max(refl.predicted_y - radius, 0); const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); @@ -967,25 +1019,21 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & for (int x = min_x; x <= max_x; ++x) { const size_t npixel = xpixel * y + x; - // Pure Bragg signal: Ibkg = 0 so Model() returns signal only; the - // background is already laid down above. Same code path as Run. PixelObs obs{ .x = static_cast(x), .y = static_cast(y), .Iobs = 0.0, - .Ibkg = 0.0, - .weight = 1.0, - .A_recip = recip_area(x, y), - .angle_rad = angle_rad + .Ibkg = have_bkg ? Ibkg : 0.0, + .weight = 1.0 }; PixelResidual pr(obs, Itrue, lambda, pixel_size, - refl.h, refl.k, refl.l, R_bw_sq, data.crystal_system); + refl.h, refl.k, refl.l, R_bw_sq, pol, data.crystal_system); - double signal = 0.0; - if (pr.Model(beam, &dist_mm, detector_rot, rot_vec, + double Ipred = 0.0; // raw counts: signal (+ local background) + if (pr.Model(beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R, signal)) - img[npixel] += to_raw(npixel, signal); + &data.scale_factor, &data.B_factor, data.R, Ipred)) + img[npixel] += static_cast(Ipred); } } } @@ -995,26 +1043,22 @@ std::vector PixelRefine::PredictImage(const AzimuthalIntegrationProfile & template std::vector PixelRefine::ChiSquaredImage(const T *image, - const AzimuthalIntegrationProfile &profile, BraggPrediction &prediction, const PixelRefineData &data) const { std::vector img(xpixel * ypixel, 0.0f); const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); - const auto azim_result = profile.GetResult(); - const auto azim_std = profile.GetStd(); - const auto &pixel_to_bin = mapping.GetPixelToBin(); - const auto &corrections = mapping.Corrections(); - const int total_bin_count = static_cast(azim_result.size()); - const double angle_rad = data.angle_deg * M_PI / 180.0; const int radius = data.shoebox_radius; + const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); const double bw = data.bandwidth; - auto recip_area = [&](double x, double y) -> double { - const Coord qx = data.geom.DetectorToRecip(x + 0.5, y) - data.geom.DetectorToRecip(x - 0.5, y); - const Coord qy = data.geom.DetectorToRecip(x, y + 0.5) - data.geom.DetectorToRecip(x, y - 0.5); - return (qx % qy).Length(); + const auto pol_factor = experiment.GetPolarizationFactor(); + auto polarization = [&](double x, double y) -> double { + if (!pol_factor) + return 1.0; + return data.geom.CalcAzIntPolarizationCorr(static_cast(x), static_cast(y), + pol_factor.value()); }; auto bandwidth_radial_sq = [&](double d) -> double { if (bw <= 0.0 || d <= 0.0) @@ -1023,9 +1067,9 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, return bl * bl / (2.0 * d * d * d * d); }; - double beam[2], dist_mm, detector_rot[2], rot_vec[3]; + double beam[2], dist_mm, detector_rot[2]; double latt_vec0[3], latt_vec1[3], latt_vec2[3]; - BuildParameterBlocks(data, beam, dist_mm, detector_rot, rot_vec, latt_vec0, latt_vec1, latt_vec2); + BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); DiffractionExperiment exp_iter = experiment; exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) @@ -1042,6 +1086,7 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, }; const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); for (int ri = 0; ri < nrefl; ++ri) { const auto &refl = predicted[ri]; @@ -1051,6 +1096,15 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, const double Itrue = it->second; const double R_bw_sq = bandwidth_radial_sq(refl.d); + const double pol = polarization(refl.predicted_x, refl.predicted_y); + + // Local flat background, identical to Run(); skip the reflection if it + // cannot be estimated (matches Run() dropping the reflection). + double Ibkg = 0.0; + if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg)) + continue; const int min_y = std::max(refl.predicted_y - radius, 0); const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); @@ -1060,23 +1114,16 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, for (int y = min_y; y <= max_y; ++y) { for (int x = min_x; x <= max_x; ++x) { const size_t npixel = xpixel * y + x; - const int azim_bin = pixel_to_bin[npixel]; // Same gating as Run(): only pixels that actually enter the fit. - if (azim_bin >= total_bin_count) - continue; if (image[npixel] == std::numeric_limits::max()) continue; if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) continue; - const double correction = corrections[npixel]; - const double Ibkg = azim_result[azim_bin]; - const double Ibkg_sigma = azim_std[azim_bin]; - const double raw = static_cast(image[npixel]); - const double Iobs = raw * correction; + const double Iobs = static_cast(image[npixel]); // raw counts - double var = correction * std::max(Iobs, 0.0) + Ibkg_sigma * Ibkg_sigma; + double var = std::max(Iobs, 0.0); if (!(var > 1.0)) var = 1.0; const double weight = 1.0 / std::sqrt(var); @@ -1086,15 +1133,13 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, .y = static_cast(y), .Iobs = Iobs, .Ibkg = Ibkg, - .weight = weight, - .A_recip = recip_area(x, y), - .angle_rad = angle_rad + .weight = weight }; PixelResidual pr(obs, Itrue, lambda, pixel_size, - refl.h, refl.k, refl.l, R_bw_sq, data.crystal_system); + refl.h, refl.k, refl.l, R_bw_sq, pol, data.crystal_system); double Ipred = 0.0; - if (pr.Model(beam, &dist_mm, detector_rot, rot_vec, + if (pr.Model(beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, &data.scale_factor, &data.B_factor, data.R, Ipred)) { // residual_i = (I_pred - I_obs) * weight (== Ceres residual); @@ -1110,16 +1155,23 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, } // Explicit instantiations for the supported (uncompressed) image pixel types. -template void PixelRefine::Run(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); -template void PixelRefine::Run(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); -template void PixelRefine::Run(const int32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); -template void PixelRefine::Run(const uint8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); -template void PixelRefine::Run(const uint16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); -template void PixelRefine::Run(const uint32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const int8_t *, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const int16_t *, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const int32_t *, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint8_t *, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint16_t *, BraggPrediction &, PixelRefineData &); +template void PixelRefine::Run(const uint32_t *, BraggPrediction &, PixelRefineData &); -template std::vector PixelRefine::ChiSquaredImage(const int8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const int16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const int32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint8_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint16_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint32_t *, const AzimuthalIntegrationProfile &, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::PredictImage(const int8_t *, BraggPrediction &, const PixelRefineData &, bool) const; +template std::vector PixelRefine::PredictImage(const int16_t *, BraggPrediction &, const PixelRefineData &, bool) const; +template std::vector PixelRefine::PredictImage(const int32_t *, BraggPrediction &, const PixelRefineData &, bool) const; +template std::vector PixelRefine::PredictImage(const uint8_t *, BraggPrediction &, const PixelRefineData &, bool) const; +template std::vector PixelRefine::PredictImage(const uint16_t *, BraggPrediction &, const PixelRefineData &, bool) const; +template std::vector PixelRefine::PredictImage(const uint32_t *, BraggPrediction &, const PixelRefineData &, bool) const; + +template std::vector PixelRefine::ChiSquaredImage(const int8_t *, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const int16_t *, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const int32_t *, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint8_t *, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint16_t *, BraggPrediction &, const PixelRefineData &) const; +template std::vector PixelRefine::ChiSquaredImage(const uint32_t *, BraggPrediction &, const PixelRefineData &) const; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 73a63094..8d28e107 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -5,8 +5,6 @@ #include "../bragg_prediction/BraggPrediction.h" #include "../common/DiffractionExperiment.h" -#include "../common/AzimuthalIntegrationMapping.h" -#include "../common/AzimuthalIntegrationProfile.h" #include "../scale_merge/HKLKey.h" // ============================================================================= @@ -33,7 +31,10 @@ // squares problem. We write down, for every pixel in a reflection's shoebox, the // expected counts as an explicit forward model // -// I_pred(pixel) = G * I_true * B_term * P_radial * P_tangential + I_bkg +// I_pred(pixel) = G * I_true * B_term * P_radial * P_tangential * pol + I_bkg +// +// in raw detector counts (pol = per-reflection polarization correction, I_bkg = +// local per-shoebox background read from the image). // // and let Ceres autodiff back-propagate the per-pixel residuals into ALL of: // * detector geometry (beam centre, distance, tilt) @@ -95,8 +96,11 @@ // (bandwidth = 0, monochromatic); set it for DMM-type data, leave it for Si. // // Status: experimental prototype. The forward model (esp. the still-image -// Lorentz/partiality normalization) is deliberately simple and expected to -// evolve. See PixelRefine.cpp for the physics conventions and known caveats. +// partiality normalization) is deliberately simple and expected to evolve: it +// works in raw detector counts with a local per-shoebox background and a +// per-reflection polarization correction (no per-pixel solid-angle/Lorentz +// weighting), matching the classical integrator. See PixelRefine.cpp for the +// physics conventions and known caveats. // ============================================================================= struct PixelRefineData { @@ -115,23 +119,23 @@ struct PixelRefineData { // to R[0]. 0 = monochromatic (the term switches off entirely). double bandwidth = 0.0; - // Goniometer: for a still image keep angle_deg = 0. For a wedge, pass the - // angle (deg) of the slice centre relative to the reference (phi=0) frame. - double angle_deg = 0.0; - // --- what to refine --- bool refine_orientation = true; // crystal orientation (p0) bool refine_unit_cell = false; // cell lengths + angles bool refine_beam_center = false; bool refine_distance = false; bool refine_detector_angles = false; - bool refine_rotation_axis = false; bool refine_scale = true; bool refine_B = false; bool refine_R = true; double max_time_s = 5.0; - int shoebox_radius = 3; // half-size of the per-reflection pixel box + int shoebox_radius = 3; // half-size of the per-reflection signal box (peak region that enters the fit) + // Half-size of the local-background sampling box. Background is estimated from + // the ring shoebox_radius < |dx|,|dy| <= bkg_outer_radius around each spot + // (excluding pixels belonging to any predicted spot core), mirroring the local + // shoebox background of BraggIntegrate2D. Must be > shoebox_radius. + int bkg_outer_radius = 6; int max_iterations = 3; // inner predict<->refine cycles (re-predict with refined geom/latt) // --- output --- @@ -144,7 +148,6 @@ struct PixelRefineData { }; class PixelRefine { - const AzimuthalIntegrationMapping &mapping; const size_t xpixel, ypixel; const DiffractionExperiment &experiment; @@ -156,42 +159,43 @@ class PixelRefine { // PredictImage so both walk identical geometry/lattice code. void BuildParameterBlocks(const PixelRefineData &data, double beam[2], double &dist_mm, - double detector_rot[2], double rot_vec[3], + double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; public: PixelRefine(const DiffractionExperiment &experiment, - const AzimuthalIntegrationMapping &mapping, const std::vector &reference); // The BraggPrediction is supplied per call (it is mutated): this keeps a // single PixelRefine instance usable from several threads, each passing its // own prediction buffer. Only `data` is written; PixelRefine state is const. + // The image is in raw detector counts (masked/saturated pixels carry the type + // sentinel); background is estimated locally per shoebox from the image itself. template void Run(const T *image, - const AzimuthalIntegrationProfile &profile, BraggPrediction &prediction, PixelRefineData &data); - // Render the forward model as a full detector image (raw detector units, so + // Render the forward model as a full detector image (raw detector counts, so // it overlays directly on the original image). Uses the *same* per-pixel // model path (PixelResidual::Model) as the optimizer, evaluated in double // precision - slow but exact. For each reference reflection it adds the Bragg - // signal over its shoebox; with include_background it also lays down the - // azimuthal background. Diagnostic tool, not on the hot path. - std::vector PredictImage(const AzimuthalIntegrationProfile &profile, + // signal over its shoebox; with include_background it also lays down the local + // per-shoebox background read from the supplied image - the same background the + // fit uses. Diagnostic tool, not on the hot path. + template + std::vector PredictImage(const T *image, BraggPrediction &prediction, const PixelRefineData &data, bool include_background = true) const; // Render the per-pixel chi-square (cost density) that the optimizer actually // minimizes: for every shoebox pixel that enters the fit it stores the squared - // weighted residual ((I_pred - I_obs)/sigma)^2 in *corrected* units - identical - // to the Ceres residual_i^2 - accumulating where shoeboxes overlap. Pixels that - // are not part of any shoebox stay 0; masked/saturated pixels (skipped by the - // fit) also stay 0. Summing the image gives ~2*final_cost. Diagnostic tool. + // weighted residual ((I_pred - I_obs)/sigma)^2 in raw counts - identical to the + // Ceres residual_i^2 - accumulating where shoeboxes overlap. Pixels that are not + // part of any shoebox stay 0; masked/saturated pixels (skipped by the fit) also + // stay 0. Summing the image gives ~2*final_cost. Diagnostic tool. template std::vector ChiSquaredImage(const T *image, - const AzimuthalIntegrationProfile &profile, BraggPrediction &prediction, const PixelRefineData &data) const; }; diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 8f90cb11..17a71934 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -457,8 +457,8 @@ void JFJochImageReadingWorker::ReanalyzeImage_i() { new_image_dataset->azimuthal_bins = azint_mapping->GetAzimuthalBinCount(); new_image_dataset->q_bins = azint_mapping->GetQBinCount(); - // Retain the profile (PixelRefine needs it). AzimuthalIntegrationProfile holds - // a mutex (non-copyable), so keep it via unique_ptr re-created each analysis. + // Azimuthal profile for the analysis/display pipeline. AzimuthalIntegrationProfile + // holds a mutex (non-copyable), so keep it via unique_ptr re-created each analysis. last_profile_ = std::make_unique(*azint_mapping); image_analysis->Analyze(new_image->ImageData(), *last_profile_, spot_finding_settings); @@ -725,8 +725,8 @@ void JFJochImageReadingWorker::LoadSpots(int64_t start_image, int64_t end_image, // Experimental PixelRefine // --------------------------------------------------------------------------- void JFJochImageReadingWorker::EnsurePixelRefine_i() { - if (!pixel_refine_ && azint_mapping && !pixel_reference_.empty()) - pixel_refine_ = std::make_unique(curr_experiment, *azint_mapping, pixel_reference_); + if (!pixel_refine_ && !pixel_reference_.empty()) + pixel_refine_ = std::make_unique(curr_experiment, pixel_reference_); if (!pixel_pred_) pixel_pred_ = CreateBraggPrediction(curr_experiment.IsRotationIndexing()); } @@ -854,15 +854,15 @@ QVector JFJochImageReadingWorker::BuildShoeboxes_i(const PixelRefineData std::vector JFJochImageReadingWorker::BuildDisplayImage_i(const PixelRefineData &data, int display_mode) const { + const auto &img32 = current_image_ptr->Image(); if (display_mode == PixelRefineParams::ChiSquared) { // The cost density the optimizer actually minimizes (weighted residual^2). - const auto &img32 = current_image_ptr->Image(); - auto chi2 = pixel_refine_->ChiSquaredImage(img32.data(), *last_profile_, *pixel_pred_, data); + auto chi2 = pixel_refine_->ChiSquaredImage(img32.data(), *pixel_pred_, data); MaskMeasuredSentinels_i(chi2); return chi2; } - auto pred = pixel_refine_->PredictImage(*last_profile_, *pixel_pred_, data, true); + auto pred = pixel_refine_->PredictImage(img32.data(), *pixel_pred_, data, true); if (display_mode == PixelRefineParams::SquaredDifference) SquaredResidualWithImage_i(pred); return pred; @@ -907,7 +907,7 @@ void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { try { const auto &img32 = current_image_ptr->Image(); - pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); + pixel_refine_->Run(img32.data(), *pixel_pred_, d); emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); auto display = BuildDisplayImage_i(d, params.display_mode); @@ -941,7 +941,7 @@ void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { try { const auto &img32 = current_image_ptr->Image(); - pixel_refine_->Run(img32.data(), *last_profile_, *pixel_pred_, d); + pixel_refine_->Run(img32.data(), *pixel_pred_, d); // Push refined values back so the sliders follow the optimizer. PixelRefineParams out = params; -- 2.52.0 From 48d4fb0d0f73e8f95184d4498695f3a6beffe46f Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 11 Jun 2026 18:35:02 +0200 Subject: [PATCH 031/228] XDS plugin: Fix mutex --- xds-plugin/plugin.cpp | 64 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/xds-plugin/plugin.cpp b/xds-plugin/plugin.cpp index a5bb92e5..699d2c97 100644 --- a/xds-plugin/plugin.cpp +++ b/xds-plugin/plugin.cpp @@ -219,50 +219,52 @@ void plugin_get_header(int *nx, int *ny, int *nbytes, float *qx, float *qy, void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, int info[1024], int *error_flag) { std::shared_lock sl(plugin_mutex); + std::vector tmp; + CompressionAlgorithm algorithm = CompressionAlgorithm::NO_COMPRESSION; try { - if (!hdf5_file) - throw PluginError(-1, "HDF5 file not open"); + { + std::unique_lock h5l(hdf5_mutex); + if (!hdf5_file) + throw PluginError(-1, "HDF5 file not open"); - FillInfoArray(info); + FillInfoArray(info); - if (*frame_number <= 0 || *frame_number > total_image_number) - throw PluginError(-1, "Frame number out of range"); + if (*frame_number <= 0 || *frame_number > total_image_number) + throw PluginError(-1, "Frame number out of range"); - std::string dataset_name; - hsize_t image_id; + std::string dataset_name; + hsize_t image_id; - if (format == FileWriterFormat::NXmxLegacy) { - char str[256]; - size_t dataset_index = (*frame_number - 1) / images_per_file + 1; - snprintf(str, sizeof(str), "/entry/data/data_%06ld", dataset_index); - dataset_name = std::string(str); - image_id = (*frame_number - 1) % images_per_file; - } else { - dataset_name = "/entry/data/data"; - image_id = *frame_number - 1; - } + if (format == FileWriterFormat::NXmxLegacy) { + char str[256]; + size_t dataset_index = (*frame_number - 1) / images_per_file + 1; + snprintf(str, sizeof(str), "/entry/data/data_%06ld", dataset_index); + dataset_name = std::string(str); + image_id = (*frame_number - 1) % images_per_file; + } else { + dataset_name = "/entry/data/data"; + image_id = *frame_number - 1; + } - HDF5DataSet dataset(*hdf5_file, dataset_name); - HDF5Dcpl dcpl(dataset); + HDF5DataSet dataset(*hdf5_file, dataset_name); + HDF5Dcpl dcpl(dataset); - std::vector tmp; - std::vector start = {image_id, 0, 0}; + std::vector start = {image_id, 0, 0}; - CompressionAlgorithm algorithm = CompressionAlgorithm::NO_COMPRESSION; - auto chunk_size = dcpl.GetChunking(); + auto chunk_size = dcpl.GetChunking(); - if ((chunk_size.size() == 3) - && (chunk_size[0] == 1) - && (chunk_size[1] == image_size_y) - && (chunk_size[2] == image_size_x)) { - dataset.ReadDirectChunk(tmp, start); - algorithm = dcpl.GetCompression(); + if ((chunk_size.size() == 3) + && (chunk_size[0] == 1) + && (chunk_size[1] == image_size_y) + && (chunk_size[2] == image_size_x)) { + dataset.ReadDirectChunk(tmp, start); + algorithm = dcpl.GetCompression(); } else { dataset.ReadVectorToU8(tmp, start, {1, image_size_y, image_size_x}); algorithm = CompressionAlgorithm::NO_COMPRESSION; } - + } if (algorithm != CompressionAlgorithm::NO_COMPRESSION) { std::vector decompressed_image(image_size_x * image_size_y * pixel_byte_depth); JFJochDecompressPtr(decompressed_image.data(), @@ -300,4 +302,4 @@ void plugin_close(int *error_flag) { *error_flag = -1; } } -} /* extern "C" */ \ No newline at end of file +} /* extern "C" */ -- 2.52.0 From d31063ca3f87aca8287afdc43b135ff60c12a6bf Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 11 Jun 2026 21:25:05 +0200 Subject: [PATCH 032/228] PixelRefine: Some improvements --- common/Reflection.h | 3 +- .../pixel_refinement/PixelRefine.cpp | 156 ++++++++++++------ image_analysis/pixel_refinement/PixelRefine.h | 12 ++ viewer/CMakeLists.txt | 2 + viewer/JFJochImageReadingWorker.cpp | 38 +++++ viewer/JFJochImageReadingWorker.h | 3 + viewer/JFJochViewerWindow.cpp | 9 + viewer/windows/JFJochPixelRefineWindow.cpp | 6 + viewer/windows/JFJochPixelRefineWindow.h | 2 + viewer/windows/PixelRefineParams.h | 25 +++ 10 files changed, 206 insertions(+), 50 deletions(-) diff --git a/common/Reflection.h b/common/Reflection.h index 08959c62..b46e38e6 100644 --- a/common/Reflection.h +++ b/common/Reflection.h @@ -25,7 +25,8 @@ struct Reflection { float sigma; float dist_ewald; float rlp; - float partiality; + float partiality; // fraction of the reflection recorded in the sampled (rocking) slice + float completeness = 1.0f; // fraction of the spot footprint on live pixels (1 = not clipped by edge/gap/mask) float zeta; float image_scale_corr; // I_true = scaling_correction * I; scaling_correction = rlp / (partiality * image_scale) bool observed = false; diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index f41925b3..12540b3c 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -33,6 +33,7 @@ struct ReflGroup { double Itrue; // reference intensity (held fixed) double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) double pol; // per-reflection polarization correction (raw = true * pol) + double Ibkg; // local flat background (raw counts, constant over the shoebox) double predicted_x, predicted_y; std::vector pixels; }; @@ -83,6 +84,23 @@ std::vector BuildSpotMask(const std::vector &predicted, int return mask; } +// Square shoebox bounds (inclusive) around a predicted spot, clamped to the +// detector. The centre is rounded to the nearest pixel with std::lround so the +// signal box is centred identically to the spot-core mask (BuildSpotMask) and +// the local-background ring (EstimateLocalBackground), which also lround. Used by +// Run and the diagnostic renderers so all three share one shoebox definition. +struct ShoeboxBox { int min_x, max_x, min_y, max_y; }; +ShoeboxBox ShoeboxBounds(double px, double py, int radius, size_t xpixel, size_t ypixel) { + const int cx = static_cast(std::lround(px)); + const int cy = static_cast(std::lround(py)); + return { + std::max(cx - radius, 0), + std::min(cx + radius, static_cast(xpixel) - 1), + std::max(cy - radius, 0), + std::min(cy + radius, static_cast(ypixel) - 1) + }; +} + // Local flat background around one shoebox, in raw detector counts. Samples the // square ring shoebox_radius < max(|dx|,|dy|) <= bkg_outer_radius centred on the // spot, dropping pixels that belong to any spot core (spot_mask) or carry a @@ -633,16 +651,14 @@ void PixelRefine::Run(const T *image, g.Itrue = reference_data[hkl]; g.R_bw_sq = bandwidth_radial_sq(refl.d); g.pol = polarization(refl.predicted_x, refl.predicted_y); + g.Ibkg = Ibkg; g.predicted_x = refl.predicted_x; g.predicted_y = refl.predicted_y; - const int min_y = std::max(refl.predicted_y - radius, 0); - const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); - const int min_x = std::max(refl.predicted_x - radius, 0); - const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - for (int y = min_y; y <= max_y; ++y) { - for (int x = min_x; x <= max_x; ++x) { + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { const size_t npixel = xpixel * y + x; // Skip sentinel (masked / saturated) pixels. We assume the pixel @@ -826,52 +842,100 @@ void PixelRefine::Run(const T *image, } // predict<->refine iterations // ---- Extract integrated reflections --------------------------------------- - // Profile fitting gives the recorded amplitude (against the tangential profile - // P_t): - // J = sum_p[ P_t,p (Iobs_p - Ibkg_p)/v_p ] / sum_p[ P_t,p^2 / v_p ] + // Profile fitting gives the recorded amplitude (fitting the tangential profile + // P_t against the background-subtracted pixels): + // J = sum_p[ P_t,p (Iobs_p - Ibkg)/v_p ] / sum_p[ P_t,p^2 / v_p ] // ~ G * Itrue * B_term * partiality * pol (recorded raw counts) // var(J) = 1 / sum_p[ P_t,p^2 / v_p ] // + // Two SEPARATE fractions reduce the full intensity to what these pixels record: + // + // partiality - the radial / rocking dimension that a still does NOT sample. + // Only the slice of the reflection that crosses the Ewald + // sphere on this shot is recorded; <= 1. We DIVIDE it out to + // recover the full intensity. = profile-weighted P_radial. + // + // completeness - the fraction of the spot's detector footprint that landed on + // live pixels (= profile captured by live pixels / profile over + // the whole shoebox). 1.0 when the spot sits fully on the + // detector; < 1.0 only when a detector edge, gap or mask clips + // it. Profile fitting already extrapolates over the missing + // pixels, so this is NOT applied to r.I - it is a quality flag. + // // Output split (Merge multiplies r.I * image_scale_corr and weights by // 1/(sigma*image_scale_corr)^2 - see Merge.cpp): // r.I = J / (B_term * partiality * pol) = G * Itrue // r.sigma = sqrt(var(J)) / (B_term * partiality * pol) - // r.partiality = profile-weighted peak radial factor in (0,1] (Merge filter only) + // r.partiality = profile-weighted P_radial in (0,1] (the rocking fraction) + // r.completeness = live/total tangential profile in (0,1] (detector clipping) // r.image_scale_corr = 1/G (per-image scale ONLY) // so r.I * image_scale_corr = Itrue. B, partiality and polarization live on the // intensity, G lives on image_scale_corr - one clean meaning per field. + // + // We walk the full (unclamped) shoebox once: every grid point feeds the total + // tangential profile (completeness denominator); points that are real, live + // detector pixels also feed the profile fit and the captured profile. data.reflections.reserve(groups.size()); for (const auto &g : groups) { - double num = 0.0, den = 0.0, bkg_sum = 0.0; - double radial_sum = 0.0, radial_w = 0.0; + const int cx = static_cast(std::lround(g.predicted_x)); + const int cy = static_cast(std::lround(g.predicted_y)); + + // Debye-Waller factor for this reflection (constant over its shoebox). + const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); + + double num = 0.0, den = 0.0, bkg_sum = 0.0, radial_sum = 0.0; + double prof_live = 0.0, prof_full = 0.0; // tangential profile: captured / total size_t n = 0; - for (const auto &obs : g.pixels) { - PixelResidual pr(obs, 1.0, lambda, pixel_size, g.h, g.k, g.l, g.R_bw_sq, g.pol, data.crystal_system); - double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, - eps_r, eps_t_sq)) - continue; - if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) - continue; + for (int y = cy - radius; y <= cy + radius; ++y) { + for (int x = cx - radius; x <= cx + radius; ++x) { + // Geometry/profile for this grid point (valid even off the detector). + PixelObs probe{static_cast(x), static_cast(y), 0.0, g.Ibkg, 1.0}; + PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, + g.R_bw_sq, g.pol, data.crystal_system); + double q_sq, eps_r, eps_t_sq; + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, + latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + continue; + if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) + continue; - // Tangential profile shape -> fit weight (every pixel counts equally). - const double P_t = std::exp(-eps_t_sq / (data.R[1] * data.R[1])) - / (M_PI * data.R[1] * data.R[1]); - // Peak-normalized radial factor (the partiality), in (0,1]. - // Bandwidth-broadened radial width, matching the model in Model(). - const double R0_eff_sq = data.R[0] * data.R[0] + g.R_bw_sq; - const double P_radial = std::exp(-eps_r * eps_r / R0_eff_sq); + // Tangential profile shape (area-normalized) -> the fit template. + const double P_t = std::exp(-eps_t_sq / (data.R[1] * data.R[1])) + / (M_PI * data.R[1] * data.R[1]); + prof_full += P_t; // whole shoebox, on- or off-detector - const double v = SafeInv(obs.weight * obs.weight, 1.0); // pixel variance - const double signal = obs.Iobs - obs.Ibkg; + // Only real, unmasked detector pixels carry signal. + if (x < 0 || x >= static_cast(xpixel) || y < 0 || y >= static_cast(ypixel)) + continue; + const size_t np = static_cast(xpixel) * y + x; + if (image[np] == std::numeric_limits::max()) + continue; + if (std::is_signed_v && image[np] == std::numeric_limits::min()) + continue; - num += P_t * signal / v; - den += P_t * P_t / v; - radial_sum += P_radial * P_t; // weight partiality by the spot core - radial_w += P_t; - bkg_sum += obs.Ibkg; - ++n; + const double Iobs = static_cast(image[np]); // raw counts + double v = std::max(Iobs, 0.0); // Poisson variance + if (!(v > 1.0)) + v = 1.0; + + // Peak-normalized radial factor (the partiality), in (0,1]. The + // bandwidth-broadened radial width matches the model in Model(). + const double R0_eff_sq = data.R[0] * data.R[0] + g.R_bw_sq; + const double P_radial = std::exp(-eps_r * eps_r / R0_eff_sq); + + // Profile-fit accumulators. The amplitude estimator weights pixels by + // P_t^2/v, so the partiality (which de-scales that amplitude) MUST use + // the SAME weights - otherwise an R0_eff-dependent (resolution- + // dependent) factor is left behind in r.I. + const double w = P_t * P_t / v; + num += P_t * (Iobs - g.Ibkg) / v; + den += w; + radial_sum += P_radial * w; // partiality weighted exactly like num/den + prof_live += P_t; // captured tangential profile + bkg_sum += g.Ibkg; + ++n; + } } Reflection r{}; @@ -884,12 +948,12 @@ void PixelRefine::Run(const T *image, r.observed_x = NAN; r.observed_y = NAN; r.rlp = 1.0f; - r.partiality = (radial_w > 0.0) ? static_cast(radial_sum / radial_w) : 1.0f; + r.partiality = (den > 0.0) ? static_cast(radial_sum / den) : 1.0f; + r.completeness = (prof_full > 0.0) ? static_cast(prof_live / prof_full) : 1.0f; if (den > 0.0 && n > 0) { const double I_amp = num / den; // ~ G*Itrue*B_term*partiality*pol const double sigma_amp = std::sqrt(1.0 / den); - const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); const double corr = static_cast(r.partiality) * B_term * g.pol; // B, partiality & pol r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; @@ -1010,13 +1074,10 @@ std::vector PixelRefine::PredictImage(const T *image, refl.predicted_x, refl.predicted_y, radius, bkg_outer_radius, Ibkg); - const int min_y = std::max(refl.predicted_y - radius, 0); - const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); - const int min_x = std::max(refl.predicted_x - radius, 0); - const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - for (int y = min_y; y <= max_y; ++y) { - for (int x = min_x; x <= max_x; ++x) { + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { const size_t npixel = xpixel * y + x; PixelObs obs{ @@ -1106,13 +1167,10 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, radius, bkg_outer_radius, Ibkg)) continue; - const int min_y = std::max(refl.predicted_y - radius, 0); - const int max_y = std::min(refl.predicted_y + radius, ypixel - 1); - const int min_x = std::max(refl.predicted_x - radius, 0); - const int max_x = std::min(refl.predicted_x + radius, xpixel - 1); + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - for (int y = min_y; y <= max_y; ++y) { - for (int x = min_x; x <= max_x; ++x) { + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { const size_t npixel = xpixel * y + x; // Same gating as Run(): only pixels that actually enter the fit. diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 8d28e107..1a2535c1 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -3,6 +3,8 @@ #pragma once +#include + #include "../bragg_prediction/BraggPrediction.h" #include "../common/DiffractionExperiment.h" #include "../scale_merge/HKLKey.h" @@ -198,4 +200,14 @@ public: std::vector ChiSquaredImage(const T *image, BraggPrediction &prediction, const PixelRefineData &data) const; + + // Reference (merged) intensity used as the fixed hypothesis for a reflection, + // or nullopt if this hkl is not in the reference. Lets callers show the fitted + // estimate next to the reference it was scaled against. + std::optional ReferenceIntensity(const Reflection &r) const { + const auto it = reference_data.find(hkl_key_generator(r)); + if (it == reference_data.end()) + return std::nullopt; + return it->second; + } }; diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 70981081..6bfbd37e 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -88,6 +88,8 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView windows/JFJochViewerReciprocalSpaceWindow.h windows/JFJochPixelRefineWindow.cpp windows/JFJochPixelRefineWindow.h + windows/JFJochPixelRefineTableWindow.cpp + windows/JFJochPixelRefineTableWindow.h windows/JFJochMagnifierWindow.cpp windows/JFJochMagnifierWindow.h windows/PixelRefineParams.h diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 17a71934..1e92cfed 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -70,6 +70,7 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { qRegisterMetaType("PixelRefineParams"); + qRegisterMetaType("PixelRefineReport"); qRegisterMetaType>("QVector"); spot_finding_settings = settings;; @@ -852,6 +853,41 @@ QVector JFJochImageReadingWorker::BuildShoeboxes_i(const PixelRefineData return boxes; } +PixelRefineReport JFJochImageReadingWorker::BuildReport_i(const PixelRefineData &data) const { + PixelRefineReport report; + report.pr_G = data.scale_factor; + report.pr_B = data.B_factor; + report.pr_cc = data.cc; + report.pr_cc_n = data.cc_n; + + // Standard ScaleOnTheFly pipeline result for the same image, as a baseline. + if (current_image_ptr) { + const auto &d = current_image_ptr->ImageData(); + if (d.image_scale_factor) report.pipe_G = d.image_scale_factor.value(); + if (d.image_scale_b_factor) report.pipe_B = d.image_scale_b_factor.value(); + if (d.image_scale_cc) report.pipe_cc = d.image_scale_cc.value(); + } + + report.rows.reserve(data.reflections.size()); + for (const auto &r : data.reflections) { + if (!r.observed) + continue; + PixelRefineReport::Row row; + row.h = r.h; row.k = r.k; row.l = r.l; + row.d = r.d; + row.completeness = r.completeness; + row.partiality = r.partiality; + row.I = r.I; + row.sigma = r.sigma; + if (std::isfinite(r.image_scale_corr)) + row.I_true_est = static_cast(r.I) * static_cast(r.image_scale_corr); + if (pixel_refine_) + row.I_true_ref = pixel_refine_->ReferenceIntensity(r).value_or(NAN); + report.rows.push_back(row); + } + return report; +} + std::vector JFJochImageReadingWorker::BuildDisplayImage_i(const PixelRefineData &data, int display_mode) const { const auto &img32 = current_image_ptr->Image(); @@ -909,6 +945,7 @@ void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { const auto &img32 = current_image_ptr->Image(); pixel_refine_->Run(img32.data(), *pixel_pred_, d); emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); + emit pixelRefineReport(BuildReport_i(d)); auto display = BuildDisplayImage_i(d, params.display_mode); emit predictedImageReady(WrapFloatImage_i(display)); @@ -954,6 +991,7 @@ void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { out.beam_y = d.geom.GetBeamY_pxl(); emit pixelRefineParamsRefined(out); emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); + emit pixelRefineReport(BuildReport_i(d)); auto display = BuildDisplayImage_i(d, params.display_mode); emit predictedImageReady(WrapFloatImage_i(display)); diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index 501b557d..50cd77ca 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -80,6 +80,8 @@ private: void MaskMeasuredSentinels_i(std::vector &img) const; // Build the per-reflection shoebox rectangles for the last refine/preview. QVector BuildShoeboxes_i(const PixelRefineData &data) const; + // Assemble the per-reflection table + per-image summary for the table window. + PixelRefineReport BuildReport_i(const PixelRefineData &data) const; // Build the float image to display for the given PixelRefineParams::DisplayMode. std::vector BuildDisplayImage_i(const PixelRefineData &data, int display_mode) const; @@ -148,6 +150,7 @@ signals: void predictedShoeboxes(QVector boxes); // per-reflection optimization windows void pixelRefineResidual(double cost, double cc, int64_t n_reflections); void pixelRefineParamsRefined(PixelRefineParams params); + void pixelRefineReport(PixelRefineReport report); // per-reflection table + summary void pixelRefineStatus(QString message); public: diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index 9fa6a302..1eb2c3c7 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -26,6 +26,7 @@ #include "windows/JFJoch2DAzintImageWindow.h" #include "windows/JFJochAzIntWindow.h" #include "windows/JFJochPixelRefineWindow.h" +#include "windows/JFJochPixelRefineTableWindow.h" #include "windows/JFJochMagnifierWindow.h" #include "image_viewer/JFJochImage.h" #include "image_viewer/JFJochSimpleImage.h" @@ -108,6 +109,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString auto azintWindow = new JFJochAzIntWindow(experiment.GetAzimuthalIntegrationSettings(), this); auto azintImageWindow = new JFJoch2DAzintImageWindow(this); auto pixelRefineWindow = new JFJochPixelRefineWindow(this); + auto pixelRefineTableWindow = new JFJochPixelRefineTableWindow(this); auto magnifierWindow = new JFJochMagnifierWindow(this); menuBar->AddWindowEntry(tableWindow, "Image list"); @@ -120,6 +122,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString menuBar->AddWindowEntry(azintWindow, "Azimuthal integration settings"); menuBar->AddWindowEntry(azintImageWindow, "Azimuthal integration 2D image"); menuBar->AddWindowEntry(pixelRefineWindow, "PixelRefine (experimental)"); + menuBar->AddWindowEntry(pixelRefineTableWindow, "PixelRefine reflections"); menuBar->AddWindowEntry(magnifierWindow, "Magnifier"); if (dbus) { @@ -361,6 +364,12 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString connect(reading_worker, &JFJochImageReadingWorker::pixelRefineStatus, pixelRefineWindow, &JFJochPixelRefineWindow::setStatus); + // Reflection-table window: refreshed on every preview/refine, raised by button. + connect(reading_worker, &JFJochImageReadingWorker::pixelRefineReport, + pixelRefineTableWindow, &JFJochPixelRefineTableWindow::setReport); + connect(pixelRefineWindow, &JFJochPixelRefineWindow::showTableRequested, + pixelRefineTableWindow, &JFJochHelperWindow::open); + // Lock the predicted-image viewport to the original image (both directions). connect(viewer, &JFJochImage::viewportChanged, pixelRefineWindow->imageView(), &JFJochImage::applyViewport); diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp index 7f7a64ec..0e8b4a16 100644 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ b/viewer/windows/JFJochPixelRefineWindow.cpp @@ -82,8 +82,10 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) // --- buttons + readouts ------------------------------------------------- m_loadRef = new QPushButton(tr("Load reference MTZ…"), this); m_refine = new QPushButton(tr("Refine"), this); + m_showTable = new QPushButton(tr("Reflection table…"), this); controlsLayout->addWidget(m_loadRef); controlsLayout->addWidget(m_refine); + controlsLayout->addWidget(m_showTable); m_residual = new QLabel(tr("Residual: —"), this); m_pipelineCC = new QLabel(tr("Pipeline CC (ref): —"), this); @@ -121,6 +123,10 @@ JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) emit loadReferenceRequested(path); }); + connect(m_showTable, &QPushButton::clicked, this, [this] { + emit showTableRequested(); + }); + connect(m_refine, &QPushButton::clicked, this, [this] { // Cancel any pending live-preview: otherwise a debounce armed by a slider // move just before this click fires after the refine and overwrites the diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h index 99c628c2..9398389d 100644 --- a/viewer/windows/JFJochPixelRefineWindow.h +++ b/viewer/windows/JFJochPixelRefineWindow.h @@ -49,6 +49,7 @@ class JFJochPixelRefineWindow : public JFJochHelperWindow { QLabel *m_status; QPushButton *m_loadRef; QPushButton *m_refine; + QPushButton *m_showTable; QTimer *m_debounce; bool m_suppress = false; // guard while pushing refined params into sliders @@ -68,6 +69,7 @@ signals: void paramsChanged(PixelRefineParams params); // debounced live preview void refineRequested(PixelRefineParams params); // "Refine" button void loadReferenceRequested(QString path); // "Load reference" button + void showTableRequested(); // "Reflection table" button public slots: void setPredictedImage(std::shared_ptr image); diff --git a/viewer/windows/PixelRefineParams.h b/viewer/windows/PixelRefineParams.h index 64f6e27b..e180847e 100644 --- a/viewer/windows/PixelRefineParams.h +++ b/viewer/windows/PixelRefineParams.h @@ -4,6 +4,8 @@ #pragma once #include +#include +#include #include // Parameters exchanged between the PixelRefine window (sliders/buttons) and the @@ -39,3 +41,26 @@ struct PixelRefineParams { }; Q_DECLARE_METATYPE(PixelRefineParams) + +// One PixelRefine result, shipped from the worker to the reflection-table window: +// a per-image summary that puts PixelRefine's scale/B/CC next to the standard +// ScaleOnTheFly pipeline's, plus one row per matched reflection. +struct PixelRefineReport { + // Per-image summary (NaN = not available). + double pr_G = NAN, pr_B = NAN, pr_cc = NAN; // PixelRefine + int64_t pr_cc_n = 0; + double pipe_G = NAN, pipe_B = NAN, pipe_cc = NAN; // ScaleOnTheFly pipeline baseline + + struct Row { + int h = 0, k = 0, l = 0; + double d = 0.0; + double completeness = 1.0; // spot footprint on live pixels (1 = not clipped) + double partiality = 1.0; // recorded rocking fraction + double I = 0.0, sigma = 0.0; + double I_true_est = NAN; // r.I * image_scale_corr (this image's estimate) + double I_true_ref = NAN; // reference (merged) intensity + }; + std::vector rows; +}; + +Q_DECLARE_METATYPE(PixelRefineReport) -- 2.52.0 From c8db50ab41352eac46066b1264d7c3a5ac116aa6 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 07:47:28 +0200 Subject: [PATCH 033/228] jfjoch_viewer: missing pixel refine table --- .../windows/JFJochPixelRefineTableWindow.cpp | 101 ++++++++++++++++++ viewer/windows/JFJochPixelRefineTableWindow.h | 34 ++++++ 2 files changed, 135 insertions(+) create mode 100644 viewer/windows/JFJochPixelRefineTableWindow.cpp create mode 100644 viewer/windows/JFJochPixelRefineTableWindow.h diff --git a/viewer/windows/JFJochPixelRefineTableWindow.cpp b/viewer/windows/JFJochPixelRefineTableWindow.cpp new file mode 100644 index 00000000..f656164a --- /dev/null +++ b/viewer/windows/JFJochPixelRefineTableWindow.cpp @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochPixelRefineTableWindow.h" + +#include +#include +#include + +JFJochPixelRefineTableWindow::JFJochPixelRefineTableWindow(QWidget *parent) + : JFJochHelperWindow(parent) { + setWindowTitle("PixelRefine reflections"); + resize(950, 600); + + auto central = new QWidget(this); + setCentralWidget(central); + auto layout = new QVBoxLayout(central); + + m_summary = new QLabel(tr("No PixelRefine result yet."), this); + m_summary->setTextFormat(Qt::RichText); + m_summary->setWordWrap(true); + layout->addWidget(m_summary); + + m_table = new QTableView(this); + m_model = new QStandardItemModel(this); + setupModel(); + + m_proxy = new QSortFilterProxyModel(this); + m_proxy->setSourceModel(m_model); + m_proxy->setSortRole(Qt::UserRole); // numeric sort on the underlying values + + m_table->setModel(m_proxy); + m_table->setEditTriggers(QAbstractItemView::NoEditTriggers); + m_table->setSortingEnabled(true); + m_table->sortByColumn(6, Qt::DescendingOrder); // default: I desc + m_table->verticalHeader()->setVisible(false); + m_table->horizontalHeader()->setSortIndicatorShown(true); + m_table->horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch); + m_table->setStyleSheet("background-color: white;"); + layout->addWidget(m_table); +} + +void JFJochPixelRefineTableWindow::setupModel() { + const QStringList headers = { + "h", "k", "l", "d [Å]", "Compl.", "Part.", + "I", "Sigma", "Est. I_true", "Ref. I_true" + }; + m_model->setColumnCount(headers.size()); + for (int i = 0; i < headers.size(); ++i) + m_model->setHeaderData(i, Qt::Horizontal, headers[i]); +} + +static QStandardItem *numItem(double value, int decimals) { + auto *it = new QStandardItem(); + const QString text = std::isfinite(value) ? QString::number(value, 'f', decimals) + : QStringLiteral("—"); + it->setData(text, Qt::DisplayRole); + it->setData(value, Qt::UserRole); + it->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter); + return it; +} + +static QStandardItem *intItem(int value) { + auto *it = new QStandardItem(); + it->setData(static_cast(value), Qt::DisplayRole); + it->setData(static_cast(value), Qt::UserRole); + it->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter); + return it; +} + +void JFJochPixelRefineTableWindow::setReport(PixelRefineReport report) { + // --- per-image summary: PixelRefine vs the standard pipeline --------------- + auto fmt = [](double v, int dec) { + return std::isfinite(v) ? QString::number(v, 'f', dec) : QStringLiteral("—"); + }; + auto pct = [](double v) { + return std::isfinite(v) ? QString::number(v * 100.0, 'f', 1) + "%" : QStringLiteral("—"); + }; + m_summary->setText( + tr("PixelRefine: scale G = %1, B = %2 Ų, CC = %3 (%4 refl)" + "  |  " + "Pipeline: scale G = %5, B = %6 Ų, CC = %7") + .arg(fmt(report.pr_G, 4), fmt(report.pr_B, 1), pct(report.pr_cc)) + .arg(report.pr_cc_n) + .arg(fmt(report.pipe_G, 4), fmt(report.pipe_B, 1), pct(report.pipe_cc))); + + // --- per-reflection rows --------------------------------------------------- + m_model->removeRows(0, m_model->rowCount()); + for (const auto &r : report.rows) { + QList row; + row << intItem(r.h) << intItem(r.k) << intItem(r.l); + row << numItem(r.d, 3); + row << numItem(r.completeness, 2); + row << numItem(r.partiality, 2); + row << numItem(r.I, 1); + row << numItem(r.sigma, 1); + row << numItem(r.I_true_est, 1); + row << numItem(r.I_true_ref, 1); + m_model->appendRow(row); + } +} diff --git a/viewer/windows/JFJochPixelRefineTableWindow.h b/viewer/windows/JFJochPixelRefineTableWindow.h new file mode 100644 index 00000000..5187b546 --- /dev/null +++ b/viewer/windows/JFJochPixelRefineTableWindow.h @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include "JFJochHelperWindow.h" +#include "PixelRefineParams.h" + +#include +#include +#include +#include + +// Reflection table for the experimental PixelRefine path. Opened from a button on +// the PixelRefine window and refreshed on every preview/refine. Each row is one +// matched reflection (the two partiality-style fractions, the fitted intensity, +// and the estimated vs reference full intensity); the header line compares +// PixelRefine's per-image scale/B/CC with the standard ScaleOnTheFly pipeline. +class JFJochPixelRefineTableWindow : public JFJochHelperWindow { + Q_OBJECT + + QLabel *m_summary = nullptr; + QTableView *m_table = nullptr; + QStandardItemModel *m_model = nullptr; + QSortFilterProxyModel *m_proxy = nullptr; + + void setupModel(); + +public: + explicit JFJochPixelRefineTableWindow(QWidget *parent = nullptr); + +public slots: + void setReport(PixelRefineReport report); +}; -- 2.52.0 From 47dc19dd030fedf6938f003cdfc6e6ca143e22bd Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 10:12:23 +0200 Subject: [PATCH 034/228] PixelRefine: Improvements to accept more reasonable count of reflections --- .../pixel_refinement/PixelRefine.cpp | 50 ++++++++++++------- image_analysis/pixel_refinement/PixelRefine.h | 7 +++ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 12540b3c..98f0a7af 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -551,6 +551,35 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, } } +BraggPredictionSettings PixelRefine::BuildPredictionSettings(const PixelRefineData &data) const { + // Radial Ewald-acceptance band: predict exactly the reflections the merge's + // partiality floor would still keep, and no tighter. A reflection's partiality + // is exp(-s^2 / R0_eff^2) (s = excitation error), so it survives min_partiality + // when |s| <= R0 * sqrt(-ln(min_partiality)). Using that as the cutoff keeps + // prediction and the partiality cut consistent. The struct default (0.0005) is + // far narrower than R0 (~0.005), so previously most keepable reflections were + // never predicted and per-reflection multiplicity collapsed ~4x. + const double min_part = std::clamp(experiment.GetScalingSettings().GetMinPartiality(), 1e-6, 0.999); + const double r0 = std::max(data.R[0], 1e-4); + const double cutoff = r0 * std::sqrt(-std::log(min_part)); + + // Relative bandwidth (sigma of dlambda/lambda): the explicit PixelRefine model + // value if set, else the experiment's nominal bandwidth (FWHM -> sigma). >0 + // thickens the band radially at high resolution (the 1/d^2 pink-beam smear), + // matching the integrator and preventing the outer shells from being clipped. + float bw_sigma = static_cast(data.bandwidth); + if (bw_sigma <= 0.0f) + bw_sigma = experiment.GetBandwidthFWHM().value_or(0.0f) / 2.3548f; + + return BraggPredictionSettings{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .ewald_dist_cutoff = static_cast(cutoff), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = bw_sigma + }; +} + template void PixelRefine::Run(const T *image, BraggPrediction &prediction, @@ -561,12 +590,7 @@ void PixelRefine::Run(const T *image, const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); - const BraggPredictionSettings settings_prediction{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma - }; + const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); const int radius = data.shoebox_radius; const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); @@ -1045,12 +1069,7 @@ std::vector PixelRefine::PredictImage(const T *image, .PoniRot1_rad(data.geom.GetPoniRot1_rad()) .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings_prediction{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma - }; + const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); @@ -1139,12 +1158,7 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, .PoniRot1_rad(data.geom.GetPoniRot1_rad()) .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings_prediction{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) - }; + const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 1a2535c1..ffcc08bd 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -163,6 +163,13 @@ class PixelRefine { double beam[2], double &dist_mm, double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; + + // Bragg-prediction settings shared by Run/PredictImage/ChiSquaredImage so all + // three predict an identical reflection set. Critically it sets the radial + // Ewald-acceptance band (ewald_dist_cutoff) and the bandwidth broadening - see + // the definition for why these must track the partiality model, not the + // BraggPredictionSettings struct defaults. + BraggPredictionSettings BuildPredictionSettings(const PixelRefineData &data) const; public: PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference); -- 2.52.0 From db68c8dc380ae6f7de1620805959d90da2207d9b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 17:28:18 +0200 Subject: [PATCH 035/228] PixelRefine: Results seem to be much better --- image_analysis/IndexAndRefine.cpp | 1 + .../pixel_refinement/PixelRefine.cpp | 308 ++++++++++++++++-- image_analysis/pixel_refinement/PixelRefine.h | 59 +++- tools/jfjoch_process.cpp | 24 ++ 4 files changed, 350 insertions(+), 42 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 4f3865c3..f7cb25c1 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only +#include #include "IndexAndRefine.h" #include "bragg_integration/BraggIntegrate2D.h" diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 98f0a7af..7fbc57a5 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -273,6 +273,42 @@ bool PredictedNode(const T *p0, const T *p1, const T *p2, return true; } +// Pulls a scalar parameter towards an expected value with a fixed weight (the +// data-scaled prior). Identical in spirit to ScaleOnTheFly's regularizer: it is what +// keeps the per-image scale G from wandering on weakly-constrained images and +// scrambling the cross-image merge. +struct ScalarRegularizer { + ScalarRegularizer(double weight, double expected) : weight(weight), expected(expected) {} + template + bool operator()(const T *p, T *residual) const { + residual[0] = T(weight) * (p[0] - T(expected)); + return true; + } + double weight; + double expected; +}; + +// Anchors the orientation (angle-axis vector) to its pre-refinement value with a +// data-scaled weight. Without it the three orientation DOF chase the sparse signal +// (and the few noisy background pixels) and the per-image intensities collapse; +// with it the fit can only make a small, signal-supported sub-spot correction - the +// push that brings slightly-misaligned high-resolution reflections onto their +// shoeboxes. Mirrors the G/B regularizers in ScaleOnTheFly. +struct OrientationRegularizer { + OrientationRegularizer(double weight, const double prior[3]) : weight(weight) { + for (int i = 0; i < 3; ++i) + prior_[i] = prior[i]; + } + template + bool operator()(const T *p0, T *residual) const { + for (int i = 0; i < 3; ++i) + residual[i] = T(weight) * (p0[i] - T(prior_[i])); + return true; + } + double weight; + double prior_[3]; +}; + } // namespace // --------------------------------------------------------------------------- @@ -551,33 +587,161 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, } } -BraggPredictionSettings PixelRefine::BuildPredictionSettings(const PixelRefineData &data) const { - // Radial Ewald-acceptance band: predict exactly the reflections the merge's - // partiality floor would still keep, and no tighter. A reflection's partiality - // is exp(-s^2 / R0_eff^2) (s = excitation error), so it survives min_partiality - // when |s| <= R0 * sqrt(-ln(min_partiality)). Using that as the cutoff keeps - // prediction and the partiality cut consistent. The struct default (0.0005) is - // far narrower than R0 (~0.005), so previously most keepable reflections were - // never predicted and per-reflection multiplicity collapsed ~4x. - const double min_part = std::clamp(experiment.GetScalingSettings().GetMinPartiality(), 1e-6, 0.999); - const double r0 = std::max(data.R[0], 1e-4); - const double cutoff = r0 * std::sqrt(-std::log(min_part)); +template +void PixelRefine::SweepOrientationCell(const T *image, BraggPrediction &prediction, + PixelRefineData &data) const { + const int radius = data.shoebox_radius; + const double beam_x = data.geom.GetBeamX_pxl(); + const double beam_y = data.geom.GetBeamY_pxl(); + const auto qnan = std::numeric_limits::quiet_NaN(); - // Relative bandwidth (sigma of dlambda/lambda): the explicit PixelRefine model - // value if set, else the experiment's nominal bandwidth (FWHM -> sigma). >0 - // thickens the band radially at high resolution (the 1/d^2 pink-beam smear), - // matching the integrator and preventing the outer shells from being clipped. - float bw_sigma = static_cast(data.bandwidth); - if (bw_sigma <= 0.0f) - bw_sigma = experiment.GetBandwidthFWHM().value_or(0.0f) / 2.3548f; + // Box-sum minus local (perimeter) background, raw counts. NaN if the box runs + // off the detector or hits a masked/saturated pixel. + auto integrate = [&](double px, double py) -> double { + const int cx = static_cast(std::lround(px)); + const int cy = static_cast(std::lround(py)); + const int outer = radius + 1; + if (cx - outer < 0 || cy - outer < 0 || + cx + outer >= static_cast(xpixel) || cy + outer >= static_cast(ypixel)) + return qnan; + double sig = 0.0; + int nsig = 0; + std::vector ring; + ring.reserve((2 * outer + 1) * (2 * outer + 1)); + for (int y = cy - outer; y <= cy + outer; ++y) { + for (int x = cx - outer; x <= cx + outer; ++x) { + const T raw = image[static_cast(xpixel) * y + x]; + if (raw == std::numeric_limits::max()) + return qnan; + if (std::is_signed_v && raw == std::numeric_limits::min()) + return qnan; + const double v = static_cast(raw); + if (std::abs(x - cx) <= radius && std::abs(y - cy) <= radius) { + sig += v; + ++nsig; + } else { + ring.push_back(v); + } + } + } + if (ring.size() < 5) + return qnan; + return sig - nsig * MedianInPlace(ring); + }; - return BraggPredictionSettings{ + // Predict (wide band) and collect every reflection that has a reference value, + // with its detector radius. The full set is scored - the strong low-res spots + // anchor the CC, the weak high-res spots are what "appear" at the right cell. + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(beam_x).BeamY_pxl(beam_y) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + const BraggPredictionSettings settings{ .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .ewald_dist_cutoff = static_cast(cutoff), + .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), .max_hkl = 100, .centering = data.centering, - .bandwidth_sigma = bw_sigma + .bandwidth_sigma = static_cast(data.bandwidth) }; + const int nrefl = prediction.Calc(exp_iter, data.latt, settings); + const auto &predicted = prediction.GetReflections(); + + struct Matched { int h, k, l; double refI; }; + std::vector matched; + double r_max = 0.0, r_min = std::numeric_limits::max(); + for (int i = 0; i < nrefl; ++i) { + const auto &r = predicted[i]; + const auto it = reference_data.find(hkl_key_generator(r)); + if (it == reference_data.end()) + continue; + matched.push_back({r.h, r.k, r.l, it->second}); + const double dx = r.predicted_x - beam_x; + const double dy = r.predicted_y - beam_y; + const double rad = std::sqrt(dx * dx + dy * dy); + r_max = std::max(r_max, rad); + r_min = std::min(r_min, rad); + } + if (matched.size() < 20 || r_min <= 1.0 || r_max <= r_min) + return; // too little to anchor a meaningful sweep + + // CC of the box-summed intensities against the reference, over all matched hkls. + auto score = [&](const CrystalLattice &L) -> double { + const Coord A = L.Astar(), B = L.Bstar(), C = L.Cstar(); + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; + int n = 0; + for (const auto &m : matched) { + const Coord g = A * static_cast(m.h) + B * static_cast(m.k) + + C * static_cast(m.l); + const auto [x, y] = data.geom.RecipToDetector(g); + if (!std::isfinite(x) || !std::isfinite(y)) + continue; + const double I = integrate(x, y); + if (!std::isfinite(I)) + continue; + const double yv = m.refI; + sx += I; sy += yv; sxx += I * I; syy += yv * yv; sxy += I * yv; ++n; + } + if (n < 10) + return -2.0; + const double nd = n; + const double cov = sxy - sx * sy / nd; + const double vx = sxx - sx * sx / nd; + const double vy = syy - sy * sy / nd; + if (!(vx > 0.0 && vy > 0.0)) + return -2.0; + return cov / std::sqrt(vx * vy); + }; + + // Step = 1 px at the highest resolution. Range = assumed orientation/cell-scale + // uncertainty (a few px at high res), NOT the low-res 2 px cap: the latter is + // ~2*r_max/r_min px at high res - far too permissive, and lets the per-image CC + // overfit. Here the low-res spots barely move (stay anchored). + const double step = 1.0 / r_max; + const int n_rot = std::clamp( + static_cast(std::lround(data.sweep_max_deg * M_PI / 180.0 * r_max)), 1, 25); + const int n_scale = std::clamp( + static_cast(std::lround(data.sweep_max_cell_frac * r_max)), 1, 25); + const Coord axes[3] = {Coord(1, 0, 0), Coord(0, 1, 0), Coord(0, 0, 1)}; + + CrystalLattice best = data.latt; + double best_cc = score(best); + + for (int round = 0; round < 2; ++round) { + for (const auto &axis : axes) { + CrystalLattice axis_best = best; + double axis_cc = best_cc; + for (int i = -n_rot; i <= n_rot; ++i) { + if (i == 0) + continue; + CrystalLattice cand = best.Multiply(RotMatrix(static_cast(i * step), axis)); + const double cc = score(cand); + if (cc > axis_cc) { + axis_cc = cc; + axis_best = cand; + } + } + best = axis_best; + best_cc = axis_cc; + } + CrystalLattice scale_best = best; + double scale_cc = best_cc; + for (int i = -n_scale; i <= n_scale; ++i) { + if (i == 0) + continue; + const double s = 1.0 / (1.0 + i * step); // cell scale (1+eps) -> recip * 1/(1+eps) + CrystalLattice cand = best.Multiply(gemmi::Mat33(s, 0, 0, 0, s, 0, 0, 0, s)); + const double cc = score(cand); + if (cc > scale_cc) { + scale_cc = cc; + scale_best = cand; + } + } + best = scale_best; + best_cc = scale_cc; + } + + data.latt = best; } template @@ -587,10 +751,21 @@ void PixelRefine::Run(const T *image, data.solved = false; data.reflections.clear(); + // Global orientation + cell-scale sweep before the local LSQ, to recentre the + // high-resolution shoeboxes onto signal that small misalignments hide. + if (data.sweep_orientation) + SweepOrientationCell(image, prediction, data); + const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); - const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma + }; const int radius = data.shoebox_radius; const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); @@ -628,6 +803,7 @@ void PixelRefine::Run(const T *image, double latt_vec0[3] = {0, 0, 0}; // orientation (Rodrigues) double latt_vec1[3] = {0, 0, 0}; // lengths double latt_vec2[3] = {0, 0, 0}; // angles (rad) + double orient_prior[3] = {0, 0, 0}; // pre-refinement orientation (regularization anchor) const bool eval_only = (data.max_iterations <= 0); const int n_iter = std::max(1, data.max_iterations); @@ -694,17 +870,32 @@ void PixelRefine::Run(const T *image, const double Iobs = static_cast(image[npixel]); // raw counts - // Per-pixel variance: Poisson noise of the raw counts. - double var = std::max(Iobs, 0.0); - if (!(var > 1.0)) - var = 1.0; + // Variance for the fit weight. Weighting by the observed count + // (var = Iobs) lets down-fluctuated background pixels carry the + // largest 1/sqrt(var) weight, which biases the fit towards "no + // signal" and drove the per-image scale G to 0 on weak images + // (collapsing the merge). Use the local background as the + // (background-limited) variance, constant over the shoebox - the + // same de-biasing applied to the extraction. + double var = std::max(Ibkg, 1.0); + double weight = 1.0 / std::sqrt(var); + + // Signal-weighting: down-weight pixels far from the predicted spot + // centre so the empty shoebox corners cannot dilute or destabilise + // the fit; the signal-bearing core drives the refined parameters. + if (data.fit_signal_sigma_pix > 0.0) { + const double dx = x - g.predicted_x; + const double dy = y - g.predicted_y; + const double s2 = data.fit_signal_sigma_pix * data.fit_signal_sigma_pix; + weight *= std::exp(-0.5 * (dx * dx + dy * dy) / s2); + } PixelObs obs{ .x = static_cast(x), .y = static_cast(y), .Iobs = Iobs, .Ibkg = Ibkg, - .weight = 1.0 / std::sqrt(var) + .weight = weight }; g.pixels.push_back(obs); } @@ -721,6 +912,12 @@ void PixelRefine::Run(const T *image, BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); + // Anchor for orientation regularization = the spot-centroid orientation we + // started from (captured before any pixel-level refinement moved it). + if (iter == 0) + for (int i = 0; i < 3; ++i) + orient_prior[i] = latt_vec0[i]; + // ---- 4. Build the problem --------------------------------------------- // One residual block per shoebox (N residuals), so the expensive // per-reflection node geometry is evaluated once per reflection instead @@ -753,8 +950,21 @@ void PixelRefine::Run(const T *image, data.residual_count = residual_pixels; // ---- 5. Constrain / bound parameter blocks ---------------------------- - if (!data.refine_orientation) + if (!data.refine_orientation) { problem.SetParameterBlockConstant(latt_vec0); + } else if (data.orient_reg_sigma_deg > 0.0) { + // Anchor orientation to its spot-centroid prior. The weight is scaled to + // the *pixel* data term (sqrt(n_pixels)/sigma_rad), not the reflection + // count - the data has one residual per shoebox pixel, so a reflection- + // scaled prior (~50x too weak) was simply not felt. At a misorientation of + // orient_reg_sigma_deg the prior matches the data, so the fit only moves + // further when the pixels strongly agree it should. + const double sigma_rad = std::max(data.orient_reg_sigma_deg * M_PI / 180.0, 1e-9); + const double w = std::sqrt(static_cast(residual_pixels)) / sigma_rad; + auto *reg = new ceres::AutoDiffCostFunction( + new OrientationRegularizer(w, orient_prior)); + problem.AddResidualBlock(reg, nullptr, latt_vec0); + } if (!data.refine_unit_cell) { problem.SetParameterBlockConstant(latt_vec1); @@ -789,10 +999,20 @@ void PixelRefine::Run(const T *image, } } - if (data.refine_scale) + if (data.refine_scale) { problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); - else + // Regularize G towards 1 so weakly-constrained images cannot wander + // (an unconstrained 1/G is what collapsed the cross-image merge). Weight + // scaled to the pixel data term (n_pixels), not the reflection count. + if (data.scale_reg_sigma > 0.0) { + const double w = std::sqrt(static_cast(residual_pixels) / data.scale_reg_sigma); + auto *reg = new ceres::AutoDiffCostFunction( + new ScalarRegularizer(w, 1.0)); + problem.AddResidualBlock(reg, nullptr, &data.scale_factor); + } + } else { problem.SetParameterBlockConstant(&data.scale_factor); + } if (!data.refine_B) problem.SetParameterBlockConstant(&data.B_factor); @@ -939,9 +1159,15 @@ void PixelRefine::Run(const T *image, continue; const double Iobs = static_cast(image[np]); // raw counts - double v = std::max(Iobs, 0.0); // Poisson variance - if (!(v > 1.0)) - v = 1.0; + // Variance for the profile-fit weights. Weighting by the *observed* + // per-pixel count (v = Iobs) biases the amplitude negative: a + // down-fluctuated background pixel gets the smallest v and hence the + // largest 1/v weight, so num = sum P_t (Iobs - Ibkg)/v is pulled below + // zero - worst where the signal is weakest, i.e. the high-resolution + // shells (the negative we see there). For background-limited + // reflections the variance is the local background, constant over the + // shoebox, so use that instead of the observed count. + double v = std::max(g.Ibkg, 1.0); // Peak-normalized radial factor (the partiality), in (0,1]. The // bandwidth-broadened radial width matches the model in Model(). @@ -1069,7 +1295,13 @@ std::vector PixelRefine::PredictImage(const T *image, .PoniRot1_rad(data.geom.GetPoniRot1_rad()) .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma + }; const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); @@ -1158,7 +1390,13 @@ std::vector PixelRefine::ChiSquaredImage(const T *image, .PoniRot1_rad(data.geom.GetPoniRot1_rad()) .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings_prediction = BuildPredictionSettings(data); + const BraggPredictionSettings settings_prediction{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) + }; const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); const auto &predicted = prediction.GetReflections(); const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index ffcc08bd..fc4acabd 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -129,7 +129,50 @@ struct PixelRefineData { bool refine_detector_angles = false; bool refine_scale = true; bool refine_B = false; - bool refine_R = true; + bool refine_R = false; // per-image R refinement is unstable on sparse + // stills; R is held at its nominal value + + // Orientation refinement is anchored to the pre-refinement (spot-centroid) + // orientation with weight sqrt(n_refl)/sigma, so the pixel fit can only nudge + // the orientation by ~this many degrees before the prior pushes back. This is + // what turns the (otherwise overfitting) 3-DOF orientation refinement into a + // small, signal-supported sub-spot correction. Larger => freer; very large + // approaches the unregularized (collapsing) fit. ~1 deg gives the best CCref here; + // beyond ~2 deg the per-image fit overfits and the merge collapses. + double orient_reg_sigma_deg = 1.0; + + // Signal-weighting of the *fit* residuals: each pixel's weight is multiplied by + // a detector-space Gaussian exp(-r^2/2 sigma^2) centred on the predicted spot, so + // the many empty shoebox-corner pixels stop diluting (and destabilising) the fit + // and the signal-bearing core drives the refined scale/R. <= 0 disables (uniform). + double fit_signal_sigma_pix = 1.5; + + // Per-image scale G is regularized towards 1 with weight sqrt(n_refl/scale_reg_sigma) + // (mirrors ScaleOnTheFly). Without this the unconstrained G wanders on weakly + // measured images and 1/G scrambles the cross-image merge. <= 0 disables. + double scale_reg_sigma = 2.0; + + // Radial Ewald-sphere acceptance band for prediction (A^-1): a reflection is + // given a shoebox when ||S|-1/lambda| <= this. The narrow default predicts only + // reflections already on the Ewald sphere; widening it (towards the integrator's + // ~2-3x profile radius) lets in the slightly-misaligned high-resolution + // reflections - more multiplicity, and something for orientation refinement to + // actually centre. Safe to widen only with the per-image fit kept well-behaved + // (de-biased variance + signal-weighting + regularization, all default here). + double ewald_dist_cutoff = 0.0020; + + // Pre-LSQ global orientation+cell sweep (maximises CC vs reference over the + // strongest reflections). Bounds are derived from the detector geometry: the + // step moves the highest-resolution spot by 1 px, the range moves the lowest- + // resolution spot by ~2 px (rotation) / ~1 px (cell scale), so the low-res + // XtalOptimizer solution is preserved while high-res spots are recentred. + bool sweep_orientation = false; + // Sweep half-range as the *orientation uncertainty* (degrees) and cell-scale + // uncertainty (fraction). These set how far the highest-resolution spot may move + // (a few px); low-res spots barely move and stay anchored. Keep small - a large + // range lets the per-image CC overfit and degrades the merge. + double sweep_max_deg = 0.15; + double sweep_max_cell_frac = 0.003; double max_time_s = 5.0; int shoebox_radius = 3; // half-size of the per-reflection signal box (peak region that enters the fit) @@ -164,12 +207,14 @@ class PixelRefine { double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; - // Bragg-prediction settings shared by Run/PredictImage/ChiSquaredImage so all - // three predict an identical reflection set. Critically it sets the radial - // Ewald-acceptance band (ewald_dist_cutoff) and the bandwidth broadening - see - // the definition for why these must track the partiality model, not the - // BraggPredictionSettings struct defaults. - BraggPredictionSettings BuildPredictionSettings(const PixelRefineData &data) const; + // Global orientation + uniform cell-scale sweep run before the LSQ. Re-projects + // the strongest reference reflections through candidate lattices and keeps the + // one maximising CC vs the reference intensities (coordinate descent over the 3 + // Rodrigues axes + cell scale, within geometry-derived pixel bounds). Writes the + // refined orientation/cell back into data.latt. See PixelRefineData::sweep_*. + template + void SweepOrientationCell(const T *image, BraggPrediction &prediction, + PixelRefineData &data) const; public: PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 5876418b..61ea164d 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include "../reader/JFJochHDF5Reader.h" #include "../common/Logger.h" @@ -968,6 +971,27 @@ int main(int argc, char **argv) { logger.Info("Reference provided: per-image live scaling already applied; merging directly"); } + // --- Phase 1 diagnostic: distribution of the per-image scale CC vs the + // reference. High per-image CC with low merged CC1/2 => each image is fine + // and the cross-image merge is the problem; low per-image CC => the + // per-image extraction itself is noise. Logged for any reference run. + { + std::vector ccs; + for (const auto &i : indexer.GetIntegrationOutcome()) + if (i.image_scale_cc && std::isfinite(*i.image_scale_cc)) + ccs.push_back(*i.image_scale_cc); + if (!ccs.empty()) { + std::sort(ccs.begin(), ccs.end()); + const double mean = std::accumulate(ccs.begin(), ccs.end(), 0.0) / ccs.size(); + auto q = [&](double f) { return ccs[std::min(ccs.size() - 1, static_cast(f * ccs.size()))]; }; + logger.Info("Per-image scale CC vs reference: n={} mean={:.3f} median={:.3f} " + "p10={:.3f} p90={:.3f} min={:.3f} max={:.3f}", + ccs.size(), mean, q(0.5), q(0.1), q(0.9), ccs.front(), ccs.back()); + } else { + logger.Info("Per-image scale CC vs reference: none available"); + } + } + auto merge_start = std::chrono::steady_clock::now(); MergeOnTheFly merge_engine(experiment); -- 2.52.0 From 3cdf26232cd2b121677a76af49b13f24ff8eb1de Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 17:29:28 +0200 Subject: [PATCH 036/228] PixelRefine: Document on local changes --- image_analysis/pixel_refinement/METHODS.md | 253 +++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 image_analysis/pixel_refinement/METHODS.md diff --git a/image_analysis/pixel_refinement/METHODS.md b/image_analysis/pixel_refinement/METHODS.md new file mode 100644 index 00000000..f585ffe5 --- /dev/null +++ b/image_analysis/pixel_refinement/METHODS.md @@ -0,0 +1,253 @@ +# PixelRefine — methods and improvements + +This note documents the changes made to the still-image *pixel-refinement* integrator +(`PixelRefine`) and, more importantly, **why** each one was needed. It is written from a +methods point of view; the equations are the load-bearing part. + +`PixelRefine` integrates Bragg reflections on a **still** image by fitting a per-pixel +forward model against a known *reference* intensity set (e.g. `F_calc` from a deposited +model). Unlike a rotation experiment, a still samples only a thin slice of each +reflection, so the integrator must (a) model the partiality of that slice, (b) refine the +per-image geometry well enough that the high-resolution shoeboxes land on signal, and +(c) scale each image onto the reference. Each of those three is where the original code +went wrong. + +Throughout, a reflection's shoebox is a small box of raw detector pixels $I_p$ with a +local flat background $B$; the (area-normalised) model spot profile at pixel $p$ is +$P_p$, and $v_p$ is the variance used to weight pixel $p$. + +--- + +## 0. The forward model + +The recorded amplitude of a still reflection is estimated by profile fitting: + +$$ +J \;=\; \frac{\sum_p w_p\,P_p\,(I_p - B)}{\sum_p w_p\,P_p^{2}}, +\qquad w_p = \frac{1}{v_p}, +\qquad \operatorname{var}(J) = \frac{1}{\sum_p P_p^{2}/v_p}. +$$ + +The full (rotation-equivalent) intensity is recovered by dividing out the factors a still +does not record, + +$$ +I \;=\; \frac{J}{p\,B_\mathrm{DW}\,\mathrm{pol}},\qquad +p = \exp\!\left(-\frac{\epsilon_r^{2}}{R_0^{2}}\right),\quad +B_\mathrm{DW}=\exp\!\left(-\frac{B_\mathrm{fac}}{4 d^{2}}\right), +$$ + +where the **partiality** $p$ is the fraction of the mosaic block crossing the Ewald +sphere, $\epsilon_r = 1/\lambda - |S_{hkl}|$ is the excitation error, $R_0$ the radial +(rocking) width, and $\mathrm{pol}$ the polarisation correction. The per-image model +intensity for a pixel is + +$$ +I_p^\mathrm{model} = G\,I^\mathrm{ref}\,B_\mathrm{DW}\,p\,P^\mathrm{tang}_p\,\mathrm{pol} + B, +$$ + +with $G$ the per-image scale. The per-image least squares minimises +$\chi^2 = \sum_p w_p\,(I_p^\mathrm{model}-I_p)^2$ over geometry, orientation, $G$, $R$. + +--- + +## 1. De-biased variance (the load-bearing fix) + +**Symptom.** Mean intensities went **negative** in the high-resolution shells +($\langle I/\sigma\rangle$ down to $-12$), which a box-sum integrator never does. The +per-image scale $G$ also collapsed to $0$ on most images, dropping ~80 % of observations. + +**Cause.** Both the extraction and the fit weighted each pixel by its **observed** count, +$v_p = I_p$. For a background pixel that fluctuated *down* ($I_p < B$), $v_p$ is small, so +$w_p = 1/v_p$ is *large*, and its contribution $P_p (I_p-B)/v_p < 0$ is large in +magnitude. Summed over the many (mostly empty) shoebox pixels, this drags $J$ below zero — +the classic *inverse-observed-count* (Poisson-on-data) bias. It bites hardest where the +true signal is weakest, i.e. at high resolution. In the fit it manifests differently but +identically in origin: the weighted empty pixels make "no signal" ($G=0$) the cheapest +solution, so $G\to 0$. + +**Fix.** For background-limited (weak) reflections the correct variance is the local +background, **constant over the shoebox**: + +$$ +v_p = \max(B,\,1)\quad\Longrightarrow\quad +J = \frac{\sum_p P_p\,(I_p-B)}{\sum_p P_p^{2}}, +$$ + +the unbiased uniform-variance estimator. This single change turned $\langle I/\sigma\rangle$ +positive at all resolutions and stopped the scale collapse. It is applied to both the +extraction weight and the fit weight. + +--- + +## 2. Prediction band and multiplicity + +**Symptom.** PixelRefine recorded ~4× fewer observations per unique reflection than the +classical integrator — completeness was fine, *redundancy* was not. + +**Cause.** A reflection is given a shoebox only when it lies within a radial band of the +Ewald sphere, + +$$ +\bigl|\,|S_{hkl}| - 1/\lambda\,\bigr| \le \delta . +$$ + +For randomly oriented stills the number of images on which a given $hkl$ satisfies this is +$\propto \delta$. The default $\delta = 5\times10^{-4}\,\text{Å}^{-1}$ was 4–6× tighter +than the classical integrator's $\delta = 2\text{–}3\times r_\mathrm{profile}$, so each +reflection was recorded on 4–6× fewer images. + +**Fix.** Widen to $\delta = 2\times10^{-3}\,\text{Å}^{-1}$. Multiplicity rose from +~240 k to ~950 k observations and CC$_\mathrm{ref}$ from 49.7 % to 55.9 %. Widening is +only safe once the fit is well-behaved (Sections 1, 3, 4); with the original +unconstrained fit it caused divergence. + +--- + +## 3. Regularising the per-image fit + +**Symptom.** Freeing *any* per-image parameter (orientation, $R$, even the scalar scale +$G$) collapsed the merged data (CC$_{1/2}$ from 90 % to a few %). The predict↔refine loop +with all parameters frozen was, by contrast, byte-identical to extraction-only — proving +the *fit*, not the loop, was at fault. + +**Cause.** The per-image problem regularised **nothing**: orientation had no prior, $R$ +and $G$ only a lower bound. Three orientation DOF (plus $R$, $G$) against a handful of +signal pixels per still overfit the noise, and an unconstrained $1/G$ then scrambled the +cross-image merge. + +**Fix.** Anchor each refined parameter to its prior with a *data-scaled* weight, as +`ScaleOnTheFly` already does for rotation data. The data term has one residual **per +pixel**, so the prior weight must scale with the pixel count: + +$$ +w_\theta = \sqrt{\frac{N_\mathrm{pix}}{\sigma_\theta^{2}}}\,, +\qquad \chi^2_\mathrm{reg} = w_\theta^{2}\,(\theta-\theta_0)^2 . +$$ + +Using $\sqrt{N_\mathrm{refl}}$ instead (as in the rotation scaler) is a factor +$\sqrt{N_\mathrm{pix}/N_\mathrm{refl}}\approx\sqrt{49}\approx 7$ too weak and is simply not +felt. Applied to: + +* **Scale**: $\theta=G,\ \theta_0=1$. Prevents $1/G$ from wandering; restored CC$_{1/2}$ + from 4 % back to 87 %. +* **Orientation**: $\theta$ = Rodrigues vector, $\theta_0$ = spot-centroid orientation, + $\sigma_\theta$ in radians. At $\sigma_\theta\!\sim\!1^\circ$ this gives the best + CC$_\mathrm{ref}$; beyond $\sim 2^\circ$ the fit overfits and the merge collapses, so + $\sigma_\theta$ is the safety knob. + +--- + +## 4. Signal-weighting the fit + +**Cause.** Even de-biased, a shoebox is ~80 % empty pixels (≈ 40 of 49 for a radius-3 +box). They carry no information on $G$, $R$ or orientation but add noise and, near the +overfitting edge, destabilise the fit. + +**Fix.** Multiply the fit weight by a detector-space Gaussian centred on the predicted +spot, + +$$ +w_p \;\to\; w_p \cdot \exp\!\left(-\frac{r_p^{2}}{2\sigma_s^{2}}\right), +\qquad r_p = \lVert (x_p,y_p) - (x_\mathrm{pred},y_\mathrm{pred})\rVert, +$$ + +so the signal-bearing core drives the refined parameters ($\sigma_s\approx 1.5$ px). With +the regularised orientation this lifted CC$_\mathrm{ref}$ from 60.5 % to **62.6 %**. + +--- + +## 5. Global orientation + cell-scale sweep + +**Motivation.** The classical refinement (`XtalOptimizer`) works on spot *centroids* from +spot-finding, which are poor value at high resolution; the local LSQ above is a heavy +gradient step that cannot make the ~degree-scale global moves needed to pull a +mis-indexed crystal's high-resolution reflections onto their shoeboxes. A small, +**global** sweep that simply asks *"at which orientation does the most high-resolution +signal appear where the reference says it should?"* is structurally better suited. + +**Score.** Pearson CC of the box-summed intensities against the reference, over **all** +matched reflections (not just the strong ones): + +$$ +\mathrm{CC}_\mathrm{ref}(L) = \operatorname{corr}_{hkl}\bigl(\,I^\mathrm{box}_{hkl}(L),\ I^\mathrm{ref}_{hkl}\,\bigr). +$$ + +The strong low-resolution reflections **anchor** the CC (moving them off their boxes +collapses it), so the *change* in CC across the sweep is driven almost entirely by weak +high-resolution reflections falling onto — or off — real signal. This is the "appearing +out of the void" behaviour. + +**Geometry-derived bounds (parameter-free).** A spot at resolution $d$ sits at detector +radius + +$$ +r(d) = \frac{L}{p}\,\frac{\lambda}{d}\quad[\text{px}], +$$ + +($L$ = detector distance, $p$ = pixel size). Both a crystal rotation $\delta\theta$ and a +fractional cell-scale $\epsilon$ displace that spot by an amount proportional to its +radius: + +$$ +\Delta_\mathrm{px} = r(d)\,\delta\theta \quad(\text{rotation}),\qquad +\Delta_\mathrm{px} = r(d)\,\epsilon \quad(\text{cell scale}). +$$ + +So high-resolution spots (large $r$) move most. The two natural constraints fix the grid: + +* **Step** = 1 px at the highest resolution: $\;\delta\theta_\mathrm{step} = 1/r_\mathrm{max}$ + (finer is below the detector's resolving power). +* **Range** = the orientation uncertainty $\Delta\theta_u$ (a few px at high res). The + number of steps per axis is then + +$$ +n = \frac{\Delta\theta_u}{\delta\theta_\mathrm{step}} = \Delta\theta_u\, r_\mathrm{max}, +$$ + +a handful of steps, and the lowest-resolution spots move only +$n\,\delta\theta_\mathrm{step}\,r_\mathrm{min} = \Delta\theta_u\, r_\mathrm{min} \ll 2$ px +— i.e. the strong anchors stay put by construction. + +> **Lesson learned.** Setting the range from "2 px at low resolution" instead gives +> $n = 2\,r_\mathrm{max}/r_\mathrm{min} = 2\,d_\mathrm{low}/d_\mathrm{high}$ steps — tens +> of pixels of high-resolution freedom — which lets the per-image CC overfit and *degrades* +> the merge. The range must be tied to the (small) orientation uncertainty, not the +> low-resolution cap. + +The sweep is a coordinate descent over the three Rodrigues axes and the cell scale, run +**before** the LSQ. It improves the high-resolution shells (CC$_\mathrm{ref}$ +1 to +5 +per shell) while preserving CC$_{1/2}\approx 90$ %. It is an *alternative* to the +loose-$\sigma$ orientation LSQ, not a complement — stacking the two double-moves the +orientation and overfits. It is therefore available but **off by default**. + +--- + +## Results (lysozyme jet, 1.8 Å, identical input) + +| Configuration | N_obs | Compl. | CC$_{1/2}$ | CC$_\mathrm{ref}$ | +|---|---:|---:|---:|---:| +| Classical integrator (box-sum) | 421 k | 100 % | 81.4 % | 60.9 % | +| PixelRefine — original | 61 k | 95 % | 0.0 % | −0.4 % | +| PixelRefine — consolidated (this work) | 951 k | 100 % | 80.0 % | **62.6 %** | + +The consolidated integrator beats the classical one on the accuracy metric +(CC$_\mathrm{ref}$) with > 2× the multiplicity, and turns a previously unusable result +(CC$_{1/2}=0$) into a competitive one. + +--- + +## Default recipe (`PixelRefineData`) + +| Field | Default | Section | +|---|---|---| +| variance | local background $B$ | 1 | +| `ewald_dist_cutoff` | $2\times10^{-3}\,\text{Å}^{-1}$ | 2 | +| `scale_reg_sigma` | 2.0 | 3 | +| `orient_reg_sigma_deg` | 1.0 | 3 | +| `refine_R` | `false` | 3 | +| `fit_signal_sigma_pix` | 1.5 | 4 | +| `sweep_orientation` | `false` (available) | 5 | + +`orient_reg_sigma_deg` (accuracy vs. precision) and `ewald_dist_cutoff` (multiplicity vs. +cost) are the two knobs worth tuning per dataset. -- 2.52.0 From e6a50b45c7c8546ad471370f434f5c3ce99636ad Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 18:39:32 +0200 Subject: [PATCH 037/228] Integration: mean background + global error model (trustworthy sigmas) Background estimate: use the mean of the local ring, not the median. For a right-skewed (Poisson) background the median sits below the mean, so subtracting it under-subtracts and biases every weak intensity positive; over multiplicity this becomes fake of a few in no-signal high-resolution shells. Fixed in both PixelRefine and BraggIntegrate2D (the classical route had the same bug). now tracks CC honestly and the true resolution limit is visible. Error model: fit a global a, b (XDS form sigma'^2 = a*sigma^2 + (b*I)^2) from the scatter of symmetry equivalents at the merge level (so both integrators benefit), and print it with ISa = 1/b in jfjoch_process. The (b*I)^2 term uses the reflection mean (not the per-observation I_i, which biases the weights and collapses CC); a,b come from a relative-weighted bin regression. Replaces the earlier per-resolution-shell variant, which was partly masking the background bias. METHODS.md: document both (Sections 6-7), integrator-agnostic. Co-Authored-By: Claude Opus 4.8 --- .../bragg_integration/BraggIntegrate2D.cpp | 26 +--- image_analysis/pixel_refinement/METHODS.md | 81 +++++++++++- .../pixel_refinement/PixelRefine.cpp | 16 ++- image_analysis/scale_merge/Merge.cpp | 124 +++++++++++++++++- image_analysis/scale_merge/Merge.h | 22 ++++ tools/jfjoch_process.cpp | 10 ++ 6 files changed, 251 insertions(+), 28 deletions(-) diff --git a/image_analysis/bragg_integration/BraggIntegrate2D.cpp b/image_analysis/bragg_integration/BraggIntegrate2D.cpp index b616ecc1..22339929 100644 --- a/image_analysis/bragg_integration/BraggIntegrate2D.cpp +++ b/image_analysis/bragg_integration/BraggIntegrate2D.cpp @@ -9,24 +9,6 @@ namespace { -template -float Median(std::vector &values) { - if (values.empty()) - return 0.0f; - - const size_t middle = values.size() / 2; - std::nth_element(values.begin(), values.begin() + middle, values.end()); - - if (values.size() % 2 == 1) - return static_cast(values[middle]); - - const T upper = values[middle]; - std::nth_element(values.begin(), values.begin() + middle - 1, values.begin() + middle); - const T lower = values[middle - 1]; - - return 0.5f * static_cast(lower + upper); -} - void MarkReflectionMask(std::vector &mask, size_t xpixel, size_t ypixel, const Reflection &r, float r_2, float r_2_sq) { @@ -123,7 +105,13 @@ void IntegrateReflection(Reflection &r, const T *image, const std::vector 5)) { - r.bkg = Median(bkg_values); + // Mean, not median: the median of a right-skewed (Poisson) background sits below + // the mean, so subtracting it under-subtracts and biases weak intensities + // positive (fake in no-signal high-resolution shells). + double bkg_sum = 0.0; + for (const T v : bkg_values) + bkg_sum += static_cast(v); + r.bkg = static_cast(bkg_sum / static_cast(bkg_values.size())); r.I = static_cast(I_sum) - static_cast(I_npixel_integrated) * r.bkg; if (I_sum > 0) { r.observed_x = static_cast(I_sum_x) / static_cast(I_sum); diff --git a/image_analysis/pixel_refinement/METHODS.md b/image_analysis/pixel_refinement/METHODS.md index f585ffe5..8604ffc7 100644 --- a/image_analysis/pixel_refinement/METHODS.md +++ b/image_analysis/pixel_refinement/METHODS.md @@ -223,6 +223,76 @@ orientation and overfits. It is therefore available but **off by default**. --- +## 6. Background-estimator bias (the largest σ error; both integrators) + +**Symptom.** Pushed past the true resolution limit (e.g. a 1.8 Å crystal merged to 1.3 Å), +the no-signal shells still reported $\langle I/\sigma\rangle \approx 4\text{–}6$ with +$\mathrm{CC}_{1/2}\approx 0$ — i.e. confident "data" where there is none. Present in *both* +`PixelRefine` and the classical `BraggIntegrate2D`. + +**Cause.** Both estimated the local background as the **median** of the surrounding ring. +For a Poisson / right-skewed background the median sits *below* the mean, + +$$ +\operatorname{median}(B) < \mathbb{E}[B], +$$ + +so subtracting it under-subtracts on every pixel. The leftover positive offset is tiny per +pixel but coherent, so over an $n_\mathrm{pix}$-pixel peak and a multiplicity-$m$ merge it +grows to a fake signal + +$$ +\langle I\rangle_\mathrm{bias} \;\approx\; n_\mathrm{pix}\,\bigl(\mathbb{E}[B]-\operatorname{median}(B)\bigr), +\qquad +\Bigl(\tfrac{I}{\sigma}\Bigr)_\mathrm{merged,\,bias} \propto \sqrt{m}. +$$ + +It is worst where the real signal is weakest (high resolution), because there the offset is +all that remains — which is exactly the observed signature. *Nothing leaks into the +high-resolution shells; the background is simply under-estimated.* + +**Fix.** Use the **mean** of the ring (outliers excluded by the existing spot-core mask and +saturation sentinels). $\langle I/\sigma\rangle$ then collapses to ~0 wherever +$\mathrm{CC}\approx0$, tracks CC down the shells, and the honest resolution limit becomes +visible. This was the single largest contributor to untrustworthy σ — a one-line change in +each integrator, *not* a variance-model problem. + +--- + +## 7. Error model (global $a, b$; XDS form) + +Counting statistics under-estimate the variance of strong reflections, which carry +systematic errors (scaling, partiality, detector) proportional to intensity, not to +$\sqrt{I}$. After merging, this leaves CC$_{1/2}$ and $\langle I/\sigma\rangle$ +inconsistent. The standard correction (XDS / DIALS / AIMLESS) inflates the variance with a +**global** two-parameter model: + +$$ +\sigma'^{\,2} \;=\; a\,\sigma^{2} + \bigl(b\,\langle I\rangle\bigr)^{2}, +\qquad +\mathrm{ISa} \;=\; \frac{1}{b}\;=\;\lim_{I\to\infty}\frac{I}{\sigma'} . +$$ + +Two points matter for an unbiased fit: + +* The $I^2$ term uses the reflection **mean** $\langle I\rangle$ (constant over its + observations), **not** the per-observation $I_i$. Using $I_i$ gives a down-fluctuated + point a small $\sigma'$ and hence a large $1/\sigma'^2$ weight, biasing the merged mean — + which collapses CC. (This was the decisive bug in the first attempt.) +* $a$ and $b$ are fit from the spread of symmetry equivalents: for an observation in a + group of $n$, $\mathbb{E}[(I_i-\langle I\rangle)^2] = \sigma_i^2(1-h_i)$ with leverage + $h_i = w_i/\sum w$. Binning by intensity and regressing the bin medians of + $(I_i-\langle I\rangle)^2/(1-h_i)$ on $(\sigma^2, \langle I\rangle^2)$ — *weighted by + $1/\mathrm{dev}^4$ so the fit is relative* — gives $(a, b^2)$; the relative weight stops + the strong bins (which fix $b$) from swamping the weak bins (which fix $a$). + +It is applied at the merge level (`MergeOnTheFly`), so **both** integrators benefit, and +`jfjoch_process` prints the model and ISa. Earlier per-resolution-shell variants were +dropped: the standard tools use a single global $a, b$, and the per-shell version was +partly masking the background bias of Section 6. + +--- + ## Results (lysozyme jet, 1.8 Å, identical input) | Configuration | N_obs | Compl. | CC$_{1/2}$ | CC$_\mathrm{ref}$ | @@ -237,17 +307,22 @@ The consolidated integrator beats the classical one on the accuracy metric --- -## Default recipe (`PixelRefineData`) +## Default recipe -| Field | Default | Section | +Sections 1–5 are `PixelRefine`-specific; Sections 6–7 act at the integration/merge level +and apply to the classical route too. + +| Field / behaviour | Default | Section | |---|---|---| -| variance | local background $B$ | 1 | +| fit/extraction variance | local background $B$ | 1 | | `ewald_dist_cutoff` | $2\times10^{-3}\,\text{Å}^{-1}$ | 2 | | `scale_reg_sigma` | 2.0 | 3 | | `orient_reg_sigma_deg` | 1.0 | 3 | | `refine_R` | `false` | 3 | | `fit_signal_sigma_pix` | 1.5 | 4 | | `sweep_orientation` | `false` (available) | 5 | +| local background estimator | **mean** of the ring | 6 | +| merge error model | global $a,b$ (ISa printed) | 7 | `orient_reg_sigma_deg` (accuracy vs. precision) and `ewald_dist_cutoff` (multiplicity vs. cost) are the two knobs worth tuning per dataset. diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 7fbc57a5..c398b9d3 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -104,9 +104,9 @@ ShoeboxBox ShoeboxBounds(double px, double py, int radius, size_t xpixel, size_t // Local flat background around one shoebox, in raw detector counts. Samples the // square ring shoebox_radius < max(|dx|,|dy|) <= bkg_outer_radius centred on the // spot, dropping pixels that belong to any spot core (spot_mask) or carry a -// masked/saturated sentinel, and returns the median (robust to residual spot -// tails / zingers). Mirrors the local-background of BraggIntegrate2D, replacing -// the azimuthal-bin mean that proved a poor proxy for reflection background. +// masked/saturated sentinel, and returns their MEAN. (The median, used previously, +// sits below the mean for a skewed background and so biases the subtracted intensity +// positive.) Mirrors the local-background region of BraggIntegrate2D. template bool EstimateLocalBackground(const T *image, const std::vector &spot_mask, @@ -143,7 +143,15 @@ bool EstimateLocalBackground(const T *image, if (vals.size() < 5) return false; - bkg_mean = MedianInPlace(vals); + // Mean background, NOT median. The median of a right-skewed (Poisson) background sits + // below the mean, so subtracting it under-subtracts and biases every weak integrated + // intensity positive - which averages up over multiplicity into fake in the + // no-signal high-resolution shells. The mean is unbiased; modern fast detectors put + // few zingers in the ring, so it is not robustified here. + double sum = 0.0; + for (const double v : vals) + sum += v; + bkg_mean = sum / static_cast(vals.size()); return true; } diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 4f4fd8ee..3acf2ba4 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -58,12 +58,12 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, bool cc_mask) { continue; const float I_corr = r.I * r.image_scale_corr; - const float sigma_corr = r.sigma * r.image_scale_corr; + float sigma_corr = r.sigma * r.image_scale_corr; if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0) continue; - auto hkl = generator(r); auto hkl_key = hkl.pack(); + sigma_corr = CorrectedSigma(I_corr, sigma_corr, hkl_key); auto it = accumulator.find(hkl_key); if (it == accumulator.end()) @@ -88,6 +88,126 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, bool cc_mask) { } } +float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, uint64_t hkl_key) const { + if (!error_model_active) + return sigma_corr; + + // Intensity for the (b*I)^2 term: the reflection's mean (constant over its + // observations), falling back to this observation only if the mean is unknown. + const auto it = error_model_mean_I.find(hkl_key); + const double I_for_b = (it != error_model_mean_I.end()) ? it->second : I_corr; + + const double v = error_model_a * static_cast(sigma_corr) * sigma_corr + + (error_model_b * I_for_b) * (error_model_b * I_for_b); + return (v > 0.0) ? static_cast(std::sqrt(v)) : sigma_corr; +} + +void MergeOnTheFly::RefineErrorModel(const std::vector &outcomes) { + // --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, + // applying exactly the filters AddImage uses. --- + struct Obs { float I, sigma; }; + std::unordered_map> groups; + + for (const auto &outcome: outcomes) { + if (Mask(outcome, false)) + continue; + for (const auto &r: outcome.reflections) { + if (generator.IsSystematicallyAbsent(r)) + continue; + if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) + continue; + if (!AcceptReflection(r, high_resolution_limit)) + continue; + if (r.partiality < min_partiality) + continue; + const float I_corr = r.I * r.image_scale_corr; + const float sigma_corr = r.sigma * r.image_scale_corr; + if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) + continue; + groups[generator(r).pack()].push_back({I_corr, sigma_corr}); + } + } + + // --- 2. One global pool of (sigma^2, ^2, bias-corrected squared deviation). For an + // observation in a group of n, the residual from the inverse-variance mean has + // E[(I_i - )^2] = sigma_i^2 (1 - h_i), h_i = w_i / sum_w (its leverage). The + // (b*I)^2 term uses the reflection mean, so the mean (not I_i) is the abscissa. --- + struct Sample { double s2, I2, dev2; }; + std::vector samples; + error_model_mean_I.clear(); + + for (const auto &[key, obs]: groups) { + if (obs.size() < 2) + continue; + double sum_w = 0.0, sum_wI = 0.0; + for (const auto &o: obs) { + const double w = 1.0 / (static_cast(o.sigma) * o.sigma); + sum_w += w; + sum_wI += w * o.I; + } + if (!(sum_w > 0.0)) + continue; + const double mean = sum_wI / sum_w; + error_model_mean_I[key] = static_cast(mean); + const double I2 = mean * mean; + for (const auto &o: obs) { + const double w = 1.0 / (static_cast(o.sigma) * o.sigma); + const double factor = 1.0 - w / sum_w; + if (factor < 0.05) + continue; + const double resid = static_cast(o.I) - mean; + samples.push_back({static_cast(o.sigma) * o.sigma, I2, resid * resid / factor}); + } + } + + // --- 3. Fit global dev2 = a*sigma^2 + b^2*^2. Bin by intensity (the per-observation + // dev2 is chi-square-1 noisy) and take medians; weight the bins by 1/dev2^2 so it + // is a *relative* fit - otherwise the strong bins (which fix b) swamp the weak + // bins (which fix a) and the weak sigmas stay over-confident. --- + constexpr int n_bins = 16; + if (samples.size() < static_cast(8 * n_bins)) + return; // too little multiplicity to fit -> leave identity + + std::sort(samples.begin(), samples.end(), + [](const Sample &p, const Sample &q) { return p.I2 < q.I2; }); + + auto median = [](std::vector &v) { + std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end()); + return v[v.size() / 2]; + }; + + double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0; + const size_t per = samples.size() / n_bins; + for (int bin = 0; bin < n_bins; ++bin) { + const size_t lo = bin * per; + const size_t hi = (bin == n_bins - 1) ? samples.size() : lo + per; + std::vector vs2, vI2, vd2; + vs2.reserve(hi - lo); vI2.reserve(hi - lo); vd2.reserve(hi - lo); + for (size_t i = lo; i < hi; ++i) { + vs2.push_back(samples[i].s2); + vI2.push_back(samples[i].I2); + vd2.push_back(samples[i].dev2); + } + const double s2 = median(vs2), I2 = median(vI2), d2 = median(vd2); + const double wgt = 1.0 / std::max(d2 * d2, 1e-30); + Ass += wgt * s2 * s2; + AsI += wgt * s2 * I2; + AII += wgt * I2 * I2; + Bs += wgt * s2 * d2; + BI += wgt * I2 * d2; + } + + const double det = Ass * AII - AsI * AsI; + if (std::fabs(det) < 1e-30) + return; + const double a = std::clamp((Bs * AII - BI * AsI) / det, 0.25, 100.0); + const double b2 = std::max((Ass * BI - AsI * Bs) / det, 0.0); + + error_model_a = a; + error_model_b = std::sqrt(b2); + error_model_active = true; +} + bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome, bool cc_mask) { if (reference_cell) { auto cell = outcome.latt.GetUnitCell(); diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index 5f6ca6e8..d69ee817 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -5,6 +5,7 @@ #include #include +#include #include #include "../../common/Logger.h" @@ -72,11 +73,32 @@ class MergeOnTheFly { std::map accumulator; + // Global error model (XDS form): sigma_corr^2 = a*sigma^2 + (b*)^2. a rescales the + // (under-estimated) counting variance; the (b*)^2 term adds the intensity- + // proportional systematic error that counting statistics miss, so strong reflections + // are no longer over-weighted. ISa = 1/b is the asymptotic I/sigma. Refined from the + // scatter of symmetry equivalents (RefineErrorModel); identity until then. + bool error_model_active = false; + double error_model_a = 1.0; + double error_model_b = 0.0; + // The (b*I)^2 term uses the reflection's *mean* intensity (constant over its + // observations), so it inflates sigma without biasing the inverse-variance weights - + // using the per-observation I_i instead would over-weight down-fluctuated points. + std::unordered_map error_model_mean_I; + [[nodiscard]] float CorrectedSigma(float I_corr, float sigma_corr, uint64_t hkl_key) const; + bool Mask(const IntegrationOutcome &outcome, bool cc_mask); public: MergeOnTheFly(const DiffractionExperiment &x); MergeOnTheFly& ReferenceCell(const std::optional &cell); + // Fit the global error model from the spread of symmetry-equivalent observations. + // Call once before merging; AddImage then applies it. + void RefineErrorModel(const std::vector &outcomes); + [[nodiscard]] bool ErrorModelActive() const { return error_model_active; } + [[nodiscard]] double ErrorModelA() const { return error_model_a; } + [[nodiscard]] double ErrorModelB() const { return error_model_b; } + void AddImage(const IntegrationOutcome& outcome, bool cc_mask = false); MergeStatistics MergeStats(const std::vector &merged, diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 61ea164d..5f7671fe 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -997,6 +997,16 @@ int main(int argc, char **argv) { MergeOnTheFly merge_engine(experiment); if (consensus_cell.has_value()) merge_engine.ReferenceCell(*consensus_cell); + // Fit the global error model (sigma'^2 = a*sigma^2 + (b*I)^2, XDS form) from the + // spread of symmetry equivalents before merging, so the merged sigmas reflect the + // real scatter rather than counting statistics alone. + merge_engine.RefineErrorModel(indexer.GetIntegrationOutcome()); + if (merge_engine.ErrorModelActive()) { + const double a = merge_engine.ErrorModelA(); + const double b = merge_engine.ErrorModelB(); + const double isa = (b > 0.0) ? 1.0 / b : std::numeric_limits::infinity(); + logger.Info("Error model: sigma'^2 = {:.3f} sigma^2 + ({:.4f} I)^2 ISa = {:.1f}", a, b, isa); + } for (auto &i : indexer.GetIntegrationOutcome()) merge_engine.AddImage(i); -- 2.52.0 From c93d381dc8fbd38ff7b904f86fa862a622ea8e17 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 12 Jun 2026 18:53:02 +0200 Subject: [PATCH 038/228] Error model: harden the fit against pathological inputs (code review) Addresses code-review findings on RefineErrorModel: - Floor the 1/dev^2 bin weight relative to the data scale (1e-3 of the median bin dev^2), not an absolute 1e-30: a near-zero-scatter bin could otherwise acquire a runaway weight and hijack the global (a,b) fit. - Reject a near-collinear normal-equation system relatively (det > 1e-10*Ass*AII) instead of with an absolute threshold that an ill-conditioned fit can pass. - Reset the model to identity at entry so any early return leaves it inactive rather than keeping a stale a/b alongside a freshly-cleared mean map (which would make CorrectedSigma fall back to the per-observation I). - PixelRefine: correct the orient_prior comment - with the sweep on, the LSQ anchor is the swept orientation (intended), not the spot-centroid one. Verified unchanged on the lyso test set (ISa 1.1, CC1/2 90.3%). Co-Authored-By: Claude Opus 4.8 --- .../pixel_refinement/PixelRefine.cpp | 7 ++-- image_analysis/scale_merge/Merge.cpp | 35 ++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index c398b9d3..bef4392b 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -920,8 +920,11 @@ void PixelRefine::Run(const T *image, BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); - // Anchor for orientation regularization = the spot-centroid orientation we - // started from (captured before any pixel-level refinement moved it). + // Anchor for orientation regularization = the orientation the LSQ starts from + // (captured before the predict<->refine iterations move it). When the global + // sweep ran first this is the swept orientation, not the original spot-centroid + // one - which is intended: the regularizer keeps the LSQ near its own starting + // point, it is not meant to pull a deliberate sweep back. if (iter == 0) for (int i = 0; i < 3; ++i) orient_prior[i] = latt_vec0[i]; diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 3acf2ba4..cfb829e9 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -103,6 +103,14 @@ float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, uint64_t hkl } void MergeOnTheFly::RefineErrorModel(const std::vector &outcomes) { + // Reset to identity up front: every early return below then leaves the model + // inactive (CorrectedSigma returns sigma unchanged) rather than keeping a stale + // a/b from a previous call alongside a freshly-cleared mean map. + error_model_active = false; + error_model_a = 1.0; + error_model_b = 0.0; + error_model_mean_I.clear(); + // --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, // applying exactly the filters AddImage uses. --- struct Obs { float I, sigma; }; @@ -134,7 +142,6 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc // (b*I)^2 term uses the reflection mean, so the mean (not I_i) is the abscissa. --- struct Sample { double s2, I2, dev2; }; std::vector samples; - error_model_mean_I.clear(); for (const auto &[key, obs]: groups) { if (obs.size() < 2) @@ -176,7 +183,9 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc return v[v.size() / 2]; }; - double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0; + // Per-intensity-bin medians of (sigma^2, ^2, dev2). + std::vector bs2, bI2, bd2; + bs2.reserve(n_bins); bI2.reserve(n_bins); bd2.reserve(n_bins); const size_t per = samples.size() / n_bins; for (int bin = 0; bin < n_bins; ++bin) { const size_t lo = bin * per; @@ -188,8 +197,22 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc vI2.push_back(samples[i].I2); vd2.push_back(samples[i].dev2); } - const double s2 = median(vs2), I2 = median(vI2), d2 = median(vd2); - const double wgt = 1.0 / std::max(d2 * d2, 1e-30); + bs2.push_back(median(vs2)); + bI2.push_back(median(vI2)); + bd2.push_back(median(vd2)); + } + + // Relative-weighted (1/dev2^2) least squares for (a, b^2). Floor the weight's dev2 at a + // small fraction of the typical bin dev2: an absolute floor (1e-30) does not stop a + // near-zero-scatter bin from acquiring a runaway weight and hijacking the fit, so the + // floor must scale with the data. The regression target keeps the unfloored dev2. + std::vector bd2_sorted = bd2; + const double dev2_floor = std::max(1e-30, 1e-3 * median(bd2_sorted)); + double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0; + for (int bin = 0; bin < n_bins; ++bin) { + const double s2 = bs2[bin], I2 = bI2[bin], d2 = bd2[bin]; + const double d2w = std::max(d2, dev2_floor); + const double wgt = 1.0 / (d2w * d2w); Ass += wgt * s2 * s2; AsI += wgt * s2 * I2; AII += wgt * I2 * I2; @@ -197,8 +220,10 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc BI += wgt * I2 * d2; } + // Reject a near-collinear (ill-conditioned) system *relatively*: det lies in + // [0, Ass*AII] by Cauchy-Schwarz, so compare against that scale rather than 1e-30. const double det = Ass * AII - AsI * AsI; - if (std::fabs(det) < 1e-30) + if (!(det > 1e-10 * Ass * AII)) return; const double a = std::clamp((Bs * AII - BI * AsI) / det, 0.25, 100.0); const double b2 = std::max((Ass * BI - AsI * Bs) / det, 0.0); -- 2.52.0 From 6f2033db004899c37530bf2431f54e8c1a16054b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 13 Jun 2026 21:34:33 +0200 Subject: [PATCH 039/228] PixelRefine: checkpoint before cleanup (factored model + all diagnostic levers) Snapshot of the messy state: factored likelihood Terms 1+2+3 behind PR_* env flags (PR_INTENSITY/PR_SHAPE/PR_RECENTER) alongside the old per-pixel ShoeboxResidual, plus diagnostic scaffolding (PR_R0/R1/COV/FIX_R0/FIX_R/ ADAPT_R1/CENTROID/RECENTER) and the FACTORED_MODEL.md spec. Next commit makes Terms 1+2 the model and strips all of this. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 34 ++ .../pixel_refinement/FACTORED_MODEL.md | 140 +++++++ .../pixel_refinement/PixelRefine.cpp | 381 +++++++++++++++++- image_analysis/pixel_refinement/PixelRefine.h | 54 +++ 4 files changed, 601 insertions(+), 8 deletions(-) create mode 100644 image_analysis/pixel_refinement/FACTORED_MODEL.md diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index f7cb25c1..ccf084fe 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -446,6 +446,28 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, if (const auto bw = experiment.GetBandwidthFWHM()) prd.bandwidth = bw.value() / 2.3548; // FWHM -> sigma + // TEMPORARY diagnostic knobs to probe the effect of the (currently fixed) spot + // widths R[0] (radial/partiality) and R[1] (tangential/profile). Remove after. + if (const char *r0 = std::getenv("PR_R0")) prd.R[0] = std::stod(r0); + if (const char *r1 = std::getenv("PR_R1")) prd.R[1] = std::stod(r1); + // TEMPORARY: PR_COV refines G,B,R and dumps their per-image correlation matrix. + if (std::getenv("PR_COV")) { + prd.refine_scale = true; prd.refine_B = true; prd.refine_R = true; + prd.compute_covariance = true; + } + if (std::getenv("PR_FIX_R0")) prd.fix_R0 = true; // hold R0, refine R1 only + if (std::getenv("PR_FIX_R")) prd.refine_R = false; // hold R0 and R1: G-B correlation only + if (std::getenv("PR_ADAPT_R1")) prd.adaptive_R1 = true; // measure R1 from spot moments + if (std::getenv("PR_CENTROID")) prd.measure_centroid = true; // observed-vs-predicted offset + if (std::getenv("PR_RECENTER")) prd.recenter_profile = true; // recentre profile on centroid + if (const char *s = std::getenv("PR_RECENTER_SIGNIF")) prd.recenter_min_signif = std::stod(s); + if (std::getenv("PR_INTENSITY")) { // factored-likelihood Term 1: per-reflection intensity residual + prd.intensity_residual = true; + prd.refine_orientation = false; prd.refine_R = false; + prd.refine_scale = true; prd.refine_B = true; + } + if (std::getenv("PR_SHAPE")) prd.shape_R1 = true; // Term 2: per-resolution R1 from spot moments + std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); switch (image.GetMode()) { @@ -465,6 +487,18 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, return false; } + if (prd.covariance_valid) + fprintf(stderr, "[cov] GB=%.3f GR0=%.3f GR1=%.3f BR0=%.3f BR1=%.3f R0R1=%.3f\n", + prd.corr_GB, prd.corr_GR0, prd.corr_GR1, prd.corr_BR0, prd.corr_BR1, prd.corr_R0R1); + if (prd.adaptive_R1) + fprintf(stderr, "[R1] %.5f\n", prd.R[1]); + if (prd.shape_R1 && std::isfinite(prd.shape_R1_lores)) + fprintf(stderr, "[shapeR1] lores=%.5f hires=%.5f\n", prd.shape_R1_lores, prd.shape_R1_hires); + if (prd.measure_centroid && std::isfinite(prd.centroid_lo_tang_c)) + fprintf(stderr, "[res] lo_s=%.1f lo_tc=%.3f lo_tp=%.3f lo_rc=%.3f hi_s=%.1f hi_tc=%.3f hi_tp=%.3f hi_rc=%.3f\n", + prd.centroid_lo_signif, prd.centroid_lo_tang_c, prd.centroid_lo_tang_p, prd.centroid_lo_rad_c, + prd.centroid_hi_signif, prd.centroid_hi_tang_c, prd.centroid_hi_tang_p, prd.centroid_hi_rad_c); + // PixelRefine output flows into the normal save/merge path: the refined // geometry/lattice and the already-scaled reflections become the outcome. i_outcome.reflections = std::move(prd.reflections); diff --git a/image_analysis/pixel_refinement/FACTORED_MODEL.md b/image_analysis/pixel_refinement/FACTORED_MODEL.md new file mode 100644 index 00000000..944514a2 --- /dev/null +++ b/image_analysis/pixel_refinement/FACTORED_MODEL.md @@ -0,0 +1,140 @@ +# A factored likelihood for joint integration + scaling + geometry + +**Status: design spec (not implemented).** Goal: replace the per-pixel least-squares +of PixelRefine with a per-*reflection* likelihood that fuses profile-fit integration, +scaling against the reference, and geometry refinement into one differentiable +objective — the foundation for priors (Bayesian) and learned components (NN), and the +thing that dissolves the empty-pixel and parameter-degeneracy problems by construction +rather than by patching. + +## 0. Notation + +Per image, parameters `θ`: scale `G`, Debye-Waller `B`, orientation + cell (geometry), +profile width `R1` (tangential, possibly a 2×2 tensor), partiality width `R0` +(radial/mosaicity; `R0_eff² = R0² + R_bw²`, `R_bw² = (bλ)²/2d⁴` *known* from bandwidth). +Per reflection `h`: reference intensity `I_ref` (the hypothesis), resolution `d`, +predicted centre `c_pred`, partiality `p = exp(−ε_r²/R0_eff²)`, polarisation `pol`, +`B_term = exp(−B/4d²)`, shoebox pixels `{I_p}` with mean local background `Bg`, and the +area-normalised tangential profile template `P_p = P_tang(ε_t,p; R1)`. + +## 1. The factorisation principle + +A reflection's shoebox carries three (to first order) **orthogonal** pieces of +information — the 0th, 1st and 2nd moments of its intensity distribution: + +| moment | statistic | constrains | +|---|---|---| +| 0th — total | profile-fit amplitude `J` | scale chain `G, B` (and `p`) | +| 1st — position | centroid `c_obs` | geometry (orientation; radial→distance/cell) | +| 2nd — shape | second moment `M₂` | profile width `R1` (and anisotropy) | + +The current per-pixel residual mixes all three into one objective over shared pixels — +*that* is what couples the parameters (measured G–R0 ≈ −0.46, G–R1 ≈ +0.51) and lets the +many empty pixels dominate. Residual-ing each **moment** against its model instead gives +a block-diagonal Jacobian: the couplings vanish because each statistic carries one +parameter block's information. + +## 2. The three residual terms + +### 2.1 Intensity / scaling residual (one scalar per reflection) + +Optimal (Diamond) profile-fit amplitude and its model: +``` +J = Σ_p w_p P_p (I_p − Bg) / Σ_p w_p P_p² w_p = 1/v_p +J_model = G · B_term · p · pol · I_ref +r¹_h = (J − J_model) / σ_J +``` +`J` is ~invariant to `R1` (a well-sampled spot integrates to the same total whatever +width is assumed) → **R1 leaves this residual**. Empty pixels make no residual; they +enter only through `J` with ~zero profile weight → **the empty-pixel problem is gone by +construction.** This residual *is* the scaling residual — integration and scaling are now +one objective. + +### 2.2 Shape residual (constrains R1; decoupled from scale) + +``` +M₂_obs = Σ_p (I_p − Bg) ε_t,p² / Σ_p (I_p − Bg) (intensity-weighted variance, Å⁻²) +M₂_model = R1² / 2 (variance of exp(−ε_t²/R1²)) +r²_h = (M₂_obs − M₂_model) / σ_M2 +``` +A moment is normalised by the total → **scale-invariant → `∂r²/∂G = 0`**. The G↔R1 +degeneracy disappears. Anisotropic extension: use the 2×2 moment tensor +`Σ(I−Bg)(ε_t⊗ε_t)/Σ(I−Bg)` vs `diag(R1a²/2, R1b²/2)` → elliptical R1 (the DMM streak). +Weak spots have huge `σ_M2` → contribute ~nothing → R1 is set by strong spots +automatically (and may be made `R1(d)` per resolution). + +### 2.3 Position residual (constrains geometry; decoupled from scale and shape) + +``` +c_obs = Σ_p (I_p − Bg)(x_p, y_p) / Σ_p (I_p − Bg) +r³_h = (c_obs − c_pred(geometry)) / σ_c (2-vector; split radial / tangential) +``` +Centroid is scale- and width-invariant → `∂r³/∂G = ∂r³/∂R1 ≈ 0`. The **radial** component +constrains distance/cell, the **tangential** constrains orientation — exactly the split +the diagnostic measured (radial≈0 = no distance error; tangential∝radius = orientation). + +## 3. Fisher / expected-variance weighting (makes it a likelihood) + +Every `σ` uses the **model-expected** variance, never observed counts — this is what +makes strong *expected* reflections carry the information and makes the model "feel pain +when something that should be there is not": +``` +v_p = Bg + J_model · P_p (background + expected signal from I_ref, not I_obs) +σ_J² = 1 / Σ_p (P_p² / v_p) +σ_M2 ≈ M₂ · √(2 / N_eff), σ_c ≈ R1 / √(N_eff), N_eff = (Σ(I−Bg))² / Σ v_p +``` +Fisher information about `G` from term 1 is `∝ (B_term·p·pol·I_ref)² / σ_J²` — driven by +`I_ref`, so a noise spike (high counts, low `I_ref`) gets *no* weight while a strong +expected reflection observed absent (`J≈0`, large residual, moderate `σ_J`) gets a large +penalty. The reference enters at maximum leverage: it sets both the target and the weight. + +## 4. Joint objective and priors + +``` +L(θ) = Σ_h [ (r¹_h)² + (r²_h)² + |r³_h|² ] + priors +``` +No free λ if the σ's are correct — the relative weighting *is* the Fisher information. +Priors are the Bayesian hooks and the principled degeneracy breaks: +- **R0 (partiality/mosaicity) is GLOBAL + prior.** R0 multiplies `J_model` (`p`), so it is + still degenerate with the per-image `G` *within term 1* — the one degeneracy the + factorisation does **not** remove. Resolve it physically, not with a directional G prior + (which would bias every output intensity): `R0 ~ N(mosaicity, σ)`, `R_bw` fixed from the + known bandwidth, and `R0` fit **globally** (one per crystal, from many reflections' + partiality distribution) so per-image G can't trade against it. +- orientation `~ N(spot-centroid, σ)`; `G ~ N(1, σ_G)` or tied to the beam monitor; + distance `~ N(nominal, σ_L)` (loose, since serial/jet alignment is poorly constrained). +- Optional Bayesian intensities: treat `I_true` as a parameter with the reference as its + prior → posterior over intensities, not point estimates. + +## 5. Why the degeneracies vanish (Jacobian structure) + +`Jᵀ W J` is approximately block-diagonal in `(G,B,p | R1 | geometry)`: +``` +∂r¹/∂{G,B,p} ≠ 0 ; ∂r¹/∂R1 ≈ 0 ; ∂r¹/∂geom ≈ 0 +∂r²/∂R1 ≠ 0 ; ∂r²/∂G = 0 ; ∂r²/∂geom ≈ 0 +∂r³/∂geom ≠ 0 ; ∂r³/∂G = 0 ; ∂r³/∂R1 ≈ 0 +``` +So G↔R1 (+0.51) and all the cross-couplings drop to ~0 by construction. Only **G↔R0** +survives (R0 is a scale-multiplier, not a shape), handled by the global+physical prior of +§4. The degeneracies we measured were artifacts of projecting all information onto a +single per-pixel residual. + +## 6. Implementation notes + +- Per reflection: 1 (intensity) + 1 (shape) + 2 (position) residuals = 4, vs ~49 per-pixel + residuals → **cheaper**, and Ceres autodiffs the moment formulas through the pixels. +- The per-pixel forward model still *defines* `P_tang`, `p`, etc.; the **loss** moves to + the moments. +- Geometry (term 3) can run as the global sweep we have (it already maximises a + position/CC objective); terms 1–2 are the per-image photometry. Or solve all three + jointly per image with the global R0/mosaicity shared across images (two-level fit). +- Drop-in path: keep the current extraction, add the three residuals as a new objective + behind a flag, compare against the per-pixel loss on both test crystals. + +## 7. Why this serves the goal + +It is one differentiable likelihood, factored along the physics, that (a) maximises use of +the reference (target + Fisher weight), (b) is the substrate for priors / posteriors over +intensities (Bayesian), and (c) lets any term — profile `P`, partiality `p`, corrections — +be replaced by a learned function trained through the same likelihood. That is the +qualitative move XDS-style empirical profile fitting cannot make. diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index bef4392b..42e8772f 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include "../geom_refinement/LatticeReduction.h" @@ -35,6 +36,8 @@ struct ReflGroup { double pol; // per-reflection polarization correction (raw = true * pol) double Ibkg; // local flat background (raw counts, constant over the shoebox) double predicted_x, predicted_y; + double R1_eff = 0.0; // tangential profile width to use (Term 2; 0 => fall back to data.R[1]) + double dcx = 0.0, dcy = 0.0; // Term 3: profile recentre shift (observed centroid - predicted) std::vector pixels; }; @@ -752,6 +755,30 @@ void PixelRefine::SweepOrientationCell(const T *image, BraggPrediction &predicti data.latt = best; } +// --------------------------------------------------------------------------- +// Term 1 of the factored likelihood (FACTORED_MODEL.md): the per-reflection +// *intensity* (0th-moment) residual. The profile-fit amplitude J should equal the +// scaled reference J_model = G * exp(-B/4d^2) * partiality * pol * I_ref. One scalar +// residual per reflection, weighted by the model-expected (Fisher) sigma_J. This is +// the scaling residual - integration and scaling become one objective, and the empty +// pixels (which make no residual of their own) stop dominating the fit. With geometry +// and R held fixed, J, partiality and sigma_J are constants, so only G and B are free. +// --------------------------------------------------------------------------- +struct IntensityResidual { + IntensityResidual(double J, double sigma_J, double partiality, double pol, + double I_ref, double inv_4d2) + : J(J), inv_sigma(1.0 / sigma_J), partiality(partiality), pol(pol), + I_ref(I_ref), inv_4d2(inv_4d2) {} + template + bool operator()(const T *const G, const T *const B, T *residual) const { + const T B_term = ceres::exp(-B[0] * T(inv_4d2)); + const T J_model = G[0] * B_term * T(partiality) * T(pol) * T(I_ref); + residual[0] = (J_model - T(J)) * T(inv_sigma); + return true; + } + double J, inv_sigma, partiality, pol, I_ref, inv_4d2; +}; + template void PixelRefine::Run(const T *image, BraggPrediction &prediction, @@ -929,12 +956,146 @@ void PixelRefine::Run(const T *image, for (int i = 0; i < 3; ++i) orient_prior[i] = latt_vec0[i]; + // ---- Term 3: per-reflection recentre on the observed centroid ---------------- + // The geometry predicts the spot to ~0.4 px (per-reflection scatter a global fit + // cannot remove); a tight Term-2 template centred on the prediction then sits off + // the real spot. For confident spots, shift the profile centre to the observed + // centroid (used consistently by Term 2, Term 1 and the extraction below). Weak + // spots keep the prediction (recentring on a noise centroid would bias them). + for (auto &g : groups) { + g.dcx = 0.0; + g.dcy = 0.0; + } + if (data.recenter_profile && !groups.empty()) { + const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); + for (auto &g : groups) { + double sw = 0.0, swx = 0.0, swy = 0.0; + for (const auto &px : g.pixels) { + const double w = std::max(px.Iobs - g.Ibkg, 0.0); + sw += w; swx += w * px.x; swy += w * px.y; + } + if (sw <= 0.0) + continue; + if (sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)) < data.recenter_min_signif) + continue; + double dcx = swx / sw - g.predicted_x, dcy = swy / sw - g.predicted_y; + const double dl = std::sqrt(dcx * dcx + dcy * dcy); + if (dl > 2.0) { dcx *= 2.0 / dl; dcy *= 2.0 / dl; } // clamp runaway centroids + g.dcx = dcx; + g.dcy = dcy; + } + } + + // ---- Term 2: per-resolution tangential profile width R1 from spot shapes ------ + // Default: every reflection uses the global R1; with shape_R1 on, override it with + // R1 = sqrt(2*) from the intensity-weighted second moment of the strong + // spots, binned by resolution (low res small spots, high res larger). A shape + // statistic - normalised by the total, so decoupled from the per-image scale. + for (auto &g : groups) + g.R1_eff = data.R[1]; + if (data.shape_R1 && !groups.empty()) { + constexpr int n_bins = 6; + const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); + double s2min = 1e30, s2max = 0.0; + for (const auto &g : groups) { + const double s2 = 1.0 / (g.d * g.d); + s2min = std::min(s2min, s2); + s2max = std::max(s2max, s2); + } + const double span = std::max(s2max - s2min, 1e-12); + auto bin_of = [&](double d) { + return std::clamp(static_cast((1.0 / (d * d) - s2min) / span * n_bins), 0, n_bins - 1); + }; + std::vector> bin_M2(n_bins); + for (const auto &g : groups) { + double sw = 0.0, sw_et2 = 0.0; + for (const auto &px : g.pixels) { + PixelObs probe{px.x - g.dcx, px.y - g.dcy, 0.0, g.Ibkg, 1.0}; + PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, + g.R_bw_sq, g.pol, data.crystal_system); + double q_sq, eps_r, eps_t_sq; + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, + latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + continue; + const double w = std::max(px.Iobs - g.Ibkg, 0.0); + sw += w; + sw_et2 += w * eps_t_sq; + } + if (sw <= 0.0) + continue; + const double signif = sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)); + if (signif >= 5.0) // only well-measured spots define the shape + bin_M2[bin_of(g.d)].push_back(sw_et2 / sw); + } + std::vector bin_R1(n_bins, data.R[1]); + for (int b = 0; b < n_bins; ++b) + if (bin_M2[b].size() >= 5) { + const double r1 = std::sqrt(2.0 * std::max(MedianInPlace(bin_M2[b]), 0.0)); + if (std::isfinite(r1) && r1 > 1e-4) + bin_R1[b] = std::clamp(r1, 1e-4, 0.05); + } + for (auto &g : groups) + g.R1_eff = bin_R1[bin_of(g.d)]; + data.shape_R1_lores = bin_R1[0]; // lowest resolution bin + data.shape_R1_hires = bin_R1[n_bins - 1]; // highest resolution bin + } + // ---- 4. Build the problem --------------------------------------------- // One residual block per shoebox (N residuals), so the expensive // per-reflection node geometry is evaluated once per reflection instead // of once per pixel. ceres::Problem problem; size_t residual_pixels = 0; + if (data.intensity_residual) { + // Term-1 path: one per-reflection intensity residual. Geometry & R fixed, so + // J / partiality / sigma_J are computed here as constants and only G, B vary. + const double R0 = data.R[0]; + for (const auto &g : groups) { + const double R1 = g.R1_eff; // Term 2: per-resolution profile width + double num = 0.0, den = 0.0, rad = 0.0; + std::vector> pt_sig; // (P_t, Iobs-Bg) for Fisher pass + pt_sig.reserve(g.pixels.size()); + for (const auto &px : g.pixels) { + PixelObs probe{px.x - g.dcx, px.y - g.dcy, 0.0, g.Ibkg, 1.0}; // Term 3 recentre + PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, + g.R_bw_sq, g.pol, data.crystal_system); + double q_sq, eps_r, eps_t_sq; + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, + latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + continue; + if (!(R1 > 0.0) || !(R0 > 0.0)) + continue; + const double P_t = std::exp(-eps_t_sq / (R1 * R1)) / (M_PI * R1 * R1); + const double R0_eff_sq = R0 * R0 + g.R_bw_sq; + const double P_rad = std::exp(-eps_r * eps_r / R0_eff_sq); + const double v = std::max(g.Ibkg, 1.0); + const double sig = px.Iobs - g.Ibkg; + num += P_t * sig / v; + den += P_t * P_t / v; + rad += P_rad * P_t * P_t / v; + pt_sig.emplace_back(P_t, sig); + } + if (!(den > 0.0)) + continue; + const double J = num / den; + const double partiality = rad / den; + // Model-expected (Fisher) variance: v_p = background + expected signal J*P_t, + // not the per-pixel observed counts (which down-bias) - so the weight tracks + // information, and an expected-strong reflection that is absent hurts. + double den_f = 0.0; + for (const auto &[P_t, sig] : pt_sig) { + const double v_f = std::max(g.Ibkg + std::max(J, 0.0) * P_t, 1.0); + den_f += P_t * P_t / v_f; + } + const double sigma_J = std::sqrt(1.0 / std::max(den_f, 1e-30)); + const double inv_4d2 = (g.d > 0.0) ? 1.0 / (4.0 * g.d * g.d) : 0.0; + auto *cost = new ceres::AutoDiffCostFunction( + new IntensityResidual(J, sigma_J, partiality, g.pol, g.Itrue, inv_4d2)); + problem.AddResidualBlock(cost, nullptr, &data.scale_factor, &data.B_factor); + ++residual_pixels; + } + data.residual_count = residual_pixels; + } else { for (const auto &g : groups) { auto *cost = new ceres::DynamicAutoDiffCostFunction( new ShoeboxResidual(g, lambda, pixel_size, data.crystal_system)); @@ -959,8 +1120,23 @@ void PixelRefine::Run(const T *image, residual_pixels += g.pixels.size(); } data.residual_count = residual_pixels; + } // ---- 5. Constrain / bound parameter blocks ---------------------------- + if (data.intensity_residual) { + // Only G and B are in this problem; geometry/R are not parameters here. + problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); + if (!data.refine_B) + problem.SetParameterBlockConstant(&data.B_factor); + // Regularize G->1, weight sqrt(n_refl/sigma): commensurate because the data + // term is now one residual per reflection (unlike the per-pixel path). + if (data.scale_reg_sigma > 0.0 && !groups.empty()) { + const double w = std::sqrt(static_cast(groups.size()) / data.scale_reg_sigma); + auto *reg = new ceres::AutoDiffCostFunction( + new ScalarRegularizer(w, 1.0)); + problem.AddResidualBlock(reg, nullptr, &data.scale_factor); + } + } else { if (!data.refine_orientation) { problem.SetParameterBlockConstant(latt_vec0); } else if (data.orient_reg_sigma_deg > 0.0) { @@ -1029,11 +1205,17 @@ void PixelRefine::Run(const T *image, problem.SetParameterBlockConstant(&data.B_factor); if (data.refine_R) { - problem.SetParameterLowerBound(data.R, 0, 1e-5); - problem.SetParameterLowerBound(data.R, 1, 1e-5); + if (data.fix_R0) { + // Diagnostic: hold R0 constant, refine R1 only. + problem.SetManifold(data.R, new ceres::SubsetManifold(2, {0})); + } else { + problem.SetParameterLowerBound(data.R, 0, 1e-5); + problem.SetParameterLowerBound(data.R, 1, 1e-5); + } } else { problem.SetParameterBlockConstant(data.R); } + } // end per-pixel (non-intensity_residual) constraints // ---- 6. Solve (or, for max_iterations<=0, just evaluate the cost) ----- // Evaluate-only is the live-residual path: it reports the current cost @@ -1057,6 +1239,50 @@ void PixelRefine::Run(const T *image, data.final_cost = summary.final_cost; data.solved = summary.IsSolutionUsable(); + + // Diagnostic: Pearson correlations on the final solve. Always needs G and B + // refined for G-B; the R correlations are added only when R is also refined. + if (data.compute_covariance && data.solved && iter == n_iter - 1 && + data.refine_scale && data.refine_B) { + ceres::Covariance::Options copt; + copt.algorithm_type = ceres::DENSE_SVD; + copt.null_space_rank = -1; // tolerate (and reveal) degenerate directions + ceres::Covariance cov(copt); + std::vector> blocks = { + {&data.scale_factor, &data.scale_factor}, {&data.B_factor, &data.B_factor}, + {&data.scale_factor, &data.B_factor}}; + if (data.refine_R) { + blocks.push_back({data.R, data.R}); + blocks.push_back({&data.scale_factor, data.R}); + blocks.push_back({&data.B_factor, data.R}); + } + if (cov.Compute(blocks, &problem)) { + double cGG, cBB, cGB; + cov.GetCovarianceBlock(&data.scale_factor, &data.scale_factor, &cGG); + cov.GetCovarianceBlock(&data.B_factor, &data.B_factor, &cBB); + cov.GetCovarianceBlock(&data.scale_factor, &data.B_factor, &cGB); + const double sG = std::sqrt(std::max(cGG, 0.0)); + const double sB = std::sqrt(std::max(cBB, 0.0)); + auto rho = [](double c, double s1, double s2) { + return (s1 > 0.0 && s2 > 0.0) ? c / (s1 * s2) : NAN; + }; + data.corr_GB = rho(cGB, sG, sB); + if (data.refine_R) { + double cRR[4], cGR[2], cBR[2]; + cov.GetCovarianceBlock(data.R, data.R, cRR); + cov.GetCovarianceBlock(&data.scale_factor, data.R, cGR); + cov.GetCovarianceBlock(&data.B_factor, data.R, cBR); + const double sR0 = std::sqrt(std::max(cRR[0], 0.0)); + const double sR1 = std::sqrt(std::max(cRR[3], 0.0)); + data.corr_GR0 = rho(cGR[0], sG, sR0); + data.corr_GR1 = rho(cGR[1], sG, sR1); + data.corr_BR0 = rho(cBR[0], sB, sR0); + data.corr_BR1 = rho(cBR[1], sB, sR1); + data.corr_R0R1 = rho(cRR[1], sR0, sR1); + } + data.covariance_valid = true; + } + } } // ---- 7. Write refined geometry + lattice back into data --------------- @@ -1096,6 +1322,139 @@ void PixelRefine::Run(const T *image, } } // predict<->refine iterations + // ---- Adaptive integration mask -------------------------------------------- + // Measure R1 (tangential profile width) from the intensity-weighted tangential + // second moment of the strong spots, rather than fitting it. R1 is a *shape* + // statistic: sigma_t^2 = sum_p (I_p-B) eps_t,p^2 / sum_p (I_p-B), normalised by the + // total so it is independent of the per-image scale - which is exactly what breaks + // the R1<->G degeneracy (a measured width cannot be traded against G). One value per + // image here (from the strong, mostly low-res spots); a per-resolution version is the + // natural next step for the high-res / DMM-streak shapes. + if (data.adaptive_R1 && !groups.empty()) { + std::vector itrue; + itrue.reserve(groups.size()); + for (const auto &g : groups) + itrue.push_back(g.Itrue); + const size_t cut_idx = itrue.size() * 7 / 10; // keep the strongest ~30% + std::nth_element(itrue.begin(), itrue.begin() + cut_idx, itrue.end()); + const double itrue_cut = itrue[cut_idx]; + + std::vector sigma_t2; + for (const auto &g : groups) { + if (g.Itrue < itrue_cut) + continue; + double sw = 0.0, sw_et2 = 0.0; + for (const auto &px : g.pixels) { + PixelObs probe{px.x, px.y, 0.0, g.Ibkg, 1.0}; + PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, + g.R_bw_sq, g.pol, data.crystal_system); + double q_sq, eps_r, eps_t_sq; + if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, + latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + continue; + const double w = std::max(px.Iobs - g.Ibkg, 0.0); + sw += w; + sw_et2 += w * eps_t_sq; + } + if (sw > 0.0) + sigma_t2.push_back(sw_et2 / sw); + } + if (sigma_t2.size() >= 5) { + const double r1 = std::sqrt(2.0 * MedianInPlace(sigma_t2)); // R1^2 = 2 sigma_t^2 + if (std::isfinite(r1) && r1 > 1e-5) + data.R[1] = r1; + } + } + + // ---- Centering diagnostic -------------------------------------------------- + // Observed-centroid vs predicted-position offset for the strong spots, after all + // refinement. Large rms (relative to the spot size) means a tight profile mask + // sits off the real spot - which is why a generous box can beat profile fitting. + if (data.measure_centroid && !groups.empty()) { + const double beam_x = data.geom.GetBeamX_pxl(); + const double beam_y = data.geom.GetBeamY_pxl(); + const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); + // Raw pixel value (sentinel/bounds safe) for the parabolic peak fit. A constant + // background cancels in the parabola, so no need to subtract it here. + auto pix_val = [&](int x, int y) -> double { + if (x < 0 || x >= static_cast(xpixel) || y < 0 || y >= static_cast(ypixel)) + return std::numeric_limits::quiet_NaN(); + const size_t np = static_cast(xpixel) * y + x; + if (image[np] == std::numeric_limits::max()) + return std::numeric_limits::quiet_NaN(); + if (std::is_signed_v && image[np] == std::numeric_limits::min()) + return std::numeric_limits::quiet_NaN(); + return static_cast(image[np]); + }; + struct Off { double signif, tang_c, tang_p, rad_c; }; + std::vector offs; + double sdx = 0.0, sdy = 0.0, sd2 = 0.0; + size_t nc = 0; + for (const auto &g : groups) { + double sw = 0.0, swx = 0.0, swy = 0.0, bmax = -1e30; + int bx = 0, by = 0; + for (const auto &px : g.pixels) { + const double s = px.Iobs - g.Ibkg; + const double w = std::max(s, 0.0); + sw += w; swx += w * px.x; swy += w * px.y; + if (s > bmax) { bmax = s; bx = static_cast(std::lround(px.x)); by = static_cast(std::lround(px.y)); } + } + if (sw <= 0.0) + continue; + const double signif = sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)); + if (signif < 5.0) + continue; // a measurable spot at any resolution (not just the strong low-res ones) + // Sub-pixel peak (mode): parabola through the brightest pixel and its two + // neighbours per axis. The mode tracks the prediction even when an asymmetric + // tail drags the centroid (mean) sideways - so peak vs centroid separates a + // shape asymmetry from a true position error. + double peak_x = bx, peak_y = by; + { const double l = pix_val(bx - 1, by), c = pix_val(bx, by), r = pix_val(bx + 1, by); + const double den = l - 2.0 * c + r; + if (std::isfinite(den) && den < -1e-9) + peak_x = bx + std::clamp(0.5 * (l - r) / den, -1.0, 1.0); } + { const double l = pix_val(bx, by - 1), c = pix_val(bx, by), r = pix_val(bx, by + 1); + const double den = l - 2.0 * c + r; + if (std::isfinite(den) && den < -1e-9) + peak_y = by + std::clamp(0.5 * (l - r) / den, -1.0, 1.0); } + const double dcx = swx / sw - g.predicted_x, dcy = swy / sw - g.predicted_y; + const double dpx = peak_x - g.predicted_x, dpy = peak_y - g.predicted_y; + sdx += dcx; sdy += dcy; sd2 += dcx * dcx + dcy * dcy; ++nc; + const double rx = g.predicted_x - beam_x, ry = g.predicted_y - beam_y; + const double rr = std::sqrt(rx * rx + ry * ry); + if (rr < 1.0) + continue; + const double rad_c = (dcx * rx + dcy * ry) / rr; // signed radial (outward +) + const double tang_c = (dcx * -ry + dcy * rx) / rr; // signed tangential + const double tang_p = (dpx * -ry + dpy * rx) / rr; + offs.push_back({signif, std::fabs(tang_c), std::fabs(tang_p), rad_c}); + } + if (nc >= 5) { + data.centroid_bias_px = std::sqrt((sdx / nc) * (sdx / nc) + (sdy / nc) * (sdy / nc)); + data.centroid_rms_px = std::sqrt(sd2 / nc); + } + if (offs.size() >= 10) { + std::vector sig; + sig.reserve(offs.size()); + for (const auto &o : offs) + sig.push_back(o.signif); + std::nth_element(sig.begin(), sig.begin() + sig.size() / 2, sig.end()); + const double smed = sig[sig.size() / 2]; + double slo = 0, shi = 0, tclo = 0, tchi = 0, tplo = 0, tphi = 0, rclo = 0, rchi = 0; + int nlo = 0, nhi = 0; + for (const auto &o : offs) { + if (o.signif < smed) { slo += o.signif; tclo += o.tang_c; tplo += o.tang_p; rclo += o.rad_c; ++nlo; } + else { shi += o.signif; tchi += o.tang_c; tphi += o.tang_p; rchi += o.rad_c; ++nhi; } + } + if (nlo > 0 && nhi > 0) { + data.centroid_lo_signif = slo / nlo; data.centroid_hi_signif = shi / nhi; + data.centroid_lo_tang_c = tclo / nlo; data.centroid_hi_tang_c = tchi / nhi; + data.centroid_lo_tang_p = tplo / nlo; data.centroid_hi_tang_p = tphi / nhi; + data.centroid_lo_rad_c = rclo / nlo; data.centroid_hi_rad_c = rchi / nhi; + } + } + } + // ---- Extract integrated reflections --------------------------------------- // Profile fitting gives the recorded amplitude (fitting the tangential profile // P_t against the background-subtracted pixels): @@ -1138,26 +1497,32 @@ void PixelRefine::Run(const T *image, // Debye-Waller factor for this reflection (constant over its shoebox). const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); + // Term 3 recentre shift (precomputed in the pre-pass; 0 if off or the spot was not + // confident). Profile evaluated at (x-dcx, y-dcy), data summed at (x,y). + const double dcx = g.dcx, dcy = g.dcy; + double num = 0.0, den = 0.0, bkg_sum = 0.0, radial_sum = 0.0; double prof_live = 0.0, prof_full = 0.0; // tangential profile: captured / total size_t n = 0; for (int y = cy - radius; y <= cy + radius; ++y) { for (int x = cx - radius; x <= cx + radius; ++x) { - // Geometry/profile for this grid point (valid even off the detector). - PixelObs probe{static_cast(x), static_cast(y), 0.0, g.Ibkg, 1.0}; + // Geometry/profile for this grid point (profile recentred by (dcx,dcy)). + PixelObs probe{static_cast(x) - dcx, static_cast(y) - dcy, + 0.0, g.Ibkg, 1.0}; PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, g.R_bw_sq, g.pol, data.crystal_system); double q_sq, eps_r, eps_t_sq; if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) continue; - if (!(data.R[0] > 0.0) || !(data.R[1] > 0.0)) + if (!(data.R[0] > 0.0) || !(g.R1_eff > 0.0)) continue; - // Tangential profile shape (area-normalized) -> the fit template. - const double P_t = std::exp(-eps_t_sq / (data.R[1] * data.R[1])) - / (M_PI * data.R[1] * data.R[1]); + // Tangential profile shape (area-normalized) -> the fit template. Uses the + // per-reflection R1_eff (Term 2), falling back to the global R1 by default. + const double R1 = g.R1_eff; + const double P_t = std::exp(-eps_t_sq / (R1 * R1)) / (M_PI * R1 * R1); prof_full += P_t; // whole shoebox, on- or off-detector // Only real, unmasked detector pixels carry signal. diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index fc4acabd..f8db193f 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -183,6 +183,56 @@ struct PixelRefineData { int bkg_outer_radius = 6; int max_iterations = 3; // inner predict<->refine cycles (re-predict with refined geom/latt) + // Diagnostic: compute the parameter correlation matrix (Pearson) from the final + // per-image solve, to expose degeneracies (e.g. G<->B, G<->R1) that let the fit + // lower its chi-square along directions that do not generalise across images. + // Requires G, B and R all refined; results in corr_* below. + bool compute_covariance = false; + bool fix_R0 = false; // diagnostic: with refine_R, hold R0 constant and refine R1 only + + // Factored-likelihood Term 1 (FACTORED_MODEL.md): replace the per-pixel fit with one + // per-reflection *intensity* residual J vs G*B_term*partiality*pol*I_ref, Fisher- + // weighted. Geometry & R fixed; only G (and B if refine_B) are fit. + bool intensity_residual = false; + + // Factored-likelihood Term 2: set the tangential profile width R1 from the *measured* + // per-resolution second moment of the strong spots (a shape statistic, decoupled from + // the scale) and feed it to the profile template used by Term 1 and the extraction. + bool shape_R1 = false; + double shape_R1_lores = NAN, shape_R1_hires = NAN; // measured R1 in the lowest/highest-res bin (diag) + + // Adaptive integration mask: set R1 (tangential profile width) from the *measured* + // tangential second moment of the strong spots, instead of fitting it (which is + // degenerate with the per-image scale). A shape statistic, independent of scale. + bool adaptive_R1 = false; + + // Diagnostic: residual centering error after refinement. For the strong spots, + // the offset between observed intensity centroid and predicted position - bias is + // the systematic (mean) part, rms the total. If rms is comparable to the spot size, + // a tight profile mask lands off the spot and box-summing wins. + bool measure_centroid = false; + double centroid_bias_px = NAN; + double centroid_rms_px = NAN; + // Offset split by spot significance (lo/hi about the median) and measured two ways: + // the intensity *centroid* (mean) and the sub-pixel *peak* (mode, parabolic fit). + // - centroid offset shrinking lo->hi => noise floor (offset ~ 1/sqrt(counts)); + // flat with significance => a systematic position error to model. + // - peak offset << centroid offset => the spot is asymmetric (non-Gaussian / + // parallax): the prediction sits on the peak, only the centroid is pulled, so + // recentring on the centroid would be wrong - the shape is what to model. + // - radial centroid offset ~ 0 => no distance/parallax error (parallax is radial). + double centroid_lo_signif = NAN, centroid_hi_signif = NAN; // mean significance per bin + double centroid_lo_tang_c = NAN, centroid_hi_tang_c = NAN; // mean |tangential centroid offset| + double centroid_lo_tang_p = NAN, centroid_hi_tang_p = NAN; // mean |tangential peak offset| + double centroid_lo_rad_c = NAN, centroid_hi_rad_c = NAN; // mean signed radial centroid offset + + // Test: recentre the extraction profile on each spot's observed centroid (instead of + // the predicted position) so a tight mask lands on the real spot - but only for spots + // whose in-shoebox significance exceeds recenter_min_signif (recentring on a noise + // centroid would bias weak reflections positive). + bool recenter_profile = false; + double recenter_min_signif = 5.0; + // --- output --- std::vector reflections; // profile-fitted integration result bool solved = false; @@ -190,6 +240,10 @@ struct PixelRefineData { size_t residual_count = 0; double cc = NAN; // per-image CC of scaled intensities vs reference int64_t cc_n = 0; // number of reflections in the CC + + bool covariance_valid = false; + double corr_GB = NAN, corr_GR0 = NAN, corr_GR1 = NAN; + double corr_BR0 = NAN, corr_BR1 = NAN, corr_R0R1 = NAN; }; class PixelRefine { -- 2.52.0 From 100fe7b7e7f56153f0d508f8bdfc7303077f5612 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 13 Jun 2026 22:02:18 +0200 Subject: [PATCH 040/228] PixelRefine: make factored Terms 1+2 the model, remove old wiring PixelRefine is now an intensity-only operation: geometry is fixed (refined upstream by XtalOptimizer) and the only objective is the factored per-reflection likelihood (FACTORED_MODEL.md Terms 1+2) - measured per-resolution profile width R1 plus one Fisher-weighted intensity/scaling residual per reflection, fitting the per-image scale G and B. Validated on crystal 2 (fixed_master.h5 as stills, 1.7 A): CC1/2 84-92%, CCref 77-92%, flat - reproduces the env-flag prototype and matches the rotation path from the stills path. Removed: - the per-pixel ShoeboxResidual loss and PixelResidual cost functor; - all in-PixelRefine geometry refinement (orientation/cell/beam/distance/R), the regularised-orientation LSQ, signal-weighting, and the global sweep; - Term 3 (per-spot recentring) - a confirmed no-op on both crystals; - the diagnostic scaffolding (covariance, centroid, adaptive_R1) and the PR_* env knobs + stderr dumps in IndexAndRefine; - the PredictImage/ChiSquaredImage renderers and the entire viewer PixelRefine window/table/params + worker bindings + shoebox overlay. The sweep box-integrator background median became mean (consistency) by virtue of removing the sweep. METHODS.md rewritten for the current model; findings recorded in FINDINGS-2026-06.md. Net -2200 lines. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 34 - .../pixel_refinement/FACTORED_MODEL.md | 4 +- .../pixel_refinement/FINDINGS-2026-06.md | 122 ++ image_analysis/pixel_refinement/METHODS.md | 392 ++--- .../pixel_refinement/PixelRefine.cpp | 1557 +++-------------- image_analysis/pixel_refinement/PixelRefine.h | 316 +--- viewer/CMakeLists.txt | 5 - viewer/JFJochImageReadingWorker.cpp | 294 ---- viewer/JFJochImageReadingWorker.h | 39 +- viewer/JFJochViewerWindow.cpp | 38 - viewer/image_viewer/JFJochImage.h | 5 +- viewer/image_viewer/JFJochSimpleImage.cpp | 24 - viewer/image_viewer/JFJochSimpleImage.h | 6 - .../windows/JFJochPixelRefineTableWindow.cpp | 101 -- viewer/windows/JFJochPixelRefineTableWindow.h | 34 - viewer/windows/JFJochPixelRefineWindow.cpp | 233 --- viewer/windows/JFJochPixelRefineWindow.h | 79 - viewer/windows/PixelRefineParams.h | 66 - 18 files changed, 569 insertions(+), 2780 deletions(-) create mode 100644 image_analysis/pixel_refinement/FINDINGS-2026-06.md delete mode 100644 viewer/windows/JFJochPixelRefineTableWindow.cpp delete mode 100644 viewer/windows/JFJochPixelRefineTableWindow.h delete mode 100644 viewer/windows/JFJochPixelRefineWindow.cpp delete mode 100644 viewer/windows/JFJochPixelRefineWindow.h delete mode 100644 viewer/windows/PixelRefineParams.h diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index ccf084fe..f7cb25c1 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -446,28 +446,6 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, if (const auto bw = experiment.GetBandwidthFWHM()) prd.bandwidth = bw.value() / 2.3548; // FWHM -> sigma - // TEMPORARY diagnostic knobs to probe the effect of the (currently fixed) spot - // widths R[0] (radial/partiality) and R[1] (tangential/profile). Remove after. - if (const char *r0 = std::getenv("PR_R0")) prd.R[0] = std::stod(r0); - if (const char *r1 = std::getenv("PR_R1")) prd.R[1] = std::stod(r1); - // TEMPORARY: PR_COV refines G,B,R and dumps their per-image correlation matrix. - if (std::getenv("PR_COV")) { - prd.refine_scale = true; prd.refine_B = true; prd.refine_R = true; - prd.compute_covariance = true; - } - if (std::getenv("PR_FIX_R0")) prd.fix_R0 = true; // hold R0, refine R1 only - if (std::getenv("PR_FIX_R")) prd.refine_R = false; // hold R0 and R1: G-B correlation only - if (std::getenv("PR_ADAPT_R1")) prd.adaptive_R1 = true; // measure R1 from spot moments - if (std::getenv("PR_CENTROID")) prd.measure_centroid = true; // observed-vs-predicted offset - if (std::getenv("PR_RECENTER")) prd.recenter_profile = true; // recentre profile on centroid - if (const char *s = std::getenv("PR_RECENTER_SIGNIF")) prd.recenter_min_signif = std::stod(s); - if (std::getenv("PR_INTENSITY")) { // factored-likelihood Term 1: per-reflection intensity residual - prd.intensity_residual = true; - prd.refine_orientation = false; prd.refine_R = false; - prd.refine_scale = true; prd.refine_B = true; - } - if (std::getenv("PR_SHAPE")) prd.shape_R1 = true; // Term 2: per-resolution R1 from spot moments - std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); switch (image.GetMode()) { @@ -487,18 +465,6 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, return false; } - if (prd.covariance_valid) - fprintf(stderr, "[cov] GB=%.3f GR0=%.3f GR1=%.3f BR0=%.3f BR1=%.3f R0R1=%.3f\n", - prd.corr_GB, prd.corr_GR0, prd.corr_GR1, prd.corr_BR0, prd.corr_BR1, prd.corr_R0R1); - if (prd.adaptive_R1) - fprintf(stderr, "[R1] %.5f\n", prd.R[1]); - if (prd.shape_R1 && std::isfinite(prd.shape_R1_lores)) - fprintf(stderr, "[shapeR1] lores=%.5f hires=%.5f\n", prd.shape_R1_lores, prd.shape_R1_hires); - if (prd.measure_centroid && std::isfinite(prd.centroid_lo_tang_c)) - fprintf(stderr, "[res] lo_s=%.1f lo_tc=%.3f lo_tp=%.3f lo_rc=%.3f hi_s=%.1f hi_tc=%.3f hi_tp=%.3f hi_rc=%.3f\n", - prd.centroid_lo_signif, prd.centroid_lo_tang_c, prd.centroid_lo_tang_p, prd.centroid_lo_rad_c, - prd.centroid_hi_signif, prd.centroid_hi_tang_c, prd.centroid_hi_tang_p, prd.centroid_hi_rad_c); - // PixelRefine output flows into the normal save/merge path: the refined // geometry/lattice and the already-scaled reflections become the outcome. i_outcome.reflections = std::move(prd.reflections); diff --git a/image_analysis/pixel_refinement/FACTORED_MODEL.md b/image_analysis/pixel_refinement/FACTORED_MODEL.md index 944514a2..cb398977 100644 --- a/image_analysis/pixel_refinement/FACTORED_MODEL.md +++ b/image_analysis/pixel_refinement/FACTORED_MODEL.md @@ -1,6 +1,8 @@ # A factored likelihood for joint integration + scaling + geometry -**Status: design spec (not implemented).** Goal: replace the per-pixel least-squares +**Status: Terms 1+2 implemented and shipping as the PixelRefine default** (see +`PixelRefine.cpp`, `METHODS.md`, `FINDINGS-2026-06.md`); Term 3 (geometry) and the +priors/NN extensions of §4 remain future work. Goal: replace the per-pixel least-squares of PixelRefine with a per-*reflection* likelihood that fuses profile-fit integration, scaling against the reference, and geometry refinement into one differentiable objective — the foundation for priors (Bayesian) and learned components (NN), and the diff --git a/image_analysis/pixel_refinement/FINDINGS-2026-06.md b/image_analysis/pixel_refinement/FINDINGS-2026-06.md new file mode 100644 index 00000000..a7345de4 --- /dev/null +++ b/image_analysis/pixel_refinement/FINDINGS-2026-06.md @@ -0,0 +1,122 @@ +# PixelRefine — findings, June 2026 + +A record of the main results from the still-integration investigation, validated on two +lysozyme P4₃2₁2 datasets indexed against the same reference (`6G8A_refine_001.mtz`): + +* **Crystal 1 — serial jet stills** (`LysozymeJet5-…`, MicroMAX DMM ~1 % bandwidth). +* **Crystal 2 — rotation crystal** (`fixed_master.h5`, 1800 × 0.2°, Si mono), run *as + stills* as a stress test, and as rotation for reference. XDS output of this crystal + (`CORRECT.LP`, `XDS_ASCII.HKL`) is the external ground truth. + +--- + +## 1. The factored model is a qualitative win (crystal 2, stills, 1.7 Å) + +Replacing the per-pixel least squares with the factored Terms 1+2 (intensity residual + +measured per-resolution $R_1$) turns an erratic, high-res-collapsing result into a flat one: + +| | N_obs | ⟨I/σ⟩ | CC₁/₂ (per shell) | CCref (per shell) | +|---|---:|---:|---|---| +| baseline per-pixel loss | 799 k | 7.2 | 75→81→56→32→…→0 % | erratic, →0 | +| **factored Terms 1+2** | 1.22 M | 10.7 | **84–92 %, flat to 1.7 Å** | **77–92 %, flat** | + +This *reaches the proper rotation (`-R -P rot`) path's quality from the stills path.* The +mechanism is clean and on-thesis: **Term 1 lifts the per-image scale CC vs the reference +from 0.09 → 0.40** (median 0.01 → 0.39) — the per-pixel loss produced near-random per-image +scales; the factored objective makes each image's scale agree with the reference, so the +merge coheres. The reference enters at maximum leverage (it sets both the target and the +Fisher weight). + +**Term 3 (per-spot recentring) is a no-op** on both crystals (93.5 vs 93.6 % CC₁/₂): the +generous integration aperture already contains the spot, so shifting the spotlight by the +sub-pixel prediction offset does not change the integrated total. It was removed; the +position residual's value can only come as a *geometry* term (refine orientation/distance +from the centroid), which is XtalOptimizer's job, not a per-spot mask shift. + +--- + +## 2. The residual centroid offset is a sampling floor, not a recoverable error + +On the jet, predictions land on the spot to within a ~0.4 px tangential scatter. Three +independent cuts show this is the **centroid undersampling floor of the ~2×2 spots**, not a +geometry error or a shape effect: + +* **Flat with intensity.** Binned by significance (≈6σ vs ≈39σ, a 6× span) the tangential + centroid offset is 0.41 → 0.36 px (median). A background-limited centroid would shrink + ∝ 1/significance (≈6×); it does not, so it is a real ~0.35 px floor, not counting noise. +* **Peak is a *worse* predictor than the centroid** (sub-pixel parabolic mode: 0.52 vs 0.36 + px at high signif). If an asymmetric tail were dragging the centroid off a correct peak, + the peak would be *better* — it is not. So there is **no coherent shape asymmetry** to + model; the offset is a zero-mean per-spot scatter from sub-pixel phase aliasing of a spot + only ~2 px wide. +* **Radial centroid offset ≈ 0.01–0.02 px, flat** → no distance/parallax error (parallax is + radial and grows outward). + +Consequences: recentring cannot rescue weak reflections (their prediction is already +centred; the 0.4 px is irreducible sampling scatter), which is *why* a generous box beats a +tight profile mask. A *symmetric* non-Gaussian shape would not move the centroid at all — it +would show up only in Term 2 ($R_1$), not in a position term. + +--- + +## 3. The σ gap to XDS is fulls-vs-partials, not intensities + +XDS reports **ISa = 28.3** (asymptotic relative error 3.5 %); our stills path reports +**ISa ≈ 1.1** and the rotation path **≈ 1.6**. The decisive clue is in `XDS_ASCII.HKL`: the +`PEAK` column (fraction of each reflection captured) is **≈ 100 % for 97 % of reflections**, +and the header gives mosaicity 0.091° < 0.2° oscillation. So **these are full reflections**, +recorded over the 1–3 frames each rocking curve spans — not partials. + +* **XDS** profile-fits in 3D (the third axis is the rotation/rocking direction) and *sums* + the rocking curve with profile weights → one full per reflection, counting-limited σ. +* **jfjoch `-P rot`** integrates a 2D shoebox *per frame* and recovers each full by + *dividing* by the rocking-curve fraction $R_j$ — a cheap approximation of 3D integration. + That division injects per-observation noise (it amplifies each frame's background noise, + pays N independent backgrounds, and carries a random per-observation partiality error). + +Crucially, **ISa and merged-intensity accuracy are different axes, decoupled by +multiplicity.** Our merged intensities are correct (§4), because ~60–240× multiplicity +averages the per-observation noise down; ISa measures the per-*observation* precision, which +multiplicity cannot improve. So "right intensities, wrong σ" is not a contradiction. + +**A cheap probe confirms it.** Raising `--min-partiality` from 0.02 → 0.5 on crystal 2 +(rotation) lifts ISa **1.6 → 3.8** at *zero* completeness cost (high multiplicity) and with +CCref flat — and the high-res shells *improve*. The default 0.02 keeps deep-tail partials +(2 % of a reflection, scaled back ×50) that were over-weighted and polluting the merge. So +of the ~17× rotation gap, **~2.8× is tunable tail-weighting** and **~6× is structural** (the +2D-divide vs 3D-sum difference) — the structural part needs a real rocking-curve sum, not a +knob. + +--- + +## 4. Our intensities are XDS-grade — the limit is σ, not I + +Direct CC of merged intensities (both reduced to the 4/mmm asymmetric unit; 98.8 % of +reflections matched, no manual reindex): + +| vs XDS (CC on merged I, to 1.2 Å) | overall | at 1.2 Å | +|---|---|---| +| **PixelRefine stills** (factored) | **95.9 %** | 96.9 % | +| rotation `-P rot` (min_partiality 0.3) | 98.8 % | 98.5 % | + +PixelRefine-stills intensities track XDS at **95–98 %, flat to the 1.2 Å diffraction +limit**, only ~3 % behind full rotation integration. This is exactly what the CC₁/₂→CC_true +relation predicts (stills CC₁/₂≈0.88 ⇒ 0.967), so it is real, carried by the huge stills +multiplicity averaging per-observation noise down. **Conclusion: the intensity estimation is +right; the remaining gap to XDS is entirely the per-observation σ / partiality axis** — the +3D rocking-curve integration frontier (`FACTORED_MODEL.md` §4), deliberately parked. + +--- + +## What is fixed vs. parked + +**Landed (and now the default model):** mean (not median) background in both integrators; +de-biased (background-limited) variance; widened Ewald prediction band; the factored +Terms 1+2 as the only PixelRefine objective; the global XDS-form merge error model with ISa. + +**Diagnosed dead-ends (do not re-litigate):** per-image $R$ refinement (degenerate with +scale); per-spot recentring (no-op); chasing the 0.4 px centroid floor (sampling, not +recoverable). **Parked (rotation-side):** the 3D rocking-curve sum / one-shot +post-refinement, a `min_partiality` default for high-multiplicity data, and +partiality-aware variance weighting — the path to XDS-grade ISa, but a separate axis from +the intensity work. diff --git a/image_analysis/pixel_refinement/METHODS.md b/image_analysis/pixel_refinement/METHODS.md index 8604ffc7..1c1fd9e1 100644 --- a/image_analysis/pixel_refinement/METHODS.md +++ b/image_analysis/pixel_refinement/METHODS.md @@ -1,31 +1,30 @@ -# PixelRefine — methods and improvements +# PixelRefine — methods -This note documents the changes made to the still-image *pixel-refinement* integrator -(`PixelRefine`) and, more importantly, **why** each one was needed. It is written from a -methods point of view; the equations are the load-bearing part. +`PixelRefine` is the still-image integrator. It integrates the Bragg reflections of one +image by **profile fitting against a reference intensity set** $I^\mathrm{ref}$ (e.g. +`F_calc` from a deposited model, or the current merged estimate in an EM-style outer loop) +and returns already-scaled intensities. It is an **intensity-wise** operation: the +detector geometry (orientation, cell, beam, distance) is taken as fixed — it was refined +upstream by `XtalOptimizer` (`IndexAndRefine::RefineGeometryIfNeeded`) — and PixelRefine +only measures the spot shape and fits the per-image scale. -`PixelRefine` integrates Bragg reflections on a **still** image by fitting a per-pixel -forward model against a known *reference* intensity set (e.g. `F_calc` from a deposited -model). Unlike a rotation experiment, a still samples only a thin slice of each -reflection, so the integrator must (a) model the partiality of that slice, (b) refine the -per-image geometry well enough that the high-resolution shoeboxes land on signal, and -(c) scale each image onto the reference. Each of those three is where the original code -went wrong. +The objective is the factored per-reflection likelihood of `FACTORED_MODEL.md`, **Terms 1 +and 2**. This note records the equations and the reasons behind each design choice. Throughout, a reflection's shoebox is a small box of raw detector pixels $I_p$ with a -local flat background $B$; the (area-normalised) model spot profile at pixel $p$ is -$P_p$, and $v_p$ is the variance used to weight pixel $p$. +local flat background $B$; the area-normalised tangential profile at pixel $p$ is $P_p$, +and $v_p$ is the variance used to weight pixel $p$. --- ## 0. The forward model -The recorded amplitude of a still reflection is estimated by profile fitting: +The recorded amplitude of a still reflection is the profile-fit amplitude $$ -J \;=\; \frac{\sum_p w_p\,P_p\,(I_p - B)}{\sum_p w_p\,P_p^{2}}, -\qquad w_p = \frac{1}{v_p}, -\qquad \operatorname{var}(J) = \frac{1}{\sum_p P_p^{2}/v_p}. +J \;=\; \frac{\sum_p P_p\,(I_p - B)/v_p}{\sum_p P_p^{2}/v_p}, +\qquad +\operatorname{var}(J) = \frac{1}{\sum_p P_p^{2}/v_p}. $$ The full (rotation-equivalent) intensity is recovered by dividing out the factors a still @@ -33,296 +32,183 @@ does not record, $$ I \;=\; \frac{J}{p\,B_\mathrm{DW}\,\mathrm{pol}},\qquad -p = \exp\!\left(-\frac{\epsilon_r^{2}}{R_0^{2}}\right),\quad +p = \exp\!\left(-\frac{\epsilon_r^{2}}{R_{0,\mathrm{eff}}^{2}}\right),\quad B_\mathrm{DW}=\exp\!\left(-\frac{B_\mathrm{fac}}{4 d^{2}}\right), $$ where the **partiality** $p$ is the fraction of the mosaic block crossing the Ewald -sphere, $\epsilon_r = 1/\lambda - |S_{hkl}|$ is the excitation error, $R_0$ the radial -(rocking) width, and $\mathrm{pol}$ the polarisation correction. The per-image model -intensity for a pixel is +sphere, $\epsilon_r$ is the radial excitation error, $R_0$ the radial (rocking) width, and +$\mathrm{pol}$ the polarisation correction. The tangential profile is a separable, +area-normalised Gaussian of width $R_1$: $$ -I_p^\mathrm{model} = G\,I^\mathrm{ref}\,B_\mathrm{DW}\,p\,P^\mathrm{tang}_p\,\mathrm{pol} + B, +P_p = \frac{1}{\pi R_1^{2}}\exp\!\left(-\frac{\epsilon_{t,p}^{2}}{R_1^{2}}\right). $$ -with $G$ the per-image scale. The per-image least squares minimises -$\chi^2 = \sum_p w_p\,(I_p^\mathrm{model}-I_p)^2$ over geometry, orientation, $G$, $R$. +A finite **X-ray bandwidth** thickens the Ewald shell radially, adding a fixed, +resolution-dependent term to the radial width, $R_{0,\mathrm{eff}}^2 = R_0^2 + +(b\lambda)^2/(2d^4)$ ($b$ = relative bandwidth; the pink-beam / DMM signature). $b=0$ is a +monochromatic no-op. --- ## 1. De-biased variance (the load-bearing fix) **Symptom.** Mean intensities went **negative** in the high-resolution shells -($\langle I/\sigma\rangle$ down to $-12$), which a box-sum integrator never does. The -per-image scale $G$ also collapsed to $0$ on most images, dropping ~80 % of observations. +($\langle I/\sigma\rangle$ down to $-12$), and the per-image scale $G$ collapsed to $0$ on +most images, dropping ~80 % of observations. -**Cause.** Both the extraction and the fit weighted each pixel by its **observed** count, -$v_p = I_p$. For a background pixel that fluctuated *down* ($I_p < B$), $v_p$ is small, so -$w_p = 1/v_p$ is *large*, and its contribution $P_p (I_p-B)/v_p < 0$ is large in -magnitude. Summed over the many (mostly empty) shoebox pixels, this drags $J$ below zero — -the classic *inverse-observed-count* (Poisson-on-data) bias. It bites hardest where the -true signal is weakest, i.e. at high resolution. In the fit it manifests differently but -identically in origin: the weighted empty pixels make "no signal" ($G=0$) the cheapest -solution, so $G\to 0$. +**Cause.** The extraction weighted each pixel by its **observed** count, $v_p = I_p$. A +down-fluctuated background pixel ($I_p < B$) then gets a small $v_p$, hence a large +$w_p=1/v_p$, and its contribution $P_p(I_p-B)/v_p < 0$ is large in magnitude. Summed over +the many empty shoebox pixels this drags $J$ below zero — the *inverse-observed-count* +(Poisson-on-data) bias, worst where the true signal is weakest (high resolution). -**Fix.** For background-limited (weak) reflections the correct variance is the local -background, **constant over the shoebox**: - -$$ -v_p = \max(B,\,1)\quad\Longrightarrow\quad -J = \frac{\sum_p P_p\,(I_p-B)}{\sum_p P_p^{2}}, -$$ - -the unbiased uniform-variance estimator. This single change turned $\langle I/\sigma\rangle$ -positive at all resolutions and stopped the scale collapse. It is applied to both the -extraction weight and the fit weight. +**Fix.** For background-limited reflections the correct variance is the local background, +**constant over the shoebox**, $v_p = \max(B,1)$, giving the unbiased uniform-variance +estimator $J = \sum_p P_p (I_p-B)/\sum_p P_p^2$. This turned $\langle I/\sigma\rangle$ +positive at all resolutions and stopped the scale collapse. --- ## 2. Prediction band and multiplicity -**Symptom.** PixelRefine recorded ~4× fewer observations per unique reflection than the -classical integrator — completeness was fine, *redundancy* was not. - -**Cause.** A reflection is given a shoebox only when it lies within a radial band of the -Ewald sphere, - -$$ -\bigl|\,|S_{hkl}| - 1/\lambda\,\bigr| \le \delta . -$$ - -For randomly oriented stills the number of images on which a given $hkl$ satisfies this is -$\propto \delta$. The default $\delta = 5\times10^{-4}\,\text{Å}^{-1}$ was 4–6× tighter -than the classical integrator's $\delta = 2\text{–}3\times r_\mathrm{profile}$, so each -reflection was recorded on 4–6× fewer images. - -**Fix.** Widen to $\delta = 2\times10^{-3}\,\text{Å}^{-1}$. Multiplicity rose from -~240 k to ~950 k observations and CC$_\mathrm{ref}$ from 49.7 % to 55.9 %. Widening is -only safe once the fit is well-behaved (Sections 1, 3, 4); with the original -unconstrained fit it caused divergence. +A reflection is given a shoebox only when it lies within a radial band of the Ewald +sphere, $\bigl|\,|S_{hkl}| - 1/\lambda\,\bigr| \le \delta$. For randomly oriented stills +the number of images on which a given $hkl$ qualifies is $\propto \delta$. The original +$\delta = 5\times10^{-4}\,\text{Å}^{-1}$ was 4–6× tighter than a box integrator, giving 4× +fewer observations per reflection. Widening to $\delta = 2\times10^{-3}\,\text{Å}^{-1}$ +(`ewald_dist_cutoff`) restores the multiplicity; the partiality $p$ downweights the +slightly-off-Ewald tails it admits. (Widening is only safe with the de-biased variance of +§1 and the factored objective of §§3–4 — with a per-pixel geometry fit it diverged.) --- -## 3. Regularising the per-image fit +## 3. Term 2 — measured per-resolution profile width $R_1$ -**Symptom.** Freeing *any* per-image parameter (orientation, $R$, even the scalar scale -$G$) collapsed the merged data (CC$_{1/2}$ from 90 % to a few %). The predict↔refine loop -with all parameters frozen was, by contrast, byte-identical to extraction-only — proving -the *fit*, not the loop, was at fault. - -**Cause.** The per-image problem regularised **nothing**: orientation had no prior, $R$ -and $G$ only a lower bound. Three orientation DOF (plus $R$, $G$) against a handful of -signal pixels per still overfit the noise, and an unconstrained $1/G$ then scrambled the -cross-image merge. - -**Fix.** Anchor each refined parameter to its prior with a *data-scaled* weight, as -`ScaleOnTheFly` already does for rotation data. The data term has one residual **per -pixel**, so the prior weight must scale with the pixel count: +$R_1$ is **measured, not fitted**. Fitting $R_1$ inside a per-image least squares is +degenerate with the scale $G$ (a narrower profile and a larger scale trade off), and that +degeneracy slides the per-image scale and wrecks the merge. But a **second moment** is +normalised by the total intensity, so it carries shape information *decoupled from scale*: $$ -w_\theta = \sqrt{\frac{N_\mathrm{pix}}{\sigma_\theta^{2}}}\,, -\qquad \chi^2_\mathrm{reg} = w_\theta^{2}\,(\theta-\theta_0)^2 . -$$ - -Using $\sqrt{N_\mathrm{refl}}$ instead (as in the rotation scaler) is a factor -$\sqrt{N_\mathrm{pix}/N_\mathrm{refl}}\approx\sqrt{49}\approx 7$ too weak and is simply not -felt. Applied to: - -* **Scale**: $\theta=G,\ \theta_0=1$. Prevents $1/G$ from wandering; restored CC$_{1/2}$ - from 4 % back to 87 %. -* **Orientation**: $\theta$ = Rodrigues vector, $\theta_0$ = spot-centroid orientation, - $\sigma_\theta$ in radians. At $\sigma_\theta\!\sim\!1^\circ$ this gives the best - CC$_\mathrm{ref}$; beyond $\sim 2^\circ$ the fit overfits and the merge collapses, so - $\sigma_\theta$ is the safety knob. - ---- - -## 4. Signal-weighting the fit - -**Cause.** Even de-biased, a shoebox is ~80 % empty pixels (≈ 40 of 49 for a radius-3 -box). They carry no information on $G$, $R$ or orientation but add noise and, near the -overfitting edge, destabilise the fit. - -**Fix.** Multiply the fit weight by a detector-space Gaussian centred on the predicted -spot, - -$$ -w_p \;\to\; w_p \cdot \exp\!\left(-\frac{r_p^{2}}{2\sigma_s^{2}}\right), -\qquad r_p = \lVert (x_p,y_p) - (x_\mathrm{pred},y_\mathrm{pred})\rVert, -$$ - -so the signal-bearing core drives the refined parameters ($\sigma_s\approx 1.5$ px). With -the regularised orientation this lifted CC$_\mathrm{ref}$ from 60.5 % to **62.6 %**. - ---- - -## 5. Global orientation + cell-scale sweep - -**Motivation.** The classical refinement (`XtalOptimizer`) works on spot *centroids* from -spot-finding, which are poor value at high resolution; the local LSQ above is a heavy -gradient step that cannot make the ~degree-scale global moves needed to pull a -mis-indexed crystal's high-resolution reflections onto their shoeboxes. A small, -**global** sweep that simply asks *"at which orientation does the most high-resolution -signal appear where the reference says it should?"* is structurally better suited. - -**Score.** Pearson CC of the box-summed intensities against the reference, over **all** -matched reflections (not just the strong ones): - -$$ -\mathrm{CC}_\mathrm{ref}(L) = \operatorname{corr}_{hkl}\bigl(\,I^\mathrm{box}_{hkl}(L),\ I^\mathrm{ref}_{hkl}\,\bigr). -$$ - -The strong low-resolution reflections **anchor** the CC (moving them off their boxes -collapses it), so the *change* in CC across the sweep is driven almost entirely by weak -high-resolution reflections falling onto — or off — real signal. This is the "appearing -out of the void" behaviour. - -**Geometry-derived bounds (parameter-free).** A spot at resolution $d$ sits at detector -radius - -$$ -r(d) = \frac{L}{p}\,\frac{\lambda}{d}\quad[\text{px}], -$$ - -($L$ = detector distance, $p$ = pixel size). Both a crystal rotation $\delta\theta$ and a -fractional cell-scale $\epsilon$ displace that spot by an amount proportional to its -radius: - -$$ -\Delta_\mathrm{px} = r(d)\,\delta\theta \quad(\text{rotation}),\qquad -\Delta_\mathrm{px} = r(d)\,\epsilon \quad(\text{cell scale}). -$$ - -So high-resolution spots (large $r$) move most. The two natural constraints fix the grid: - -* **Step** = 1 px at the highest resolution: $\;\delta\theta_\mathrm{step} = 1/r_\mathrm{max}$ - (finer is below the detector's resolving power). -* **Range** = the orientation uncertainty $\Delta\theta_u$ (a few px at high res). The - number of steps per axis is then - -$$ -n = \frac{\Delta\theta_u}{\delta\theta_\mathrm{step}} = \Delta\theta_u\, r_\mathrm{max}, -$$ - -a handful of steps, and the lowest-resolution spots move only -$n\,\delta\theta_\mathrm{step}\,r_\mathrm{min} = \Delta\theta_u\, r_\mathrm{min} \ll 2$ px -— i.e. the strong anchors stay put by construction. - -> **Lesson learned.** Setting the range from "2 px at low resolution" instead gives -> $n = 2\,r_\mathrm{max}/r_\mathrm{min} = 2\,d_\mathrm{low}/d_\mathrm{high}$ steps — tens -> of pixels of high-resolution freedom — which lets the per-image CC overfit and *degrades* -> the merge. The range must be tied to the (small) orientation uncertainty, not the -> low-resolution cap. - -The sweep is a coordinate descent over the three Rodrigues axes and the cell scale, run -**before** the LSQ. It improves the high-resolution shells (CC$_\mathrm{ref}$ +1 to +5 -per shell) while preserving CC$_{1/2}\approx 90$ %. It is an *alternative* to the -loose-$\sigma$ orientation LSQ, not a complement — stacking the two double-moves the -orientation and overfits. It is therefore available but **off by default**. - ---- - -## 6. Background-estimator bias (the largest σ error; both integrators) - -**Symptom.** Pushed past the true resolution limit (e.g. a 1.8 Å crystal merged to 1.3 Å), -the no-signal shells still reported $\langle I/\sigma\rangle \approx 4\text{–}6$ with -$\mathrm{CC}_{1/2}\approx 0$ — i.e. confident "data" where there is none. Present in *both* -`PixelRefine` and the classical `BraggIntegrate2D`. - -**Cause.** Both estimated the local background as the **median** of the surrounding ring. -For a Poisson / right-skewed background the median sits *below* the mean, - -$$ -\operatorname{median}(B) < \mathbb{E}[B], -$$ - -so subtracting it under-subtracts on every pixel. The leftover positive offset is tiny per -pixel but coherent, so over an $n_\mathrm{pix}$-pixel peak and a multiplicity-$m$ merge it -grows to a fake signal - -$$ -\langle I\rangle_\mathrm{bias} \;\approx\; n_\mathrm{pix}\,\bigl(\mathbb{E}[B]-\operatorname{median}(B)\bigr), +R_1^2 = 2\,\langle \epsilon_t^2\rangle, \qquad -\Bigl(\tfrac{I}{\sigma}\Bigr)_\mathrm{merged,\,bias} \propto \sqrt{m}. +\langle \epsilon_t^2\rangle = \frac{\sum_p (I_p-B)\,\epsilon_{t,p}^2}{\sum_p (I_p-B)} . $$ -It is worst where the real signal is weakest (high resolution), because there the offset is -all that remains — which is exactly the observed signature. *Nothing leaks into the -high-resolution shells; the background is simply under-estimated.* - -**Fix.** Use the **mean** of the ring (outliers excluded by the existing spot-core mask and -saturation sentinels). $\langle I/\sigma\rangle$ then collapses to ~0 wherever -$\mathrm{CC}\approx0$, tracks CC down the shells, and the honest resolution limit becomes -visible. This was the single largest contributor to untrustworthy σ — a one-line change in -each integrator, *not* a variance-model problem. +We bin the strong spots ($\mathrm{signif}\ge 5$) by resolution ($1/d^2$, 6 bins) and take +the **median** $\langle\epsilon_t^2\rangle$ per bin, so each reflection integrates with the +$R_1$ of its resolution shell (low-res spots are tight; high-res anisotropic streaks are +wider). Weak spots fall back to the global $R_1$. Measuring the width rather than fitting +it is what makes profile-width refinement stable — and it is a selling point: the mask +adapts to the data per shell. --- -## 7. Error model (global $a, b$; XDS form) +## 4. Term 1 — the intensity / scaling residual + +The per-pixel least squares is replaced by **one residual per reflection**: the profile-fit +amplitude $J$ (using the Term-2 $R_1$) should equal the scaled reference, + +$$ +r_h = \frac{J_h - G\,B_\mathrm{DW}\,p_h\,\mathrm{pol}_h\,I^\mathrm{ref}_h}{\sigma_{J,h}}, +\qquad +L = \sum_h r_h^2 + \text{(scale prior)} . +$$ + +Only the per-image scale $G$ and Debye–Waller $B$ are optimised; geometry and $R$ are +fixed. Three consequences: + +* **Integration and scaling become one objective.** $J$ *is* the integrated intensity and + the residual *is* the scaling residual. +* **The empty-pixel problem disappears by construction.** Empty pixels enter only through + $J$ (with ~zero profile weight); they make no residual of their own and cannot dominate. +* **Fisher weighting puts the reference at maximum leverage.** $\sigma_J$ uses the + *model-expected* variance $v_p = B + \max(J,0)\,P_p$ (background plus expected signal from + $I^\mathrm{ref}$), not the observed counts — so a strong *expected* reflection observed + absent is penalised, and a noise spike with low $I^\mathrm{ref}$ gets no weight. + +The scale is regularised towards 1 with a data-scaled weight $w_G=\sqrt{N_\mathrm{refl}/ +\sigma_G}$ (mirrors `ScaleOnTheFly`) so weakly-measured images cannot drift and scramble +the merge. + +**Geometry is not refined here.** PixelRefine's earlier per-image geometry refinement +(regularised orientation LSQ, signal-weighting, a global orientation/cell sweep) was +removed: on true stills the predictions are already good (radial centroid error ≈ 0, +tangential ≈ a 0.4 px sampling floor that is *not* a recoverable misprediction), and per-image +geometry refinement only overfit the sparse signal. Geometry is the job of `XtalOptimizer`. + +--- + +## 5. Background estimator — mean, not median (both integrators) + +**Symptom.** Pushed past the true resolution limit, the no-signal shells reported +$\langle I/\sigma\rangle\approx4\text{–}6$ at $\mathrm{CC}_{1/2}\approx0$ — confident "data" +where there is none. Present in *both* PixelRefine and the classical `BraggIntegrate2D`. + +**Cause.** Both used the **median** of the background ring. For a right-skewed (Poisson) +background $\operatorname{median}(B) < \mathbb{E}[B]$, so subtraction under-subtracts by a +tiny but *coherent* per-pixel offset that grows over an $n_\mathrm{pix}$ peak and a +multiplicity-$m$ merge into a fake $\langle I/\sigma\rangle\propto\sqrt{m}$, worst where the +real signal is weakest. + +**Fix.** Use the **mean** of the ring (spot cores and saturation sentinels already +excluded). $\langle I/\sigma\rangle$ then collapses to ~0 wherever $\mathrm{CC}\approx0$ and +the honest resolution limit becomes visible. This was the single largest contributor to +untrustworthy σ — a one-line change in each integrator. + +--- + +## 6. Error model (global $a,b$; XDS form) Counting statistics under-estimate the variance of strong reflections, which carry -systematic errors (scaling, partiality, detector) proportional to intensity, not to -$\sqrt{I}$. After merging, this leaves CC$_{1/2}$ and $\langle I/\sigma\rangle$ -inconsistent. The standard correction (XDS / DIALS / AIMLESS) inflates the variance with a -**global** two-parameter model: +systematic errors proportional to $I$, not $\sqrt{I}$. The standard correction inflates the +variance with a **global** two-parameter model, applied at the merge level so both +integrators benefit: $$ -\sigma'^{\,2} \;=\; a\,\sigma^{2} + \bigl(b\,\langle I\rangle\bigr)^{2}, +\sigma'^{\,2} = a\,\sigma^{2} + (b\,\langle I\rangle)^{2}, \qquad -\mathrm{ISa} \;=\; \frac{1}{b}\;=\;\lim_{I\to\infty}\frac{I}{\sigma'} . +\mathrm{ISa} = \frac{1}{b} = \lim_{I\to\infty}\frac{I}{\sigma'} . $$ -Two points matter for an unbiased fit: - -* The $I^2$ term uses the reflection **mean** $\langle I\rangle$ (constant over its - observations), **not** the per-observation $I_i$. Using $I_i$ gives a down-fluctuated - point a small $\sigma'$ and hence a large $1/\sigma'^2$ weight, biasing the merged mean — - which collapses CC. (This was the decisive bug in the first attempt.) -* $a$ and $b$ are fit from the spread of symmetry equivalents: for an observation in a - group of $n$, $\mathbb{E}[(I_i-\langle I\rangle)^2] = \sigma_i^2(1-h_i)$ with leverage - $h_i = w_i/\sum w$. Binning by intensity and regressing the bin medians of - $(I_i-\langle I\rangle)^2/(1-h_i)$ on $(\sigma^2, \langle I\rangle^2)$ — *weighted by - $1/\mathrm{dev}^4$ so the fit is relative* — gives $(a, b^2)$; the relative weight stops - the strong bins (which fix $b$) from swamping the weak bins (which fix $a$). - -It is applied at the merge level (`MergeOnTheFly`), so **both** integrators benefit, and -`jfjoch_process` prints the model and ISa. Earlier per-resolution-shell variants were -dropped: the standard tools use a single global $a, b$, and the per-shell version was -partly masking the background bias of Section 6. +The $I^2$ term uses the reflection **mean** $\langle I\rangle$ (not the per-observation +$I_i$, which would bias the merged mean and collapse CC); $a,b$ are fit from the spread of +symmetry equivalents with a relative ($1/\mathrm{dev}^4$) weight so the strong bins (which +fix $b$) do not swamp the weak bins (which fix $a$). `jfjoch_process` prints the model and +ISa. --- -## Results (lysozyme jet, 1.8 Å, identical input) +## Results (lysozyme rotation crystal `fixed_master.h5`, treated as stills, 1.7 Å) -| Configuration | N_obs | Compl. | CC$_{1/2}$ | CC$_\mathrm{ref}$ | +| Configuration | N_obs | $\langle I/\sigma\rangle$ | CC$_{1/2}$ | CC$_\mathrm{ref}$ | |---|---:|---:|---:|---:| -| Classical integrator (box-sum) | 421 k | 100 % | 81.4 % | 60.9 % | -| PixelRefine — original | 61 k | 95 % | 0.0 % | −0.4 % | -| PixelRefine — consolidated (this work) | 951 k | 100 % | 80.0 % | **62.6 %** | +| Baseline per-pixel loss | 799 k | 7.2 | erratic, →0 at 1.7 Å | erratic | +| **Factored Terms 1+2 (this model)** | 1.22 M | 10.7 | **84–92 % flat** | **77–92 % flat** | -The consolidated integrator beats the classical one on the accuracy metric -(CC$_\mathrm{ref}$) with > 2× the multiplicity, and turns a previously unusable result -(CC$_{1/2}=0$) into a competitive one. +The factored objective turns the erratic, high-res-collapsing per-pixel result into flat +~90 % CC$_{1/2}$/CC$_\mathrm{ref}$ to 1.7 Å — matching the proper rotation integration path +from the *stills* path. (See `FINDINGS-2026-06.md`.) --- ## Default recipe -Sections 1–5 are `PixelRefine`-specific; Sections 6–7 act at the integration/merge level -and apply to the classical route too. +§§1–4 are PixelRefine-specific; §§5–6 act at the integration/merge level and apply to the +classical route too. | Field / behaviour | Default | Section | |---|---|---| | fit/extraction variance | local background $B$ | 1 | | `ewald_dist_cutoff` | $2\times10^{-3}\,\text{Å}^{-1}$ | 2 | -| `scale_reg_sigma` | 2.0 | 3 | -| `orient_reg_sigma_deg` | 1.0 | 3 | -| `refine_R` | `false` | 3 | -| `fit_signal_sigma_pix` | 1.5 | 4 | -| `sweep_orientation` | `false` (available) | 5 | -| local background estimator | **mean** of the ring | 6 | -| merge error model | global $a,b$ (ISa printed) | 7 | +| tangential width $R_1$ | measured per resolution shell | 3 | +| objective | per-reflection intensity residual, Fisher-weighted | 4 | +| refined parameters | per-image $G$ (and $B$); geometry fixed | 4 | +| `scale_reg_sigma` | 2.0 | 4 | +| local background estimator | **mean** of the ring | 5 | +| merge error model | global $a,b$ (ISa printed) | 6 | -`orient_reg_sigma_deg` (accuracy vs. precision) and `ewald_dist_cutoff` (multiplicity vs. -cost) are the two knobs worth tuning per dataset. +`ewald_dist_cutoff` (multiplicity vs. cost) and `bandwidth` (Si vs. DMM) are the two knobs +worth setting per dataset. diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 42e8772f..75facde4 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -6,11 +6,11 @@ #include #include #include +#include #include #include #include -#include #include #include "../geom_refinement/LatticeReduction.h" @@ -24,7 +24,6 @@ struct PixelObs { double x, y; // detector pixel coordinate double Iobs; // raw pixel value (signal + background) double Ibkg; // local background estimate (per-shoebox level, raw counts) - double weight; // 1 / sigma_pixel }; // One reflection together with the pixels of its shoebox. @@ -36,17 +35,10 @@ struct ReflGroup { double pol; // per-reflection polarization correction (raw = true * pol) double Ibkg; // local flat background (raw counts, constant over the shoebox) double predicted_x, predicted_y; - double R1_eff = 0.0; // tangential profile width to use (Term 2; 0 => fall back to data.R[1]) - double dcx = 0.0, dcy = 0.0; // Term 3: profile recentre shift (observed centroid - predicted) + double R1_eff = 0.0; // tangential profile width to use (Term 2) std::vector pixels; }; -double SafeInv(double x, double fallback) { - if (!std::isfinite(x) || std::fabs(x) < 1e-30) - return fallback; - return 1.0 / x; -} - // Median of a vector (in place, partially reorders it). double MedianInPlace(std::vector &v) { if (v.empty()) @@ -90,8 +82,7 @@ std::vector BuildSpotMask(const std::vector &predicted, int // Square shoebox bounds (inclusive) around a predicted spot, clamped to the // detector. The centre is rounded to the nearest pixel with std::lround so the // signal box is centred identically to the spot-core mask (BuildSpotMask) and -// the local-background ring (EstimateLocalBackground), which also lround. Used by -// Run and the diagnostic renderers so all three share one shoebox definition. +// the local-background ring (EstimateLocalBackground), which also lround. struct ShoeboxBox { int min_x, max_x, min_y, max_y; }; ShoeboxBox ShoeboxBounds(double px, double py, int radius, size_t xpixel, size_t ypixel) { const int cx = static_cast(std::lround(px)); @@ -284,6 +275,30 @@ bool PredictedNode(const T *p0, const T *p1, const T *p2, return true; } +// Geometry terms for one shoebox pixel under a FIXED geometry (PixelRefine no +// longer refines geometry, so this is a plain double evaluation, not a Ceres cost): +// q_sq = |g_hkl|^2 (predicted node, for the B-factor) +// eps_radial = deviation along the Ewald normal (the partiality direction) +// eps_tang_sq = squared deviation in the tangent plane (the profile direction) +bool GeometryProbe(double obs_x, double obs_y, double lambda, double pixel_size, + int h, int k, int l, gemmi::CrystalSystem symmetry, + const double beam[2], double dist_mm, const double detector_rot[2], + const double p0[3], const double p1[3], const double p2[3], + double &q_sq, double &eps_radial, double &eps_tang_sq) { + const double inv_lambda = 1.0 / lambda; + Eigen::Vector3d e_obs; + ObservedRecip(beam, &dist_mm, detector_rot, obs_x, obs_y, pixel_size, inv_lambda, e_obs); + + Eigen::Vector3d e_pred, n_radial; + if (!PredictedNode(p0, p1, p2, h, k, l, symmetry, inv_lambda, e_pred, n_radial, q_sq)) + return false; + + const Eigen::Vector3d delta_q = e_obs - e_pred; + eps_radial = delta_q.dot(n_radial); + eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); + return true; +} + // Pulls a scalar parameter towards an expected value with a fixed weight (the // data-scaled prior). Identical in spirit to ScaleOnTheFly's regularizer: it is what // keeps the per-image scale G from wandering on weakly-constrained images and @@ -299,250 +314,30 @@ struct ScalarRegularizer { double expected; }; -// Anchors the orientation (angle-axis vector) to its pre-refinement value with a -// data-scaled weight. Without it the three orientation DOF chase the sparse signal -// (and the few noisy background pixels) and the per-image intensities collapse; -// with it the fit can only make a small, signal-supported sub-spot correction - the -// push that brings slightly-misaligned high-resolution reflections onto their -// shoeboxes. Mirrors the G/B regularizers in ScaleOnTheFly. -struct OrientationRegularizer { - OrientationRegularizer(double weight, const double prior[3]) : weight(weight) { - for (int i = 0; i < 3; ++i) - prior_[i] = prior[i]; - } +// Term 1 of the factored likelihood (FACTORED_MODEL.md): the per-reflection +// *intensity* (0th-moment) residual. The profile-fit amplitude J should equal the +// scaled reference J_model = G * exp(-B/4d^2) * partiality * pol * I_ref. One scalar +// residual per reflection, weighted by the model-expected (Fisher) sigma_J. This is +// the scaling residual - integration and scaling become one objective, and the empty +// pixels (which make no residual of their own) stop dominating the fit. Geometry is +// fixed, so J, partiality and sigma_J are constants and only G and B are free. +struct IntensityResidual { + IntensityResidual(double J, double sigma_J, double partiality, double pol, + double I_ref, double inv_4d2) + : J(J), inv_sigma(1.0 / sigma_J), partiality(partiality), pol(pol), + I_ref(I_ref), inv_4d2(inv_4d2) {} template - bool operator()(const T *p0, T *residual) const { - for (int i = 0; i < 3; ++i) - residual[i] = T(weight) * (p0[i] - T(prior_[i])); + bool operator()(const T *const G, const T *const B, T *residual) const { + const T B_term = ceres::exp(-B[0] * T(inv_4d2)); + const T J_model = G[0] * B_term * T(partiality) * T(pol) * T(I_ref); + residual[0] = (J_model - T(J)) * T(inv_sigma); return true; } - double weight; - double prior_[3]; + double J, inv_sigma, partiality, pol, I_ref, inv_4d2; }; } // namespace -// --------------------------------------------------------------------------- -// Cost functor -// -// I_pred(pixel) = G * Itrue * B_term * P_radial * P_tangential * pol + I_bkg -// -// B_term = exp(-B |q|^2 / 4) (Debye-Waller) -// P_radial = exp(-eps_r^2 / R0_eff^2) (partiality: fraction of -// the mosaic blob on the -// Ewald sphere; <= 1) -// P_tangential = exp(-eps_t^2/R1^2) / (pi R1^2) (Gaussian spatial profile -// in the Ewald tangent plane) -// pol = per-reflection polarization correction (raw = true * pol), -// evaluated once at the predicted spot position (as in -// BraggIntegrate2D). 1 if polarization is disabled. -// -// Everything is in *raw* detector counts: there is no per-pixel solid-angle or -// area (Lorentz/Jacobian) weighting - each pixel counts equally, like the normal -// integrator. The tangential factor is what makes this "profile fitting"; the -// 1/(pi R1^2) normalization keeps the profile width R1 from soaking up the -// overall scale G. -// -// X-ray bandwidth: a spread in lambda is a spread in the Ewald-sphere radius, -// i.e. a purely *radial* thickening of the shell. It adds (in quadrature) a -// resolution-dependent term to the radial width: -// R0_eff^2 = R0^2 + R_bw^2 , R_bw^2 = (b*lambda)^2 / (2 d^4) -// where b = relative bandwidth (sigma of dlambda/lambda). R_bw grows like 1/d^2, -// so bandwidth leaves low-resolution spots sharp and smears high-resolution ones -// radially - the pink-beam/DMM signature. R_bw_sq is a fixed per-reflection -// constant (b is known), so R0 keeps meaning "intrinsic" width (mosaic + -// divergence + beam). b = 0 makes R_bw = 0: a monochromatic no-op. -// --------------------------------------------------------------------------- -struct PixelResidual { - PixelResidual(const PixelObs &obs, double Itrue, - double lambda, double pixel_size, - double exp_h, double exp_k, double exp_l, - double R_bw_sq, double pol, - gemmi::CrystalSystem symmetry) - : Itrue(Itrue), Iobs(obs.Iobs), Ibkg(obs.Ibkg), weight(obs.weight), - obs_x(obs.x), obs_y(obs.y), - inv_lambda(1.0 / lambda), pixel_size(pixel_size), - exp_h(exp_h), exp_k(exp_k), exp_l(exp_l), - R_bw_sq(R_bw_sq), pol(pol), symmetry(symmetry) { - if (std::fabs(lambda) < 1e-6) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Lambda cannot be close to zero"); - } - - // Maps a detector pixel through the current geometry + lattice into the - // reference reciprocal frame and returns: - // q_sq = |g_hkl|^2 (predicted node, for B-factor) - // eps_radial = deviation along Ewald normal (partiality direction) - // eps_tang_sq = squared deviation in the detector-tangential plane (profile) - template - bool GeometryTerms(const T *const beam, - const T *const distance_mm, - const T *const detector_rot, - const T *const p0, - const T *const p1, - const T *const p2, - T &q_sq, T &eps_radial, T &eps_tang_sq) const { - Eigen::Matrix e_obs_recip; - ObservedRecip(beam, distance_mm, detector_rot, - obs_x, obs_y, pixel_size, inv_lambda, - e_obs_recip); - - Eigen::Matrix e_pred_recip, n_radial; - if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, - e_pred_recip, n_radial, q_sq)) - return false; - - const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - eps_radial = delta_q.dot(n_radial); - eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); - return true; - } - - // Assembles the full model intensity for the pixel from the geometry terms. - template - bool Model(const T *const beam, const T *const distance_mm, - const T *const detector_rot, - const T *const p0, const T *const p1, const T *const p2, - const T *const scale_factor, const T *const B, const T *const R, - T &Ipred) const { - T q_sq, eps_radial, eps_tang_sq; - if (!GeometryTerms(beam, distance_mm, detector_rot, - p0, p1, p2, q_sq, eps_radial, eps_tang_sq)) - return false; - - if (R[0] < T(1e-10) || R[1] < T(1e-10)) - return false; - - const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); - - // Separable Gaussian spot model: - // radial P_r(e) = exp(-e^2/R0_eff^2) (peak-normalized, in (0,1]) - // tangent g_t(e) = exp(-|e|^2/R1^2) / (pi R1^2) [1/A^-2] - // Every pixel counts equally (no area/Lorentz weighting); the radial factor - // is the still-image partiality (how far the reflection sits from the Ewald - // sphere); the overall scale is carried by the free G. - // - // IMPORTANT: the radial factor MUST use the same convention here as the - // extraction's `partiality` (peak-normalized), otherwise image_scale_corr - // = 1/(partiality*G*B) does not invert the model and a leftover, R0_eff- - // dependent (hence resolution-dependent) factor biases the intensities. - // R0_eff folds in the energy-bandwidth broadening via R_bw_sq. - const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); - const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); - const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) - / (T(M_PI) * R[1] * R[1]); - - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); - Ipred = signal + T(Ibkg); - return true; - } - - template - bool operator()(const T *const beam, - const T *const distance_mm, - const T *const detector_rot, - const T *const p0, - const T *const p1, - const T *const p2, - const T *const scale_factor, - const T *const B, - const T *const R, - T *residual) const { - T Ipred; - if (!Model(beam, distance_mm, detector_rot, p0, p1, p2, scale_factor, B, R, Ipred)) - return false; - - residual[0] = (Ipred - T(Iobs)) * T(weight); - return true; - } - - const double Itrue, Iobs, Ibkg, weight; - const double obs_x, obs_y; - const double inv_lambda; - const double pixel_size; - const double exp_h, exp_k, exp_l; - const double R_bw_sq; // bandwidth radial-width^2 contribution (0 = monochromatic) - const double pol; // per-reflection polarization correction - gemmi::CrystalSystem symmetry; -}; - -// --------------------------------------------------------------------------- -// Per-shoebox cost functor -// -// One residual block per reflection emitting N residuals (one per shoebox pixel). -// The expensive per-reflection geometry (PredictedNode: symmetry-aware B matrix, -// three rotations, cross products) is computed ONCE; only the cheap per-pixel -// ObservedRecip + Gaussian profile run in the pixel loop. This is identical in -// value to the old one-block-per-pixel formulation but ~(pixels-per-shoebox)x -// fewer evaluations of the costly node computation. Uses the same shared helpers -// (and hence the same conventions) as PixelResidual. -// --------------------------------------------------------------------------- -struct ShoeboxResidual { - ShoeboxResidual(const ReflGroup &g, double lambda, double pixel_size, - gemmi::CrystalSystem symmetry) - : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), pol(g.pol), - exp_h(g.h), exp_k(g.k), exp_l(g.l), - inv_lambda(1.0 / lambda), pixel_size(pixel_size), - symmetry(symmetry) {} - - template - bool operator()(const T *const *params, T *residual) const { - // Parameter blocks (order matches AddParameterBlock in Run): - // 0 beam[2] 1 distance[1] 2 detector_rot[2] - // 3 p0[3] 4 p1[3] 5 p2[3] 6 scale[1] 7 B[1] 8 R[2] - const T *beam = params[0]; - const T *distance_mm = params[1]; - const T *detector_rot = params[2]; - const T *p0 = params[3]; - const T *p1 = params[4]; - const T *p2 = params[5]; - const T *scale_factor = params[6]; - const T *B = params[7]; - const T *R = params[8]; - - if (R[0] < T(1e-10) || R[1] < T(1e-10)) - return false; - - // --- per-reflection: computed once --------------------------------- - Eigen::Matrix e_pred_recip, n_radial; - T q_sq; - if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, - e_pred_recip, n_radial, q_sq)) - return false; - - const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); - const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); - - // --- per-pixel loop ------------------------------------------------- - for (size_t i = 0; i < pixels.size(); ++i) { - const PixelObs &obs = pixels[i]; - - Eigen::Matrix e_obs_recip; - ObservedRecip(beam, distance_mm, detector_rot, - obs.x, obs.y, pixel_size, inv_lambda, e_obs_recip); - - const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - const T eps_radial = delta_q.dot(n_radial); - const T eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); - - const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); - const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) - / (T(M_PI) * R[1] * R[1]); - - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); - const T Ipred = signal + T(obs.Ibkg); - residual[i] = (Ipred - T(obs.Iobs)) * T(obs.weight); - } - return true; - } - - std::vector pixels; - const double Itrue, R_bw_sq, pol; - const double exp_h, exp_k, exp_l; - const double inv_lambda, pixel_size; - gemmi::CrystalSystem symmetry; -}; - PixelRefine::PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference) : xpixel(experiment.GetXPixelsNum()), @@ -598,187 +393,6 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, } } -template -void PixelRefine::SweepOrientationCell(const T *image, BraggPrediction &prediction, - PixelRefineData &data) const { - const int radius = data.shoebox_radius; - const double beam_x = data.geom.GetBeamX_pxl(); - const double beam_y = data.geom.GetBeamY_pxl(); - const auto qnan = std::numeric_limits::quiet_NaN(); - - // Box-sum minus local (perimeter) background, raw counts. NaN if the box runs - // off the detector or hits a masked/saturated pixel. - auto integrate = [&](double px, double py) -> double { - const int cx = static_cast(std::lround(px)); - const int cy = static_cast(std::lround(py)); - const int outer = radius + 1; - if (cx - outer < 0 || cy - outer < 0 || - cx + outer >= static_cast(xpixel) || cy + outer >= static_cast(ypixel)) - return qnan; - double sig = 0.0; - int nsig = 0; - std::vector ring; - ring.reserve((2 * outer + 1) * (2 * outer + 1)); - for (int y = cy - outer; y <= cy + outer; ++y) { - for (int x = cx - outer; x <= cx + outer; ++x) { - const T raw = image[static_cast(xpixel) * y + x]; - if (raw == std::numeric_limits::max()) - return qnan; - if (std::is_signed_v && raw == std::numeric_limits::min()) - return qnan; - const double v = static_cast(raw); - if (std::abs(x - cx) <= radius && std::abs(y - cy) <= radius) { - sig += v; - ++nsig; - } else { - ring.push_back(v); - } - } - } - if (ring.size() < 5) - return qnan; - return sig - nsig * MedianInPlace(ring); - }; - - // Predict (wide band) and collect every reflection that has a reference value, - // with its detector radius. The full set is scored - the strong low-res spots - // anchor the CC, the weak high-res spots are what "appear" at the right cell. - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(beam_x).BeamY_pxl(beam_y) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) - }; - const int nrefl = prediction.Calc(exp_iter, data.latt, settings); - const auto &predicted = prediction.GetReflections(); - - struct Matched { int h, k, l; double refI; }; - std::vector matched; - double r_max = 0.0, r_min = std::numeric_limits::max(); - for (int i = 0; i < nrefl; ++i) { - const auto &r = predicted[i]; - const auto it = reference_data.find(hkl_key_generator(r)); - if (it == reference_data.end()) - continue; - matched.push_back({r.h, r.k, r.l, it->second}); - const double dx = r.predicted_x - beam_x; - const double dy = r.predicted_y - beam_y; - const double rad = std::sqrt(dx * dx + dy * dy); - r_max = std::max(r_max, rad); - r_min = std::min(r_min, rad); - } - if (matched.size() < 20 || r_min <= 1.0 || r_max <= r_min) - return; // too little to anchor a meaningful sweep - - // CC of the box-summed intensities against the reference, over all matched hkls. - auto score = [&](const CrystalLattice &L) -> double { - const Coord A = L.Astar(), B = L.Bstar(), C = L.Cstar(); - double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; - int n = 0; - for (const auto &m : matched) { - const Coord g = A * static_cast(m.h) + B * static_cast(m.k) - + C * static_cast(m.l); - const auto [x, y] = data.geom.RecipToDetector(g); - if (!std::isfinite(x) || !std::isfinite(y)) - continue; - const double I = integrate(x, y); - if (!std::isfinite(I)) - continue; - const double yv = m.refI; - sx += I; sy += yv; sxx += I * I; syy += yv * yv; sxy += I * yv; ++n; - } - if (n < 10) - return -2.0; - const double nd = n; - const double cov = sxy - sx * sy / nd; - const double vx = sxx - sx * sx / nd; - const double vy = syy - sy * sy / nd; - if (!(vx > 0.0 && vy > 0.0)) - return -2.0; - return cov / std::sqrt(vx * vy); - }; - - // Step = 1 px at the highest resolution. Range = assumed orientation/cell-scale - // uncertainty (a few px at high res), NOT the low-res 2 px cap: the latter is - // ~2*r_max/r_min px at high res - far too permissive, and lets the per-image CC - // overfit. Here the low-res spots barely move (stay anchored). - const double step = 1.0 / r_max; - const int n_rot = std::clamp( - static_cast(std::lround(data.sweep_max_deg * M_PI / 180.0 * r_max)), 1, 25); - const int n_scale = std::clamp( - static_cast(std::lround(data.sweep_max_cell_frac * r_max)), 1, 25); - const Coord axes[3] = {Coord(1, 0, 0), Coord(0, 1, 0), Coord(0, 0, 1)}; - - CrystalLattice best = data.latt; - double best_cc = score(best); - - for (int round = 0; round < 2; ++round) { - for (const auto &axis : axes) { - CrystalLattice axis_best = best; - double axis_cc = best_cc; - for (int i = -n_rot; i <= n_rot; ++i) { - if (i == 0) - continue; - CrystalLattice cand = best.Multiply(RotMatrix(static_cast(i * step), axis)); - const double cc = score(cand); - if (cc > axis_cc) { - axis_cc = cc; - axis_best = cand; - } - } - best = axis_best; - best_cc = axis_cc; - } - CrystalLattice scale_best = best; - double scale_cc = best_cc; - for (int i = -n_scale; i <= n_scale; ++i) { - if (i == 0) - continue; - const double s = 1.0 / (1.0 + i * step); // cell scale (1+eps) -> recip * 1/(1+eps) - CrystalLattice cand = best.Multiply(gemmi::Mat33(s, 0, 0, 0, s, 0, 0, 0, s)); - const double cc = score(cand); - if (cc > scale_cc) { - scale_cc = cc; - scale_best = cand; - } - } - best = scale_best; - best_cc = scale_cc; - } - - data.latt = best; -} - -// --------------------------------------------------------------------------- -// Term 1 of the factored likelihood (FACTORED_MODEL.md): the per-reflection -// *intensity* (0th-moment) residual. The profile-fit amplitude J should equal the -// scaled reference J_model = G * exp(-B/4d^2) * partiality * pol * I_ref. One scalar -// residual per reflection, weighted by the model-expected (Fisher) sigma_J. This is -// the scaling residual - integration and scaling become one objective, and the empty -// pixels (which make no residual of their own) stop dominating the fit. With geometry -// and R held fixed, J, partiality and sigma_J are constants, so only G and B are free. -// --------------------------------------------------------------------------- -struct IntensityResidual { - IntensityResidual(double J, double sigma_J, double partiality, double pol, - double I_ref, double inv_4d2) - : J(J), inv_sigma(1.0 / sigma_J), partiality(partiality), pol(pol), - I_ref(I_ref), inv_4d2(inv_4d2) {} - template - bool operator()(const T *const G, const T *const B, T *residual) const { - const T B_term = ceres::exp(-B[0] * T(inv_4d2)); - const T J_model = G[0] * B_term * T(partiality) * T(pol) * T(I_ref); - residual[0] = (J_model - T(J)) * T(inv_sigma); - return true; - } - double J, inv_sigma, partiality, pol, I_ref, inv_4d2; -}; - template void PixelRefine::Run(const T *image, BraggPrediction &prediction, @@ -786,11 +400,6 @@ void PixelRefine::Run(const T *image, data.solved = false; data.reflections.clear(); - // Global orientation + cell-scale sweep before the local LSQ, to recentre the - // high-resolution shoeboxes onto signal that small misalignments hide. - if (data.sweep_orientation) - SweepOrientationCell(image, prediction, data); - const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); @@ -826,701 +435,244 @@ void PixelRefine::Run(const T *image, return bl * bl / (2.0 * d * d * d * d); }; - // Mutable experiment whose geometry is re-synced from the refined data.geom - // before each prediction, so shoeboxes track the refined geometry/cell. + // Geometry is FIXED here: orientation/cell/detector were already refined upstream + // by XtalOptimizer (IndexAndRefine::RefineGeometryIfNeeded). PixelRefine is an + // intensity-only operation - it predicts shoeboxes with this geometry, measures the + // tangential profile width, and fits the per-image scale G (and B) to the reference. + double beam[2], dist_mm, detector_rot[2]; + double latt_vec0[3], latt_vec1[3], latt_vec2[3]; + BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); + + // ---- 1. Predict shoeboxes for the current geometry ------------------------ DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + + // ---- 2. Collect per-reflection shoebox pixels + local background ---------- + // GetReflections() returns the full pre-sized buffer; only the first nrefl + // entries are valid for this image. A spot-core mask over ALL predictions keeps + // each reflection's background ring from picking up a neighbour's signal. + const auto &predicted = prediction.GetReflections(); + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - // State retained after the loop for the final reflection extraction. std::vector groups; - double beam[2] = {0, 0}; - double dist_mm = data.geom.GetDetectorDistance_mm(); - double detector_rot[2] = {0, 0}; - double latt_vec0[3] = {0, 0, 0}; // orientation (Rodrigues) - double latt_vec1[3] = {0, 0, 0}; // lengths - double latt_vec2[3] = {0, 0, 0}; // angles (rad) - double orient_prior[3] = {0, 0, 0}; // pre-refinement orientation (regularization anchor) + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; + const auto hkl = hkl_key_generator(refl); + if (!reference_data.contains(hkl)) + continue; - const bool eval_only = (data.max_iterations <= 0); - const int n_iter = std::max(1, data.max_iterations); - for (int iter = 0; iter < n_iter; ++iter) { - // ---- 1. Re-sync prediction geometry from the (refined) model ---------- - exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) - .BeamY_pxl(data.geom.GetBeamY_pxl()) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + // Local flat background from the ring around the shoebox (raw counts). If we + // cannot estimate a clean local background the reflection is dropped, exactly + // as BraggIntegrate2D marks it unobserved when too few background pixels survive. + double Ibkg = 0.0; + if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg)) + continue; - const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + ReflGroup g; + g.h = refl.h; + g.k = refl.k; + g.l = refl.l; + g.d = refl.d; + g.Itrue = reference_data[hkl]; + g.R_bw_sq = bandwidth_radial_sq(refl.d); + g.pol = polarization(refl.predicted_x, refl.predicted_y); + g.Ibkg = Ibkg; + g.predicted_x = refl.predicted_x; + g.predicted_y = refl.predicted_y; - // ---- 2. Collect per-reflection shoebox pixels ------------------------- - // GetReflections() returns the full pre-sized buffer; only the first - // nrefl entries are valid for this image (the rest are stale/zeroed). - groups.clear(); - const auto &predicted = prediction.GetReflections(); - - // Spot-core mask over ALL predicted reflections, so each reflection's - // local background ignores pixels that belong to a neighbouring spot. - const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - - for (int ri = 0; ri < nrefl; ++ri) { - const auto &refl = predicted[ri]; - const auto hkl = hkl_key_generator(refl); - if (!reference_data.contains(hkl)) - continue; - - // Local flat background from the ring around the shoebox (raw counts). - // No azimuthal fallback: if we cannot estimate a clean local background - // the reflection is dropped, exactly as BraggIntegrate2D marks it - // unobserved when fewer than a handful of background pixels survive. - double Ibkg = 0.0; - if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, - refl.predicted_x, refl.predicted_y, - radius, bkg_outer_radius, Ibkg)) - continue; - - ReflGroup g; - g.h = refl.h; - g.k = refl.k; - g.l = refl.l; - g.d = refl.d; - g.Itrue = reference_data[hkl]; - g.R_bw_sq = bandwidth_radial_sq(refl.d); - g.pol = polarization(refl.predicted_x, refl.predicted_y); - g.Ibkg = Ibkg; - g.predicted_x = refl.predicted_x; - g.predicted_y = refl.predicted_y; - - const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - - for (int y = box.min_y; y <= box.max_y; ++y) { - for (int x = box.min_x; x <= box.max_x; ++x) { - const size_t npixel = xpixel * y + x; - - // Skip sentinel (masked / saturated) pixels. We assume the pixel - // mask is already applied upstream (encoded as the sentinel). - if (image[npixel] == std::numeric_limits::max()) - continue; - if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) - continue; - - const double Iobs = static_cast(image[npixel]); // raw counts - - // Variance for the fit weight. Weighting by the observed count - // (var = Iobs) lets down-fluctuated background pixels carry the - // largest 1/sqrt(var) weight, which biases the fit towards "no - // signal" and drove the per-image scale G to 0 on weak images - // (collapsing the merge). Use the local background as the - // (background-limited) variance, constant over the shoebox - the - // same de-biasing applied to the extraction. - double var = std::max(Ibkg, 1.0); - double weight = 1.0 / std::sqrt(var); - - // Signal-weighting: down-weight pixels far from the predicted spot - // centre so the empty shoebox corners cannot dilute or destabilise - // the fit; the signal-bearing core drives the refined parameters. - if (data.fit_signal_sigma_pix > 0.0) { - const double dx = x - g.predicted_x; - const double dy = y - g.predicted_y; - const double s2 = data.fit_signal_sigma_pix * data.fit_signal_sigma_pix; - weight *= std::exp(-0.5 * (dx * dx + dy * dy) / s2); - } - - PixelObs obs{ - .x = static_cast(x), - .y = static_cast(y), - .Iobs = Iobs, - .Ibkg = Ibkg, - .weight = weight - }; - g.pixels.push_back(obs); - } - } - - if (!g.pixels.empty()) - groups.push_back(std::move(g)); - } - - if (groups.empty()) - return; - - // ---- 3. Set up parameter blocks (geometry part mirrors XtalOptimizer) - - BuildParameterBlocks(data, beam, dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2); - - // Anchor for orientation regularization = the orientation the LSQ starts from - // (captured before the predict<->refine iterations move it). When the global - // sweep ran first this is the swept orientation, not the original spot-centroid - // one - which is intended: the regularizer keeps the LSQ near its own starting - // point, it is not meant to pull a deliberate sweep back. - if (iter == 0) - for (int i = 0; i < 3; ++i) - orient_prior[i] = latt_vec0[i]; - - // ---- Term 3: per-reflection recentre on the observed centroid ---------------- - // The geometry predicts the spot to ~0.4 px (per-reflection scatter a global fit - // cannot remove); a tight Term-2 template centred on the prediction then sits off - // the real spot. For confident spots, shift the profile centre to the observed - // centroid (used consistently by Term 2, Term 1 and the extraction below). Weak - // spots keep the prediction (recentring on a noise centroid would bias them). - for (auto &g : groups) { - g.dcx = 0.0; - g.dcy = 0.0; - } - if (data.recenter_profile && !groups.empty()) { - const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); - for (auto &g : groups) { - double sw = 0.0, swx = 0.0, swy = 0.0; - for (const auto &px : g.pixels) { - const double w = std::max(px.Iobs - g.Ibkg, 0.0); - sw += w; swx += w * px.x; swy += w * px.y; - } - if (sw <= 0.0) + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { + const size_t npixel = xpixel * y + x; + // Skip sentinel (masked / saturated) pixels. + if (image[npixel] == std::numeric_limits::max()) continue; - if (sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)) < data.recenter_min_signif) + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) continue; - double dcx = swx / sw - g.predicted_x, dcy = swy / sw - g.predicted_y; - const double dl = std::sqrt(dcx * dcx + dcy * dcy); - if (dl > 2.0) { dcx *= 2.0 / dl; dcy *= 2.0 / dl; } // clamp runaway centroids - g.dcx = dcx; - g.dcy = dcy; + g.pixels.push_back({static_cast(x), static_cast(y), + static_cast(image[npixel]), Ibkg}); } } + if (!g.pixels.empty()) + groups.push_back(std::move(g)); + } + if (groups.empty()) + return; - // ---- Term 2: per-resolution tangential profile width R1 from spot shapes ------ - // Default: every reflection uses the global R1; with shape_R1 on, override it with - // R1 = sqrt(2*) from the intensity-weighted second moment of the strong - // spots, binned by resolution (low res small spots, high res larger). A shape - // statistic - normalised by the total, so decoupled from the per-image scale. - for (auto &g : groups) - g.R1_eff = data.R[1]; - if (data.shape_R1 && !groups.empty()) { - constexpr int n_bins = 6; - const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); - double s2min = 1e30, s2max = 0.0; - for (const auto &g : groups) { - const double s2 = 1.0 / (g.d * g.d); - s2min = std::min(s2min, s2); - s2max = std::max(s2max, s2); - } - const double span = std::max(s2max - s2min, 1e-12); - auto bin_of = [&](double d) { - return std::clamp(static_cast((1.0 / (d * d) - s2min) / span * n_bins), 0, n_bins - 1); - }; - std::vector> bin_M2(n_bins); - for (const auto &g : groups) { - double sw = 0.0, sw_et2 = 0.0; - for (const auto &px : g.pixels) { - PixelObs probe{px.x - g.dcx, px.y - g.dcy, 0.0, g.Ibkg, 1.0}; - PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, - g.R_bw_sq, g.pol, data.crystal_system); - double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) - continue; - const double w = std::max(px.Iobs - g.Ibkg, 0.0); - sw += w; - sw_et2 += w * eps_t_sq; - } - if (sw <= 0.0) - continue; - const double signif = sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)); - if (signif >= 5.0) // only well-measured spots define the shape - bin_M2[bin_of(g.d)].push_back(sw_et2 / sw); - } - std::vector bin_R1(n_bins, data.R[1]); - for (int b = 0; b < n_bins; ++b) - if (bin_M2[b].size() >= 5) { - const double r1 = std::sqrt(2.0 * std::max(MedianInPlace(bin_M2[b]), 0.0)); - if (std::isfinite(r1) && r1 > 1e-4) - bin_R1[b] = std::clamp(r1, 1e-4, 0.05); - } - for (auto &g : groups) - g.R1_eff = bin_R1[bin_of(g.d)]; - data.shape_R1_lores = bin_R1[0]; // lowest resolution bin - data.shape_R1_hires = bin_R1[n_bins - 1]; // highest resolution bin - } - - // ---- 4. Build the problem --------------------------------------------- - // One residual block per shoebox (N residuals), so the expensive - // per-reflection node geometry is evaluated once per reflection instead - // of once per pixel. - ceres::Problem problem; - size_t residual_pixels = 0; - if (data.intensity_residual) { - // Term-1 path: one per-reflection intensity residual. Geometry & R fixed, so - // J / partiality / sigma_J are computed here as constants and only G, B vary. - const double R0 = data.R[0]; - for (const auto &g : groups) { - const double R1 = g.R1_eff; // Term 2: per-resolution profile width - double num = 0.0, den = 0.0, rad = 0.0; - std::vector> pt_sig; // (P_t, Iobs-Bg) for Fisher pass - pt_sig.reserve(g.pixels.size()); - for (const auto &px : g.pixels) { - PixelObs probe{px.x - g.dcx, px.y - g.dcy, 0.0, g.Ibkg, 1.0}; // Term 3 recentre - PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, - g.R_bw_sq, g.pol, data.crystal_system); - double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) - continue; - if (!(R1 > 0.0) || !(R0 > 0.0)) - continue; - const double P_t = std::exp(-eps_t_sq / (R1 * R1)) / (M_PI * R1 * R1); - const double R0_eff_sq = R0 * R0 + g.R_bw_sq; - const double P_rad = std::exp(-eps_r * eps_r / R0_eff_sq); - const double v = std::max(g.Ibkg, 1.0); - const double sig = px.Iobs - g.Ibkg; - num += P_t * sig / v; - den += P_t * P_t / v; - rad += P_rad * P_t * P_t / v; - pt_sig.emplace_back(P_t, sig); - } - if (!(den > 0.0)) - continue; - const double J = num / den; - const double partiality = rad / den; - // Model-expected (Fisher) variance: v_p = background + expected signal J*P_t, - // not the per-pixel observed counts (which down-bias) - so the weight tracks - // information, and an expected-strong reflection that is absent hurts. - double den_f = 0.0; - for (const auto &[P_t, sig] : pt_sig) { - const double v_f = std::max(g.Ibkg + std::max(J, 0.0) * P_t, 1.0); - den_f += P_t * P_t / v_f; - } - const double sigma_J = std::sqrt(1.0 / std::max(den_f, 1e-30)); - const double inv_4d2 = (g.d > 0.0) ? 1.0 / (4.0 * g.d * g.d) : 0.0; - auto *cost = new ceres::AutoDiffCostFunction( - new IntensityResidual(J, sigma_J, partiality, g.pol, g.Itrue, inv_4d2)); - problem.AddResidualBlock(cost, nullptr, &data.scale_factor, &data.B_factor); - ++residual_pixels; - } - data.residual_count = residual_pixels; - } else { + // ---- 3. Term 2: per-resolution tangential profile width R1 ---------------- + // R1 = sqrt(2*) from the intensity-weighted tangential second moment of + // the strong spots, binned by resolution (low res small spots, high res larger). + // A *shape* statistic, normalised by the total intensity, so it is decoupled from + // the per-image scale - which is what makes measuring it (rather than fitting it, + // where it is degenerate with G) stable. Weak spots fall back to the global R[1]. + for (auto &g : groups) + g.R1_eff = data.R[1]; + { + constexpr int n_bins = 6; + const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); + double s2min = 1e30, s2max = 0.0; for (const auto &g : groups) { - auto *cost = new ceres::DynamicAutoDiffCostFunction( - new ShoeboxResidual(g, lambda, pixel_size, data.crystal_system)); - cost->AddParameterBlock(2); // beam - cost->AddParameterBlock(1); // distance - cost->AddParameterBlock(2); // detector_rot - cost->AddParameterBlock(3); // p0 (orientation) - cost->AddParameterBlock(3); // p1 (lengths) - cost->AddParameterBlock(3); // p2 (angles) - cost->AddParameterBlock(1); // scale G - cost->AddParameterBlock(1); // B - cost->AddParameterBlock(2); // R - cost->SetNumResiduals(static_cast(g.pixels.size())); - // No robust loss here: a per-block (whole-shoebox) Huber would act on - // the sum of ~N squared residuals and mis-scale, unlike the previous - // per-pixel Huber. Per-pixel sigma weighting is retained; per-pixel - // outlier rejection (zingers) is a TODO if needed. - problem.AddResidualBlock(cost, nullptr, - beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R); - residual_pixels += g.pixels.size(); + const double s2 = 1.0 / (g.d * g.d); + s2min = std::min(s2min, s2); + s2max = std::max(s2max, s2); } - data.residual_count = residual_pixels; - } - - // ---- 5. Constrain / bound parameter blocks ---------------------------- - if (data.intensity_residual) { - // Only G and B are in this problem; geometry/R are not parameters here. - problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); - if (!data.refine_B) - problem.SetParameterBlockConstant(&data.B_factor); - // Regularize G->1, weight sqrt(n_refl/sigma): commensurate because the data - // term is now one residual per reflection (unlike the per-pixel path). - if (data.scale_reg_sigma > 0.0 && !groups.empty()) { - const double w = std::sqrt(static_cast(groups.size()) / data.scale_reg_sigma); - auto *reg = new ceres::AutoDiffCostFunction( - new ScalarRegularizer(w, 1.0)); - problem.AddResidualBlock(reg, nullptr, &data.scale_factor); - } - } else { - if (!data.refine_orientation) { - problem.SetParameterBlockConstant(latt_vec0); - } else if (data.orient_reg_sigma_deg > 0.0) { - // Anchor orientation to its spot-centroid prior. The weight is scaled to - // the *pixel* data term (sqrt(n_pixels)/sigma_rad), not the reflection - // count - the data has one residual per shoebox pixel, so a reflection- - // scaled prior (~50x too weak) was simply not felt. At a misorientation of - // orient_reg_sigma_deg the prior matches the data, so the fit only moves - // further when the pixels strongly agree it should. - const double sigma_rad = std::max(data.orient_reg_sigma_deg * M_PI / 180.0, 1e-9); - const double w = std::sqrt(static_cast(residual_pixels)) / sigma_rad; - auto *reg = new ceres::AutoDiffCostFunction( - new OrientationRegularizer(w, orient_prior)); - problem.AddResidualBlock(reg, nullptr, latt_vec0); - } - - if (!data.refine_unit_cell) { - problem.SetParameterBlockConstant(latt_vec1); - problem.SetParameterBlockConstant(latt_vec2); - } else { - for (int i = 0; i < 3; ++i) { - problem.SetParameterLowerBound(latt_vec1, i, 5.0); - problem.SetParameterUpperBound(latt_vec1, i, 1000.0); - } - if (data.crystal_system != gemmi::CrystalSystem::Monoclinic && - data.crystal_system != gemmi::CrystalSystem::Triclinic) - problem.SetParameterBlockConstant(latt_vec2); - } - - if (!data.refine_beam_center) - problem.SetParameterBlockConstant(beam); - - if (!data.refine_distance) { - problem.SetParameterBlockConstant(&dist_mm); - } else { - problem.SetParameterLowerBound(&dist_mm, 0, dist_mm * 0.9); - problem.SetParameterUpperBound(&dist_mm, 0, dist_mm * 1.1); - } - - if (!data.refine_detector_angles) { - problem.SetParameterBlockConstant(detector_rot); - } else { - const double rng = 3.0 / 180.0 * M_PI; - for (int i = 0; i < 2; ++i) { - problem.SetParameterLowerBound(detector_rot, i, detector_rot[i] - rng); - problem.SetParameterUpperBound(detector_rot, i, detector_rot[i] + rng); - } - } - - if (data.refine_scale) { - problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); - // Regularize G towards 1 so weakly-constrained images cannot wander - // (an unconstrained 1/G is what collapsed the cross-image merge). Weight - // scaled to the pixel data term (n_pixels), not the reflection count. - if (data.scale_reg_sigma > 0.0) { - const double w = std::sqrt(static_cast(residual_pixels) / data.scale_reg_sigma); - auto *reg = new ceres::AutoDiffCostFunction( - new ScalarRegularizer(w, 1.0)); - problem.AddResidualBlock(reg, nullptr, &data.scale_factor); - } - } else { - problem.SetParameterBlockConstant(&data.scale_factor); - } - - if (!data.refine_B) - problem.SetParameterBlockConstant(&data.B_factor); - - if (data.refine_R) { - if (data.fix_R0) { - // Diagnostic: hold R0 constant, refine R1 only. - problem.SetManifold(data.R, new ceres::SubsetManifold(2, {0})); - } else { - problem.SetParameterLowerBound(data.R, 0, 1e-5); - problem.SetParameterLowerBound(data.R, 1, 1e-5); - } - } else { - problem.SetParameterBlockConstant(data.R); - } - } // end per-pixel (non-intensity_residual) constraints - - // ---- 6. Solve (or, for max_iterations<=0, just evaluate the cost) ----- - // Evaluate-only is the live-residual path: it reports the current cost - // without moving any parameter, so a UI can show how good the present - // R0/R1/bandwidth/geometry are as the user drags sliders. - if (eval_only) { - double cost = 0.0; - problem.Evaluate(ceres::Problem::EvaluateOptions(), &cost, nullptr, nullptr, nullptr); - data.final_cost = cost; - data.solved = true; - } else { - ceres::Solver::Options options; - options.linear_solver_type = ceres::DENSE_QR; - options.minimizer_progress_to_stdout = false; - options.logging_type = ceres::LoggingType::SILENT; - options.max_solver_time_in_seconds = data.max_time_s; - options.num_threads = 1; - - ceres::Solver::Summary summary; - ceres::Solve(options, &problem, &summary); - - data.final_cost = summary.final_cost; - data.solved = summary.IsSolutionUsable(); - - // Diagnostic: Pearson correlations on the final solve. Always needs G and B - // refined for G-B; the R correlations are added only when R is also refined. - if (data.compute_covariance && data.solved && iter == n_iter - 1 && - data.refine_scale && data.refine_B) { - ceres::Covariance::Options copt; - copt.algorithm_type = ceres::DENSE_SVD; - copt.null_space_rank = -1; // tolerate (and reveal) degenerate directions - ceres::Covariance cov(copt); - std::vector> blocks = { - {&data.scale_factor, &data.scale_factor}, {&data.B_factor, &data.B_factor}, - {&data.scale_factor, &data.B_factor}}; - if (data.refine_R) { - blocks.push_back({data.R, data.R}); - blocks.push_back({&data.scale_factor, data.R}); - blocks.push_back({&data.B_factor, data.R}); - } - if (cov.Compute(blocks, &problem)) { - double cGG, cBB, cGB; - cov.GetCovarianceBlock(&data.scale_factor, &data.scale_factor, &cGG); - cov.GetCovarianceBlock(&data.B_factor, &data.B_factor, &cBB); - cov.GetCovarianceBlock(&data.scale_factor, &data.B_factor, &cGB); - const double sG = std::sqrt(std::max(cGG, 0.0)); - const double sB = std::sqrt(std::max(cBB, 0.0)); - auto rho = [](double c, double s1, double s2) { - return (s1 > 0.0 && s2 > 0.0) ? c / (s1 * s2) : NAN; - }; - data.corr_GB = rho(cGB, sG, sB); - if (data.refine_R) { - double cRR[4], cGR[2], cBR[2]; - cov.GetCovarianceBlock(data.R, data.R, cRR); - cov.GetCovarianceBlock(&data.scale_factor, data.R, cGR); - cov.GetCovarianceBlock(&data.B_factor, data.R, cBR); - const double sR0 = std::sqrt(std::max(cRR[0], 0.0)); - const double sR1 = std::sqrt(std::max(cRR[3], 0.0)); - data.corr_GR0 = rho(cGR[0], sG, sR0); - data.corr_GR1 = rho(cGR[1], sG, sR1); - data.corr_BR0 = rho(cBR[0], sB, sR0); - data.corr_BR1 = rho(cBR[1], sB, sR1); - data.corr_R0R1 = rho(cRR[1], sR0, sR1); - } - data.covariance_valid = true; - } - } - } - - // ---- 7. Write refined geometry + lattice back into data --------------- - if (data.refine_beam_center) - data.geom.BeamX_pxl(beam[0]).BeamY_pxl(beam[1]); - if (data.refine_distance) - data.geom.DetectorDistance_mm(dist_mm); - if (data.refine_detector_angles) - data.geom.PoniRot1_rad(detector_rot[0]).PoniRot2_rad(detector_rot[1]); - - if (data.refine_orientation || data.refine_unit_cell) { - switch (data.crystal_system) { - case gemmi::CrystalSystem::Orthorhombic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - case gemmi::CrystalSystem::Tetragonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - case gemmi::CrystalSystem::Cubic: - latt_vec1[1] = latt_vec1[0]; - latt_vec1[2] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - case gemmi::CrystalSystem::Hexagonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); - break; - case gemmi::CrystalSystem::Monoclinic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); - break; - default: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, - latt_vec2[0], latt_vec2[1], latt_vec2[2]); - break; - } - } - } // predict<->refine iterations - - // ---- Adaptive integration mask -------------------------------------------- - // Measure R1 (tangential profile width) from the intensity-weighted tangential - // second moment of the strong spots, rather than fitting it. R1 is a *shape* - // statistic: sigma_t^2 = sum_p (I_p-B) eps_t,p^2 / sum_p (I_p-B), normalised by the - // total so it is independent of the per-image scale - which is exactly what breaks - // the R1<->G degeneracy (a measured width cannot be traded against G). One value per - // image here (from the strong, mostly low-res spots); a per-resolution version is the - // natural next step for the high-res / DMM-streak shapes. - if (data.adaptive_R1 && !groups.empty()) { - std::vector itrue; - itrue.reserve(groups.size()); - for (const auto &g : groups) - itrue.push_back(g.Itrue); - const size_t cut_idx = itrue.size() * 7 / 10; // keep the strongest ~30% - std::nth_element(itrue.begin(), itrue.begin() + cut_idx, itrue.end()); - const double itrue_cut = itrue[cut_idx]; - - std::vector sigma_t2; + const double span = std::max(s2max - s2min, 1e-12); + auto bin_of = [&](double d) { + return std::clamp(static_cast((1.0 / (d * d) - s2min) / span * n_bins), 0, n_bins - 1); + }; + std::vector> bin_M2(n_bins); for (const auto &g : groups) { - if (g.Itrue < itrue_cut) - continue; double sw = 0.0, sw_et2 = 0.0; for (const auto &px : g.pixels) { - PixelObs probe{px.x, px.y, 0.0, g.Ibkg, 1.0}; - PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, - g.R_bw_sq, g.pol, data.crystal_system); double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, + q_sq, eps_r, eps_t_sq)) continue; const double w = std::max(px.Iobs - g.Ibkg, 0.0); sw += w; sw_et2 += w * eps_t_sq; } - if (sw > 0.0) - sigma_t2.push_back(sw_et2 / sw); - } - if (sigma_t2.size() >= 5) { - const double r1 = std::sqrt(2.0 * MedianInPlace(sigma_t2)); // R1^2 = 2 sigma_t^2 - if (std::isfinite(r1) && r1 > 1e-5) - data.R[1] = r1; - } - } - - // ---- Centering diagnostic -------------------------------------------------- - // Observed-centroid vs predicted-position offset for the strong spots, after all - // refinement. Large rms (relative to the spot size) means a tight profile mask - // sits off the real spot - which is why a generous box can beat profile fitting. - if (data.measure_centroid && !groups.empty()) { - const double beam_x = data.geom.GetBeamX_pxl(); - const double beam_y = data.geom.GetBeamY_pxl(); - const double box_px = (2.0 * radius + 1.0) * (2.0 * radius + 1.0); - // Raw pixel value (sentinel/bounds safe) for the parabolic peak fit. A constant - // background cancels in the parabola, so no need to subtract it here. - auto pix_val = [&](int x, int y) -> double { - if (x < 0 || x >= static_cast(xpixel) || y < 0 || y >= static_cast(ypixel)) - return std::numeric_limits::quiet_NaN(); - const size_t np = static_cast(xpixel) * y + x; - if (image[np] == std::numeric_limits::max()) - return std::numeric_limits::quiet_NaN(); - if (std::is_signed_v && image[np] == std::numeric_limits::min()) - return std::numeric_limits::quiet_NaN(); - return static_cast(image[np]); - }; - struct Off { double signif, tang_c, tang_p, rad_c; }; - std::vector offs; - double sdx = 0.0, sdy = 0.0, sd2 = 0.0; - size_t nc = 0; - for (const auto &g : groups) { - double sw = 0.0, swx = 0.0, swy = 0.0, bmax = -1e30; - int bx = 0, by = 0; - for (const auto &px : g.pixels) { - const double s = px.Iobs - g.Ibkg; - const double w = std::max(s, 0.0); - sw += w; swx += w * px.x; swy += w * px.y; - if (s > bmax) { bmax = s; bx = static_cast(std::lround(px.x)); by = static_cast(std::lround(px.y)); } - } if (sw <= 0.0) continue; const double signif = sw / std::sqrt(std::max(box_px * g.Ibkg, 1.0)); - if (signif < 5.0) - continue; // a measurable spot at any resolution (not just the strong low-res ones) - // Sub-pixel peak (mode): parabola through the brightest pixel and its two - // neighbours per axis. The mode tracks the prediction even when an asymmetric - // tail drags the centroid (mean) sideways - so peak vs centroid separates a - // shape asymmetry from a true position error. - double peak_x = bx, peak_y = by; - { const double l = pix_val(bx - 1, by), c = pix_val(bx, by), r = pix_val(bx + 1, by); - const double den = l - 2.0 * c + r; - if (std::isfinite(den) && den < -1e-9) - peak_x = bx + std::clamp(0.5 * (l - r) / den, -1.0, 1.0); } - { const double l = pix_val(bx, by - 1), c = pix_val(bx, by), r = pix_val(bx, by + 1); - const double den = l - 2.0 * c + r; - if (std::isfinite(den) && den < -1e-9) - peak_y = by + std::clamp(0.5 * (l - r) / den, -1.0, 1.0); } - const double dcx = swx / sw - g.predicted_x, dcy = swy / sw - g.predicted_y; - const double dpx = peak_x - g.predicted_x, dpy = peak_y - g.predicted_y; - sdx += dcx; sdy += dcy; sd2 += dcx * dcx + dcy * dcy; ++nc; - const double rx = g.predicted_x - beam_x, ry = g.predicted_y - beam_y; - const double rr = std::sqrt(rx * rx + ry * ry); - if (rr < 1.0) - continue; - const double rad_c = (dcx * rx + dcy * ry) / rr; // signed radial (outward +) - const double tang_c = (dcx * -ry + dcy * rx) / rr; // signed tangential - const double tang_p = (dpx * -ry + dpy * rx) / rr; - offs.push_back({signif, std::fabs(tang_c), std::fabs(tang_p), rad_c}); + if (signif >= 5.0) // only well-measured spots define the shape + bin_M2[bin_of(g.d)].push_back(sw_et2 / sw); } - if (nc >= 5) { - data.centroid_bias_px = std::sqrt((sdx / nc) * (sdx / nc) + (sdy / nc) * (sdy / nc)); - data.centroid_rms_px = std::sqrt(sd2 / nc); - } - if (offs.size() >= 10) { - std::vector sig; - sig.reserve(offs.size()); - for (const auto &o : offs) - sig.push_back(o.signif); - std::nth_element(sig.begin(), sig.begin() + sig.size() / 2, sig.end()); - const double smed = sig[sig.size() / 2]; - double slo = 0, shi = 0, tclo = 0, tchi = 0, tplo = 0, tphi = 0, rclo = 0, rchi = 0; - int nlo = 0, nhi = 0; - for (const auto &o : offs) { - if (o.signif < smed) { slo += o.signif; tclo += o.tang_c; tplo += o.tang_p; rclo += o.rad_c; ++nlo; } - else { shi += o.signif; tchi += o.tang_c; tphi += o.tang_p; rchi += o.rad_c; ++nhi; } + std::vector bin_R1(n_bins, data.R[1]); + for (int b = 0; b < n_bins; ++b) + if (bin_M2[b].size() >= 5) { + const double r1 = std::sqrt(2.0 * std::max(MedianInPlace(bin_M2[b]), 0.0)); + if (std::isfinite(r1) && r1 > 1e-4) + bin_R1[b] = std::clamp(r1, 1e-4, 0.05); } - if (nlo > 0 && nhi > 0) { - data.centroid_lo_signif = slo / nlo; data.centroid_hi_signif = shi / nhi; - data.centroid_lo_tang_c = tclo / nlo; data.centroid_hi_tang_c = tchi / nhi; - data.centroid_lo_tang_p = tplo / nlo; data.centroid_hi_tang_p = tphi / nhi; - data.centroid_lo_rad_c = rclo / nlo; data.centroid_hi_rad_c = rchi / nhi; - } - } + for (auto &g : groups) + g.R1_eff = bin_R1[bin_of(g.d)]; } - // ---- Extract integrated reflections --------------------------------------- - // Profile fitting gives the recorded amplitude (fitting the tangential profile - // P_t against the background-subtracted pixels): + // ---- 4. Term 1: one intensity residual per reflection; fit G (and B) ------ + // J / partiality / sigma_J are computed here as constants (geometry & R fixed), + // and only the per-image scale G and Debye-Waller B are optimised. + ceres::Problem problem; + size_t n_blocks = 0; + const double R0 = data.R[0]; + for (const auto &g : groups) { + const double R1 = g.R1_eff; // Term 2: per-resolution profile width + if (!(R1 > 0.0) || !(R0 > 0.0)) + continue; + double num = 0.0, den = 0.0, rad = 0.0; + std::vector> pt_sig; // (P_t, Iobs-Bg) for the Fisher pass + pt_sig.reserve(g.pixels.size()); + for (const auto &px : g.pixels) { + double q_sq, eps_r, eps_t_sq; + if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, + q_sq, eps_r, eps_t_sq)) + continue; + const double P_t = std::exp(-eps_t_sq / (R1 * R1)) / (M_PI * R1 * R1); + const double R0_eff_sq = R0 * R0 + g.R_bw_sq; + const double P_rad = std::exp(-eps_r * eps_r / R0_eff_sq); + const double v = std::max(g.Ibkg, 1.0); + const double sig = px.Iobs - g.Ibkg; + num += P_t * sig / v; + den += P_t * P_t / v; + rad += P_rad * P_t * P_t / v; + pt_sig.emplace_back(P_t, sig); + } + if (!(den > 0.0)) + continue; + const double J = num / den; + const double partiality = rad / den; + // Model-expected (Fisher) variance: v_p = background + expected signal J*P_t, + // not the per-pixel observed counts (which down-bias) - so the weight tracks + // information, and an expected-strong reflection that is absent hurts. + double den_f = 0.0; + for (const auto &[P_t, sig] : pt_sig) { + const double v_f = std::max(g.Ibkg + std::max(J, 0.0) * P_t, 1.0); + den_f += P_t * P_t / v_f; + } + const double sigma_J = std::sqrt(1.0 / std::max(den_f, 1e-30)); + const double inv_4d2 = (g.d > 0.0) ? 1.0 / (4.0 * g.d * g.d) : 0.0; + auto *cost = new ceres::AutoDiffCostFunction( + new IntensityResidual(J, sigma_J, partiality, g.pol, g.Itrue, inv_4d2)); + problem.AddResidualBlock(cost, nullptr, &data.scale_factor, &data.B_factor); + ++n_blocks; + } + data.residual_count = n_blocks; + if (n_blocks == 0) + return; + + // G >= 0; B fixed unless requested; G regularized -> 1 with weight sqrt(n/sigma) + // (mirrors ScaleOnTheFly) so weakly-measured images cannot drift and scramble the merge. + problem.SetParameterLowerBound(&data.scale_factor, 0, 0.0); + if (!data.refine_B) + problem.SetParameterBlockConstant(&data.B_factor); + if (data.scale_reg_sigma > 0.0) { + const double w = std::sqrt(static_cast(groups.size()) / data.scale_reg_sigma); + auto *reg = new ceres::AutoDiffCostFunction( + new ScalarRegularizer(w, 1.0)); + problem.AddResidualBlock(reg, nullptr, &data.scale_factor); + } + + ceres::Solver::Options options; + options.linear_solver_type = ceres::DENSE_QR; + options.minimizer_progress_to_stdout = false; + options.logging_type = ceres::LoggingType::SILENT; + options.max_solver_time_in_seconds = data.max_time_s; + options.num_threads = 1; + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + data.final_cost = summary.final_cost; + data.solved = summary.IsSolutionUsable(); + + // ---- 5. Extract integrated reflections ------------------------------------ + // Profile fitting gives the recorded amplitude (fitting the tangential profile P_t + // against the background-subtracted pixels): // J = sum_p[ P_t,p (Iobs_p - Ibkg)/v_p ] / sum_p[ P_t,p^2 / v_p ] // ~ G * Itrue * B_term * partiality * pol (recorded raw counts) // var(J) = 1 / sum_p[ P_t,p^2 / v_p ] // - // Two SEPARATE fractions reduce the full intensity to what these pixels record: - // - // partiality - the radial / rocking dimension that a still does NOT sample. - // Only the slice of the reflection that crosses the Ewald - // sphere on this shot is recorded; <= 1. We DIVIDE it out to - // recover the full intensity. = profile-weighted P_radial. - // - // completeness - the fraction of the spot's detector footprint that landed on - // live pixels (= profile captured by live pixels / profile over - // the whole shoebox). 1.0 when the spot sits fully on the - // detector; < 1.0 only when a detector edge, gap or mask clips - // it. Profile fitting already extrapolates over the missing - // pixels, so this is NOT applied to r.I - it is a quality flag. - // // Output split (Merge multiplies r.I * image_scale_corr and weights by - // 1/(sigma*image_scale_corr)^2 - see Merge.cpp): + // 1/(sigma*image_scale_corr)^2): // r.I = J / (B_term * partiality * pol) = G * Itrue // r.sigma = sqrt(var(J)) / (B_term * partiality * pol) // r.partiality = profile-weighted P_radial in (0,1] (the rocking fraction) // r.completeness = live/total tangential profile in (0,1] (detector clipping) - // r.image_scale_corr = 1/G (per-image scale ONLY) - // so r.I * image_scale_corr = Itrue. B, partiality and polarization live on the - // intensity, G lives on image_scale_corr - one clean meaning per field. - // - // We walk the full (unclamped) shoebox once: every grid point feeds the total - // tangential profile (completeness denominator); points that are real, live - // detector pixels also feed the profile fit and the captured profile. + // r.image_scale_corr = 1/G (per-image scale ONLY) + // so r.I * image_scale_corr = Itrue: B, partiality and polarization live on the + // intensity, G lives on image_scale_corr - one clean meaning per field. We walk the + // full (unclamped) shoebox once: every grid point feeds the total tangential profile + // (completeness denominator); points that are real, live pixels also feed the fit. data.reflections.reserve(groups.size()); for (const auto &g : groups) { const int cx = static_cast(std::lround(g.predicted_x)); const int cy = static_cast(std::lround(g.predicted_y)); - - // Debye-Waller factor for this reflection (constant over its shoebox). const double B_term = std::exp(-data.B_factor / (4.0 * g.d * g.d)); - // Term 3 recentre shift (precomputed in the pre-pass; 0 if off or the spot was not - // confident). Profile evaluated at (x-dcx, y-dcy), data summed at (x,y). - const double dcx = g.dcx, dcy = g.dcy; - double num = 0.0, den = 0.0, bkg_sum = 0.0, radial_sum = 0.0; double prof_live = 0.0, prof_full = 0.0; // tangential profile: captured / total size_t n = 0; for (int y = cy - radius; y <= cy + radius; ++y) { for (int x = cx - radius; x <= cx + radius; ++x) { - // Geometry/profile for this grid point (profile recentred by (dcx,dcy)). - PixelObs probe{static_cast(x) - dcx, static_cast(y) - dcy, - 0.0, g.Ibkg, 1.0}; - PixelResidual pr(probe, 1.0, lambda, pixel_size, g.h, g.k, g.l, - g.R_bw_sq, g.pol, data.crystal_system); double q_sq, eps_r, eps_t_sq; - if (!pr.GeometryTerms(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) + if (!GeometryProbe(static_cast(x), static_cast(y), + lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, + q_sq, eps_r, eps_t_sq)) continue; if (!(data.R[0] > 0.0) || !(g.R1_eff > 0.0)) continue; - // Tangential profile shape (area-normalized) -> the fit template. Uses the - // per-reflection R1_eff (Term 2), falling back to the global R1 by default. + // Tangential profile shape (area-normalized) -> the fit template, using the + // per-reflection R1_eff (Term 2). const double R1 = g.R1_eff; const double P_t = std::exp(-eps_t_sq / (R1 * R1)) / (M_PI * R1 * R1); prof_full += P_t; // whole shoebox, on- or off-detector @@ -1535,30 +687,21 @@ void PixelRefine::Run(const T *image, continue; const double Iobs = static_cast(image[np]); // raw counts - // Variance for the profile-fit weights. Weighting by the *observed* - // per-pixel count (v = Iobs) biases the amplitude negative: a - // down-fluctuated background pixel gets the smallest v and hence the - // largest 1/v weight, so num = sum P_t (Iobs - Ibkg)/v is pulled below - // zero - worst where the signal is weakest, i.e. the high-resolution - // shells (the negative we see there). For background-limited - // reflections the variance is the local background, constant over the - // shoebox, so use that instead of the observed count. - double v = std::max(g.Ibkg, 1.0); + // Background-limited variance (constant over the shoebox): weighting by + // the observed count biases the amplitude negative where signal is weakest. + const double v = std::max(g.Ibkg, 1.0); - // Peak-normalized radial factor (the partiality), in (0,1]. The - // bandwidth-broadened radial width matches the model in Model(). + // Peak-normalized radial factor (the partiality), in (0,1]. MUST use the + // same P_t^2/v weights as the amplitude, else an R0_eff-dependent (hence + // resolution-dependent) factor is left behind in r.I. const double R0_eff_sq = data.R[0] * data.R[0] + g.R_bw_sq; const double P_radial = std::exp(-eps_r * eps_r / R0_eff_sq); - // Profile-fit accumulators. The amplitude estimator weights pixels by - // P_t^2/v, so the partiality (which de-scales that amplitude) MUST use - // the SAME weights - otherwise an R0_eff-dependent (resolution- - // dependent) factor is left behind in r.I. const double w = P_t * P_t / v; num += P_t * (Iobs - g.Ibkg) / v; den += w; - radial_sum += P_radial * w; // partiality weighted exactly like num/den - prof_live += P_t; // captured tangential profile + radial_sum += P_radial * w; + prof_live += P_t; bkg_sum += g.Ibkg; ++n; } @@ -1580,7 +723,7 @@ void PixelRefine::Run(const T *image, if (den > 0.0 && n > 0) { const double I_amp = num / den; // ~ G*Itrue*B_term*partiality*pol const double sigma_amp = std::sqrt(1.0 / den); - const double corr = static_cast(r.partiality) * B_term * g.pol; // B, partiality & pol + const double corr = static_cast(r.partiality) * B_term * g.pol; r.bkg = static_cast(bkg_sum / static_cast(n)); r.observed = true; @@ -1602,9 +745,9 @@ void PixelRefine::Run(const T *image, data.reflections.push_back(r); } - // ---- Per-image CC vs reference (the half/ref correlation diagnostic) ------- - // Pearson CC of the scaled estimate (r.I * image_scale_corr = Itrue_est) - // against the reference intensities, over the matched reflections. + // ---- 6. Per-image CC vs reference (diagnostic) ---------------------------- + // Pearson CC of the scaled estimate (r.I * image_scale_corr = Itrue_est) against + // the reference intensities, over the matched reflections. { double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; size_t cn = 0; @@ -1633,231 +776,9 @@ void PixelRefine::Run(const T *image, } } -template -std::vector PixelRefine::PredictImage(const T *image, - BraggPrediction &prediction, - const PixelRefineData &data, - bool include_background) const { - std::vector img(xpixel * ypixel, 0.0f); - - const double lambda = data.geom.GetWavelength_A(); - const double pixel_size = data.geom.GetPixelSize_mm(); - const int radius = data.shoebox_radius; - const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); - const double bw = data.bandwidth; - - const auto pol_factor = experiment.GetPolarizationFactor(); - auto polarization = [&](double x, double y) -> double { - if (!pol_factor) - return 1.0; - return data.geom.CalcAzIntPolarizationCorr(static_cast(x), static_cast(y), - pol_factor.value()); - }; - auto bandwidth_radial_sq = [&](double d) -> double { - if (bw <= 0.0 || d <= 0.0) - return 0.0; - const double bl = bw * lambda; - return bl * bl / (2.0 * d * d * d * d); - }; - - double beam[2], dist_mm, detector_rot[2]; - double latt_vec0[3], latt_vec1[3], latt_vec2[3]; - BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); - - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) - .BeamY_pxl(data.geom.GetBeamY_pxl()) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - - const BraggPredictionSettings settings_prediction{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) // relative Δλ/λ sigma - }; - const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); - const auto &predicted = prediction.GetReflections(); - const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - - for (int ri = 0; ri < nrefl; ++ri) { - const auto &refl = predicted[ri]; - const auto it = reference_data.find(hkl_key_generator(refl)); - if (it == reference_data.end()) - continue; - - const double Itrue = it->second; - const double R_bw_sq = bandwidth_radial_sq(refl.d); - const double pol = polarization(refl.predicted_x, refl.predicted_y); - - // Local background straight from the actual image (flat per shoebox), laid - // into the box so the prediction overlays the real frame - the same model - // path Run() fits, now reproduced faithfully because we have the image. - double Ibkg = 0.0; - const bool have_bkg = include_background && - EstimateLocalBackground(image, spot_mask, xpixel, ypixel, - refl.predicted_x, refl.predicted_y, - radius, bkg_outer_radius, Ibkg); - - const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - - for (int y = box.min_y; y <= box.max_y; ++y) { - for (int x = box.min_x; x <= box.max_x; ++x) { - const size_t npixel = xpixel * y + x; - - PixelObs obs{ - .x = static_cast(x), - .y = static_cast(y), - .Iobs = 0.0, - .Ibkg = have_bkg ? Ibkg : 0.0, - .weight = 1.0 - }; - PixelResidual pr(obs, Itrue, lambda, pixel_size, - refl.h, refl.k, refl.l, R_bw_sq, pol, data.crystal_system); - - double Ipred = 0.0; // raw counts: signal (+ local background) - if (pr.Model(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R, Ipred)) - img[npixel] += static_cast(Ipred); - } - } - } - - return img; -} - -template -std::vector PixelRefine::ChiSquaredImage(const T *image, - BraggPrediction &prediction, - const PixelRefineData &data) const { - std::vector img(xpixel * ypixel, 0.0f); - - const double lambda = data.geom.GetWavelength_A(); - const double pixel_size = data.geom.GetPixelSize_mm(); - const int radius = data.shoebox_radius; - const int bkg_outer_radius = std::max(radius + 1, data.bkg_outer_radius); - const double bw = data.bandwidth; - - const auto pol_factor = experiment.GetPolarizationFactor(); - auto polarization = [&](double x, double y) -> double { - if (!pol_factor) - return 1.0; - return data.geom.CalcAzIntPolarizationCorr(static_cast(x), static_cast(y), - pol_factor.value()); - }; - auto bandwidth_radial_sq = [&](double d) -> double { - if (bw <= 0.0 || d <= 0.0) - return 0.0; - const double bl = bw * lambda; - return bl * bl / (2.0 * d * d * d * d); - }; - - double beam[2], dist_mm, detector_rot[2]; - double latt_vec0[3], latt_vec1[3], latt_vec2[3]; - BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); - - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) - .BeamY_pxl(data.geom.GetBeamY_pxl()) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - - const BraggPredictionSettings settings_prediction{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) - }; - const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); - const auto &predicted = prediction.GetReflections(); - const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - - for (int ri = 0; ri < nrefl; ++ri) { - const auto &refl = predicted[ri]; - const auto it = reference_data.find(hkl_key_generator(refl)); - if (it == reference_data.end()) - continue; - - const double Itrue = it->second; - const double R_bw_sq = bandwidth_radial_sq(refl.d); - const double pol = polarization(refl.predicted_x, refl.predicted_y); - - // Local flat background, identical to Run(); skip the reflection if it - // cannot be estimated (matches Run() dropping the reflection). - double Ibkg = 0.0; - if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, - refl.predicted_x, refl.predicted_y, - radius, bkg_outer_radius, Ibkg)) - continue; - - const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - - for (int y = box.min_y; y <= box.max_y; ++y) { - for (int x = box.min_x; x <= box.max_x; ++x) { - const size_t npixel = xpixel * y + x; - - // Same gating as Run(): only pixels that actually enter the fit. - if (image[npixel] == std::numeric_limits::max()) - continue; - if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) - continue; - - const double Iobs = static_cast(image[npixel]); // raw counts - - double var = std::max(Iobs, 0.0); - if (!(var > 1.0)) - var = 1.0; - const double weight = 1.0 / std::sqrt(var); - - PixelObs obs{ - .x = static_cast(x), - .y = static_cast(y), - .Iobs = Iobs, - .Ibkg = Ibkg, - .weight = weight - }; - PixelResidual pr(obs, Itrue, lambda, pixel_size, - refl.h, refl.k, refl.l, R_bw_sq, pol, data.crystal_system); - - double Ipred = 0.0; - if (pr.Model(beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R, Ipred)) { - // residual_i = (I_pred - I_obs) * weight (== Ceres residual); - // its square is this pixel's contribution to the cost. - const double rw = (Ipred - Iobs) * weight; - img[npixel] += static_cast(rw * rw); - } - } - } - } - - return img; -} - -// Explicit instantiations for the supported (uncompressed) image pixel types. template void PixelRefine::Run(const int8_t *, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const int16_t *, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const int32_t *, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const uint8_t *, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const uint16_t *, BraggPrediction &, PixelRefineData &); template void PixelRefine::Run(const uint32_t *, BraggPrediction &, PixelRefineData &); - -template std::vector PixelRefine::PredictImage(const int8_t *, BraggPrediction &, const PixelRefineData &, bool) const; -template std::vector PixelRefine::PredictImage(const int16_t *, BraggPrediction &, const PixelRefineData &, bool) const; -template std::vector PixelRefine::PredictImage(const int32_t *, BraggPrediction &, const PixelRefineData &, bool) const; -template std::vector PixelRefine::PredictImage(const uint8_t *, BraggPrediction &, const PixelRefineData &, bool) const; -template std::vector PixelRefine::PredictImage(const uint16_t *, BraggPrediction &, const PixelRefineData &, bool) const; -template std::vector PixelRefine::PredictImage(const uint32_t *, BraggPrediction &, const PixelRefineData &, bool) const; - -template std::vector PixelRefine::ChiSquaredImage(const int8_t *, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const int16_t *, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const int32_t *, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint8_t *, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint16_t *, BraggPrediction &, const PixelRefineData &) const; -template std::vector PixelRefine::ChiSquaredImage(const uint32_t *, BraggPrediction &, const PixelRefineData &) const; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index f8db193f..dfc560c5 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -3,247 +3,100 @@ #pragma once -#include - #include "../bragg_prediction/BraggPrediction.h" #include "../common/DiffractionExperiment.h" #include "../scale_merge/HKLKey.h" // ============================================================================= -// PixelRefine — one optimization to rule geometry, integration and scaling +// PixelRefine — reference-driven profile-fit integration + scaling for stills // ============================================================================= // -// Intent -// ------ -// Classical crystallographic data processing is a one-way pipeline: +// PixelRefine is the still-image integrator: given a reference set of merged +// intensities I_ref (the current best hypothesis for each reflection's full +// intensity), it integrates one image and returns already-scaled intensities. It +// is an *intensity-wise* operation - the detector geometry is taken as fixed (it +// was refined upstream by XtalOptimizer in IndexAndRefine::RefineGeometryIfNeeded); +// PixelRefine does not touch orientation, cell or detector parameters. // -// spot finding -> indexing -> geometry refinement -> integration -> scaling -> merging +// The objective is the factored per-reflection likelihood of FACTORED_MODEL.md, +// Terms 1 + 2: // -// Each stage consumes the previous stage's output and never talks back. The -// integrator trusts the refined geometry; the scaler trusts the integrated -// intensities; nothing downstream is ever allowed to correct an upstream -// parameter. Post-refinement and profile fitting were the field's partial -// answers to this: post-refinement lets merged intensities nudge per-image -// orientation/cell/mosaicity, and profile fitting lets a learned spot shape -// improve weak-reflection intensities. But both are narrow back-channels bolted -// onto a feed-forward pipeline — there is no end-to-end gradient that flows from -// the raw detector pixels all the way back to every parameter at once. +// Term 2 (shape) — for each resolution shell, the tangential profile width R1 is +// *measured* from the intensity-weighted second moment of the strong spots: +// R1 = sqrt(2*). A second moment is normalised by the total intensity, +// so it is decoupled from the per-image scale - which is why measuring R1 is +// stable where *fitting* it (degenerate with G) is not. // -// PixelRefine is an experiment in doing the whole thing as a *single* least -// squares problem. We write down, for every pixel in a reflection's shoebox, the -// expected counts as an explicit forward model +// Term 1 (intensity / scaling) — one residual per reflection: the profile-fit +// amplitude J (using the Term-2 R1) should equal the scaled reference +// J_model = G * exp(-B/4d^2) * partiality * pol * I_ref, +// weighted by the model-expected (Fisher) sigma_J. Only the per-image scale G +// and Debye-Waller B are optimised. Integration and scaling become one objective; +// the many empty shoebox pixels enter only through J (with ~zero profile weight) +// instead of dominating a per-pixel loss. // -// I_pred(pixel) = G * I_true * B_term * P_radial * P_tangential * pol + I_bkg +// I_ref is NOT refined here - it is a fixed hypothesis for the pass. The intended +// outer loop is EM-like: run PixelRefine on every image against the current I_ref, +// re-merge to a new I_ref, repeat. // -// in raw detector counts (pol = per-reflection polarization correction, I_bkg = -// local per-shoebox background read from the image). +// Forward model per pixel (raw detector counts, no per-pixel solid-angle/Lorentz +// weighting - same units as the classical integrator): +// signal = G * I_ref * B_term * P_radial * P_tangential * pol , + I_bkg +// B_term = exp(-B |q|^2 / 4) (Debye-Waller) +// P_radial = exp(-eps_r^2 / R0_eff^2) (still partiality, <= 1) +// P_tangential = exp(-eps_t^2 / R1^2) / (pi R1^2) (area-normalized profile) +// where eps_r / eps_t are the radial / tangential deviations of the pixel from the +// predicted node, and pol is the per-reflection polarization correction. // -// and let Ceres autodiff back-propagate the per-pixel residuals into ALL of: -// * detector geometry (beam centre, distance, tilt) -// * crystal orientation + unit cell -// * overall scale G and Debye-Waller B -// * the reciprocal-space spot widths R = (radial, tangential) -// simultaneously. Geometry refinement, profile-fitted integration and scaling -// then stop being separate stages: they are different parameters of one model, -// coupled through the same pixels, with full backpropagation between them. Once -// the model is differentiable end-to-end, things that used to need bespoke code -// — mosaicity refinement, profile fitting, partiality — fall out "for free" as -// extra parameters of the same forward model. -// -// How I_true enters -// ----------------- -// I_true is NOT refined here. It is a *fixed hypothesis* for the duration of a -// pass: the current best merged estimate of each reflection's full intensity. -// The intended outer loop is iterative, like EM / self-consistent field: -// -// repeat over the whole dataset: -// run PixelRefine on every image with the current I_true reference -// re-merge the resulting intensities -> new I_true -// until the reference stops changing -// -// So a single PixelRefine call answers "given this intensity hypothesis, what -// geometry/scale/profile best explains these pixels, and what intensities do I -// read back out?", and the dataset-level loop refines the hypothesis itself. -// -// Inner predict<->refine loop -// --------------------------- -// Within one image we also iterate (max_iterations): Bragg prediction places the -// shoeboxes, we refine, the refined geometry/cell feed the next prediction, etc. -// Initially the model geometry equals the experiment's DiffractionGeometry, but -// as refinement proceeds it diverges, so later predictions must use the *refined* -// data.geom rather than the static experiment geometry. -// -// On shoeboxes, tails, and gatekeeping -// ------------------------------------ -// We deliberately do NOT chase the full spot. A shoebox only needs to cover -// enough of a reflection for the fit to be meaningful; clipped tails are fine, -// because the partiality term already downweights whatever falls outside the -// well-modelled core. The premise - especially for serial crystallography - is -// that the problem is rarely *missing* information; it is *failing to gate out* -// information that is not meaningful. As long as the model knows a piece is -// missing (low partiality), it is safe to leave it missing. That flips the usual -// trade-off: rather than shrinking boxes to avoid contamination, we can grow them -// and let partiality decide, per pixel and per reflection, what actually carries -// signal. (For downstream integration this pixel-level gating is the point - keep -// only meaningful pixels, instead of a fixed geometric mask.) The bandwidth term -// below is part of the same idea: it tells the model where the radial tails are -// *expected* to be, so it can weight rather than blindly include them. -// -// X-ray bandwidth (optional) -// -------------------------- -// A finite bandwidth thickens the Ewald shell radially and smears spots along the -// radial direction, growing like 1/d^2 (the pink-beam/DMM signature). It enters -// as a fixed, resolution-dependent addition to the radial width R0 (see -// PixelRefineData::bandwidth and PixelRefine.cpp). It is OFF by default -// (bandwidth = 0, monochromatic); set it for DMM-type data, leave it for Si. -// -// Status: experimental prototype. The forward model (esp. the still-image -// partiality normalization) is deliberately simple and expected to evolve: it -// works in raw detector counts with a local per-shoebox background and a -// per-reflection polarization correction (no per-pixel solid-angle/Lorentz -// weighting), matching the classical integrator. See PixelRefine.cpp for the -// physics conventions and known caveats. +// X-ray bandwidth (optional): a finite bandwidth thickens the Ewald shell radially, +// adding a fixed, resolution-dependent term to R0 that grows like 1/d^2 (the +// pink-beam/DMM signature): R0_eff^2 = R0^2 + (b*lambda)^2/(2 d^4). b = 0 (the +// default) is a monochromatic no-op; set it for DMM-type data, leave it for Si. // ============================================================================= struct PixelRefineData { // --- model state (input as initial guess, output as refined result) --- - DiffractionGeometry geom; - CrystalLattice latt; + DiffractionGeometry geom; // fixed (refined upstream by XtalOptimizer) + CrystalLattice latt; // fixed gemmi::CrystalSystem crystal_system = gemmi::CrystalSystem::Triclinic; char centering = 'P'; - double B_factor = 0.0; // Debye-Waller B (A^2) - double scale_factor = 1.0; // overall scale G - double R[2] = {0.005, 0.005}; // R[0] = radial (partiality) width, R[1] = tangential (profile) width (A^-1) + double B_factor = 0.0; // Debye-Waller B (A^2), refined + double scale_factor = 1.0; // overall per-image scale G, refined + double R[2] = {0.005, 0.005}; // R[0] = radial (partiality) width; R[1] = fallback + // tangential profile width before Term 2 measures it (A^-1) + bool refine_B = true; // refine the per-image B-factor along with G - // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% - // FWHM DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening - // to R[0]. 0 = monochromatic (the term switches off entirely). + // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% FWHM + // DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening to R[0]. + // 0 = monochromatic (the term switches off entirely). double bandwidth = 0.0; - // --- what to refine --- - bool refine_orientation = true; // crystal orientation (p0) - bool refine_unit_cell = false; // cell lengths + angles - bool refine_beam_center = false; - bool refine_distance = false; - bool refine_detector_angles = false; - bool refine_scale = true; - bool refine_B = false; - bool refine_R = false; // per-image R refinement is unstable on sparse - // stills; R is held at its nominal value - - // Orientation refinement is anchored to the pre-refinement (spot-centroid) - // orientation with weight sqrt(n_refl)/sigma, so the pixel fit can only nudge - // the orientation by ~this many degrees before the prior pushes back. This is - // what turns the (otherwise overfitting) 3-DOF orientation refinement into a - // small, signal-supported sub-spot correction. Larger => freer; very large - // approaches the unregularized (collapsing) fit. ~1 deg gives the best CCref here; - // beyond ~2 deg the per-image fit overfits and the merge collapses. - double orient_reg_sigma_deg = 1.0; - - // Signal-weighting of the *fit* residuals: each pixel's weight is multiplied by - // a detector-space Gaussian exp(-r^2/2 sigma^2) centred on the predicted spot, so - // the many empty shoebox-corner pixels stop diluting (and destabilising) the fit - // and the signal-bearing core drives the refined scale/R. <= 0 disables (uniform). - double fit_signal_sigma_pix = 1.5; - // Per-image scale G is regularized towards 1 with weight sqrt(n_refl/scale_reg_sigma) // (mirrors ScaleOnTheFly). Without this the unconstrained G wanders on weakly // measured images and 1/G scrambles the cross-image merge. <= 0 disables. double scale_reg_sigma = 2.0; - // Radial Ewald-sphere acceptance band for prediction (A^-1): a reflection is - // given a shoebox when ||S|-1/lambda| <= this. The narrow default predicts only - // reflections already on the Ewald sphere; widening it (towards the integrator's - // ~2-3x profile radius) lets in the slightly-misaligned high-resolution - // reflections - more multiplicity, and something for orientation refinement to - // actually centre. Safe to widen only with the per-image fit kept well-behaved - // (de-biased variance + signal-weighting + regularization, all default here). + // Radial Ewald-sphere acceptance band for prediction (A^-1): a reflection is given + // a shoebox when ||S|-1/lambda| <= this. Widened from the on-sphere default towards + // the integrator's profile radius so slightly-misaligned high-resolution reflections + // are still integrated (multiplicity), while the partiality downweights their tails. double ewald_dist_cutoff = 0.0020; - // Pre-LSQ global orientation+cell sweep (maximises CC vs reference over the - // strongest reflections). Bounds are derived from the detector geometry: the - // step moves the highest-resolution spot by 1 px, the range moves the lowest- - // resolution spot by ~2 px (rotation) / ~1 px (cell scale), so the low-res - // XtalOptimizer solution is preserved while high-res spots are recentred. - bool sweep_orientation = false; - // Sweep half-range as the *orientation uncertainty* (degrees) and cell-scale - // uncertainty (fraction). These set how far the highest-resolution spot may move - // (a few px); low-res spots barely move and stay anchored. Keep small - a large - // range lets the per-image CC overfit and degrades the merge. - double sweep_max_deg = 0.15; - double sweep_max_cell_frac = 0.003; - double max_time_s = 5.0; - int shoebox_radius = 3; // half-size of the per-reflection signal box (peak region that enters the fit) - // Half-size of the local-background sampling box. Background is estimated from - // the ring shoebox_radius < |dx|,|dy| <= bkg_outer_radius around each spot - // (excluding pixels belonging to any predicted spot core), mirroring the local - // shoebox background of BraggIntegrate2D. Must be > shoebox_radius. + int shoebox_radius = 3; // half-size of the per-reflection signal box + // Half-size of the local-background sampling box. Background is the MEAN of the ring + // shoebox_radius < |dx|,|dy| <= bkg_outer_radius (excluding spot cores), like + // BraggIntegrate2D. Must be > shoebox_radius. int bkg_outer_radius = 6; - int max_iterations = 3; // inner predict<->refine cycles (re-predict with refined geom/latt) - - // Diagnostic: compute the parameter correlation matrix (Pearson) from the final - // per-image solve, to expose degeneracies (e.g. G<->B, G<->R1) that let the fit - // lower its chi-square along directions that do not generalise across images. - // Requires G, B and R all refined; results in corr_* below. - bool compute_covariance = false; - bool fix_R0 = false; // diagnostic: with refine_R, hold R0 constant and refine R1 only - - // Factored-likelihood Term 1 (FACTORED_MODEL.md): replace the per-pixel fit with one - // per-reflection *intensity* residual J vs G*B_term*partiality*pol*I_ref, Fisher- - // weighted. Geometry & R fixed; only G (and B if refine_B) are fit. - bool intensity_residual = false; - - // Factored-likelihood Term 2: set the tangential profile width R1 from the *measured* - // per-resolution second moment of the strong spots (a shape statistic, decoupled from - // the scale) and feed it to the profile template used by Term 1 and the extraction. - bool shape_R1 = false; - double shape_R1_lores = NAN, shape_R1_hires = NAN; // measured R1 in the lowest/highest-res bin (diag) - - // Adaptive integration mask: set R1 (tangential profile width) from the *measured* - // tangential second moment of the strong spots, instead of fitting it (which is - // degenerate with the per-image scale). A shape statistic, independent of scale. - bool adaptive_R1 = false; - - // Diagnostic: residual centering error after refinement. For the strong spots, - // the offset between observed intensity centroid and predicted position - bias is - // the systematic (mean) part, rms the total. If rms is comparable to the spot size, - // a tight profile mask lands off the spot and box-summing wins. - bool measure_centroid = false; - double centroid_bias_px = NAN; - double centroid_rms_px = NAN; - // Offset split by spot significance (lo/hi about the median) and measured two ways: - // the intensity *centroid* (mean) and the sub-pixel *peak* (mode, parabolic fit). - // - centroid offset shrinking lo->hi => noise floor (offset ~ 1/sqrt(counts)); - // flat with significance => a systematic position error to model. - // - peak offset << centroid offset => the spot is asymmetric (non-Gaussian / - // parallax): the prediction sits on the peak, only the centroid is pulled, so - // recentring on the centroid would be wrong - the shape is what to model. - // - radial centroid offset ~ 0 => no distance/parallax error (parallax is radial). - double centroid_lo_signif = NAN, centroid_hi_signif = NAN; // mean significance per bin - double centroid_lo_tang_c = NAN, centroid_hi_tang_c = NAN; // mean |tangential centroid offset| - double centroid_lo_tang_p = NAN, centroid_hi_tang_p = NAN; // mean |tangential peak offset| - double centroid_lo_rad_c = NAN, centroid_hi_rad_c = NAN; // mean signed radial centroid offset - - // Test: recentre the extraction profile on each spot's observed centroid (instead of - // the predicted position) so a tight mask lands on the real spot - but only for spots - // whose in-shoebox significance exceeds recenter_min_signif (recentring on a noise - // centroid would bias weak reflections positive). - bool recenter_profile = false; - double recenter_min_signif = 5.0; // --- output --- - std::vector reflections; // profile-fitted integration result + std::vector reflections; // profile-fitted, scaled integration result bool solved = false; double final_cost = NAN; size_t residual_count = 0; double cc = NAN; // per-image CC of scaled intensities vs reference int64_t cc_n = 0; // number of reflections in the CC - - bool covariance_valid = false; - double corr_GB = NAN, corr_GR0 = NAN, corr_GR1 = NAN; - double corr_BR0 = NAN, corr_BR1 = NAN, corr_R0R1 = NAN; }; class PixelRefine { @@ -253,67 +106,24 @@ class PixelRefine { const HKLKeyGenerator hkl_key_generator; std::map reference_data; - // Fills the Ceres parameter blocks (geometry + symmetry-aware lattice - // parametrization) from the current model state. Shared by Run and - // PredictImage so both walk identical geometry/lattice code. + // Fills the fixed geometry + symmetry-aware lattice parametrization (beam, + // distance, detector tilt, and the Rodrigues orientation / cell-length / angle + // vectors) from the current model state, for the per-pixel geometry evaluation. void BuildParameterBlocks(const PixelRefineData &data, double beam[2], double &dist_mm, double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; - - // Global orientation + uniform cell-scale sweep run before the LSQ. Re-projects - // the strongest reference reflections through candidate lattices and keeps the - // one maximising CC vs the reference intensities (coordinate descent over the 3 - // Rodrigues axes + cell scale, within geometry-derived pixel bounds). Writes the - // refined orientation/cell back into data.latt. See PixelRefineData::sweep_*. - template - void SweepOrientationCell(const T *image, BraggPrediction &prediction, - PixelRefineData &data) const; public: PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference); - // The BraggPrediction is supplied per call (it is mutated): this keeps a - // single PixelRefine instance usable from several threads, each passing its - // own prediction buffer. Only `data` is written; PixelRefine state is const. - // The image is in raw detector counts (masked/saturated pixels carry the type - // sentinel); background is estimated locally per shoebox from the image itself. + // The BraggPrediction is supplied per call (it is mutated): this keeps a single + // PixelRefine instance usable from several threads, each passing its own prediction + // buffer. Only `data` is written; PixelRefine state is const. The image is in raw + // detector counts (masked/saturated pixels carry the type sentinel); background is + // estimated locally per shoebox from the image itself. template void Run(const T *image, BraggPrediction &prediction, PixelRefineData &data); - - // Render the forward model as a full detector image (raw detector counts, so - // it overlays directly on the original image). Uses the *same* per-pixel - // model path (PixelResidual::Model) as the optimizer, evaluated in double - // precision - slow but exact. For each reference reflection it adds the Bragg - // signal over its shoebox; with include_background it also lays down the local - // per-shoebox background read from the supplied image - the same background the - // fit uses. Diagnostic tool, not on the hot path. - template - std::vector PredictImage(const T *image, - BraggPrediction &prediction, - const PixelRefineData &data, - bool include_background = true) const; - - // Render the per-pixel chi-square (cost density) that the optimizer actually - // minimizes: for every shoebox pixel that enters the fit it stores the squared - // weighted residual ((I_pred - I_obs)/sigma)^2 in raw counts - identical to the - // Ceres residual_i^2 - accumulating where shoeboxes overlap. Pixels that are not - // part of any shoebox stay 0; masked/saturated pixels (skipped by the fit) also - // stay 0. Summing the image gives ~2*final_cost. Diagnostic tool. - template - std::vector ChiSquaredImage(const T *image, - BraggPrediction &prediction, - const PixelRefineData &data) const; - - // Reference (merged) intensity used as the fixed hypothesis for a reflection, - // or nullopt if this hkl is not in the reference. Lets callers show the fitted - // estimate next to the reference it was scaled against. - std::optional ReferenceIntensity(const Reflection &r) const { - const auto it = reference_data.find(hkl_key_generator(r)); - if (it == reference_data.end()) - return std::nullopt; - return it->second; - } }; diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 6bfbd37e..d7f0875c 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -86,13 +86,8 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView ${APP_RESOURCES} windows/JFJochViewerReciprocalSpaceWindow.cpp windows/JFJochViewerReciprocalSpaceWindow.h - windows/JFJochPixelRefineWindow.cpp - windows/JFJochPixelRefineWindow.h - windows/JFJochPixelRefineTableWindow.cpp - windows/JFJochPixelRefineTableWindow.h windows/JFJochMagnifierWindow.cpp windows/JFJochMagnifierWindow.h - windows/PixelRefineParams.h ) TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::DBus Qt6::Concurrent diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 1e92cfed..a10ded30 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -9,8 +9,6 @@ #include "JFJochImageReadingWorker.h" #include "../reader/JFJochReaderImage.h" // JFJochReaderImage + GAP/ERROR/SATURATED sentinels -#include "../image_analysis/LoadFCalcFromMtz.h" -#include "../image_analysis/bragg_prediction/BraggPredictionFactory.h" #include "../image_analysis/geom_refinement/AssignSpotsToRings.h" #include "../image_analysis/spot_finding/StrongPixelSet.h" #include "../image_analysis/spot_finding/SpotUtils.h" @@ -69,8 +67,6 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se : QObject(parent), indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { - qRegisterMetaType("PixelRefineParams"); - qRegisterMetaType("PixelRefineReport"); qRegisterMetaType>("QVector"); spot_finding_settings = settings;; @@ -307,13 +303,7 @@ void JFJochImageReadingWorker::UpdateAzint_i(const JFJochReaderDataset *dataset) image_analysis = std::make_unique(curr_experiment, *azint_mapping, dataset->pixel_mask, *index_and_refine.get()); - // PixelRefine state is tied to the experiment/mapping; rebuild lazily. - pixel_refine_.reset(); - pixel_pred_.reset(); last_profile_.reset(); - // Keep scale-on-the-fly alive across dataset reloads. - if (!pixel_reference_.empty()) - index_and_refine->ReferenceIntensities(pixel_reference_); } } @@ -722,287 +712,3 @@ void JFJochImageReadingWorker::LoadSpots(int64_t start_image, int64_t end_image, emit spotsLoaded(result); } -// --------------------------------------------------------------------------- -// Experimental PixelRefine -// --------------------------------------------------------------------------- -void JFJochImageReadingWorker::EnsurePixelRefine_i() { - if (!pixel_refine_ && !pixel_reference_.empty()) - pixel_refine_ = std::make_unique(curr_experiment, pixel_reference_); - if (!pixel_pred_) - pixel_pred_ = CreateBraggPrediction(curr_experiment.IsRotationIndexing()); -} - -bool JFJochImageReadingWorker::BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p, QString &reason) const { - if (!current_image_ptr || !index_and_refine || !current_image.has_value()) { - reason = "No image loaded"; - return false; - } - - const auto &outcomes = index_and_refine->GetIntegrationOutcome(); - const int64_t n = current_image.value(); - if (n < 0 || n >= static_cast(outcomes.size())) { - reason = "No analysis result for current image"; - return false; - } - - const auto &io = outcomes[n]; - if (io.reflections.empty()) { - // The "indexed" badge in the viewer comes from indexing (lattice_type); - // PixelRefine instead seeds from the *integration* outcome, which is only - // populated when Quick integration is enabled and succeeds for this image. - // Distinguish the two so the user knows what to turn on. - if (current_image_ptr->ImageData().lattice_type.has_value()) - reason = "Image is indexed but not integrated - enable Quick integration"; - else - reason = "Current image is not indexed"; - return false; - } - - d.geom = io.geom; - d.latt = io.latt; - - const auto < = current_image_ptr->ImageData().lattice_type; - if (lt) { - d.crystal_system = lt->crystal_system; - d.centering = lt->centering; - } - if (d.crystal_system == gemmi::CrystalSystem::Trigonal) - d.crystal_system = gemmi::CrystalSystem::Hexagonal; - - d.R[0] = p.R0; - d.R[1] = p.R1; - d.bandwidth = p.bandwidth_fwhm / 2.3548; // FWHM -> sigma - d.scale_factor = p.scale_factor; - d.B_factor = p.B_factor; - if (std::isfinite(p.beam_x) && std::isfinite(p.beam_y)) - d.geom.BeamX_pxl(static_cast(p.beam_x)).BeamY_pxl(static_cast(p.beam_y)); - - d.refine_orientation = p.refine_orientation; - d.refine_unit_cell = p.refine_unit_cell; - d.refine_beam_center = p.refine_beam_center; - d.refine_scale = p.refine_scale; - d.refine_B = p.refine_B; - d.refine_R = p.refine_R; - d.max_iterations = p.max_iterations; - return true; -} - -std::shared_ptr JFJochImageReadingWorker::WrapFloatImage_i(const std::vector &img) const { - auto si = std::make_shared(); - // CompressedImage is a non-owning view over its data pointer. predictedImageReady - // is a queued cross-thread connection, so the source float vector must outlive the - // emit: copy it into SimpleImage::buffer (which the shared_ptr keeps alive) instead - // of aliasing the caller's temporary, otherwise loadImageInternal() reads freed - // memory in the GUI thread (SIGSEGV). - si->buffer.resize(img.size() * sizeof(float)); - std::memcpy(si->buffer.data(), img.data(), si->buffer.size()); - si->image = CompressedImage(si->buffer, curr_experiment.GetXPixelsNum(), curr_experiment.GetYPixelsNum(), - CompressedImageMode::Float32); - return si; -} - -void JFJochImageReadingWorker::SquaredResidualWithImage_i(std::vector &pred) const { - // PredictImage() returns raw detector units (same as the measured counts), so - // pred - measured is the per-pixel residual the model fails to explain. We plot - // |pred - measured|^2: sign-free, so it needs no diverging colour scale and just - // highlights where the model disagrees most. Masked / saturated pixels carry - // sentinels rather than counts, so no comparison is possible -> NaN (gap). - if (!current_image_ptr) - return; - const auto &img = current_image_ptr->Image(); - const size_t n = std::min(pred.size(), img.size()); - for (size_t i = 0; i < n; ++i) { - const int32_t v = img[i]; - if (v == GAP_PXL_VALUE || v == ERROR_PXL_VALUE || v == SATURATED_PXL_VALUE) { - pred[i] = NAN; - } else { - const float diff = pred[i] - static_cast(v); - pred[i] = diff * diff; - } - } -} - -void JFJochImageReadingWorker::MaskMeasuredSentinels_i(std::vector &img) const { - // The chi^2 image is 0 outside shoeboxes; show masked/saturated pixels as a gap - // (NaN) instead, so they read as "not comparable" rather than "zero cost". - if (!current_image_ptr) - return; - const auto &measured = current_image_ptr->Image(); - const size_t n = std::min(img.size(), measured.size()); - for (size_t i = 0; i < n; ++i) { - const int32_t v = measured[i]; - if (v == GAP_PXL_VALUE || v == ERROR_PXL_VALUE || v == SATURATED_PXL_VALUE) - img[i] = NAN; - } -} - -QVector JFJochImageReadingWorker::BuildShoeboxes_i(const PixelRefineData &data) const { - // One rectangle per fitted reflection: the shoebox the optimizer summed over, - // centred on the predicted position with half-size data.shoebox_radius. - QVector boxes; - boxes.reserve(static_cast(data.reflections.size())); - const int r = data.shoebox_radius; - const int side = 2 * r + 1; - for (const auto &refl : data.reflections) { - if (!std::isfinite(refl.predicted_x) || !std::isfinite(refl.predicted_y)) - continue; - const int cx = static_cast(std::lround(refl.predicted_x)); - const int cy = static_cast(std::lround(refl.predicted_y)); - boxes.push_back(QRect(cx - r, cy - r, side, side)); - } - return boxes; -} - -PixelRefineReport JFJochImageReadingWorker::BuildReport_i(const PixelRefineData &data) const { - PixelRefineReport report; - report.pr_G = data.scale_factor; - report.pr_B = data.B_factor; - report.pr_cc = data.cc; - report.pr_cc_n = data.cc_n; - - // Standard ScaleOnTheFly pipeline result for the same image, as a baseline. - if (current_image_ptr) { - const auto &d = current_image_ptr->ImageData(); - if (d.image_scale_factor) report.pipe_G = d.image_scale_factor.value(); - if (d.image_scale_b_factor) report.pipe_B = d.image_scale_b_factor.value(); - if (d.image_scale_cc) report.pipe_cc = d.image_scale_cc.value(); - } - - report.rows.reserve(data.reflections.size()); - for (const auto &r : data.reflections) { - if (!r.observed) - continue; - PixelRefineReport::Row row; - row.h = r.h; row.k = r.k; row.l = r.l; - row.d = r.d; - row.completeness = r.completeness; - row.partiality = r.partiality; - row.I = r.I; - row.sigma = r.sigma; - if (std::isfinite(r.image_scale_corr)) - row.I_true_est = static_cast(r.I) * static_cast(r.image_scale_corr); - if (pixel_refine_) - row.I_true_ref = pixel_refine_->ReferenceIntensity(r).value_or(NAN); - report.rows.push_back(row); - } - return report; -} - -std::vector JFJochImageReadingWorker::BuildDisplayImage_i(const PixelRefineData &data, - int display_mode) const { - const auto &img32 = current_image_ptr->Image(); - if (display_mode == PixelRefineParams::ChiSquared) { - // The cost density the optimizer actually minimizes (weighted residual^2). - auto chi2 = pixel_refine_->ChiSquaredImage(img32.data(), *pixel_pred_, data); - MaskMeasuredSentinels_i(chi2); - return chi2; - } - - auto pred = pixel_refine_->PredictImage(img32.data(), *pixel_pred_, data, true); - if (display_mode == PixelRefineParams::SquaredDifference) - SquaredResidualWithImage_i(pred); - return pred; -} - -void JFJochImageReadingWorker::LoadReference(QString path) { - QMutexLocker ul(&m); - try { - pixel_reference_ = LoadFCalcFromMtz(path.toStdString()); - if (index_and_refine) - index_and_refine->ReferenceIntensities(pixel_reference_); // enables scale-on-the-fly too - pixel_refine_.reset(); // rebuild with new reference - emit pixelRefineStatus(QString("Loaded %1 reference reflections").arg(pixel_reference_.size())); - } catch (const std::exception &e) { - emit pixelRefineStatus(QString("Failed to load reference: %1").arg(e.what())); - } -} - -void JFJochImageReadingWorker::PixelRefinePreview(PixelRefineParams params) { - QMutexLocker ul(&m); - if (!last_profile_) { - emit pixelRefineStatus("Analyze an image first"); - return; - } - EnsurePixelRefine_i(); - if (!pixel_refine_) { - emit pixelRefineStatus("Load reference data first"); - return; - } - - PixelRefineData d; - QString seed_reason; - if (!BuildPixelSeed_i(d, params, seed_reason)) { - emit pixelRefineStatus(seed_reason); - return; - } - - // Preview = evaluate only: do not move any parameter. - d.refine_orientation = d.refine_unit_cell = d.refine_beam_center = false; - d.refine_scale = d.refine_B = d.refine_R = false; - d.max_iterations = 0; - - try { - const auto &img32 = current_image_ptr->Image(); - pixel_refine_->Run(img32.data(), *pixel_pred_, d); - emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); - emit pixelRefineReport(BuildReport_i(d)); - - auto display = BuildDisplayImage_i(d, params.display_mode); - emit predictedImageReady(WrapFloatImage_i(display)); - emit predictedShoeboxes(BuildShoeboxes_i(d)); - } catch (const std::exception &e) { - emit pixelRefineStatus(QString("PixelRefine preview failed: %1").arg(e.what())); - } -} - -void JFJochImageReadingWorker::PixelRefineRun(PixelRefineParams params) { - QMutexLocker ul(&m); - if (!last_profile_) { - emit pixelRefineStatus("Analyze an image first"); - return; - } - EnsurePixelRefine_i(); - if (!pixel_refine_) { - emit pixelRefineStatus("Load reference data first"); - return; - } - - PixelRefineData d; - QString seed_reason; - if (!BuildPixelSeed_i(d, params, seed_reason)) { - emit pixelRefineStatus(seed_reason); - return; - } - if (d.max_iterations <= 0) - d.max_iterations = 3; - - try { - const auto &img32 = current_image_ptr->Image(); - pixel_refine_->Run(img32.data(), *pixel_pred_, d); - - // Push refined values back so the sliders follow the optimizer. - PixelRefineParams out = params; - out.R0 = d.R[0]; - out.R1 = d.R[1]; - out.bandwidth_fwhm = d.bandwidth * 2.3548; // sigma -> FWHM - out.scale_factor = d.scale_factor; - out.B_factor = d.B_factor; - out.beam_x = d.geom.GetBeamX_pxl(); - out.beam_y = d.geom.GetBeamY_pxl(); - emit pixelRefineParamsRefined(out); - emit pixelRefineResidual(d.final_cost, d.cc, static_cast(d.reflections.size())); - emit pixelRefineReport(BuildReport_i(d)); - - auto display = BuildDisplayImage_i(d, params.display_mode); - emit predictedImageReady(WrapFloatImage_i(display)); - emit predictedShoeboxes(BuildShoeboxes_i(d)); - - // Show the refined predictions on the main image too. - auto new_image = std::make_shared(*current_image_ptr); - new_image->ImageData().reflections = d.reflections; - current_image_ptr = new_image; - emit imageLoaded(current_image_ptr); - } catch (const std::exception &e) { - emit pixelRefineStatus(QString("PixelRefine failed: %1").arg(e.what())); - } -} diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index 50cd77ca..fcf74d5d 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -15,10 +15,8 @@ #include "../common/Logger.h" #include "../reader/JFJochHttpReader.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" -#include "../image_analysis/pixel_refinement/PixelRefine.h" #include "../image_analysis/bragg_prediction/BraggPrediction.h" #include "SimpleImage.h" -#include "windows/PixelRefineParams.h" #include "../common/MovingAverage.h" Q_DECLARE_METATYPE(std::shared_ptr) @@ -60,30 +58,8 @@ private: std::unique_ptr image_analysis; std::unique_ptr index_and_refine; - // Experimental PixelRefine support. last_profile_ keeps the azimuthal profile - // of the most recently analyzed image (PixelRefine needs it); the engine and a - // dedicated prediction buffer are built lazily once a reference is loaded. + // Azimuthal profile buffer of the most recently analyzed image (filled by Analyze). std::unique_ptr last_profile_; - std::vector pixel_reference_; - std::unique_ptr pixel_refine_; - std::unique_ptr pixel_pred_; - - void EnsurePixelRefine_i(); - bool BuildPixelSeed_i(PixelRefineData &d, const PixelRefineParams &p, QString &reason) const; - std::shared_ptr WrapFloatImage_i(const std::vector &img) const; - // Turn a predicted image into the squared residual |predicted - measured|^2 in - // place. Masked/saturated pixels become NaN (rendered as a gap: no comparison - // possible), not 0. - void SquaredResidualWithImage_i(std::vector &pred) const; - // Mark masked/saturated pixels of the current image as NaN (gap) in a float - // image, leaving the rest untouched (used for the chi^2 view). - void MaskMeasuredSentinels_i(std::vector &img) const; - // Build the per-reflection shoebox rectangles for the last refine/preview. - QVector BuildShoeboxes_i(const PixelRefineData &data) const; - // Assemble the per-reflection table + per-image summary for the table window. - PixelRefineReport BuildReport_i(const PixelRefineData &data) const; - // Build the float image to display for the given PixelRefineParams::DisplayMode. - std::vector BuildDisplayImage_i(const PixelRefineData &data, int display_mode) const; std::unique_ptr roi; @@ -145,14 +121,6 @@ signals: void fileLoadError(QString title, QString message); void fileLoadRetryStatus(bool active, QString message); - // PixelRefine (experimental) - void predictedImageReady(std::shared_ptr image); - void predictedShoeboxes(QVector boxes); // per-reflection optimization windows - void pixelRefineResidual(double cost, double cc, int64_t n_reflections); - void pixelRefineParamsRefined(PixelRefineParams params); - void pixelRefineReport(PixelRefineReport report); // per-reflection table + summary - void pixelRefineStatus(QString message); - public: JFJochImageReadingWorker(const SpotFindingSettings &settings, const DiffractionExperiment& experiment, QObject *parent = nullptr); ~JFJochImageReadingWorker() override = default; @@ -189,9 +157,4 @@ public slots: void LoadCalibration(QString dataset); void setAutoLoadMode(AutoloadMode mode); void setAutoLoadJump(int64_t val); - - // PixelRefine (experimental) - void LoadReference(QString path); - void PixelRefinePreview(PixelRefineParams params); - void PixelRefineRun(PixelRefineParams params); }; diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index 1eb2c3c7..903b7528 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -25,8 +25,6 @@ #include "toolbar/JFJochViewerToolbarImage.h" #include "windows/JFJoch2DAzintImageWindow.h" #include "windows/JFJochAzIntWindow.h" -#include "windows/JFJochPixelRefineWindow.h" -#include "windows/JFJochPixelRefineTableWindow.h" #include "windows/JFJochMagnifierWindow.h" #include "image_viewer/JFJochImage.h" #include "image_viewer/JFJochSimpleImage.h" @@ -108,8 +106,6 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString auto azintWindow = new JFJochAzIntWindow(experiment.GetAzimuthalIntegrationSettings(), this); auto azintImageWindow = new JFJoch2DAzintImageWindow(this); - auto pixelRefineWindow = new JFJochPixelRefineWindow(this); - auto pixelRefineTableWindow = new JFJochPixelRefineTableWindow(this); auto magnifierWindow = new JFJochMagnifierWindow(this); menuBar->AddWindowEntry(tableWindow, "Image list"); @@ -121,8 +117,6 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString menuBar->AddWindowEntry(reciprocalWindow, "Reciprocal space viewer"); menuBar->AddWindowEntry(azintWindow, "Azimuthal integration settings"); menuBar->AddWindowEntry(azintImageWindow, "Azimuthal integration 2D image"); - menuBar->AddWindowEntry(pixelRefineWindow, "PixelRefine (experimental)"); - menuBar->AddWindowEntry(pixelRefineTableWindow, "PixelRefine reflections"); menuBar->AddWindowEntry(magnifierWindow, "Magnifier"); if (dbus) { @@ -344,38 +338,6 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString connect(azintImageWindow, &JFJoch2DAzintImageWindow::zoomOnBin, viewer, &JFJochDiffractionImage::centerOnSpot); - // --- PixelRefine (experimental) --- - connect(reading_worker, &JFJochImageReadingWorker::imageLoaded, - pixelRefineWindow, &JFJochHelperWindow::imageLoaded); - connect(pixelRefineWindow, &JFJochPixelRefineWindow::paramsChanged, - reading_worker, &JFJochImageReadingWorker::PixelRefinePreview); - connect(pixelRefineWindow, &JFJochPixelRefineWindow::refineRequested, - reading_worker, &JFJochImageReadingWorker::PixelRefineRun); - connect(pixelRefineWindow, &JFJochPixelRefineWindow::loadReferenceRequested, - reading_worker, &JFJochImageReadingWorker::LoadReference); - connect(reading_worker, &JFJochImageReadingWorker::predictedImageReady, - pixelRefineWindow, &JFJochPixelRefineWindow::setPredictedImage); - connect(reading_worker, &JFJochImageReadingWorker::predictedShoeboxes, - pixelRefineWindow->imageView(), &JFJochSimpleImage::setShoeboxes); - connect(reading_worker, &JFJochImageReadingWorker::pixelRefineResidual, - pixelRefineWindow, &JFJochPixelRefineWindow::setResidual); - connect(reading_worker, &JFJochImageReadingWorker::pixelRefineParamsRefined, - pixelRefineWindow, &JFJochPixelRefineWindow::setRefinedParams); - connect(reading_worker, &JFJochImageReadingWorker::pixelRefineStatus, - pixelRefineWindow, &JFJochPixelRefineWindow::setStatus); - - // Reflection-table window: refreshed on every preview/refine, raised by button. - connect(reading_worker, &JFJochImageReadingWorker::pixelRefineReport, - pixelRefineTableWindow, &JFJochPixelRefineTableWindow::setReport); - connect(pixelRefineWindow, &JFJochPixelRefineWindow::showTableRequested, - pixelRefineTableWindow, &JFJochHelperWindow::open); - - // Lock the predicted-image viewport to the original image (both directions). - connect(viewer, &JFJochImage::viewportChanged, - pixelRefineWindow->imageView(), &JFJochImage::applyViewport); - connect(pixelRefineWindow->imageView(), &JFJochImage::viewportChanged, - viewer, &JFJochImage::applyViewport); - // --- Magnifier --- connect(reading_worker, &JFJochImageReadingWorker::imageLoaded, magnifierWindow, &JFJochHelperWindow::imageLoaded); diff --git a/viewer/image_viewer/JFJochImage.h b/viewer/image_viewer/JFJochImage.h index 5b3816ea..5e98e599 100644 --- a/viewer/image_viewer/JFJochImage.h +++ b/viewer/image_viewer/JFJochImage.h @@ -55,9 +55,8 @@ protected: QColor feature_color = Qt::magenta; - // Decimal places for non-integer per-pixel value labels. Float images (e.g. the - // PixelRefine prediction) are unreadable with many decimals, so subclasses can - // lower this. + // Decimal places for non-integer per-pixel value labels. Float images are + // unreadable with many decimals, so subclasses can lower this. int label_decimals_ = 3; float foreground = 10.0; diff --git a/viewer/image_viewer/JFJochSimpleImage.cpp b/viewer/image_viewer/JFJochSimpleImage.cpp index d3494a8f..bfa029e7 100644 --- a/viewer/image_viewer/JFJochSimpleImage.cpp +++ b/viewer/image_viewer/JFJochSimpleImage.cpp @@ -40,30 +40,6 @@ void JFJochSimpleImage::setImage(std::shared_ptr img) { } } -void JFJochSimpleImage::setShoeboxes(QVector boxes) { - shoeboxes_ = std::move(boxes); - // Redraw overlays on the current image (no-op if no image yet). - updateOverlay(); -} - -void JFJochSimpleImage::addCustomOverlay() { - if (shoeboxes_.isEmpty() || !scene()) - return; - - // Cosmetic 1-px outline so the box edges stay thin at any zoom; only draw the - // ones currently in view (there can be hundreds of reflections). - const QRectF visibleRect = mapToScene(viewport()->geometry()).boundingRect(); - QPen pen(QColor(0, 220, 255), 0); // cyan, distinct from the prediction colours - pen.setCosmetic(true); - - for (const QRect &b : shoeboxes_) { - const QRectF r(b.x(), b.y(), b.width(), b.height()); - if (!visibleRect.intersects(r)) - continue; - auto *item = scene()->addRect(r, pen); - addOverlayItem(item); - } -} void JFJochSimpleImage::mouseHover(QMouseEvent *event) { if (image_) { diff --git a/viewer/image_viewer/JFJochSimpleImage.h b/viewer/image_viewer/JFJochSimpleImage.h index 40abac06..3232bc7c 100644 --- a/viewer/image_viewer/JFJochSimpleImage.h +++ b/viewer/image_viewer/JFJochSimpleImage.h @@ -20,20 +20,14 @@ class JFJochSimpleImage : public JFJochImage { std::shared_ptr image_; - // Per-reflection shoebox rectangles (pixel coordinates) to overlay: the pixels - // PixelRefine actually summed over. Empty = nothing drawn. - QVector shoeboxes_; - // Prepare image template void loadImageInternal(const uint8_t *input); void loadImageInternal(); void mouseHover(QMouseEvent *event) override; - void addCustomOverlay() override; public: explicit JFJochSimpleImage(QWidget *parent = nullptr); public slots: void setImage(std::shared_ptr img); - void setShoeboxes(QVector boxes); }; diff --git a/viewer/windows/JFJochPixelRefineTableWindow.cpp b/viewer/windows/JFJochPixelRefineTableWindow.cpp deleted file mode 100644 index f656164a..00000000 --- a/viewer/windows/JFJochPixelRefineTableWindow.cpp +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "JFJochPixelRefineTableWindow.h" - -#include -#include -#include - -JFJochPixelRefineTableWindow::JFJochPixelRefineTableWindow(QWidget *parent) - : JFJochHelperWindow(parent) { - setWindowTitle("PixelRefine reflections"); - resize(950, 600); - - auto central = new QWidget(this); - setCentralWidget(central); - auto layout = new QVBoxLayout(central); - - m_summary = new QLabel(tr("No PixelRefine result yet."), this); - m_summary->setTextFormat(Qt::RichText); - m_summary->setWordWrap(true); - layout->addWidget(m_summary); - - m_table = new QTableView(this); - m_model = new QStandardItemModel(this); - setupModel(); - - m_proxy = new QSortFilterProxyModel(this); - m_proxy->setSourceModel(m_model); - m_proxy->setSortRole(Qt::UserRole); // numeric sort on the underlying values - - m_table->setModel(m_proxy); - m_table->setEditTriggers(QAbstractItemView::NoEditTriggers); - m_table->setSortingEnabled(true); - m_table->sortByColumn(6, Qt::DescendingOrder); // default: I desc - m_table->verticalHeader()->setVisible(false); - m_table->horizontalHeader()->setSortIndicatorShown(true); - m_table->horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch); - m_table->setStyleSheet("background-color: white;"); - layout->addWidget(m_table); -} - -void JFJochPixelRefineTableWindow::setupModel() { - const QStringList headers = { - "h", "k", "l", "d [Å]", "Compl.", "Part.", - "I", "Sigma", "Est. I_true", "Ref. I_true" - }; - m_model->setColumnCount(headers.size()); - for (int i = 0; i < headers.size(); ++i) - m_model->setHeaderData(i, Qt::Horizontal, headers[i]); -} - -static QStandardItem *numItem(double value, int decimals) { - auto *it = new QStandardItem(); - const QString text = std::isfinite(value) ? QString::number(value, 'f', decimals) - : QStringLiteral("—"); - it->setData(text, Qt::DisplayRole); - it->setData(value, Qt::UserRole); - it->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter); - return it; -} - -static QStandardItem *intItem(int value) { - auto *it = new QStandardItem(); - it->setData(static_cast(value), Qt::DisplayRole); - it->setData(static_cast(value), Qt::UserRole); - it->setTextAlignment(Qt::AlignRight | Qt::AlignVCenter); - return it; -} - -void JFJochPixelRefineTableWindow::setReport(PixelRefineReport report) { - // --- per-image summary: PixelRefine vs the standard pipeline --------------- - auto fmt = [](double v, int dec) { - return std::isfinite(v) ? QString::number(v, 'f', dec) : QStringLiteral("—"); - }; - auto pct = [](double v) { - return std::isfinite(v) ? QString::number(v * 100.0, 'f', 1) + "%" : QStringLiteral("—"); - }; - m_summary->setText( - tr("PixelRefine: scale G = %1, B = %2 Ų, CC = %3 (%4 refl)" - "  |  " - "Pipeline: scale G = %5, B = %6 Ų, CC = %7") - .arg(fmt(report.pr_G, 4), fmt(report.pr_B, 1), pct(report.pr_cc)) - .arg(report.pr_cc_n) - .arg(fmt(report.pipe_G, 4), fmt(report.pipe_B, 1), pct(report.pipe_cc))); - - // --- per-reflection rows --------------------------------------------------- - m_model->removeRows(0, m_model->rowCount()); - for (const auto &r : report.rows) { - QList row; - row << intItem(r.h) << intItem(r.k) << intItem(r.l); - row << numItem(r.d, 3); - row << numItem(r.completeness, 2); - row << numItem(r.partiality, 2); - row << numItem(r.I, 1); - row << numItem(r.sigma, 1); - row << numItem(r.I_true_est, 1); - row << numItem(r.I_true_ref, 1); - m_model->appendRow(row); - } -} diff --git a/viewer/windows/JFJochPixelRefineTableWindow.h b/viewer/windows/JFJochPixelRefineTableWindow.h deleted file mode 100644 index 5187b546..00000000 --- a/viewer/windows/JFJochPixelRefineTableWindow.h +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include "JFJochHelperWindow.h" -#include "PixelRefineParams.h" - -#include -#include -#include -#include - -// Reflection table for the experimental PixelRefine path. Opened from a button on -// the PixelRefine window and refreshed on every preview/refine. Each row is one -// matched reflection (the two partiality-style fractions, the fitted intensity, -// and the estimated vs reference full intensity); the header line compares -// PixelRefine's per-image scale/B/CC with the standard ScaleOnTheFly pipeline. -class JFJochPixelRefineTableWindow : public JFJochHelperWindow { - Q_OBJECT - - QLabel *m_summary = nullptr; - QTableView *m_table = nullptr; - QStandardItemModel *m_model = nullptr; - QSortFilterProxyModel *m_proxy = nullptr; - - void setupModel(); - -public: - explicit JFJochPixelRefineTableWindow(QWidget *parent = nullptr); - -public slots: - void setReport(PixelRefineReport report); -}; diff --git a/viewer/windows/JFJochPixelRefineWindow.cpp b/viewer/windows/JFJochPixelRefineWindow.cpp deleted file mode 100644 index 0e8b4a16..00000000 --- a/viewer/windows/JFJochPixelRefineWindow.cpp +++ /dev/null @@ -1,233 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "JFJochPixelRefineWindow.h" -#include "../image_viewer/JFJochSimpleImage.h" - -#include -#include -#include -#include -#include -#include -#include - -JFJochPixelRefineWindow::JFJochPixelRefineWindow(QWidget *parent) - : JFJochHelperWindow(parent) { - setWindowTitle("PixelRefine (experimental)"); - - auto central = new QWidget(this); - setCentralWidget(central); - auto layout = new QHBoxLayout(central); - - // --- predicted image (left, expanding) --------------------------------- - m_image = new JFJochSimpleImage(this); - m_image->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding); - layout->addWidget(m_image, 1); - - // --- control panel (right) --------------------------------------------- - auto controls = new QWidget(this); - controls->setMinimumWidth(320); - auto controlsLayout = new QVBoxLayout(controls); - layout->addWidget(controls, 0); - - // --- what the left image shows ------------------------------------------ - m_displayMode = new QComboBox(this); - m_displayMode->addItem(tr("Prediction")); - m_displayMode->addItem(tr("Squared difference |pred - image|²")); - m_displayMode->addItem(tr("χ² (weighted residual = LSQ cost)")); - auto displayForm = new QFormLayout(); - displayForm->addRow(tr("Display:"), m_displayMode); - controlsLayout->addLayout(displayForm); - - auto paramBox = new QGroupBox(tr("Model parameters"), this); - auto form = new QFormLayout(paramBox); - - m_R0 = new SliderPlusBox(1e-4, 0.05, 1e-4, 4, this); m_R0->setValue(0.005); - m_R1 = new SliderPlusBox(1e-4, 0.05, 1e-4, 4, this); m_R1->setValue(0.005); - m_bw = new SliderPlusBox(0.0, 0.05, 1e-4, 4, this); m_bw->setValue(0.0); - m_scale = new SliderPlusBox(1e-3, 1e4, 1e-3, 3, this, SliderPlusBox::Logarithmic); m_scale->setValue(1.0); - m_B = new SliderPlusBox(0.0, 200.0, 0.1, 1, this); m_B->setValue(0.0); - m_beamx = new SliderPlusBox(0.0, 5000.0, 0.5, 1, this); m_beamx->setValue(0.0); - m_beamy = new SliderPlusBox(0.0, 5000.0, 0.5, 1, this); m_beamy->setValue(0.0); - - form->addRow(tr("R0 radial [Å⁻¹]:"), m_R0); - form->addRow(tr("R1 tangential [Å⁻¹]:"), m_R1); - form->addRow(tr("Bandwidth FWHM (Δλ/λ):"), m_bw); - form->addRow(tr("Scale G:"), m_scale); - form->addRow(tr("B-factor [Ų]:"), m_B); - - m_overrideBeam = new QCheckBox(tr("Override beam centre"), this); - form->addRow(QString(), m_overrideBeam); - form->addRow(tr("Beam X [px]:"), m_beamx); - form->addRow(tr("Beam Y [px]:"), m_beamy); - m_beamx->setEnabled(false); - m_beamy->setEnabled(false); - - controlsLayout->addWidget(paramBox); - - // --- what "Refine" is allowed to move ---------------------------------- - auto refBox = new QGroupBox(tr("Refine (Ceres)"), this); - auto refLayout = new QVBoxLayout(refBox); - m_refOrientation = new QCheckBox(tr("Orientation"), this); m_refOrientation->setChecked(true); - m_refCell = new QCheckBox(tr("Unit cell"), this); - m_refBeam = new QCheckBox(tr("Beam centre"), this); - m_refScale = new QCheckBox(tr("Scale G"), this); m_refScale->setChecked(true); - m_refB = new QCheckBox(tr("B-factor"), this); - m_refR = new QCheckBox(tr("Widths R0/R1"), this); m_refR->setChecked(true); - for (auto *cb : {m_refOrientation, m_refCell, m_refBeam, m_refScale, m_refB, m_refR}) - refLayout->addWidget(cb); - controlsLayout->addWidget(refBox); - - // --- buttons + readouts ------------------------------------------------- - m_loadRef = new QPushButton(tr("Load reference MTZ…"), this); - m_refine = new QPushButton(tr("Refine"), this); - m_showTable = new QPushButton(tr("Reflection table…"), this); - controlsLayout->addWidget(m_loadRef); - controlsLayout->addWidget(m_refine); - controlsLayout->addWidget(m_showTable); - - m_residual = new QLabel(tr("Residual: —"), this); - m_pipelineCC = new QLabel(tr("Pipeline CC (ref): —"), this); - m_status = new QLabel(QString(), this); - m_status->setWordWrap(true); - m_status->setStyleSheet("color: rgb(80, 80, 80);"); - controlsLayout->addWidget(m_residual); - controlsLayout->addWidget(m_pipelineCC); - controlsLayout->addWidget(m_status); - controlsLayout->addStretch(1); - - // --- debounce timer for live preview ----------------------------------- - m_debounce = new QTimer(this); - m_debounce->setSingleShot(true); - m_debounce->setInterval(150); - connect(m_debounce, &QTimer::timeout, this, [this] { - emit paramsChanged(currentParams()); - }); - - for (auto *s : {m_R0, m_R1, m_bw, m_scale, m_B, m_beamx, m_beamy}) - connect(s, &SliderPlusBox::valueChanged, this, [this](double) { onControlChanged(); }); - - connect(m_displayMode, &QComboBox::currentIndexChanged, this, [this](int) { onControlChanged(); }); - - connect(m_overrideBeam, &QCheckBox::toggled, this, [this](bool on) { - m_beamx->setEnabled(on); - m_beamy->setEnabled(on); - onControlChanged(); - }); - - connect(m_loadRef, &QPushButton::clicked, this, [this] { - const QString path = QFileDialog::getOpenFileName( - this, tr("Load reference MTZ"), QString(), tr("MTZ files (*.mtz);;All files (*)")); - if (!path.isEmpty()) - emit loadReferenceRequested(path); - }); - - connect(m_showTable, &QPushButton::clicked, this, [this] { - emit showTableRequested(); - }); - - connect(m_refine, &QPushButton::clicked, this, [this] { - // Cancel any pending live-preview: otherwise a debounce armed by a slider - // move just before this click fires after the refine and overwrites the - // refined residual/preview with the stale pre-refine slider values. - m_debounce->stop(); - PixelRefineParams p = currentParams(); - p.max_iterations = 5; - emit refineRequested(p); - }); -} - -PixelRefineParams JFJochPixelRefineWindow::currentParams() const { - PixelRefineParams p; - p.R0 = m_R0->value(); - p.R1 = m_R1->value(); - p.bandwidth_fwhm = m_bw->value(); - p.scale_factor = m_scale->value(); - p.B_factor = m_B->value(); - if (m_overrideBeam->isChecked()) { - p.beam_x = m_beamx->value(); - p.beam_y = m_beamy->value(); - } else { - p.beam_x = NAN; - p.beam_y = NAN; - } - p.refine_orientation = m_refOrientation->isChecked(); - p.refine_unit_cell = m_refCell->isChecked(); - p.refine_beam_center = m_refBeam->isChecked(); - p.refine_scale = m_refScale->isChecked(); - p.refine_B = m_refB->isChecked(); - p.refine_R = m_refR->isChecked(); - p.display_mode = m_displayMode->currentIndex(); - return p; -} - -void JFJochPixelRefineWindow::onControlChanged() { - if (m_suppress) - return; - m_debounce->start(); -} - -void JFJochPixelRefineWindow::imageLoaded(std::shared_ptr image) { - if (!image) - return; - - // Initialise the beam-centre sliders from the geometry once. - if (!m_beamInit) { - const auto geom = image->Dataset().experiment.GetDiffractionGeometry(); - m_beamx->setMax(static_cast(image->Dataset().experiment.GetXPixelsNum())); - m_beamy->setMax(static_cast(image->Dataset().experiment.GetYPixelsNum())); - m_suppress = true; - m_beamx->setValue(geom.GetBeamX_pxl()); - m_beamy->setValue(geom.GetBeamY_pxl()); - m_suppress = false; - m_beamInit = true; - } - - // Show the standard ScaleOnTheFly pipeline's per-image CC vs the reference (set - // on the message during analysis when a reference is loaded), as a baseline to - // compare PixelRefine against. - const auto pipeline_cc = image->ImageData().image_scale_cc; - if (pipeline_cc.has_value() && std::isfinite(pipeline_cc.value())) - m_pipelineCC->setText(tr("Pipeline CC (ref): %1%") - .arg(pipeline_cc.value() * 100.0, 0, 'f', 1)); - else - m_pipelineCC->setText(tr("Pipeline CC (ref): —")); - - // Request a predicted-image preview for the (re)loaded image. Without this the - // preview only refreshed on a slider change, so opening/reanalyzing an image - // left the predicted view empty. The worker no-ops if there is no reference or - // the image is not integrated yet. - onControlChanged(); -} - -void JFJochPixelRefineWindow::setPredictedImage(std::shared_ptr image) { - m_image->setImage(std::move(image)); -} - -void JFJochPixelRefineWindow::setResidual(double cost, double cc, int64_t n_reflections) { - const QString cc_str = std::isfinite(cc) ? QString::number(cc * 100.0, 'f', 1) + "%" - : QStringLiteral("—"); - m_residual->setText(tr("Residual: %1 CC: %2 (%3 reflections)") - .arg(cost, 0, 'g', 6) - .arg(cc_str) - .arg(n_reflections)); -} - -void JFJochPixelRefineWindow::setRefinedParams(PixelRefineParams params) { - m_suppress = true; - m_R0->setValue(params.R0); - m_R1->setValue(params.R1); - m_bw->setValue(params.bandwidth_fwhm); - m_scale->setValue(params.scale_factor); - m_B->setValue(params.B_factor); - if (std::isfinite(params.beam_x) && std::isfinite(params.beam_y)) { - m_beamx->setValue(params.beam_x); - m_beamy->setValue(params.beam_y); - } - m_suppress = false; -} - -void JFJochPixelRefineWindow::setStatus(QString message) { - m_status->setText(message); -} diff --git a/viewer/windows/JFJochPixelRefineWindow.h b/viewer/windows/JFJochPixelRefineWindow.h deleted file mode 100644 index 9398389d..00000000 --- a/viewer/windows/JFJochPixelRefineWindow.h +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include "JFJochHelperWindow.h" -#include "PixelRefineParams.h" -#include "../SimpleImage.h" -#include "../widgets/SliderPlusBox.h" - -#include -#include -#include -#include -#include -#include - -class JFJochSimpleImage; - -// Experimental PixelRefine control window: sliders for the forward-model -// parameters, a live (debounced) predicted-image preview, a residual readout, -// "Load reference" and "Refine" buttons. The predicted-image view is exposed so -// the main window can lock its viewport to the original image. -class JFJochPixelRefineWindow : public JFJochHelperWindow { - Q_OBJECT - - JFJochSimpleImage *m_image; - - SliderPlusBox *m_R0; - SliderPlusBox *m_R1; - SliderPlusBox *m_bw; - SliderPlusBox *m_scale; - SliderPlusBox *m_B; - SliderPlusBox *m_beamx; - SliderPlusBox *m_beamy; - - QComboBox *m_displayMode; // Prediction vs. Difference (prediction - image) - - QCheckBox *m_overrideBeam; - QCheckBox *m_refOrientation; - QCheckBox *m_refCell; - QCheckBox *m_refBeam; - QCheckBox *m_refScale; - QCheckBox *m_refB; - QCheckBox *m_refR; - - QLabel *m_residual; - QLabel *m_pipelineCC; // first-image CC vs reference from the standard ScaleOnTheFly pipeline - QLabel *m_status; - QPushButton *m_loadRef; - QPushButton *m_refine; - QPushButton *m_showTable; - - QTimer *m_debounce; - bool m_suppress = false; // guard while pushing refined params into sliders - bool m_beamInit = false; // beam-centre sliders initialised from geometry - - PixelRefineParams currentParams() const; - void onControlChanged(); - -public: - explicit JFJochPixelRefineWindow(QWidget *parent = nullptr); - - JFJochSimpleImage *imageView() const { return m_image; } - - void imageLoaded(std::shared_ptr image) override; - -signals: - void paramsChanged(PixelRefineParams params); // debounced live preview - void refineRequested(PixelRefineParams params); // "Refine" button - void loadReferenceRequested(QString path); // "Load reference" button - void showTableRequested(); // "Reflection table" button - -public slots: - void setPredictedImage(std::shared_ptr image); - void setResidual(double cost, double cc, int64_t n_reflections); - void setRefinedParams(PixelRefineParams params); - void setStatus(QString message); -}; diff --git a/viewer/windows/PixelRefineParams.h b/viewer/windows/PixelRefineParams.h deleted file mode 100644 index e180847e..00000000 --- a/viewer/windows/PixelRefineParams.h +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include -#include -#include -#include - -// Parameters exchanged between the PixelRefine window (sliders/buttons) and the -// reading worker. bandwidth here is the *FWHM* of dlambda/lambda (user-facing); -// the worker converts it to the sigma that PixelRefineData expects. beam_x/beam_y -// are NaN to mean "keep the current refined geometry". -struct PixelRefineParams { - double R0 = 0.005; // radial / partiality width (A^-1) - double R1 = 0.005; // tangential / profile width (A^-1) - double bandwidth_fwhm = 0.0; // relative bandwidth FWHM (dlambda/lambda) - double scale_factor = 1.0; // overall scale G - double B_factor = 0.0; // Debye-Waller B (A^2) - double beam_x = NAN; // detector beam centre X (px); NaN = keep current - double beam_y = NAN; // detector beam centre Y (px); NaN = keep current - - // What the "Refine" button is allowed to move (ignored by the preview path). - bool refine_orientation = true; - bool refine_unit_cell = false; - bool refine_beam_center = false; - bool refine_scale = true; - bool refine_B = false; - bool refine_R = true; - - int max_iterations = 3; // <=0 means evaluate-only (preview / residual) - - // Display only (no effect on the fit): what the preview/refine image shows. - enum DisplayMode : int { - Prediction = 0, // forward-model image - SquaredDifference = 1, // |prediction - measured|^2 (raw, unweighted) - ChiSquared = 2 // ((prediction - measured)/sigma)^2 = the LSQ cost density - }; - int display_mode = Prediction; -}; - -Q_DECLARE_METATYPE(PixelRefineParams) - -// One PixelRefine result, shipped from the worker to the reflection-table window: -// a per-image summary that puts PixelRefine's scale/B/CC next to the standard -// ScaleOnTheFly pipeline's, plus one row per matched reflection. -struct PixelRefineReport { - // Per-image summary (NaN = not available). - double pr_G = NAN, pr_B = NAN, pr_cc = NAN; // PixelRefine - int64_t pr_cc_n = 0; - double pipe_G = NAN, pipe_B = NAN, pipe_cc = NAN; // ScaleOnTheFly pipeline baseline - - struct Row { - int h = 0, k = 0, l = 0; - double d = 0.0; - double completeness = 1.0; // spot footprint on live pixels (1 = not clipped) - double partiality = 1.0; // recorded rocking fraction - double I = 0.0, sigma = 0.0; - double I_true_est = NAN; // r.I * image_scale_corr (this image's estimate) - double I_true_ref = NAN; // reference (merged) intensity - }; - std::vector rows; -}; - -Q_DECLARE_METATYPE(PixelRefineReport) -- 2.52.0 From bf6efc7fe9d72d8773e51083bef2ca6d935a2b13 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 13 Jun 2026 23:27:36 +0200 Subject: [PATCH 041/228] Integration radius: default r1=4, CLI flag, PixelRefine shares the knob Bumped the default signal-box radius to r1=4 (r2=6, r3=8): on the lysozyme jet (1% DMM) it lifts CCref 50.1->52.2% and CC1/2 90.8->92.5% (its broadened spots spill past a radius-3 box), is neutral on the mono crystal and on the classical integrator. Added 'jfjoch_process --integration-radius ' (a single value derives r2=r1+2, r3=r1+4) and wired PixelRefine's shoebox radius to BraggIntegrationSettings::GetR1() so it shares the classical integrator's knob. (Explored but rejected: an elliptical/anisotropic Term-2 profile. The jet's tangential spots are mildly anisotropic - axis ratio 1.15->1.44 low->high res, azimuthal mosaic, separate from the radial DMM bandwidth - but using the measured 2x2 covariance as the profile DEGRADED the jet, CC1/2 92.5->83.5: the tight measured width loses to a generous aperture, same lesson as the radius bump.) Co-Authored-By: Claude Opus 4.8 --- common/BraggIntegrationSettings.h | 4 ++-- image_analysis/IndexAndRefine.cpp | 3 +++ tools/jfjoch_process.cpp | 29 ++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/common/BraggIntegrationSettings.h b/common/BraggIntegrationSettings.h index 60ab4059..dbe75ae6 100644 --- a/common/BraggIntegrationSettings.h +++ b/common/BraggIntegrationSettings.h @@ -6,8 +6,8 @@ #include class BraggIntegrationSettings { - float r_1 = 3; - float r_2 = 5; + float r_1 = 4; + float r_2 = 6; float r_3 = 8; float d_min_limit_A = 1.0; std::optional fixed_profile_radius; diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index f7cb25c1..479d5ac1 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -446,6 +446,9 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, if (const auto bw = experiment.GetBandwidthFWHM()) prd.bandwidth = bw.value() / 2.3548; // FWHM -> sigma + // Signal-box radius from the shared integration setting (same knob as BraggIntegrate2D). + prd.shoebox_radius = static_cast(std::lround(experiment.GetBraggIntegrationSettings().GetR1())); + std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); switch (image.GetMode()) { diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 5f7671fe..01ed1bf6 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -80,6 +80,7 @@ void print_usage() { std::cout << " Pixel refinement (experimental, select via -r pixelrefine, needs --reference-mtz)" << std::endl; std::cout << " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" << std::endl; + std::cout << " --integration-radius Signal-box radius r1, or r1,r2,r3 (px). One value => r2=r1+2, r3=r1+4" << std::endl; } enum { @@ -95,7 +96,8 @@ enum { OPT_SINGLE_PASS_ROTATION, OPT_REDO_ROTATION_SPOTS, OPT_FORCE_ROTATION_LATTICE, - OPT_BANDWIDTH + OPT_BANDWIDTH, + OPT_INTEGRATION_RADIUS }; static option long_options[] = { @@ -132,6 +134,7 @@ static option long_options[] = { {"scaling-high-resolution", required_argument, nullptr, OPT_SCALING_HIGH_RESOLUTION}, {"scaling-output", required_argument, nullptr, OPT_SCALING_OUTPUT}, {"bandwidth", required_argument, nullptr, OPT_BANDWIDTH}, + {"integration-radius", required_argument, nullptr, OPT_INTEGRATION_RADIUS}, {nullptr, 0, nullptr, 0} }; @@ -312,6 +315,7 @@ int main(int argc, char **argv) { float d_min_spot_finding = 1.5; std::optional d_min_scale_merge; + std::optional integration_radius_arg; // "r1" or "r1,r2,r3" if (argc == 1) { print_usage(); @@ -501,6 +505,9 @@ int main(int argc, char **argv) { case OPT_MIN_PARTIALITY: min_partiality = std::stod(optarg); break; + case OPT_INTEGRATION_RADIUS: + integration_radius_arg = optarg; + break; case OPT_MIN_IMAGE_CC: min_image_cc = std::stod(optarg); break; @@ -671,6 +678,26 @@ int main(int argc, char **argv) { experiment.ImportScalingSettings(scaling_settings); + // Integration radii: r1 (signal box), r2/r3 (background annulus). PixelRefine reads + // r1 as its shoebox radius; the classical integrator uses all three. + if (integration_radius_arg) { + std::vector rr; + std::stringstream ss(*integration_radius_arg); + std::string tok; + while (std::getline(ss, tok, ',')) { + trim_in_place(tok); + if (!tok.empty()) rr.push_back(std::stof(tok)); + } + float r1, r2, r3; + if (rr.size() == 1) { r1 = rr[0]; r2 = r1 + 2.0f; r3 = r1 + 4.0f; } + else if (rr.size() == 3) { r1 = rr[0]; r2 = rr[1]; r3 = rr[2]; } + else { logger.Error("--integration-radius expects r1 or r1,r2,r3"); return 1; } + BraggIntegrationSettings bis = experiment.GetBraggIntegrationSettings(); + bis.R1(r1).R2(r2).R3(r3); + experiment.ImportBraggIntegrationSettings(bis); + logger.Info("Integration radii set to r1={:.1f} r2={:.1f} r3={:.1f}", r1, r2, r3); + } + SpotFindingSettings spot_settings; spot_settings.enable = true; spot_settings.indexing = true; -- 2.52.0 From cfcd4c9e56491a38848e902eef1e545d7420cddd Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sun, 14 Jun 2026 09:18:09 +0200 Subject: [PATCH 042/228] PixelRefine: R1 multiplier (fix tight-Term2 regression) + env-gated orientation refine Full-jet R-free showed the factored model regressed vs the traditional pipeline (R-free 0.34 vs 0.26, CCref 58 vs 76), traced to Term 2 using the raw measured profile width: it is too tight (~0.002 A^-1), so the template sits off the ~0.4 px centroid-floor scatter and underestimates the intensity (box-sum is immune). Fix, XDS-style (integrate over ~6 sigma): multiply the measured R1 by r1_multiplier (default 6) before use. On the jet at 1.3 A this recovers CCref 55.2 -> 59.3 (~ traditional's 60.5); crystal 2 -0.7. Tunable via env PR_RMULT for R-free. Also reinstates the pre-factored per-image orientation refinement (per-pixel ShoeboxResidual against the shoebox, regularised to the spot-centroid orientation, re-predict), behind env PR_ORIENT (off by default), to A/B its effect on serial-data R-free. On the jet CCref it is a no-op (XtalOptimizer orientation already at the optimum), but CCref is a weak R-free proxy here, so it is left for R-free validation. NOTE: PR_RMULT / PR_ORIENT are temporary diagnostic env knobs; the final multiplier value and the orientation-refine decision are to be fixed against R-free, then baked. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 2 + .../pixel_refinement/PixelRefine.cpp | 265 ++++++++++++++---- image_analysis/pixel_refinement/PixelRefine.h | 16 ++ 3 files changed, 225 insertions(+), 58 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 479d5ac1..34244a4a 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -448,6 +448,8 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, // Signal-box radius from the shared integration setting (same knob as BraggIntegrate2D). prd.shoebox_radius = static_cast(std::lround(experiment.GetBraggIntegrationSettings().GetR1())); + if (const char *m = std::getenv("PR_RMULT")) prd.r1_multiplier = std::stod(m); // TEMP: Term-2 R1 multiplier sweep + if (std::getenv("PR_ORIENT")) prd.refine_orientation = true; // A/B: per-image orientation refinement std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 75facde4..843d6360 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -24,6 +24,7 @@ struct PixelObs { double x, y; // detector pixel coordinate double Iobs; // raw pixel value (signal + background) double Ibkg; // local background estimate (per-shoebox level, raw counts) + double weight; // 1/sigma_pixel (used only by the optional orientation refinement) }; // One reflection together with the pixels of its shoebox. @@ -336,6 +337,69 @@ struct IntensityResidual { double J, inv_sigma, partiality, pol, I_ref, inv_4d2; }; +// Per-shoebox per-pixel forward-model cost (raw counts), used ONLY by the optional +// orientation refinement: I_pred = G*Itrue*B_term*P_radial*P_tang*pol + Ibkg. The +// expensive node geometry is computed once per reflection; the cheap ObservedRecip + +// Gaussian profile run per pixel. Shares ObservedRecip/PredictedNode with GeometryProbe. +struct ShoeboxResidual { + ShoeboxResidual(const ReflGroup &g, double lambda, double pixel_size, + gemmi::CrystalSystem symmetry) + : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), pol(g.pol), + exp_h(g.h), exp_k(g.k), exp_l(g.l), + inv_lambda(1.0 / lambda), pixel_size(pixel_size), symmetry(symmetry) {} + template + bool operator()(const T *const *params, T *residual) const { + // 0 beam 1 dist 2 detector_rot 3 p0 4 p1 5 p2 6 scale 7 B 8 R + const T *beam = params[0]; const T *distance_mm = params[1]; const T *detector_rot = params[2]; + const T *p0 = params[3]; const T *p1 = params[4]; const T *p2 = params[5]; + const T *scale_factor = params[6]; const T *B = params[7]; const T *R = params[8]; + if (R[0] < T(1e-10) || R[1] < T(1e-10)) + return false; + Eigen::Matrix e_pred_recip, n_radial; + T q_sq; + if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, + e_pred_recip, n_radial, q_sq)) + return false; + const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); + const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); + for (size_t i = 0; i < pixels.size(); ++i) { + const PixelObs &obs = pixels[i]; + Eigen::Matrix e_obs_recip; + ObservedRecip(beam, distance_mm, detector_rot, obs.x, obs.y, pixel_size, inv_lambda, e_obs_recip); + const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; + const T eps_radial = delta_q.dot(n_radial); + const T eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); + const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); + const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); + const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); + residual[i] = (signal + T(obs.Ibkg) - T(obs.Iobs)) * T(obs.weight); + } + return true; + } + std::vector pixels; + const double Itrue, R_bw_sq, pol; + const double exp_h, exp_k, exp_l; + const double inv_lambda, pixel_size; + gemmi::CrystalSystem symmetry; +}; + +// Anchors the orientation (angle-axis vector) to its prior with a data-scaled weight, so +// the per-image fit can only make a small, signal-supported sub-spot correction. +struct OrientationRegularizer { + OrientationRegularizer(double weight, const double prior[3]) : weight(weight) { + for (int i = 0; i < 3; ++i) + prior_[i] = prior[i]; + } + template + bool operator()(const T *p0, T *residual) const { + for (int i = 0; i < 3; ++i) + residual[i] = T(weight) * (p0[i] - T(prior_[i])); + return true; + } + double weight; + double prior_[3]; +}; + } // namespace PixelRefine::PixelRefine(const DiffractionExperiment &experiment, @@ -443,69 +507,154 @@ void PixelRefine::Run(const T *image, double latt_vec0[3], latt_vec1[3], latt_vec2[3]; BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); - // ---- 1. Predict shoeboxes for the current geometry ------------------------ - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) - .BeamY_pxl(data.geom.GetBeamY_pxl()) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); - - // ---- 2. Collect per-reflection shoebox pixels + local background ---------- - // GetReflections() returns the full pre-sized buffer; only the first nrefl - // entries are valid for this image. A spot-core mask over ALL predictions keeps - // each reflection's background ring from picking up a neighbour's signal. - const auto &predicted = prediction.GetReflections(); - const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - + // ---- 1-2. Predict shoeboxes + collect pixels (a lambda so it can be re-run after + // an orientation refinement moves the predicted positions). ---------- + // A spot-core mask over ALL predictions keeps each reflection's background ring from + // picking up a neighbour's signal. Pixels carry a fit weight (background-limited + // variance, signal-weighted toward the predicted centre) used only by the optional + // orientation refinement - the factored integration weights by v = Ibkg directly. std::vector groups; - for (int ri = 0; ri < nrefl; ++ri) { - const auto &refl = predicted[ri]; - const auto hkl = hkl_key_generator(refl); - if (!reference_data.contains(hkl)) - continue; - - // Local flat background from the ring around the shoebox (raw counts). If we - // cannot estimate a clean local background the reflection is dropped, exactly - // as BraggIntegrate2D marks it unobserved when too few background pixels survive. - double Ibkg = 0.0; - if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, - refl.predicted_x, refl.predicted_y, - radius, bkg_outer_radius, Ibkg)) - continue; - - ReflGroup g; - g.h = refl.h; - g.k = refl.k; - g.l = refl.l; - g.d = refl.d; - g.Itrue = reference_data[hkl]; - g.R_bw_sq = bandwidth_radial_sq(refl.d); - g.pol = polarization(refl.predicted_x, refl.predicted_y); - g.Ibkg = Ibkg; - g.predicted_x = refl.predicted_x; - g.predicted_y = refl.predicted_y; - - const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - for (int y = box.min_y; y <= box.max_y; ++y) { - for (int x = box.min_x; x <= box.max_x; ++x) { - const size_t npixel = xpixel * y + x; - // Skip sentinel (masked / saturated) pixels. - if (image[npixel] == std::numeric_limits::max()) - continue; - if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) - continue; - g.pixels.push_back({static_cast(x), static_cast(y), - static_cast(image[npixel]), Ibkg}); + auto build_groups = [&]() { + groups.clear(); + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + const auto &predicted = prediction.GetReflections(); + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; + const auto hkl = hkl_key_generator(refl); + if (!reference_data.contains(hkl)) + continue; + double Ibkg = 0.0; + if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg)) + continue; + ReflGroup g; + g.h = refl.h; + g.k = refl.k; + g.l = refl.l; + g.d = refl.d; + g.Itrue = reference_data[hkl]; + g.R_bw_sq = bandwidth_radial_sq(refl.d); + g.pol = polarization(refl.predicted_x, refl.predicted_y); + g.Ibkg = Ibkg; + g.predicted_x = refl.predicted_x; + g.predicted_y = refl.predicted_y; + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { + const size_t npixel = xpixel * y + x; + if (image[npixel] == std::numeric_limits::max()) + continue; + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) + continue; + double weight = 1.0 / std::sqrt(std::max(Ibkg, 1.0)); + if (data.fit_signal_sigma_pix > 0.0) { + const double dx = x - refl.predicted_x, dy = y - refl.predicted_y; + const double s2 = data.fit_signal_sigma_pix * data.fit_signal_sigma_pix; + weight *= std::exp(-0.5 * (dx * dx + dy * dy) / s2); + } + g.pixels.push_back({static_cast(x), static_cast(y), + static_cast(image[npixel]), Ibkg, weight}); + } } + if (!g.pixels.empty()) + groups.push_back(std::move(g)); } - if (!g.pixels.empty()) - groups.push_back(std::move(g)); - } + }; + build_groups(); if (groups.empty()) return; + // ---- Optional per-image orientation refinement (off by default) ----------------- + // The pre-factored per-pixel step: refine the orientation (and a nuisance scale) + // against the shoebox pixels via ShoeboxResidual, regularised to the spot-centroid + // orientation, then re-predict. Dropped from the factored model (geometry fixed to + // XtalOptimizer); behind PixelRefineData::refine_orientation to A/B its R-free effect. + if (data.refine_orientation) { + const double orient_prior[3] = {latt_vec0[0], latt_vec0[1], latt_vec0[2]}; + ceres::Problem oprob; + size_t npix = 0; + for (const auto &g : groups) { + auto *cost = new ceres::DynamicAutoDiffCostFunction( + new ShoeboxResidual(g, lambda, pixel_size, data.crystal_system)); + for (int b : {2, 1, 2, 3, 3, 3, 1, 1, 2}) + cost->AddParameterBlock(b); + cost->SetNumResiduals(static_cast(g.pixels.size())); + oprob.AddResidualBlock(cost, nullptr, beam, &dist_mm, detector_rot, + latt_vec0, latt_vec1, latt_vec2, + &data.scale_factor, &data.B_factor, data.R); + npix += g.pixels.size(); + } + // Refine only orientation (latt_vec0) + the nuisance scale G; everything else fixed. + oprob.SetParameterBlockConstant(beam); + oprob.SetParameterBlockConstant(&dist_mm); + oprob.SetParameterBlockConstant(detector_rot); + oprob.SetParameterBlockConstant(latt_vec1); + oprob.SetParameterBlockConstant(latt_vec2); + oprob.SetParameterBlockConstant(&data.B_factor); + oprob.SetParameterBlockConstant(data.R); + oprob.SetParameterLowerBound(&data.scale_factor, 0, 0.0); + if (data.orient_reg_sigma_deg > 0.0 && npix > 0) { + const double sigma_rad = std::max(data.orient_reg_sigma_deg * M_PI / 180.0, 1e-9); + const double w = std::sqrt(static_cast(npix)) / sigma_rad; + oprob.AddResidualBlock(new ceres::AutoDiffCostFunction( + new OrientationRegularizer(w, orient_prior)), nullptr, latt_vec0); + } + if (data.scale_reg_sigma > 0.0) { + const double w = std::sqrt(static_cast(groups.size()) / data.scale_reg_sigma); + oprob.AddResidualBlock(new ceres::AutoDiffCostFunction( + new ScalarRegularizer(w, 1.0)), nullptr, &data.scale_factor); + } + ceres::Solver::Options oopt; + oopt.linear_solver_type = ceres::DENSE_QR; + oopt.logging_type = ceres::LoggingType::SILENT; + oopt.minimizer_progress_to_stdout = false; + oopt.max_solver_time_in_seconds = data.max_time_s; + oopt.num_threads = 1; + ceres::Solver::Summary osum; + ceres::Solve(oopt, &oprob, &osum); + + // Write the refined orientation back into data.latt (cell held at latt_vec1/2), + // then re-predict + rebuild groups at the new orientation. Scale is reset; Term 1 + // re-fits it properly below. + switch (data.crystal_system) { + case gemmi::CrystalSystem::Tetragonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Cubic: + latt_vec1[1] = latt_vec1[0]; + latt_vec1[2] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + case gemmi::CrystalSystem::Hexagonal: + latt_vec1[1] = latt_vec1[0]; + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); + break; + case gemmi::CrystalSystem::Monoclinic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); + break; + case gemmi::CrystalSystem::Orthorhombic: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); + break; + default: + data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, + latt_vec2[0], latt_vec2[1], latt_vec2[2]); + break; + } + data.scale_factor = 1.0; + build_groups(); + if (groups.empty()) + return; + } + // ---- 3. Term 2: per-resolution tangential profile width R1 ---------------- // R1 = sqrt(2*) from the intensity-weighted tangential second moment of // the strong spots, binned by resolution (low res small spots, high res larger). @@ -549,7 +698,7 @@ void PixelRefine::Run(const T *image, std::vector bin_R1(n_bins, data.R[1]); for (int b = 0; b < n_bins; ++b) if (bin_M2[b].size() >= 5) { - const double r1 = std::sqrt(2.0 * std::max(MedianInPlace(bin_M2[b]), 0.0)); + const double r1 = data.r1_multiplier * std::sqrt(2.0 * std::max(MedianInPlace(bin_M2[b]), 0.0)); if (std::isfinite(r1) && r1 > 1e-4) bin_R1[b] = std::clamp(r1, 1e-4, 0.05); } diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index dfc560c5..1e7a2dac 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -67,6 +67,22 @@ struct PixelRefineData { // tangential profile width before Term 2 measures it (A^-1) bool refine_B = true; // refine the per-image B-factor along with G + // Term 2 measures the physical tangential width R1, but the *integration* profile must + // be generous (XDS-style: integrate over ~6-10 sigma), or a tight template centred on + // the prediction sits off the ~0.4 px centroid-floor scatter and underestimates the + // intensity. The measured R1 is multiplied by this before use. 1.0 = the raw physical + // width (too tight - regressed R-free on the serial jet); ~6 recovers it (closes the + // CCref gap to box-sum). Tune the final value against R-free. + double r1_multiplier = 6.0; + + // Optional per-image orientation refinement (env-gated, off by default): the + // pre-factored per-pixel step that refines the crystal orientation against the + // shoebox pixels, regularised to the spot-centroid orientation, before integration. + // Reinstated to A/B its effect on serial-data R-free vs the fixed-geometry default. + bool refine_orientation = false; + double orient_reg_sigma_deg = 1.0; // orientation-prior strength (deg) + double fit_signal_sigma_pix = 1.5; // signal-weighting sigma for the orientation fit (px) + // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% FWHM // DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening to R[0]. // 0 = monochromatic (the term switches off entirely). -- 2.52.0 From 45ee8c2b40204ceaa4126f004e5024830b17e01b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sun, 14 Jun 2026 19:59:50 +0200 Subject: [PATCH 043/228] PixelRefine: env-gated orientation + cell-scale sweep (PR_SWEEP) for R-free A/B R-free validation (full jet, 1.5 A) confirmed the r1_multiplier fix: PR x6 beats traditional (R-free 0.2625 vs 0.2802), the multiplier optimum is ~6 (x6==x9 on R-free; x9 only buys CC1/2 internal consistency), and per-image orientation *refinement* is a no-op (0.2618 vs 0.2625). Re-adds the reference-driven orientation + uniform cell-scale SWEEP behind PR_SWEEP (off by default) to test whether the cell-scale degree of freedom - which the gradient orientation refinement lacks, and which moves the high-res shells radially - helps R-free at 1.5 A on serial data. CCref is a near-no-op on the jet (as for the gradient path), but that does not certify R-free, so it is left for validation. NOTE: PR_RMULT / PR_ORIENT / PR_SWEEP remain temporary diagnostic env knobs; once the sweep R-free is in, bake r1_multiplier=6, drop the no-op paths, strip the knobs. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 1 + .../pixel_refinement/PixelRefine.cpp | 142 ++++++++++++++++++ image_analysis/pixel_refinement/PixelRefine.h | 12 ++ 3 files changed, 155 insertions(+) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 34244a4a..a37a6962 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -450,6 +450,7 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, prd.shoebox_radius = static_cast(std::lround(experiment.GetBraggIntegrationSettings().GetR1())); if (const char *m = std::getenv("PR_RMULT")) prd.r1_multiplier = std::stod(m); // TEMP: Term-2 R1 multiplier sweep if (std::getenv("PR_ORIENT")) prd.refine_orientation = true; // A/B: per-image orientation refinement + if (std::getenv("PR_SWEEP")) prd.sweep_orientation = true; // A/B: orientation + cell-scale sweep std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 843d6360..b2f2161f 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -457,6 +457,144 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, } } +// Optional pre-pass (env-gated): a small GLOBAL orientation + uniform cell-scale sweep that +// maximises CC of the box-summed intensities against the reference. Unlike the per-pixel +// orientation refinement it also adjusts a per-image cell scale (a radial degree of freedom), +// and makes coarse global moves the local gradient cannot. Coordinate descent over the three +// Rodrigues axes + cell scale within geometry-derived pixel bounds (highest-resolution spot +// moves ~1 px/step, lowest barely moves). Writes the best orientation/cell into data.latt. +template +void PixelRefine::SweepOrientationCell(const T *image, BraggPrediction &prediction, + PixelRefineData &data) const { + const int radius = data.shoebox_radius; + const double beam_x = data.geom.GetBeamX_pxl(); + const double beam_y = data.geom.GetBeamY_pxl(); + const auto qnan = std::numeric_limits::quiet_NaN(); + + // Box-sum minus local (perimeter) background MEAN, raw counts; NaN off-detector/masked. + auto integrate = [&](double px, double py) -> double { + const int cx = static_cast(std::lround(px)); + const int cy = static_cast(std::lround(py)); + const int outer = radius + 1; + if (cx - outer < 0 || cy - outer < 0 || + cx + outer >= static_cast(xpixel) || cy + outer >= static_cast(ypixel)) + return qnan; + double sig = 0.0; + int nsig = 0; + std::vector ring; + ring.reserve((2 * outer + 1) * (2 * outer + 1)); + for (int y = cy - outer; y <= cy + outer; ++y) { + for (int x = cx - outer; x <= cx + outer; ++x) { + const T raw = image[static_cast(xpixel) * y + x]; + if (raw == std::numeric_limits::max()) + return qnan; + if (std::is_signed_v && raw == std::numeric_limits::min()) + return qnan; + const double v = static_cast(raw); + if (std::abs(x - cx) <= radius && std::abs(y - cy) <= radius) { + sig += v; ++nsig; + } else { + ring.push_back(v); + } + } + } + if (ring.size() < 5) + return qnan; + double rsum = 0.0; + for (const double v : ring) + rsum += v; + return sig - nsig * (rsum / static_cast(ring.size())); + }; + + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(beam_x).BeamY_pxl(beam_y) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + const BraggPredictionSettings settings{ + .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), + .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), + .max_hkl = 100, + .centering = data.centering, + .bandwidth_sigma = static_cast(data.bandwidth) + }; + const int nrefl = prediction.Calc(exp_iter, data.latt, settings); + const auto &predicted = prediction.GetReflections(); + + struct Matched { int h, k, l; double refI; }; + std::vector matched; + double r_max = 0.0, r_min = std::numeric_limits::max(); + for (int i = 0; i < nrefl; ++i) { + const auto &r = predicted[i]; + const auto it = reference_data.find(hkl_key_generator(r)); + if (it == reference_data.end()) + continue; + matched.push_back({r.h, r.k, r.l, it->second}); + const double dx = r.predicted_x - beam_x, dy = r.predicted_y - beam_y; + const double rad = std::sqrt(dx * dx + dy * dy); + r_max = std::max(r_max, rad); + r_min = std::min(r_min, rad); + } + if (matched.size() < 20 || r_min <= 1.0 || r_max <= r_min) + return; // too little to anchor a meaningful sweep + + auto score = [&](const CrystalLattice &L) -> double { + const Coord A = L.Astar(), B = L.Bstar(), C = L.Cstar(); + double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; + int n = 0; + for (const auto &m : matched) { + const Coord g = A * static_cast(m.h) + B * static_cast(m.k) + + C * static_cast(m.l); + const auto [x, y] = data.geom.RecipToDetector(g); + if (!std::isfinite(x) || !std::isfinite(y)) + continue; + const double I = integrate(x, y); + if (!std::isfinite(I)) + continue; + sx += I; sy += m.refI; sxx += I * I; syy += m.refI * m.refI; sxy += I * m.refI; ++n; + } + if (n < 10) + return -2.0; + const double nd = n; + const double cov = sxy - sx * sy / nd, vx = sxx - sx * sx / nd, vy = syy - sy * sy / nd; + return (vx > 0.0 && vy > 0.0) ? cov / std::sqrt(vx * vy) : -2.0; + }; + + const double step = 1.0 / r_max; + const int n_rot = std::clamp( + static_cast(std::lround(data.sweep_max_deg * M_PI / 180.0 * r_max)), 1, 25); + const int n_scale = std::clamp( + static_cast(std::lround(data.sweep_max_cell_frac * r_max)), 1, 25); + const Coord axes[3] = {Coord(1, 0, 0), Coord(0, 1, 0), Coord(0, 0, 1)}; + + CrystalLattice best = data.latt; + double best_cc = score(best); + for (int round = 0; round < 2; ++round) { + for (const auto &axis : axes) { + CrystalLattice axis_best = best; + double axis_cc = best_cc; + for (int i = -n_rot; i <= n_rot; ++i) { + if (i == 0) continue; + CrystalLattice cand = best.Multiply(RotMatrix(static_cast(i * step), axis)); + const double cc = score(cand); + if (cc > axis_cc) { axis_cc = cc; axis_best = cand; } + } + best = axis_best; best_cc = axis_cc; + } + CrystalLattice scale_best = best; + double scale_cc = best_cc; + for (int i = -n_scale; i <= n_scale; ++i) { + if (i == 0) continue; + const double s = 1.0 / (1.0 + i * step); + CrystalLattice cand = best.Multiply(gemmi::Mat33(s, 0, 0, 0, s, 0, 0, 0, s)); + const double cc = score(cand); + if (cc > scale_cc) { scale_cc = cc; scale_best = cand; } + } + best = scale_best; best_cc = scale_cc; + } + data.latt = best; +} + template void PixelRefine::Run(const T *image, BraggPrediction &prediction, @@ -464,6 +602,10 @@ void PixelRefine::Run(const T *image, data.solved = false; data.reflections.clear(); + // Optional reference-driven orientation + cell-scale sweep before prediction (env-gated). + if (data.sweep_orientation) + SweepOrientationCell(image, prediction, data); + const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 1e7a2dac..40dba239 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -83,6 +83,13 @@ struct PixelRefineData { double orient_reg_sigma_deg = 1.0; // orientation-prior strength (deg) double fit_signal_sigma_pix = 1.5; // signal-weighting sigma for the orientation fit (px) + // Optional reference-driven orientation + per-image cell-scale sweep before prediction + // (env-gated, off by default). Coarse global moves the gradient refinement cannot make, + // plus a radial (cell-scale) DOF the gradient path lacks. See SweepOrientationCell. + bool sweep_orientation = false; + double sweep_max_deg = 0.15; + double sweep_max_cell_frac = 0.003; + // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% FWHM // DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening to R[0]. // 0 = monochromatic (the term switches off entirely). @@ -129,6 +136,11 @@ class PixelRefine { double beam[2], double &dist_mm, double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; + + // Reference-driven global orientation + cell-scale sweep (see PixelRefineData::sweep_*). + template + void SweepOrientationCell(const T *image, BraggPrediction &prediction, + PixelRefineData &data) const; public: PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference); -- 2.52.0 From a47b376dc39f77d0ca4581e0a47fc7197b872952 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sun, 14 Jun 2026 20:56:53 +0200 Subject: [PATCH 044/228] Merge: per-observation outlier rejection (env-gated PR_REJECT) At the jet's ~1000x multiplicity R-free is bias-limited, and the merge had NO outlier rejection - serial data zingers/overlaps/mis-indexed frames bias every merged mean. Add a robust per-observation cut: drop observations whose corrected intensity lies > reject_nsigma error-model sigmas from the reflection's MEDIAN. The error-model sigma already captures the genuine (partiality) scatter, and the median is a robust centre, so only the tail beyond the real spread is removed - not good partials. The median is computed in RefineErrorModel (which already pools the observations per reflection); AddImage applies the cut. Env-gated via PR_REJECT= (off by default); logs the count removed. On the jet (CC proxy) it lifts CCref +8 (nsigma 6, 0.6% cut) to +11 (nsigma 3, 7.4% cut) - the cut is vs the data's own median, not the reference, so the gain is real cleaner means. R-free validation + the nsigma sweet spot (over-rejection risk at low nsigma) are for Filip's full-jet R-free. Co-Authored-By: Claude Opus 4.8 --- image_analysis/scale_merge/Merge.cpp | 25 +++++++++++++++++++++++++ image_analysis/scale_merge/Merge.h | 14 ++++++++++++++ tools/jfjoch_process.cpp | 7 +++++++ 3 files changed, 46 insertions(+) diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index cfb829e9..08f875ba 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -65,6 +65,19 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, bool cc_mask) { auto hkl_key = hkl.pack(); sigma_corr = CorrectedSigma(I_corr, sigma_corr, hkl_key); + // Robust outlier rejection: drop this observation if it sits more than + // reject_nsigma error-model sigmas from the reflection's median. Needs the active + // error model so sigma_corr reflects the real scatter (else the threshold is the + // bare counting sigma and would cull good partials). + if (reject_outliers && error_model_active) { + const auto mit = reject_median_I.find(hkl_key); + if (mit != reject_median_I.end() && + std::fabs(I_corr - mit->second) > reject_nsigma * sigma_corr) { + ++reject_count; + continue; + } + } + auto it = accumulator.find(hkl_key); if (it == accumulator.end()) it = accumulator.emplace(hkl_key, MergeAccum{ @@ -110,6 +123,8 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc error_model_a = 1.0; error_model_b = 0.0; error_model_mean_I.clear(); + reject_median_I.clear(); + reject_count = 0; // --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, // applying exactly the filters AddImage uses. --- @@ -156,6 +171,16 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc continue; const double mean = sum_wI / sum_w; error_model_mean_I[key] = static_cast(mean); + // Robust centre for outlier rejection: the median intensity (resists the very + // outliers the inverse-variance mean is being protected from). + { + std::vector iv; + iv.reserve(obs.size()); + for (const auto &o: obs) + iv.push_back(o.I); + std::nth_element(iv.begin(), iv.begin() + iv.size() / 2, iv.end()); + reject_median_I[key] = iv[iv.size() / 2]; + } const double I2 = mean * mean; for (const auto &o: obs) { const double w = 1.0 / (static_cast(o.sigma) * o.sigma); diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index d69ee817..a1ea2776 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -87,6 +87,16 @@ class MergeOnTheFly { std::unordered_map error_model_mean_I; [[nodiscard]] float CorrectedSigma(float I_corr, float sigma_corr, uint64_t hkl_key) const; + // Optional per-observation outlier rejection: drop observations whose corrected + // intensity lies more than reject_nsigma error-model sigmas from the reflection's + // *median* (a robust centre). The error-model sigma already captures the genuine + // (e.g. partiality) scatter, so this removes only the tail beyond it - zingers, + // overlaps, mis-indexed frames - not good partials. Populated by RefineErrorModel. + bool reject_outliers = false; + double reject_nsigma = 6.0; + std::unordered_map reject_median_I; + size_t reject_count = 0; + bool Mask(const IntegrationOutcome &outcome, bool cc_mask); public: MergeOnTheFly(const DiffractionExperiment &x); @@ -99,6 +109,10 @@ public: [[nodiscard]] double ErrorModelA() const { return error_model_a; } [[nodiscard]] double ErrorModelB() const { return error_model_b; } + // Enable per-observation outlier rejection (call after RefineErrorModel, before AddImage). + void SetRejectOutliers(double nsigma) { reject_outliers = true; reject_nsigma = nsigma; } + [[nodiscard]] size_t RejectedCount() const { return reject_count; } + void AddImage(const IntegrationOutcome& outcome, bool cc_mask = false); MergeStatistics MergeStats(const std::vector &merged, diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 01ed1bf6..ef847d86 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -1034,8 +1034,15 @@ int main(int argc, char **argv) { const double isa = (b > 0.0) ? 1.0 / b : std::numeric_limits::infinity(); logger.Info("Error model: sigma'^2 = {:.3f} sigma^2 + ({:.4f} I)^2 ISa = {:.1f}", a, b, isa); } + if (const char *rj = std::getenv("PR_REJECT")) { // TEMP A/B: per-observation outlier rejection + const double nsig = std::atof(rj); + merge_engine.SetRejectOutliers(nsig); + logger.Info("Outlier rejection enabled: dropping observations > {:.1f} sigma from the per-reflection median", nsig); + } for (auto &i : indexer.GetIntegrationOutcome()) merge_engine.AddImage(i); + if (merge_engine.RejectedCount() > 0) + logger.Info("Outlier rejection removed {} observations", merge_engine.RejectedCount()); auto merged_reflections = merge_engine.ExportReflections(); auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), reference_data); -- 2.52.0 From 6f733d74c2555d6243ebb6767b989282e379b3ce Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 12:42:25 +0200 Subject: [PATCH 045/228] Merge: make outlier rejection a ScalingSettings parameter + CLI flag, default 6 Promote the per-observation merge outlier rejection from the temporary PR_REJECT env knob to a real setting: ScalingSettings::OutlierRejectNsigma (default 6, <=0 disables), driven into MergeOnTheFly via the constructor, with a --reject-outliers CLI flag in jfjoch_process. Default-on at 6 sigma matches XDS (MISFITS/REJECT) and DIALS (normalised-deviation test); it validated on the full-jet R-free (0.2625 -> 0.2585). Applies to both the PixelRefine and classical merge paths. Co-Authored-By: Claude Opus 4.8 --- common/ScalingSettings.cpp | 9 +++++++++ common/ScalingSettings.h | 3 +++ image_analysis/scale_merge/Merge.cpp | 8 +++++--- image_analysis/scale_merge/Merge.h | 3 +-- tools/jfjoch_process.cpp | 19 ++++++++++++------- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/common/ScalingSettings.cpp b/common/ScalingSettings.cpp index a91aa1ff..866d48a4 100644 --- a/common/ScalingSettings.cpp +++ b/common/ScalingSettings.cpp @@ -115,6 +115,15 @@ ScalingSettings &ScalingSettings::MinCCForImage(double input) { return *this; } +double ScalingSettings::GetOutlierRejectNsigma() const { + return outlier_reject_nsigma; +} + +ScalingSettings &ScalingSettings::OutlierRejectNsigma(double input) { + outlier_reject_nsigma = input; // <= 0 disables; no upper bound (large = effectively off) + return *this; +} + double ScalingSettings::GetMinPartiality() const { return min_partiality; } diff --git a/common/ScalingSettings.h b/common/ScalingSettings.h index f7c724f8..6d548cfc 100644 --- a/common/ScalingSettings.h +++ b/common/ScalingSettings.h @@ -23,6 +23,7 @@ class ScalingSettings { std::optional wedge_for_scaling; double min_partiality = 0.02; double min_cc_for_image = 0.0; + double outlier_reject_nsigma = 6.0; // per-observation merge outlier rejection (XDS/DIALS-style); <= 0 disables double rfree_fraction = 0.05; IntensityFormat intensity_format = IntensityFormat::MTZ; @@ -37,6 +38,7 @@ public: ScalingSettings& HighResolutionLimit_A(double limit); ScalingSettings& MinPartiality(double min_partiality); ScalingSettings& MinCCForImage(double min_cc_for_image); + ScalingSettings& OutlierRejectNsigma(double input); ScalingSettings& RfreeFraction(double input); ScalingSettings& FileFormat(IntensityFormat input); @@ -63,6 +65,7 @@ public: [[nodiscard]] double GetMinPartiality() const; [[nodiscard]] double GetMinCCForImage() const; + [[nodiscard]] double GetOutlierRejectNsigma() const; [[nodiscard]] double GetRfreeFraction() const; [[nodiscard]] IntensityFormat GetFileFormat() const; diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 08f875ba..d604e092 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -31,7 +31,9 @@ MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x) ? std::optional(scaling_settings.GetMinCCForImage()) : std::nullopt), min_partiality(scaling_settings.GetMinPartiality()), - generator(scaling_settings.GetMergeFriedel(), space_group_number) { + generator(scaling_settings.GetMergeFriedel(), space_group_number), + reject_outliers(scaling_settings.GetOutlierRejectNsigma() > 0.0), + reject_nsigma(scaling_settings.GetOutlierRejectNsigma()) { } MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional &cell) { @@ -172,8 +174,8 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc const double mean = sum_wI / sum_w; error_model_mean_I[key] = static_cast(mean); // Robust centre for outlier rejection: the median intensity (resists the very - // outliers the inverse-variance mean is being protected from). - { + // outliers the inverse-variance mean is being protected from). Only when active. + if (reject_outliers) { std::vector iv; iv.reserve(obs.size()); for (const auto &o: obs) diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index a1ea2776..ab9877f4 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -109,8 +109,7 @@ public: [[nodiscard]] double ErrorModelA() const { return error_model_a; } [[nodiscard]] double ErrorModelB() const { return error_model_b; } - // Enable per-observation outlier rejection (call after RefineErrorModel, before AddImage). - void SetRejectOutliers(double nsigma) { reject_outliers = true; reject_nsigma = nsigma; } + // Outlier rejection (driven by ScalingSettings::GetOutlierRejectNsigma) reports its count. [[nodiscard]] size_t RejectedCount() const { return reject_count; } void AddImage(const IntegrationOutcome& outcome, bool cc_mask = false); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index ef847d86..93ade850 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -72,6 +72,7 @@ void print_usage() { std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << std::endl; std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << std::endl; std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; + std::cout << " --reject-outliers Per-observation merge outlier rejection, N sigma from the per-reflection median (default: 6; 0 disables)" << std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; @@ -97,7 +98,8 @@ enum { OPT_REDO_ROTATION_SPOTS, OPT_FORCE_ROTATION_LATTICE, OPT_BANDWIDTH, - OPT_INTEGRATION_RADIUS + OPT_INTEGRATION_RADIUS, + OPT_REJECT_OUTLIERS }; static option long_options[] = { @@ -135,6 +137,7 @@ static option long_options[] = { {"scaling-output", required_argument, nullptr, OPT_SCALING_OUTPUT}, {"bandwidth", required_argument, nullptr, OPT_BANDWIDTH}, {"integration-radius", required_argument, nullptr, OPT_INTEGRATION_RADIUS}, + {"reject-outliers", required_argument, nullptr, OPT_REJECT_OUTLIERS}, {nullptr, 0, nullptr, 0} }; @@ -316,6 +319,7 @@ int main(int argc, char **argv) { float d_min_spot_finding = 1.5; std::optional d_min_scale_merge; std::optional integration_radius_arg; // "r1" or "r1,r2,r3" + std::optional outlier_reject_nsigma; // merge per-observation outlier rejection if (argc == 1) { print_usage(); @@ -508,6 +512,9 @@ int main(int argc, char **argv) { case OPT_INTEGRATION_RADIUS: integration_radius_arg = optarg; break; + case OPT_REJECT_OUTLIERS: + outlier_reject_nsigma = std::stod(optarg); + break; case OPT_MIN_IMAGE_CC: min_image_cc = std::stod(optarg); break; @@ -674,6 +681,8 @@ int main(int argc, char **argv) { scaling_settings.RotationWedgeForScaling(wedge_for_scaling); scaling_settings.MinPartiality(min_partiality); scaling_settings.MinCCForImage(min_image_cc); + if (outlier_reject_nsigma) + scaling_settings.OutlierRejectNsigma(*outlier_reject_nsigma); scaling_settings.FileFormat(intensity_format); experiment.ImportScalingSettings(scaling_settings); @@ -1034,15 +1043,11 @@ int main(int argc, char **argv) { const double isa = (b > 0.0) ? 1.0 / b : std::numeric_limits::infinity(); logger.Info("Error model: sigma'^2 = {:.3f} sigma^2 + ({:.4f} I)^2 ISa = {:.1f}", a, b, isa); } - if (const char *rj = std::getenv("PR_REJECT")) { // TEMP A/B: per-observation outlier rejection - const double nsig = std::atof(rj); - merge_engine.SetRejectOutliers(nsig); - logger.Info("Outlier rejection enabled: dropping observations > {:.1f} sigma from the per-reflection median", nsig); - } for (auto &i : indexer.GetIntegrationOutcome()) merge_engine.AddImage(i); if (merge_engine.RejectedCount() > 0) - logger.Info("Outlier rejection removed {} observations", merge_engine.RejectedCount()); + logger.Info("Outlier rejection (>{:.1f} sigma from the per-reflection median) removed {} observations", + experiment.GetScalingSettings().GetOutlierRejectNsigma(), merge_engine.RejectedCount()); auto merged_reflections = merge_engine.ExportReflections(); auto merged_statistics = merge_engine.MergeStats(merged_reflections, indexer.GetIntegrationOutcome(), reference_data); -- 2.52.0 From 102a2a7c814c8a54a14c95d888e3ed4220ef7095 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 13:34:54 +0200 Subject: [PATCH 046/228] PixelRefine: strip experimental env knobs (orientation, sweep, Lorentz, ML census) Remove the env-gated experiments that A/B'd to dead-ends or are no longer needed, returning PixelRefine to the clean factored Terms 1+2 plus the one validated keeper (r1_multiplier, default 6): - PR_ORIENT (per-image orientation refinement): R-free no-op (0.2618 vs 0.2625) - XtalOptimizer's orientation is already optimal. Removes ShoeboxResidual, OrientationRegularizer, PixelObs::weight, the refinement block and its fields. - PR_SWEEP (orientation + cell-scale sweep): R-free no-op, degraded high-res CC1/2 (per-image overfit). Removes SweepOrientationCell and its fields. - PR_LORENTZ (rotation Lorentz/zeta): hurt both directions (the factored partiality already subsumes it); was already reverted. - PR_MLCENSUS (multi-lattice census in AnalyzeIndexing): served its purpose (~3-5% of jet frames are multi-lattice; shelved). PR_RMULT (the validated Term-2 multiplier knob) is kept. Defaults unchanged: crystal 2 / jet / hybrid -R -r pixelrefine all reproduce. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 4 +- .../pixel_refinement/PixelRefine.cpp | 405 +++--------------- image_analysis/pixel_refinement/PixelRefine.h | 29 +- 3 files changed, 62 insertions(+), 376 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index a37a6962..c4c0e831 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -448,9 +448,7 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, // Signal-box radius from the shared integration setting (same knob as BraggIntegrate2D). prd.shoebox_radius = static_cast(std::lround(experiment.GetBraggIntegrationSettings().GetR1())); - if (const char *m = std::getenv("PR_RMULT")) prd.r1_multiplier = std::stod(m); // TEMP: Term-2 R1 multiplier sweep - if (std::getenv("PR_ORIENT")) prd.refine_orientation = true; // A/B: per-image orientation refinement - if (std::getenv("PR_SWEEP")) prd.sweep_orientation = true; // A/B: orientation + cell-scale sweep + if (const char *m = std::getenv("PR_RMULT")) prd.r1_multiplier = std::stod(m); // Term-2 R1 multiplier (default 6) std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index b2f2161f..4f45c7aa 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -24,7 +24,6 @@ struct PixelObs { double x, y; // detector pixel coordinate double Iobs; // raw pixel value (signal + background) double Ibkg; // local background estimate (per-shoebox level, raw counts) - double weight; // 1/sigma_pixel (used only by the optional orientation refinement) }; // One reflection together with the pixels of its shoebox. @@ -337,69 +336,6 @@ struct IntensityResidual { double J, inv_sigma, partiality, pol, I_ref, inv_4d2; }; -// Per-shoebox per-pixel forward-model cost (raw counts), used ONLY by the optional -// orientation refinement: I_pred = G*Itrue*B_term*P_radial*P_tang*pol + Ibkg. The -// expensive node geometry is computed once per reflection; the cheap ObservedRecip + -// Gaussian profile run per pixel. Shares ObservedRecip/PredictedNode with GeometryProbe. -struct ShoeboxResidual { - ShoeboxResidual(const ReflGroup &g, double lambda, double pixel_size, - gemmi::CrystalSystem symmetry) - : pixels(g.pixels), Itrue(g.Itrue), R_bw_sq(g.R_bw_sq), pol(g.pol), - exp_h(g.h), exp_k(g.k), exp_l(g.l), - inv_lambda(1.0 / lambda), pixel_size(pixel_size), symmetry(symmetry) {} - template - bool operator()(const T *const *params, T *residual) const { - // 0 beam 1 dist 2 detector_rot 3 p0 4 p1 5 p2 6 scale 7 B 8 R - const T *beam = params[0]; const T *distance_mm = params[1]; const T *detector_rot = params[2]; - const T *p0 = params[3]; const T *p1 = params[4]; const T *p2 = params[5]; - const T *scale_factor = params[6]; const T *B = params[7]; const T *R = params[8]; - if (R[0] < T(1e-10) || R[1] < T(1e-10)) - return false; - Eigen::Matrix e_pred_recip, n_radial; - T q_sq; - if (!PredictedNode(p0, p1, p2, exp_h, exp_k, exp_l, symmetry, inv_lambda, - e_pred_recip, n_radial, q_sq)) - return false; - const T B_term = ceres::exp(-B[0] * q_sq / T(4.0)); - const T R0_eff_sq = R[0] * R[0] + T(R_bw_sq); - for (size_t i = 0; i < pixels.size(); ++i) { - const PixelObs &obs = pixels[i]; - Eigen::Matrix e_obs_recip; - ObservedRecip(beam, distance_mm, detector_rot, obs.x, obs.y, pixel_size, inv_lambda, e_obs_recip); - const Eigen::Matrix delta_q = e_obs_recip - e_pred_recip; - const T eps_radial = delta_q.dot(n_radial); - const T eps_tang_sq = (delta_q - eps_radial * n_radial).squaredNorm(); - const T P_radial = ceres::exp(-eps_radial * eps_radial / R0_eff_sq); - const T P_tang = ceres::exp(-eps_tang_sq / (R[1] * R[1])) / (T(M_PI) * R[1] * R[1]); - const T signal = scale_factor[0] * T(Itrue) * B_term * P_radial * P_tang * T(pol); - residual[i] = (signal + T(obs.Ibkg) - T(obs.Iobs)) * T(obs.weight); - } - return true; - } - std::vector pixels; - const double Itrue, R_bw_sq, pol; - const double exp_h, exp_k, exp_l; - const double inv_lambda, pixel_size; - gemmi::CrystalSystem symmetry; -}; - -// Anchors the orientation (angle-axis vector) to its prior with a data-scaled weight, so -// the per-image fit can only make a small, signal-supported sub-spot correction. -struct OrientationRegularizer { - OrientationRegularizer(double weight, const double prior[3]) : weight(weight) { - for (int i = 0; i < 3; ++i) - prior_[i] = prior[i]; - } - template - bool operator()(const T *p0, T *residual) const { - for (int i = 0; i < 3; ++i) - residual[i] = T(weight) * (p0[i] - T(prior_[i])); - return true; - } - double weight; - double prior_[3]; -}; - } // namespace PixelRefine::PixelRefine(const DiffractionExperiment &experiment, @@ -457,144 +393,6 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, } } -// Optional pre-pass (env-gated): a small GLOBAL orientation + uniform cell-scale sweep that -// maximises CC of the box-summed intensities against the reference. Unlike the per-pixel -// orientation refinement it also adjusts a per-image cell scale (a radial degree of freedom), -// and makes coarse global moves the local gradient cannot. Coordinate descent over the three -// Rodrigues axes + cell scale within geometry-derived pixel bounds (highest-resolution spot -// moves ~1 px/step, lowest barely moves). Writes the best orientation/cell into data.latt. -template -void PixelRefine::SweepOrientationCell(const T *image, BraggPrediction &prediction, - PixelRefineData &data) const { - const int radius = data.shoebox_radius; - const double beam_x = data.geom.GetBeamX_pxl(); - const double beam_y = data.geom.GetBeamY_pxl(); - const auto qnan = std::numeric_limits::quiet_NaN(); - - // Box-sum minus local (perimeter) background MEAN, raw counts; NaN off-detector/masked. - auto integrate = [&](double px, double py) -> double { - const int cx = static_cast(std::lround(px)); - const int cy = static_cast(std::lround(py)); - const int outer = radius + 1; - if (cx - outer < 0 || cy - outer < 0 || - cx + outer >= static_cast(xpixel) || cy + outer >= static_cast(ypixel)) - return qnan; - double sig = 0.0; - int nsig = 0; - std::vector ring; - ring.reserve((2 * outer + 1) * (2 * outer + 1)); - for (int y = cy - outer; y <= cy + outer; ++y) { - for (int x = cx - outer; x <= cx + outer; ++x) { - const T raw = image[static_cast(xpixel) * y + x]; - if (raw == std::numeric_limits::max()) - return qnan; - if (std::is_signed_v && raw == std::numeric_limits::min()) - return qnan; - const double v = static_cast(raw); - if (std::abs(x - cx) <= radius && std::abs(y - cy) <= radius) { - sig += v; ++nsig; - } else { - ring.push_back(v); - } - } - } - if (ring.size() < 5) - return qnan; - double rsum = 0.0; - for (const double v : ring) - rsum += v; - return sig - nsig * (rsum / static_cast(ring.size())); - }; - - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(beam_x).BeamY_pxl(beam_y) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const BraggPredictionSettings settings{ - .high_res_A = experiment.GetBraggIntegrationSettings().GetDMinLimit_A(), - .ewald_dist_cutoff = static_cast(data.ewald_dist_cutoff), - .max_hkl = 100, - .centering = data.centering, - .bandwidth_sigma = static_cast(data.bandwidth) - }; - const int nrefl = prediction.Calc(exp_iter, data.latt, settings); - const auto &predicted = prediction.GetReflections(); - - struct Matched { int h, k, l; double refI; }; - std::vector matched; - double r_max = 0.0, r_min = std::numeric_limits::max(); - for (int i = 0; i < nrefl; ++i) { - const auto &r = predicted[i]; - const auto it = reference_data.find(hkl_key_generator(r)); - if (it == reference_data.end()) - continue; - matched.push_back({r.h, r.k, r.l, it->second}); - const double dx = r.predicted_x - beam_x, dy = r.predicted_y - beam_y; - const double rad = std::sqrt(dx * dx + dy * dy); - r_max = std::max(r_max, rad); - r_min = std::min(r_min, rad); - } - if (matched.size() < 20 || r_min <= 1.0 || r_max <= r_min) - return; // too little to anchor a meaningful sweep - - auto score = [&](const CrystalLattice &L) -> double { - const Coord A = L.Astar(), B = L.Bstar(), C = L.Cstar(); - double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; - int n = 0; - for (const auto &m : matched) { - const Coord g = A * static_cast(m.h) + B * static_cast(m.k) - + C * static_cast(m.l); - const auto [x, y] = data.geom.RecipToDetector(g); - if (!std::isfinite(x) || !std::isfinite(y)) - continue; - const double I = integrate(x, y); - if (!std::isfinite(I)) - continue; - sx += I; sy += m.refI; sxx += I * I; syy += m.refI * m.refI; sxy += I * m.refI; ++n; - } - if (n < 10) - return -2.0; - const double nd = n; - const double cov = sxy - sx * sy / nd, vx = sxx - sx * sx / nd, vy = syy - sy * sy / nd; - return (vx > 0.0 && vy > 0.0) ? cov / std::sqrt(vx * vy) : -2.0; - }; - - const double step = 1.0 / r_max; - const int n_rot = std::clamp( - static_cast(std::lround(data.sweep_max_deg * M_PI / 180.0 * r_max)), 1, 25); - const int n_scale = std::clamp( - static_cast(std::lround(data.sweep_max_cell_frac * r_max)), 1, 25); - const Coord axes[3] = {Coord(1, 0, 0), Coord(0, 1, 0), Coord(0, 0, 1)}; - - CrystalLattice best = data.latt; - double best_cc = score(best); - for (int round = 0; round < 2; ++round) { - for (const auto &axis : axes) { - CrystalLattice axis_best = best; - double axis_cc = best_cc; - for (int i = -n_rot; i <= n_rot; ++i) { - if (i == 0) continue; - CrystalLattice cand = best.Multiply(RotMatrix(static_cast(i * step), axis)); - const double cc = score(cand); - if (cc > axis_cc) { axis_cc = cc; axis_best = cand; } - } - best = axis_best; best_cc = axis_cc; - } - CrystalLattice scale_best = best; - double scale_cc = best_cc; - for (int i = -n_scale; i <= n_scale; ++i) { - if (i == 0) continue; - const double s = 1.0 / (1.0 + i * step); - CrystalLattice cand = best.Multiply(gemmi::Mat33(s, 0, 0, 0, s, 0, 0, 0, s)); - const double cc = score(cand); - if (cc > scale_cc) { scale_cc = cc; scale_best = cand; } - } - best = scale_best; best_cc = scale_cc; - } - data.latt = best; -} - template void PixelRefine::Run(const T *image, BraggPrediction &prediction, @@ -602,10 +400,6 @@ void PixelRefine::Run(const T *image, data.solved = false; data.reflections.clear(); - // Optional reference-driven orientation + cell-scale sweep before prediction (env-gated). - if (data.sweep_orientation) - SweepOrientationCell(image, prediction, data); - const double lambda = data.geom.GetWavelength_A(); const double pixel_size = data.geom.GetPixelSize_mm(); @@ -649,154 +443,69 @@ void PixelRefine::Run(const T *image, double latt_vec0[3], latt_vec1[3], latt_vec2[3]; BuildParameterBlocks(data, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2); - // ---- 1-2. Predict shoeboxes + collect pixels (a lambda so it can be re-run after - // an orientation refinement moves the predicted positions). ---------- - // A spot-core mask over ALL predictions keeps each reflection's background ring from - // picking up a neighbour's signal. Pixels carry a fit weight (background-limited - // variance, signal-weighted toward the predicted centre) used only by the optional - // orientation refinement - the factored integration weights by v = Ibkg directly. + // ---- 1. Predict shoeboxes for the current geometry ------------------------ + DiffractionExperiment exp_iter = experiment; + exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) + .BeamY_pxl(data.geom.GetBeamY_pxl()) + .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) + .PoniRot1_rad(data.geom.GetPoniRot1_rad()) + .PoniRot2_rad(data.geom.GetPoniRot2_rad()); + const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); + + // ---- 2. Collect per-reflection shoebox pixels + local background ---------- + // GetReflections() returns the full pre-sized buffer; only the first nrefl + // entries are valid for this image. A spot-core mask over ALL predictions keeps + // each reflection's background ring from picking up a neighbour's signal. + const auto &predicted = prediction.GetReflections(); + const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); + std::vector groups; - auto build_groups = [&]() { - groups.clear(); - DiffractionExperiment exp_iter = experiment; - exp_iter.BeamX_pxl(data.geom.GetBeamX_pxl()) - .BeamY_pxl(data.geom.GetBeamY_pxl()) - .DetectorDistance_mm(data.geom.GetDetectorDistance_mm()) - .PoniRot1_rad(data.geom.GetPoniRot1_rad()) - .PoniRot2_rad(data.geom.GetPoniRot2_rad()); - const int nrefl = prediction.Calc(exp_iter, data.latt, settings_prediction); - const auto &predicted = prediction.GetReflections(); - const auto spot_mask = BuildSpotMask(predicted, nrefl, xpixel, ypixel, radius); - for (int ri = 0; ri < nrefl; ++ri) { - const auto &refl = predicted[ri]; - const auto hkl = hkl_key_generator(refl); - if (!reference_data.contains(hkl)) - continue; - double Ibkg = 0.0; - if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, - refl.predicted_x, refl.predicted_y, - radius, bkg_outer_radius, Ibkg)) - continue; - ReflGroup g; - g.h = refl.h; - g.k = refl.k; - g.l = refl.l; - g.d = refl.d; - g.Itrue = reference_data[hkl]; - g.R_bw_sq = bandwidth_radial_sq(refl.d); - g.pol = polarization(refl.predicted_x, refl.predicted_y); - g.Ibkg = Ibkg; - g.predicted_x = refl.predicted_x; - g.predicted_y = refl.predicted_y; - const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); - for (int y = box.min_y; y <= box.max_y; ++y) { - for (int x = box.min_x; x <= box.max_x; ++x) { - const size_t npixel = xpixel * y + x; - if (image[npixel] == std::numeric_limits::max()) - continue; - if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) - continue; - double weight = 1.0 / std::sqrt(std::max(Ibkg, 1.0)); - if (data.fit_signal_sigma_pix > 0.0) { - const double dx = x - refl.predicted_x, dy = y - refl.predicted_y; - const double s2 = data.fit_signal_sigma_pix * data.fit_signal_sigma_pix; - weight *= std::exp(-0.5 * (dx * dx + dy * dy) / s2); - } - g.pixels.push_back({static_cast(x), static_cast(y), - static_cast(image[npixel]), Ibkg, weight}); - } + for (int ri = 0; ri < nrefl; ++ri) { + const auto &refl = predicted[ri]; + const auto hkl = hkl_key_generator(refl); + if (!reference_data.contains(hkl)) + continue; + + // Local flat background from the ring around the shoebox (raw counts). If we + // cannot estimate a clean local background the reflection is dropped, exactly + // as BraggIntegrate2D marks it unobserved when too few background pixels survive. + double Ibkg = 0.0; + if (!EstimateLocalBackground(image, spot_mask, xpixel, ypixel, + refl.predicted_x, refl.predicted_y, + radius, bkg_outer_radius, Ibkg)) + continue; + + ReflGroup g; + g.h = refl.h; + g.k = refl.k; + g.l = refl.l; + g.d = refl.d; + g.Itrue = reference_data[hkl]; + g.R_bw_sq = bandwidth_radial_sq(refl.d); + g.pol = polarization(refl.predicted_x, refl.predicted_y); + g.Ibkg = Ibkg; + g.predicted_x = refl.predicted_x; + g.predicted_y = refl.predicted_y; + + const auto box = ShoeboxBounds(refl.predicted_x, refl.predicted_y, radius, xpixel, ypixel); + for (int y = box.min_y; y <= box.max_y; ++y) { + for (int x = box.min_x; x <= box.max_x; ++x) { + const size_t npixel = xpixel * y + x; + // Skip sentinel (masked / saturated) pixels. + if (image[npixel] == std::numeric_limits::max()) + continue; + if (std::is_signed_v && (image[npixel] == std::numeric_limits::min())) + continue; + g.pixels.push_back({static_cast(x), static_cast(y), + static_cast(image[npixel]), Ibkg}); } - if (!g.pixels.empty()) - groups.push_back(std::move(g)); } - }; - build_groups(); + if (!g.pixels.empty()) + groups.push_back(std::move(g)); + } if (groups.empty()) return; - // ---- Optional per-image orientation refinement (off by default) ----------------- - // The pre-factored per-pixel step: refine the orientation (and a nuisance scale) - // against the shoebox pixels via ShoeboxResidual, regularised to the spot-centroid - // orientation, then re-predict. Dropped from the factored model (geometry fixed to - // XtalOptimizer); behind PixelRefineData::refine_orientation to A/B its R-free effect. - if (data.refine_orientation) { - const double orient_prior[3] = {latt_vec0[0], latt_vec0[1], latt_vec0[2]}; - ceres::Problem oprob; - size_t npix = 0; - for (const auto &g : groups) { - auto *cost = new ceres::DynamicAutoDiffCostFunction( - new ShoeboxResidual(g, lambda, pixel_size, data.crystal_system)); - for (int b : {2, 1, 2, 3, 3, 3, 1, 1, 2}) - cost->AddParameterBlock(b); - cost->SetNumResiduals(static_cast(g.pixels.size())); - oprob.AddResidualBlock(cost, nullptr, beam, &dist_mm, detector_rot, - latt_vec0, latt_vec1, latt_vec2, - &data.scale_factor, &data.B_factor, data.R); - npix += g.pixels.size(); - } - // Refine only orientation (latt_vec0) + the nuisance scale G; everything else fixed. - oprob.SetParameterBlockConstant(beam); - oprob.SetParameterBlockConstant(&dist_mm); - oprob.SetParameterBlockConstant(detector_rot); - oprob.SetParameterBlockConstant(latt_vec1); - oprob.SetParameterBlockConstant(latt_vec2); - oprob.SetParameterBlockConstant(&data.B_factor); - oprob.SetParameterBlockConstant(data.R); - oprob.SetParameterLowerBound(&data.scale_factor, 0, 0.0); - if (data.orient_reg_sigma_deg > 0.0 && npix > 0) { - const double sigma_rad = std::max(data.orient_reg_sigma_deg * M_PI / 180.0, 1e-9); - const double w = std::sqrt(static_cast(npix)) / sigma_rad; - oprob.AddResidualBlock(new ceres::AutoDiffCostFunction( - new OrientationRegularizer(w, orient_prior)), nullptr, latt_vec0); - } - if (data.scale_reg_sigma > 0.0) { - const double w = std::sqrt(static_cast(groups.size()) / data.scale_reg_sigma); - oprob.AddResidualBlock(new ceres::AutoDiffCostFunction( - new ScalarRegularizer(w, 1.0)), nullptr, &data.scale_factor); - } - ceres::Solver::Options oopt; - oopt.linear_solver_type = ceres::DENSE_QR; - oopt.logging_type = ceres::LoggingType::SILENT; - oopt.minimizer_progress_to_stdout = false; - oopt.max_solver_time_in_seconds = data.max_time_s; - oopt.num_threads = 1; - ceres::Solver::Summary osum; - ceres::Solve(oopt, &oprob, &osum); - - // Write the refined orientation back into data.latt (cell held at latt_vec1/2), - // then re-predict + rebuild groups at the new orientation. Scale is reset; Term 1 - // re-fits it properly below. - switch (data.crystal_system) { - case gemmi::CrystalSystem::Tetragonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - case gemmi::CrystalSystem::Cubic: - latt_vec1[1] = latt_vec1[0]; - latt_vec1[2] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - case gemmi::CrystalSystem::Hexagonal: - latt_vec1[1] = latt_vec1[0]; - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, 2.0*M_PI/3.0); - break; - case gemmi::CrystalSystem::Monoclinic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, latt_vec2[0], M_PI/2); - break; - case gemmi::CrystalSystem::Orthorhombic: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, M_PI/2, M_PI/2, M_PI/2); - break; - default: - data.latt = AngleAxisAndCellToLattice(latt_vec0, latt_vec1, - latt_vec2[0], latt_vec2[1], latt_vec2[2]); - break; - } - data.scale_factor = 1.0; - build_groups(); - if (groups.empty()) - return; - } - // ---- 3. Term 2: per-resolution tangential profile width R1 ---------------- // R1 = sqrt(2*) from the intensity-weighted tangential second moment of // the strong spots, binned by resolution (low res small spots, high res larger). diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 40dba239..366d8ca3 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -68,28 +68,12 @@ struct PixelRefineData { bool refine_B = true; // refine the per-image B-factor along with G // Term 2 measures the physical tangential width R1, but the *integration* profile must - // be generous (XDS-style: integrate over ~6-10 sigma), or a tight template centred on - // the prediction sits off the ~0.4 px centroid-floor scatter and underestimates the - // intensity. The measured R1 is multiplied by this before use. 1.0 = the raw physical - // width (too tight - regressed R-free on the serial jet); ~6 recovers it (closes the - // CCref gap to box-sum). Tune the final value against R-free. + // be generous (XDS-style: integrate over ~6 sigma) or a tight template centred on the + // prediction sits off the ~0.4 px centroid-floor scatter and underestimates the + // intensity (validated on the jet R-free: 0.34 with the raw width -> 0.26 at x6). The + // measured R1 is multiplied by this before use. double r1_multiplier = 6.0; - // Optional per-image orientation refinement (env-gated, off by default): the - // pre-factored per-pixel step that refines the crystal orientation against the - // shoebox pixels, regularised to the spot-centroid orientation, before integration. - // Reinstated to A/B its effect on serial-data R-free vs the fixed-geometry default. - bool refine_orientation = false; - double orient_reg_sigma_deg = 1.0; // orientation-prior strength (deg) - double fit_signal_sigma_pix = 1.5; // signal-weighting sigma for the orientation fit (px) - - // Optional reference-driven orientation + per-image cell-scale sweep before prediction - // (env-gated, off by default). Coarse global moves the gradient refinement cannot make, - // plus a radial (cell-scale) DOF the gradient path lacks. See SweepOrientationCell. - bool sweep_orientation = false; - double sweep_max_deg = 0.15; - double sweep_max_cell_frac = 0.003; - // Relative X-ray bandwidth (sigma of dlambda/lambda), e.g. ~0.004 for a 1% FWHM // DMM, ~1e-4 for Si(111). Adds a resolution-dependent radial broadening to R[0]. // 0 = monochromatic (the term switches off entirely). @@ -136,11 +120,6 @@ class PixelRefine { double beam[2], double &dist_mm, double detector_rot[2], double latt_vec0[3], double latt_vec1[3], double latt_vec2[3]) const; - - // Reference-driven global orientation + cell-scale sweep (see PixelRefineData::sweep_*). - template - void SweepOrientationCell(const T *image, BraggPrediction &prediction, - PixelRefineData &data) const; public: PixelRefine(const DiffractionExperiment &experiment, const std::vector &reference); -- 2.52.0 From 8a9d80eb71311345ef37aca0c08b78d286c97dcd Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 13:48:11 +0200 Subject: [PATCH 047/228] PixelRefine: profile multiplier as --profile-multiplier flag; outlier rejection default off (1) Promote the Term-2 profile-width multiplier from the PR_RMULT env knob to a real BraggIntegrationSettings::profile_multiplier (default 6) + jfjoch_process --profile-multiplier flag. Removes the last PR_* env knob; IndexAndRefine reads the setting. (2) Flip the merge outlier rejection default to OFF (ScalingSettings outlier_reject_nsigma 6 -> 0); it stays available via --reject-outliers , since it helps the jet R-free but is dataset-dependent (drops crystal-2 CCref). Co-Authored-By: Claude Opus 4.8 --- common/BraggIntegrationSettings.cpp | 11 +++++++++++ common/BraggIntegrationSettings.h | 6 ++++++ common/ScalingSettings.h | 2 +- image_analysis/IndexAndRefine.cpp | 2 +- tools/jfjoch_process.cpp | 18 ++++++++++++++++-- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/common/BraggIntegrationSettings.cpp b/common/BraggIntegrationSettings.cpp index 3fe0a861..6ea3381e 100644 --- a/common/BraggIntegrationSettings.cpp +++ b/common/BraggIntegrationSettings.cpp @@ -72,6 +72,17 @@ float BraggIntegrationSettings::GetR1() const { return r_1; } +float BraggIntegrationSettings::GetProfileMultiplier() const { + return profile_multiplier; +} + +BraggIntegrationSettings &BraggIntegrationSettings::ProfileMultiplier(float input) { + if (input <= 0.0f) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Profile multiplier must be positive"); + profile_multiplier = input; + return *this; +} + float BraggIntegrationSettings::GetR2() const { return r_2; } diff --git a/common/BraggIntegrationSettings.h b/common/BraggIntegrationSettings.h index dbe75ae6..02f30782 100644 --- a/common/BraggIntegrationSettings.h +++ b/common/BraggIntegrationSettings.h @@ -12,6 +12,10 @@ class BraggIntegrationSettings { float d_min_limit_A = 1.0; std::optional fixed_profile_radius; float minimum_sigma_in_regards_to_i = 0.02; + // PixelRefine only: the measured (physical) tangential profile width is multiplied by + // this for the integration template, so the aperture is generous (XDS-style ~6 sigma) + // and tolerant of the centroid-floor scatter. Ignored by the classical integrator. + float profile_multiplier = 6.0; public: BraggIntegrationSettings& R1(float input); @@ -19,11 +23,13 @@ public: BraggIntegrationSettings& R3(float input); BraggIntegrationSettings& DMinLimit_A(float input); BraggIntegrationSettings& FixedProfileRadius_recipA(std::optional input); + BraggIntegrationSettings& ProfileMultiplier(float input); [[nodiscard]] float GetR1() const; [[nodiscard]] float GetR2() const; [[nodiscard]] float GetR3() const; + [[nodiscard]] float GetProfileMultiplier() const; [[nodiscard]] std::optional GetFixedProfileRadius_recipA() const; [[nodiscard]] float GetDMinLimit_A() const; diff --git a/common/ScalingSettings.h b/common/ScalingSettings.h index 6d548cfc..09d591c1 100644 --- a/common/ScalingSettings.h +++ b/common/ScalingSettings.h @@ -23,7 +23,7 @@ class ScalingSettings { std::optional wedge_for_scaling; double min_partiality = 0.02; double min_cc_for_image = 0.0; - double outlier_reject_nsigma = 6.0; // per-observation merge outlier rejection (XDS/DIALS-style); <= 0 disables + double outlier_reject_nsigma = 0.0; // per-observation merge outlier rejection (XDS/DIALS-style); 0 = off, e.g. 6 enables double rfree_fraction = 0.05; IntensityFormat intensity_format = IntensityFormat::MTZ; diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index c4c0e831..4800e8bb 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -448,7 +448,7 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, // Signal-box radius from the shared integration setting (same knob as BraggIntegrate2D). prd.shoebox_radius = static_cast(std::lround(experiment.GetBraggIntegrationSettings().GetR1())); - if (const char *m = std::getenv("PR_RMULT")) prd.r1_multiplier = std::stod(m); // Term-2 R1 multiplier (default 6) + prd.r1_multiplier = experiment.GetBraggIntegrationSettings().GetProfileMultiplier(); std::vector buffer; const uint8_t *ptr = image.GetUncompressedPtr(buffer); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 93ade850..7748ccdd 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -72,7 +72,7 @@ void print_usage() { std::cout << " -w, --wedge[=num] Refine image wedge during scaling with starting wedge value" << std::endl; std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << std::endl; std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; - std::cout << " --reject-outliers Per-observation merge outlier rejection, N sigma from the per-reflection median (default: 6; 0 disables)" << std::endl; + std::cout << " --reject-outliers Per-observation merge outlier rejection, N sigma from the per-reflection median (default: off; e.g. 6, XDS/DIALS-style)" << std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; @@ -82,6 +82,7 @@ void print_usage() { std::cout << " Pixel refinement (experimental, select via -r pixelrefine, needs --reference-mtz)" << std::endl; std::cout << " --bandwidth Relative X-ray bandwidth FWHM (e.g. 0.01 for 1% DMM); default from file or 0" << std::endl; std::cout << " --integration-radius Signal-box radius r1, or r1,r2,r3 (px). One value => r2=r1+2, r3=r1+4" << std::endl; + std::cout << " --profile-multiplier PixelRefine: scale the measured tangential profile width R1 (default: 6; XDS-style generous aperture)" << std::endl; } enum { @@ -99,7 +100,8 @@ enum { OPT_FORCE_ROTATION_LATTICE, OPT_BANDWIDTH, OPT_INTEGRATION_RADIUS, - OPT_REJECT_OUTLIERS + OPT_REJECT_OUTLIERS, + OPT_PROFILE_MULTIPLIER }; static option long_options[] = { @@ -138,6 +140,7 @@ static option long_options[] = { {"bandwidth", required_argument, nullptr, OPT_BANDWIDTH}, {"integration-radius", required_argument, nullptr, OPT_INTEGRATION_RADIUS}, {"reject-outliers", required_argument, nullptr, OPT_REJECT_OUTLIERS}, + {"profile-multiplier", required_argument, nullptr, OPT_PROFILE_MULTIPLIER}, {nullptr, 0, nullptr, 0} }; @@ -320,6 +323,7 @@ int main(int argc, char **argv) { std::optional d_min_scale_merge; std::optional integration_radius_arg; // "r1" or "r1,r2,r3" std::optional outlier_reject_nsigma; // merge per-observation outlier rejection + std::optional profile_multiplier; // PixelRefine Term-2 profile-width multiplier if (argc == 1) { print_usage(); @@ -515,6 +519,9 @@ int main(int argc, char **argv) { case OPT_REJECT_OUTLIERS: outlier_reject_nsigma = std::stod(optarg); break; + case OPT_PROFILE_MULTIPLIER: + profile_multiplier = std::stof(optarg); + break; case OPT_MIN_IMAGE_CC: min_image_cc = std::stod(optarg); break; @@ -707,6 +714,13 @@ int main(int argc, char **argv) { logger.Info("Integration radii set to r1={:.1f} r2={:.1f} r3={:.1f}", r1, r2, r3); } + if (profile_multiplier) { + BraggIntegrationSettings bis = experiment.GetBraggIntegrationSettings(); + bis.ProfileMultiplier(*profile_multiplier); + experiment.ImportBraggIntegrationSettings(bis); + logger.Info("PixelRefine profile-width multiplier set to {:.1f}", *profile_multiplier); + } + SpotFindingSettings spot_settings; spot_settings.enable = true; spot_settings.indexing = true; -- 2.52.0 From ecdb7048a018a2dbf33bf27088045401ddcbb728 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 14:11:26 +0200 Subject: [PATCH 048/228] PixelRefine: drop crystal-system idealisation, use the indexed cell as-is PixelRefine no longer refines the cell, so the symmetry-constrained cell parameterisation inherited from XtalOptimizer has no manifold to constrain. The decompose->rebuild round-trip (Gram-Schmidt orientation + symmetry B matrix) merely reconstructed the lattice columns it started from, and for non-triclinic systems it re-idealised the indexed cell (averaging a,b for tetragonal; a,b,c for cubic; forcing alpha=gamma=90). Replace both six-way switches (PredictedNode and BuildParameterBlocks) with a single path: take the three real-space lattice columns (latt.Vec0/1/2()) directly and form the reciprocal node via the general cross-product inverse. This reproduces every crystal system exactly from the actual cell, is more faithful (no re-idealisation), and removes the crystal_system field plus two now-unused includes. PredictedNode de-templated (only ever called with double). Crystal symmetry still lives where it belongs: indexing (upstream) and merging (downstream via the space-group HKL key). A/B (both lyso P4_3 2_1 2 crystals, refined under tetragonal constraint, so the old idealisation was already a no-op): stat tables bit-identical across all shells -- crystal 2 CC1/2 94.6% / CCref 92.5%, jet CC1/2 91.9% / CCref 55.8% -- the only delta is 4th-digit float-ordering noise in the error-model b coefficient (0.8143->0.8141, same ISa). Same merged intensities => R-free unchanged. Net -81 lines. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 3 - .../pixel_refinement/PixelRefine.cpp | 153 +++++------------- image_analysis/pixel_refinement/PixelRefine.h | 7 +- 3 files changed, 41 insertions(+), 122 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 4800e8bb..ebe85b0c 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -439,9 +439,6 @@ bool IndexAndRefine::PixelRefineIntegrate(DataMessage &msg, PixelRefineData prd; prd.geom = outcome.experiment.GetDiffractionGeometry(); prd.latt = *outcome.lattice_candidate; - prd.crystal_system = outcome.symmetry.crystal_system; - if (prd.crystal_system == gemmi::CrystalSystem::Trigonal) - prd.crystal_system = gemmi::CrystalSystem::Hexagonal; prd.centering = outcome.symmetry.centering; if (const auto bw = experiment.GetBandwidthFWHM()) prd.bandwidth = bw.value() / 2.3548; // FWHM -> sigma diff --git a/image_analysis/pixel_refinement/PixelRefine.cpp b/image_analysis/pixel_refinement/PixelRefine.cpp index 4f45c7aa..b9014d82 100644 --- a/image_analysis/pixel_refinement/PixelRefine.cpp +++ b/image_analysis/pixel_refinement/PixelRefine.cpp @@ -11,9 +11,6 @@ #include #include -#include - -#include "../geom_refinement/LatticeReduction.h" namespace { @@ -186,89 +183,36 @@ void ObservedRecip(const T *beam, const T *distance_mm, const T *detector_rot, } // Per-reflection: predicted node g_hkl, |g_hkl|^2, and the Ewald-sphere normal. -// This is the expensive part (symmetry-aware B matrix, three rotations, cross -// products) - it depends only on the lattice (p0,p1,p2) and hkl, so for a whole -// shoebox it can be computed once. Convention identical to XtalOptimizer. -template -bool PredictedNode(const T *p0, const T *p1, const T *p2, - double exp_h, double exp_k, double exp_l, - gemmi::CrystalSystem symmetry, double inv_lambda, - Eigen::Matrix &e_pred_recip, - Eigen::Matrix &n_radial, T &q_sq) { - Eigen::Matrix e_uc_len = Eigen::Matrix::Zero(); - Eigen::Matrix Bmat = Eigen::Matrix::Identity(); +// (a0,a1,a2) are the three real-space lattice column vectors in the lab frame +// (latt.Vec0/1/2()), used exactly as indexed: PixelRefine does not refine the cell, +// so there is no symmetry manifold to constrain - a general (triclinic) reciprocal +// inverse reproduces every crystal system from the actual cell. Depends only on the +// lattice and hkl, so for a whole shoebox it is computed once. +bool PredictedNode(const double *a0, const double *a1, const double *a2, + double exp_h, double exp_k, double exp_l, double inv_lambda, + Eigen::Vector3d &e_pred_recip, + Eigen::Vector3d &n_radial, double &q_sq) { + const Eigen::Vector3d A(a0[0], a0[1], a0[2]); + const Eigen::Vector3d Bv(a1[0], a1[1], a1[2]); + const Eigen::Vector3d C(a2[0], a2[1], a2[2]); - if (symmetry == gemmi::CrystalSystem::Hexagonal) { - e_uc_len << p1[0], p1[0], p1[2]; - Bmat(0, 1) = T(-0.5); - Bmat(1, 1) = T(sqrt(3.0) / 2.0); - } else if (symmetry == gemmi::CrystalSystem::Orthorhombic) { - e_uc_len << p1[0], p1[1], p1[2]; - } else if (symmetry == gemmi::CrystalSystem::Tetragonal) { - e_uc_len << p1[0], p1[0], p1[2]; - } else if (symmetry == gemmi::CrystalSystem::Cubic) { - e_uc_len << p1[0], p1[0], p1[0]; - } else if (symmetry == gemmi::CrystalSystem::Monoclinic) { - e_uc_len << p1[0], p1[1], p1[2]; - Bmat(0, 2) = ceres::cos(p2[0]); - Bmat(2, 2) = ceres::sin(p2[0]); - } else { - const T ca = ceres::cos(p2[0]); - const T cb = ceres::cos(p2[1]); - const T cg = ceres::cos(p2[2]); - const T sg = ceres::sin(p2[2]); + const Eigen::Vector3d BxC = Bv.cross(C); + const Eigen::Vector3d CxA = C.cross(A); + const Eigen::Vector3d AxB = A.cross(Bv); - e_uc_len << p1[0], p1[1], p1[2]; - - Bmat(0, 1) = cg; - Bmat(1, 1) = sg; - - const T cx = cb; - const T cy = (ca - cb * cg) / sg; - const T v = T(1) - cx * cx - cy * cy; - const T cz = (v >= T(0)) ? ceres::sqrt(v) : T(0); - - Bmat(0, 2) = cx; - Bmat(1, 2) = cy; - Bmat(2, 2) = cz; - } - - const T L0 = e_uc_len[0]; - const T L1 = e_uc_len[1]; - const T L2 = e_uc_len[2]; - - T col0_unrot[3] = {Bmat(0, 0) * L0, Bmat(1, 0) * L0, Bmat(2, 0) * L0}; - T col1_unrot[3] = {Bmat(0, 1) * L1, Bmat(1, 1) * L1, Bmat(2, 1) * L1}; - T col2_unrot[3] = {Bmat(0, 2) * L2, Bmat(1, 2) * L2, Bmat(2, 2) * L2}; - - T col0_rot[3], col1_rot[3], col2_rot[3]; - ceres::AngleAxisRotatePoint(p0, col0_unrot, col0_rot); - ceres::AngleAxisRotatePoint(p0, col1_unrot, col1_rot); - ceres::AngleAxisRotatePoint(p0, col2_unrot, col2_rot); - - const Eigen::Matrix A(col0_rot[0], col0_rot[1], col0_rot[2]); - const Eigen::Matrix Bv(col1_rot[0], col1_rot[1], col1_rot[2]); - const Eigen::Matrix C(col2_rot[0], col2_rot[1], col2_rot[2]); - - const Eigen::Matrix BxC = Bv.cross(C); - const Eigen::Matrix CxA = C.cross(A); - const Eigen::Matrix AxB = A.cross(Bv); - - const T Vol = A.dot(BxC); - if (ceres::abs(Vol) < T(1e-12)) + const double Vol = A.dot(BxC); + if (std::abs(Vol) < 1e-12) return false; - const T invV = T(1) / Vol; + const double invV = 1.0 / Vol; - e_pred_recip = (BxC * T(exp_h) + CxA * T(exp_k) + AxB * T(exp_l)) * invV; + e_pred_recip = (BxC * exp_h + CxA * exp_k + AxB * exp_l) * invV; q_sq = e_pred_recip.squaredNorm(); // Ewald sphere centre at -k_i = (0,0,-inv_lambda); radial normal at g_hkl. - const Eigen::Matrix S_pred( - e_pred_recip[0], - e_pred_recip[1], - e_pred_recip[2] + T(inv_lambda)); - const T S_pred_norm = S_pred.norm(); - if (S_pred_norm < T(1e-10)) + const Eigen::Vector3d S_pred( + e_pred_recip[0], e_pred_recip[1], e_pred_recip[2] + inv_lambda); + const double S_pred_norm = S_pred.norm(); + if (S_pred_norm < 1e-10) return false; n_radial = S_pred / S_pred_norm; @@ -281,7 +225,7 @@ bool PredictedNode(const T *p0, const T *p1, const T *p2, // eps_radial = deviation along the Ewald normal (the partiality direction) // eps_tang_sq = squared deviation in the tangent plane (the profile direction) bool GeometryProbe(double obs_x, double obs_y, double lambda, double pixel_size, - int h, int k, int l, gemmi::CrystalSystem symmetry, + int h, int k, int l, const double beam[2], double dist_mm, const double detector_rot[2], const double p0[3], const double p1[3], const double p2[3], double &q_sq, double &eps_radial, double &eps_tang_sq) { @@ -290,7 +234,7 @@ bool GeometryProbe(double obs_x, double obs_y, double lambda, double pixel_size, ObservedRecip(beam, &dist_mm, detector_rot, obs_x, obs_y, pixel_size, inv_lambda, e_obs); Eigen::Vector3d e_pred, n_radial; - if (!PredictedNode(p0, p1, p2, h, k, l, symmetry, inv_lambda, e_pred, n_radial, q_sq)) + if (!PredictedNode(p0, p1, p2, h, k, l, inv_lambda, e_pred, n_radial, q_sq)) return false; const Eigen::Vector3d delta_q = e_obs - e_pred; @@ -359,37 +303,16 @@ void PixelRefine::BuildParameterBlocks(const PixelRefineData &data, detector_rot[0] = data.geom.GetPoniRot1_rad(); detector_rot[1] = data.geom.GetPoniRot2_rad(); - for (int i = 0; i < 3; ++i) - latt_vec0[i] = latt_vec1[i] = latt_vec2[i] = 0.0; - - double beta = data.latt.GetUnitCell().beta; - switch (data.crystal_system) { - case gemmi::CrystalSystem::Orthorhombic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Tetragonal: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1]) / 2.0; - break; - case gemmi::CrystalSystem::Cubic: - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - latt_vec1[0] = (latt_vec1[0] + latt_vec1[1] + latt_vec1[2]) / 3.0; - break; - case gemmi::CrystalSystem::Hexagonal: - LatticeToRodriguesAndLengths_Hex(data.latt, latt_vec0, latt_vec1); - break; - case gemmi::CrystalSystem::Monoclinic: - LatticeToRodriguesLengthsBeta_Mono(data.latt, latt_vec0, latt_vec1, beta); - latt_vec2[0] = beta; - break; - default: { - LatticeToRodriguesAndLengths_GS(data.latt, latt_vec0, latt_vec1); - const auto uc = data.latt.GetUnitCell(); - latt_vec2[0] = uc.alpha * M_PI / 180.0; - latt_vec2[1] = uc.beta * M_PI / 180.0; - latt_vec2[2] = uc.gamma * M_PI / 180.0; - break; - } + // The three real-space lattice columns in the lab frame, used as-is. PixelRefine + // does not refine the cell, so there is no symmetry manifold to constrain; the + // indexed cell is taken exactly, with no re-idealisation to ideal symmetry. + const Coord &a = data.latt.Vec0(); + const Coord &b = data.latt.Vec1(); + const Coord &c = data.latt.Vec2(); + for (int i = 0; i < 3; ++i) { + latt_vec0[i] = a[i]; + latt_vec1[i] = b[i]; + latt_vec2[i] = c[i]; } } @@ -532,7 +455,7 @@ void PixelRefine::Run(const T *image, double sw = 0.0, sw_et2 = 0.0; for (const auto &px : g.pixels) { double q_sq, eps_r, eps_t_sq; - if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) continue; @@ -572,7 +495,7 @@ void PixelRefine::Run(const T *image, pt_sig.reserve(g.pixels.size()); for (const auto &px : g.pixels) { double q_sq, eps_r, eps_t_sq; - if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + if (!GeometryProbe(px.x, px.y, lambda, pixel_size, g.h, g.k, g.l, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) continue; @@ -664,7 +587,7 @@ void PixelRefine::Run(const T *image, for (int x = cx - radius; x <= cx + radius; ++x) { double q_sq, eps_r, eps_t_sq; if (!GeometryProbe(static_cast(x), static_cast(y), - lambda, pixel_size, g.h, g.k, g.l, data.crystal_system, + lambda, pixel_size, g.h, g.k, g.l, beam, dist_mm, detector_rot, latt_vec0, latt_vec1, latt_vec2, q_sq, eps_r, eps_t_sq)) continue; diff --git a/image_analysis/pixel_refinement/PixelRefine.h b/image_analysis/pixel_refinement/PixelRefine.h index 366d8ca3..b180e1de 100644 --- a/image_analysis/pixel_refinement/PixelRefine.h +++ b/image_analysis/pixel_refinement/PixelRefine.h @@ -58,7 +58,6 @@ struct PixelRefineData { // --- model state (input as initial guess, output as refined result) --- DiffractionGeometry geom; // fixed (refined upstream by XtalOptimizer) CrystalLattice latt; // fixed - gemmi::CrystalSystem crystal_system = gemmi::CrystalSystem::Triclinic; char centering = 'P'; double B_factor = 0.0; // Debye-Waller B (A^2), refined @@ -113,9 +112,9 @@ class PixelRefine { const HKLKeyGenerator hkl_key_generator; std::map reference_data; - // Fills the fixed geometry + symmetry-aware lattice parametrization (beam, - // distance, detector tilt, and the Rodrigues orientation / cell-length / angle - // vectors) from the current model state, for the per-pixel geometry evaluation. + // Fills the fixed geometry (beam, distance, detector tilt) and the three + // real-space lattice column vectors from the current model state, for the + // per-pixel geometry evaluation. void BuildParameterBlocks(const PixelRefineData &data, double beam[2], double &dist_mm, double detector_rot[2], -- 2.52.0 From 40da1aab13966e23401783cfae9858bf62312fc5 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 20:35:03 +0200 Subject: [PATCH 049/228] IndexAndRefine: conventionalise de-novo lattice when cell+SG are given Providing both a unit cell (-C) and space group (-S) silently broke the de-novo indexers (FFT/FFTW) -> 0% indexing, while each flag alone worked. Root cause: the `sg && GetUnitCell()` branch fed the indexer's raw lattice straight into symmetry-constrained refinement. FFBIDX returns the lattice already in the reference setting, but FFT/FFTW return an arbitrarily-oriented Niggli-primitive cell; enforcing the crystal system on its mis-assigned axes rejects every frame. Fix: for de-novo indexers only, reduce the lattice to the conventional setting (LatticeSearch) before refinement. FFBIDX keeps using its raw lattice as-is, so it is byte-identical to before (no regression). niggli_class stays unassigned (0) in this branch - it is a property of the primitive cell incl. centering, which LatticeSearch cannot recover from a user-supplied (possibly centered, e.g. C2) cell. A proper primitive-cell indexing path (CrystFEL-style) is deferred. Validation (lyso, -C79,79,38 -S96): crystal 2 FFT : 0% -> 94.9% (CC1/2 95.8, CCref 92.7) = de-novo quality crystal 2 FFBIDX: 71.4% (CC1/2 94.6) - byte-identical jet FFBIDX: 27.2% (CC1/2 91.9) - byte-identical (Jet FFT stays 0% - that is a separate, still-open issue: de-novo finds no consensus on the weak serial-still frames, 0% even without -S; to investigate.) Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index ebe85b0c..7b9563f8 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -101,16 +101,27 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetry(Data auto latt = indexer_result.lattice[0]; if (latt.CalcVolume() > 1.0) { auto sg = experiment.GetGemmiSpaceGroup(); + const auto algorithm = experiment.GetIndexingAlgorithm(); + const bool de_novo = (algorithm == IndexingAlgorithmEnum::FFT + || algorithm == IndexingAlgorithmEnum::FFTW); - // If space group and cell provided => always enforce symmetry in refinement - // If space group not provided => guess symmetry + // If space group and cell provided => enforce that symmetry in refinement. + // If not => detect the symmetry from the lattice. if (sg && experiment.GetUnitCell()) { + // FFBIDX returns the lattice already in the reference setting, so use it + // as-is. De-novo indexers (FFT/FFTW) return an arbitrarily-oriented primitive + // cell that must first be reduced to the conventional setting, else enforcing + // the crystal system on mis-assigned axes rejects every frame. Centering is + // taken from the user's space group; niggli_class is left unassigned (0) - it + // is a property of the primitive cell incl. centering, which LatticeSearch + // cannot recover from a cell the user may have given centered (e.g. C2). A + // proper primitive-cell indexing path (CrystFEL-style) is deferred. outcome.symmetry = LatticeMessage{ .centering = sg->centring_type(), .niggli_class = 0, .crystal_system = sg->crystal_system() }; - outcome.lattice_candidate = latt; + outcome.lattice_candidate = de_novo ? LatticeSearch(latt).conventional : latt; } else { auto sym_result = LatticeSearch(latt); outcome.symmetry = LatticeMessage{ -- 2.52.0 From e1e2ca8e4921ba2a9398d176152079e1a0a33f3c Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 22:50:56 +0200 Subject: [PATCH 050/228] IndexAndRefine: unify cell setting across indexers, keep FFBIDX neutral Follow-up to 40da1aab. Conventionalise the indexed lattice for BOTH indexers when cell+SG are given, so every frame reports its cell in ONE consistent setting per space group (mixed axis orders like [78,78,38] vs [38,78,78] index the same reflection as different HKLs and cannot be merged). Use LatticeSearch's conventional cell when its detected symmetry agrees with the user's space group. On noisy frames LatticeSearch can pick an alternative Bravais setting (e.g. the sqrt2 C-centred description of a primitive tetragonal cell, [78,78,38]->[110,111,38]) whose system disagrees; there: - FFBIDX already returns the reference setting (c-last), consistent with the conventional frames, so its raw lattice is safe -> use it. This recovers the ~3% of frames the unconditional version lost: FFBIDX is byte-for-byte neutral. - de-novo (FFT/FFTW) return a Niggli-primitive cell with a DIFFERENT axis order (c-first); merging it with the conventional (c-last) frames would corrupt the data -> reject the frame instead. Validation (lyso): FFBIDX c2 71.44% / jet 27.15% (both unchanged vs raw-lattice baseline, CC1/2 94.6 / 91.9); FFT c2 +C+S 92.72% (CC1/2 95.8, all c-last, consistent); FFT c2 de-novo 95.44% (unchanged). Verified per-frame that the c-axis (~38A) is axis 2 in every merged frame - no axis-order mixing. Co-Authored-By: Claude Opus 4.8 --- image_analysis/IndexAndRefine.cpp | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/image_analysis/IndexAndRefine.cpp b/image_analysis/IndexAndRefine.cpp index 7b9563f8..61b30861 100644 --- a/image_analysis/IndexAndRefine.cpp +++ b/image_analysis/IndexAndRefine.cpp @@ -108,20 +108,31 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetry(Data // If space group and cell provided => enforce that symmetry in refinement. // If not => detect the symmetry from the lattice. if (sg && experiment.GetUnitCell()) { - // FFBIDX returns the lattice already in the reference setting, so use it - // as-is. De-novo indexers (FFT/FFTW) return an arbitrarily-oriented primitive - // cell that must first be reduced to the conventional setting, else enforcing - // the crystal system on mis-assigned axes rejects every frame. Centering is - // taken from the user's space group; niggli_class is left unassigned (0) - it - // is a property of the primitive cell incl. centering, which LatticeSearch - // cannot recover from a cell the user may have given centered (e.g. C2). A - // proper primitive-cell indexing path (CrystFEL-style) is deferred. outcome.symmetry = LatticeMessage{ .centering = sg->centring_type(), .niggli_class = 0, .crystal_system = sg->crystal_system() }; - outcome.lattice_candidate = de_novo ? LatticeSearch(latt).conventional : latt; + // Place every frame's cell in ONE consistent setting for the whole dataset: + // mixed axis orders (e.g. [78,78,38] vs [38,78,78]) index the same reflection + // as different HKLs and cannot be merged. LatticeSearch gives the conventional + // setting when its detected symmetry agrees with the user's space group. On + // noisy frames it can instead pick an alternative Bravais setting (e.g. the + // sqrt2 C-centred description of a primitive tetragonal cell, + // [78,78,38]->[110,111,38]); there: + // - FFBIDX already returns the reference setting (c-last), consistent with the + // conventional frames, so its raw lattice is safe -> use it (FFBIDX neutral); + // - de-novo indexers (FFT/FFTW) return a Niggli-primitive cell with a DIFFERENT + // axis order (c-first) that would corrupt the merge -> reject the frame. + // niggli_class is left unassigned (0): it needs the primitive cell incl. + // centering, which LatticeSearch cannot recover from a (possibly centred, e.g. + // C2) user cell. A proper primitive-cell indexing path (CrystFEL-style) is deferred. + auto sym_result = LatticeSearch(latt); + if (sym_result.system == sg->crystal_system()) + outcome.lattice_candidate = sym_result.conventional; + else if (!de_novo) + outcome.lattice_candidate = latt; + // else: de-novo + symmetry mismatch -> leave unset, frame is not indexed } else { auto sym_result = LatticeSearch(latt); outcome.symmetry = LatticeMessage{ @@ -136,7 +147,7 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetry(Data // lattice to each accepted extra lattice. Candidates are materialized later // in RefineGeometryIfNeeded so they're rooted in the refined (and, for // monoclinic, reordered) main lattice. - if (indexer_result.lattice.size() > 1) { + if (outcome.lattice_candidate && indexer_result.lattice.size() > 1) { auto ml_latt = MultiLatticeSearch(indexer_result.lattice); for (auto &ml : ml_latt) { if (outcome.extra_lattice_rotations.size() >= experiment.GetIndexingSettings().GetMaxExtraLattices()) -- 2.52.0 From f2f95c44f6e1a64b2fc71c63c312c159841d187d Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Mon, 15 Jun 2026 23:00:37 +0200 Subject: [PATCH 051/228] FFTIndexer: name the histogram axis one_over_d (1/d), not q/Q The FFT histogram extent was written as maxQ = 2*pi/HighRes (the powder Q = 2*pi/d convention) while the bin width and spot coordinates use the internal 1/d convention - a unit conflation. It is benign: len_coeff = 2*max_length/histogram_size carries the same factor, so recovered cell lengths are correct; the 2*pi only acts as a ~6.28x zero-padding of the histogram. Rewrite it transparently: one_over_d_max = 1/HighRes (1/d), with the 2*pi kept as an explicitly-named oversampling factor. In this codebase Q always denotes 2*pi/d, so 1/d is named one_over_d (never q). histogram_size is numerically identical to before, so behaviour is unchanged (FFT de-novo crystal 2 95.44%, bit-identical merge). Documented that the exact padding amount is load-bearing at the marginal-frame level (rounding 2*pi to a nearby integer shifts indexing ~0.5-1%). Co-Authored-By: Claude Opus 4.8 --- image_analysis/indexing/FFTIndexer.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/image_analysis/indexing/FFTIndexer.cpp b/image_analysis/indexing/FFTIndexer.cpp index a0f224d3..2578d589 100644 --- a/image_analysis/indexing/FFTIndexer.cpp +++ b/image_analysis/indexing/FFTIndexer.cpp @@ -13,10 +13,21 @@ FFTIndexer::FFTIndexer(const IndexingSettings &settings) nDirections(settings.GetFFT_NumVectors()), result_fft(nDirections) { - float maxQ = 2.0f * static_cast(M_PI) / settings.GetFFT_HighResolution_A(); + // Reciprocal-magnitude histogram in one_over_d = 1/d units (the internal convention - + // the spot coordinates and histogram_spacing are 1/d too; the resolution limit converts + // as 1/HighRes). NB in this code Q always means the powder 2*pi/d, so 1/d is named + // one_over_d, never q. The histogram covers the data range [0, one_over_d_max] and is + // zero-padded by OVERSAMPLING for sub-bin peak localisation (finer cell lengths than the + // raw bin width gives). len_coeff (= 2*max_length/histogram_size) cancels the factor, so + // recovered lengths are independent of it. The padding factor is 2*pi: a historical value + // from when the extent was mistakenly written as 2*pi/d (the Q convention); it is kept + // because the exact amount sets which marginal frames index - rounding it to a nearby + // integer shifts the indexing rate ~0.5-1% (validated on the lyso datasets). + const float oversampling = 2.0f * static_cast(M_PI); + const float one_over_d_max = 1.0f / settings.GetFFT_HighResolution_A(); histogram_spacing = 1.0f / (2.0f * max_length_A); - histogram_size = std::ceil(maxQ / histogram_spacing); + histogram_size = std::ceil(oversampling * one_over_d_max / histogram_spacing); input_size = histogram_size * nDirections; output_size = (histogram_size / 2 + 1) * nDirections; -- 2.52.0 From 929e6e1fa0b93c5351397501147b0a993b4d70c9 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 00:18:15 +0200 Subject: [PATCH 052/228] FFTIndexer: pick peaks by prominence above a local background, not raw magnitude The per-direction peak search took argmax|spec|. The projected-spot histogram has a broad low-frequency ENVELOPE (spots cluster near the origin) whose magnitude can exceed the true lattice peaks, so on weak / pink-beam frames every direction reported a short envelope vector (~10 A) and the real 38-79 A axes never surfaced -> 0 candidate cells -> 0% indexing. (Diagnosed on the lyso jet: the FFT returned only 10-15 A vectors, the true axes entirely absent.) Subtract a running-mean background of half-width ~15 A and pick the peak by its PROMINENCE (mag - background) instead. The smooth envelope cancels to ~0 while sharp lattice peaks - fundamentals and harmonics alike - keep their height, so the real axes win. The prominence is also reported as the magnitude, so FilterFFTResults ranks directions by real-peak strength rather than envelope. Ported identically to CPU (prefix-sum window) and GPU (sliding-window in the kernel). Validation (lyso, de-novo): jet FFT 0% -> 20.5% (CPU and GPU identical; vs FFBIDX 27%); crystal 2 95.3% -> 95.5% (no regression, CC1/2 95.8 / CCref 92.7 unchanged). The ~15 A window is the validated optimum (wider over-smooths, narrower under-removes the envelope). Co-Authored-By: Claude Opus 4.8 --- image_analysis/indexing/FFTIndexerCPU.cpp | 38 +++++++++++++---- image_analysis/indexing/FFTIndexerGPU.cu | 50 ++++++++++++++++++----- 2 files changed, 69 insertions(+), 19 deletions(-) diff --git a/image_analysis/indexing/FFTIndexerCPU.cpp b/image_analysis/indexing/FFTIndexerCPU.cpp index abe6cc91..b9ac69e0 100644 --- a/image_analysis/indexing/FFTIndexerCPU.cpp +++ b/image_analysis/indexing/FFTIndexerCPU.cpp @@ -8,6 +8,7 @@ #include #include #include +#include static std::mutex fftw_plan_mutex; @@ -76,31 +77,52 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector &coord, size_t nspots) { // Plan and execute batched R2C FFT with FFTW fftwf_execute(plan); - // Post-process: pick peaks past min_length_A + // Post-process: pick the peak past min_length_A by PROMINENCE above a local background. + // The projected histogram has a broad low-frequency ENVELOPE (spots cluster near the + // origin) whose magnitude can exceed the true lattice peaks; a plain argmax|spec| then + // returns a short envelope vector (~10A) on weak/pink-beam frames and the real axes are + // lost. Subtracting a running-mean background of half-width BG_HALF bins removes that + // smooth envelope (it cancels to ~0) while sharp lattice peaks - fundamentals AND + // harmonics - keep their height. The prominence is also reported as the magnitude so + // FilterFFTResults ranks directions by real-peak strength, not by envelope. const double len_coeff = 2.0 * static_cast(max_length_A) / static_cast(H); + // Background half-window ~15 A (in length, so it is independent of the histogram sizing); + // wide enough to span the envelope yet narrow enough not to smooth real peaks. Validated + // optimum on the lyso jet (de-novo indexing 0% -> 24%, vs FFBIDX 27%); crystal 2 unchanged. + constexpr double BG_HALF_WIDTH_A = 15.0; + const int bg_half = std::max(1, static_cast(std::lround(BG_HALF_WIDTH_A / len_coeff))); + + std::vector mag(out_len), pref(out_len + 1); for (int d = 0; d < D; ++d) { const auto* spec = h_output_fft.data() + static_cast(d) * out_len; - double best_mag = 0.0; + pref[0] = 0.0; + for (int j = 0; j < out_len; ++j) { + mag[j] = std::hypot(spec[j][0], spec[j][1]); + pref[j + 1] = pref[j] + mag[j]; + } + + double best_prom = 0.0; double best_len = -1.0; for (int j = 0; j < out_len; ++j) { double len = len_coeff * static_cast(j); if (len <= static_cast(min_length_A)) continue; - const double re = spec[j][0]; - const double im = spec[j][1]; - const double mag = std::hypot(re, im); + const int lo = std::max(0, j - bg_half); + const int hi = std::min(out_len, j + bg_half + 1); + const double background = (pref[hi] - pref[lo]) / static_cast(hi - lo); + const double prominence = mag[j] - background; - if (mag > best_mag) { - best_mag = mag; + if (prominence > best_prom) { + best_prom = prominence; best_len = len; } } result_fft[d] = FFTResult{ - .magnitude = static_cast(best_mag), + .magnitude = static_cast(best_prom), .direction = d, .length = static_cast(best_len) }; diff --git a/image_analysis/indexing/FFTIndexerGPU.cu b/image_analysis/indexing/FFTIndexerGPU.cu index 0da4e91c..52ba31aa 100644 --- a/image_analysis/indexing/FFTIndexerGPU.cu +++ b/image_analysis/indexing/FFTIndexerGPU.cu @@ -3,6 +3,8 @@ #include "FFTIndexerGPU.h" #include +#include +#include __device__ __host__ inline float complex_abs(const cufftComplex &z) { return sqrtf(z.x * z.x + z.y * z.y); @@ -13,25 +15,45 @@ __global__ void calculate_fft_result( const float max_length_A, const float min_length_A, const int histogram_size, + const int bg_half, const int directions_size, FFTResult *d_results) { int i = blockIdx.x * blockDim.x + threadIdx.x; // Get thread index if (i < directions_size) { - // Ensure within bounds - size_t offset = ((histogram_size / 2) + 1) * i; - float max_magnitude = 0.0f; - FFTResult result{.direction = i, .length = -1}; - + const int out_len = (histogram_size / 2) + 1; + size_t offset = static_cast(out_len) * i; float len_coeff = 2.0f * max_length_A / static_cast(histogram_size); - for (int j = 0; j < (histogram_size / 2) + 1; j++) { - float mag = complex_abs(d_output[offset + j]); - float len = len_coeff * static_cast(j); + // Pick the peak by PROMINENCE above a running-mean background of half-width bg_half: + // the projected histogram has a broad low-frequency envelope whose magnitude can + // exceed the true lattice peaks, so a plain argmax|spec| returns a short envelope + // vector on weak/pink-beam frames. Subtracting the local mean removes that envelope + // while sharp lattice peaks keep their height (mirrors FFTIndexerCPU). + double winsum = 0.0; + int wlo = 0; + int whi = min(bg_half, out_len - 1); + for (int k = wlo; k <= whi; ++k) + winsum += complex_abs(d_output[offset + k]); - if (len > min_length_A && mag > max_magnitude) { - max_magnitude = mag; - result.magnitude = mag; + float best_prom = 0.0f; + FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1}; + + for (int j = 0; j < out_len; ++j) { + const int want_hi = min(j + bg_half, out_len - 1); + while (whi < want_hi) { ++whi; winsum += complex_abs(d_output[offset + whi]); } + const int want_lo = max(0, j - bg_half); + while (wlo < want_lo) { winsum -= complex_abs(d_output[offset + wlo]); ++wlo; } + + const float len = len_coeff * static_cast(j); + if (len <= min_length_A) continue; + + const float mag = complex_abs(d_output[offset + j]); + const float bg = static_cast(winsum / static_cast(whi - wlo + 1)); + const float prom = mag - bg; + if (prom > best_prom) { + best_prom = prom; + result.magnitude = prom; result.length = len; } } @@ -142,8 +164,14 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector &coord, size_t nspots) { cuda_err(cufftExecR2C(plan, d_input_fft, d_output_fft)); + // Background half-window ~15 A (length-based, so independent of histogram sizing); see + // FFTIndexerCPU for the prominence-vs-envelope rationale and the validated optimum. + const double len_coeff = 2.0 * static_cast(max_length_A) / static_cast(histogram_size); + const int bg_half = std::max(1, static_cast(std::lround(15.0 / len_coeff))); + calculate_fft_result<<>>(d_output_fft, max_length_A, min_length_A, histogram_size, + bg_half, direction_vectors.size(), d_result_fft); cuda_err(cudaMemcpyAsync(result_fft.data(), d_result_fft, direction_vectors.size() * sizeof(FFTResult), -- 2.52.0 From f878fb9d5d28d8efff09cfcbf82194a5a247200c Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 08:53:05 +0200 Subject: [PATCH 053/228] SpotFindingSettings: Default signal to noise ratio is 4 --- image_analysis/spot_finding/SpotFindingSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image_analysis/spot_finding/SpotFindingSettings.h b/image_analysis/spot_finding/SpotFindingSettings.h index 94de14a1..cf626254 100644 --- a/image_analysis/spot_finding/SpotFindingSettings.h +++ b/image_analysis/spot_finding/SpotFindingSettings.h @@ -8,7 +8,7 @@ struct SpotFindingSettings { bool enable = true; - float signal_to_noise_threshold = 3; // STRONG_PIXEL in XDS + float signal_to_noise_threshold = 4.0; // STRONG_PIXEL in XDS int64_t photon_count_threshold = 10; // Threshold in photon counts int64_t min_pix_per_spot = 2; // Minimum pixels per spot int64_t max_pix_per_spot = 50; // Maximum pixels per spot -- 2.52.0 From 545ebdf868035822470c4d1caffa513af72fa2f4 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 12:32:52 +0200 Subject: [PATCH 054/228] Merge: per-crystal CC1/2-delta rejection (--reject-delta-cchalf) CrystFEL deltaCChalf-style per-crystal quality filter for heterogeneous serial data. Each image is assigned to one CC1/2 half, so removing it perturbs only that half's per-reflection means; deltaCChalf_i = CC1/2(all) - CC1/2(without image i). A negative value means dropping the image RAISES CC1/2 (it disagrees with the consensus). Images whose deltaCChalf is a low-side statistical outlier (< mean - N*stddev) are skipped when merging. Reference-free. Two passes over the retained integration outcomes; per-image contributions are re-derived rather than stored, so memory stays O(unique reflections + images) for full 200k-frame runs. New CLI flag --reject-delta-cchalf (default: off). Validation (jet FFBIDX +C+S, sigma4): removing 17/4000 (3 sigma) raises CC1/2 95.1->96.1%, CCref 54.9->55.2; 2 sigma -> 96.1/55.3. Dataset-appropriate: it HELPS heterogeneous serial data (some crystals genuinely bad) but slightly trims a homogeneous single rotation crystal (c2 94.6->93.8 - no bad crystals, the relative cut still removes the tail), so it is opt-in. R-free is the real test (user's full 200k). Note: the reported overall N_obs still counts all observations; the exported merge (and CC1/2) correctly exclude the rejected images. Co-Authored-By: Claude Opus 4.8 --- image_analysis/scale_merge/Merge.cpp | 102 +++++++++++++++++++++++++++ image_analysis/scale_merge/Merge.h | 6 ++ tools/jfjoch_process.cpp | 24 ++++++- 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index d604e092..5661d66e 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -260,6 +260,108 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc error_model_active = true; } +// Per-crystal CC1/2-delta rejection (CrystFEL deltaCChalf style). Each image is assigned +// to one CC1/2 half, so removing an image only perturbs that half's per-reflection means. +// deltaCChalf_i = CC1/2(all) - CC1/2(without image i): a NEGATIVE value means removing the +// image RAISES CC1/2, i.e. it is inconsistent with the consensus. We flag images whose +// deltaCChalf is a low-side statistical outlier (< mean - nsigma*stddev). Reference-free. +// Two passes over the (retained) outcomes; per-image contributions are re-derived, not +// stored, so memory stays O(unique reflections + images) for full 200k-frame datasets. +std::vector MergeOnTheFly::DeltaCChalfReject(const std::vector &outcomes, + double nsigma) const { + struct Acc { double swI[2] = {0, 0}; double sw[2] = {0, 0}; size_t n[2] = {0, 0}; }; + std::unordered_map acc; + std::vector img_half(outcomes.size(), 0); + + std::mt19937 lrng{2026061600u}; + std::bernoulli_distribution hd{0.5}; + + // ---- pass 1: accumulate half-set sums, record each image's half ---- + auto contribution = [&](const Reflection &r, uint64_t &key, double &wI, double &w) -> bool { + if (generator.IsSystematicallyAbsent(r)) return false; + if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) return false; + if (!AcceptReflection(r, high_resolution_limit)) return false; + if (r.partiality < min_partiality) return false; + const double I = static_cast(r.I) * r.image_scale_corr; + const double s = static_cast(r.sigma) * r.image_scale_corr; + if (!std::isfinite(I) || !std::isfinite(s) || s <= 0.0) return false; + w = 1.0 / (s * s); + wI = w * I; + key = generator(r).pack(); + return true; + }; + + for (size_t i = 0; i < outcomes.size(); ++i) { + const int half = hd(lrng) ? 1 : 0; + img_half[i] = half; + for (const auto &r : outcomes[i].reflections) { + uint64_t key; double wI, w; + if (!contribution(r, key, wI, w)) continue; + auto &a = acc[key]; + a.swI[half] += wI; a.sw[half] += w; a.n[half]++; + } + } + + // ---- baseline Pearson over reflections present in BOTH halves (x=half0, y=half1) ---- + auto pearson = [](double N, double Sx, double Sy, double Sxx, double Syy, double Sxy) -> double { + const double cov = N * Sxy - Sx * Sy; + const double vx = N * Sxx - Sx * Sx, vy = N * Syy - Sy * Sy; + const double den = std::sqrt(vx * vy); + return den > 0.0 ? cov / den : 0.0; + }; + double N = 0, Sx = 0, Sy = 0, Sxx = 0, Syy = 0, Sxy = 0; + for (const auto &kv : acc) { + const auto &a = kv.second; + if (a.n[0] == 0 || a.n[1] == 0) continue; + const double x = a.swI[0] / a.sw[0], y = a.swI[1] / a.sw[1]; + N += 1; Sx += x; Sy += y; Sxx += x * x; Syy += y * y; Sxy += x * y; + } + const double cc_base = pearson(N, Sx, Sy, Sxx, Syy, Sxy); + + // ---- pass 2: leave-one-out deltaCChalf per image ---- + std::vector delta(outcomes.size(), 0.0); + for (size_t i = 0; i < outcomes.size(); ++i) { + const int h = img_half[i]; + // aggregate this image's contributions per reflection key (an image may, rarely, + // touch the same ASU reflection twice) + std::unordered_map> mine; // key -> (sum wI, sum w), count via .first + std::unordered_map mine_n; + for (const auto &r : outcomes[i].reflections) { + uint64_t key; double wI, w; + if (!contribution(r, key, wI, w)) continue; + auto &p = mine[key]; p.first += wI; p.second += w; mine_n[key]++; + } + double n = N, sx = Sx, sy = Sy, sxx = Sxx, syy = Syy, sxy = Sxy; + for (const auto &m : mine) { + const auto &a = acc.at(m.first); + if (a.n[0] == 0 || a.n[1] == 0) continue; // reflection not in CC1/2 + const double x0 = a.swI[0] / a.sw[0], y0 = a.swI[1] / a.sw[1]; + n -= 1; sx -= x0; sy -= y0; sxx -= x0 * x0; syy -= y0 * y0; sxy -= x0 * y0; + const double swI_h = a.swI[h] - m.second.first; + const double sw_h = a.sw[h] - m.second.second; + if (a.n[h] - mine_n[m.first] == 0 || sw_h <= 0.0) continue; // reflection drops half h + const double mean_h = swI_h / sw_h; + const double xnew = (h == 0) ? mean_h : x0; + const double ynew = (h == 1) ? mean_h : y0; + n += 1; sx += xnew; sy += ynew; sxx += xnew * xnew; syy += ynew * ynew; sxy += xnew * ynew; + } + delta[i] = cc_base - pearson(n, sx, sy, sxx, syy, sxy); + } + + // ---- reject low-side outliers: delta < mean - nsigma*stddev ---- + double dm = 0, dv = 0; + for (double d : delta) dm += d; + dm /= std::max(1, delta.size()); + for (double d : delta) dv += (d - dm) * (d - dm); + const double dstd = std::sqrt(dv / std::max(1, delta.size())); + const double cut = dm - nsigma * dstd; + + std::vector reject(outcomes.size(), 0); + for (size_t i = 0; i < outcomes.size(); ++i) + reject[i] = (outcomes[i].reflections.empty() ? 0 : (delta[i] < cut ? 1 : 0)); + return reject; +} + bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome, bool cc_mask) { if (reference_cell) { auto cell = outcome.latt.GetUnitCell(); diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index ab9877f4..458d9dab 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -114,6 +114,12 @@ public: void AddImage(const IntegrationOutcome& outcome, bool cc_mask = false); + // Per-crystal CC1/2-delta rejection (CrystFEL deltaCChalf): returns a per-image flag + // marking images whose removal would raise CC1/2 by a low-side outlier amount + // (deltaCChalf < mean - nsigma*stddev). Skip the flagged images when merging. + [[nodiscard]] std::vector DeltaCChalfReject(const std::vector &outcomes, + double nsigma) const; + MergeStatistics MergeStats(const std::vector &merged, const std::vector &reflections, const std::vector &reference = {}); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 7748ccdd..acadd7ba 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -73,6 +73,7 @@ void print_usage() { std::cout << " --scaling-high-resolution High resolution limit for spot finding (default: no limit)" << std::endl; std::cout << " --min-partiality Minimum partiality to accept reflection (default: 0.02)" << std::endl; std::cout << " --reject-outliers Per-observation merge outlier rejection, N sigma from the per-reflection median (default: off; e.g. 6, XDS/DIALS-style)" << std::endl; + std::cout << " --reject-delta-cchalf Per-crystal CC1/2-delta rejection: drop images with deltaCChalf below mean - N*stddev (default: off; e.g. 2.5)" << std::endl; std::cout << " --min-image-cc Per-image CC limit in percent (default: no limit)" << std::endl; std::cout << " --scaling-iterations Number of scaling iterations with no reference data (default: 3)" << std::endl; std::cout << " --scaling-output Output format for scaling results mtz|cif|txt (default: mtz)" << std::endl; @@ -101,7 +102,8 @@ enum { OPT_BANDWIDTH, OPT_INTEGRATION_RADIUS, OPT_REJECT_OUTLIERS, - OPT_PROFILE_MULTIPLIER + OPT_PROFILE_MULTIPLIER, + OPT_REJECT_DELTA_CCHALF }; static option long_options[] = { @@ -140,6 +142,7 @@ static option long_options[] = { {"bandwidth", required_argument, nullptr, OPT_BANDWIDTH}, {"integration-radius", required_argument, nullptr, OPT_INTEGRATION_RADIUS}, {"reject-outliers", required_argument, nullptr, OPT_REJECT_OUTLIERS}, + {"reject-delta-cchalf", required_argument, nullptr, OPT_REJECT_DELTA_CCHALF}, {"profile-multiplier", required_argument, nullptr, OPT_PROFILE_MULTIPLIER}, {nullptr, 0, nullptr, 0} }; @@ -323,6 +326,7 @@ int main(int argc, char **argv) { std::optional d_min_scale_merge; std::optional integration_radius_arg; // "r1" or "r1,r2,r3" std::optional outlier_reject_nsigma; // merge per-observation outlier rejection + std::optional delta_cchalf_nsigma; // per-crystal CC1/2-delta rejection std::optional profile_multiplier; // PixelRefine Term-2 profile-width multiplier if (argc == 1) { @@ -519,6 +523,9 @@ int main(int argc, char **argv) { case OPT_REJECT_OUTLIERS: outlier_reject_nsigma = std::stod(optarg); break; + case OPT_REJECT_DELTA_CCHALF: + delta_cchalf_nsigma = std::stod(optarg); + break; case OPT_PROFILE_MULTIPLIER: profile_multiplier = std::stof(optarg); break; @@ -1057,8 +1064,19 @@ int main(int argc, char **argv) { const double isa = (b > 0.0) ? 1.0 / b : std::numeric_limits::infinity(); logger.Info("Error model: sigma'^2 = {:.3f} sigma^2 + ({:.4f} I)^2 ISa = {:.1f}", a, b, isa); } - for (auto &i : indexer.GetIntegrationOutcome()) - merge_engine.AddImage(i); + const auto &merge_outcomes = indexer.GetIntegrationOutcome(); + std::vector dcch_reject; + if (delta_cchalf_nsigma) { + dcch_reject = merge_engine.DeltaCChalfReject(merge_outcomes, *delta_cchalf_nsigma); + const size_t nrej = std::count(dcch_reject.begin(), dcch_reject.end(), static_cast(1)); + logger.Info("CC1/2-delta rejection (deltaCChalf < mean - {:.1f} stddev) removed {} / {} images", + *delta_cchalf_nsigma, nrej, dcch_reject.size()); + } + for (size_t i = 0; i < merge_outcomes.size(); ++i) { + if (!dcch_reject.empty() && dcch_reject[i]) + continue; + merge_engine.AddImage(merge_outcomes[i]); + } if (merge_engine.RejectedCount() > 0) logger.Info("Outlier rejection (>{:.1f} sigma from the per-reflection median) removed {} observations", experiment.GetScalingSettings().GetOutlierRejectNsigma(), merge_engine.RejectedCount()); -- 2.52.0 From 2ba28aea0e8cbda7229ee8378249705a4f47cf43 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 14:19:28 +0200 Subject: [PATCH 055/228] LoadFCalcFromMtz: fall back to an intensity column when F-model is absent PixelRefine's reference can now be a merged/observed-intensity MTZ, not only a calculated F-model. If F-model is missing, read a mean-intensity column (IMEAN, which a jfjoch merge writes; also I/IOBS/Iobs/I-obs, or any 'J'-type column) and use it directly as I_ref (F-model is squared, intensities are not). This enables a SELF-SEEDED / EM-style workflow: run a first pass (traditional integration, or PixelRefine against an external reference) to produce a merge, then re-run PixelRefine with `-z .mtz` so it scales against the data's own intensities - free of the reference structure's non-isomorphism bias - and iterate. Verified: a jfjoch IMEAN merge loads as the reference and PixelRefine runs against it (crystal 2, 55279 reflections, CC1/2 94.3). Co-Authored-By: Claude Opus 4.8 --- image_analysis/LoadFCalcFromMtz.cpp | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/image_analysis/LoadFCalcFromMtz.cpp b/image_analysis/LoadFCalcFromMtz.cpp index b7153f32..503ed9fd 100644 --- a/image_analysis/LoadFCalcFromMtz.cpp +++ b/image_analysis/LoadFCalcFromMtz.cpp @@ -16,9 +16,27 @@ std::vector LoadFCalcFromMtz(const std::string& path) { gemmi::Mtz mtz; mtz.read_file_gz(path, true); - const gemmi::Mtz::Column* fc = mtz.column_with_label("F-model", nullptr, 'F'); - if (fc == nullptr) - throw std::runtime_error("MTZ does not contain F-model column"); + // Prefer a calculated structure factor F-model (e.g. a reference structure): I_ref = F^2. + const gemmi::Mtz::Column* col = mtz.column_with_label("F-model", nullptr, 'F'); + const bool square = (col != nullptr); + + // Otherwise fall back to a merged/observed intensity column, used directly as I_ref. This + // lets PixelRefine be SELF-SEEDED from the data's own previous merge (a jfjoch merge writes + // IMEAN) instead of an external reference structure - the EM-style outer loop, free of the + // reference's non-isomorphism bias. + if (col == nullptr) { + for (const char* label : {"IMEAN", "I", "IOBS", "Iobs", "I-obs"}) { + col = mtz.column_with_label(label, nullptr, 'J'); + if (col != nullptr) + break; + } + } + if (col == nullptr) { + for (const auto& c : mtz.columns) + if (c.type == 'J') { col = &c; break; } // any mean-intensity column + } + if (col == nullptr) + throw std::runtime_error("MTZ has no F-model or intensity (J) column to use as reference"); std::vector result; result.reserve(static_cast(mtz.nreflections)); @@ -27,16 +45,16 @@ std::vector LoadFCalcFromMtz(const std::string& path) { for (int i = 0; i < mtz.nreflections; ++i) { const std::size_t row = static_cast(i) * stride; - const float f = (*fc)[static_cast(i)]; + const float v = (*col)[static_cast(i)]; - if (std::isnan(f)) + if (std::isnan(v)) continue; MergedReflection r; r.h = static_cast(mtz.data[row + 0]); r.k = static_cast(mtz.data[row + 1]); r.l = static_cast(mtz.data[row + 2]); - r.I = f * f; + r.I = square ? v * v : v; r.sigma = NAN; r.d = 0.0f; -- 2.52.0 From 6479b91e5008c8a19d99298541967a098731982e Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 19:03:34 +0200 Subject: [PATCH 056/228] JFJochHDF5Reader: read master indexedLatticeCount (back-compat) The master-file writer emits /entry/MX/indexedLatticeCount, but the reader only looked for indexingLatticeCount (the name used by the per-file MX plugin and data-file read path). Existing master files on disk therefore returned no indexed-lattice-count vector. Read the master's indexedLatticeCount first, falling back to indexingLatticeCount, matching the niggli_class/niggliClass compatibility pattern. The writer is left unchanged so already-written files keep working. Co-Authored-By: Claude Opus 4.8 --- reader/JFJochHDF5Reader.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index cefbba4e..e6d639c8 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -308,7 +308,11 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) { dataset->bkg_estimate = master_file->ReadOptVector("/entry/MX/bkgEstimate"); dataset->resolution_estimate = master_file->ReadOptVector("/entry/MX/resolutionEstimate"); dataset->profile_radius = master_file->ReadOptVector("/entry/MX/profileRadius"); - dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexingLatticeCount"); + // Master files write indexedLatticeCount; data files / the per-file MX + // plugin use indexingLatticeCount. Accept either for backward compatibility. + dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexedLatticeCount"); + if (dataset->indexing_lattice_count.empty()) + dataset->indexing_lattice_count = master_file->ReadOptVector("/entry/MX/indexingLatticeCount"); dataset->mosaicity_deg = master_file->ReadOptVector("/entry/MX/mosaicity"); dataset->b_factor = master_file->ReadOptVector("/entry/MX/bFactor"); dataset->image_scale_factor = master_file->ReadOptVector("/entry/MX/imageScaleFactor"); -- 2.52.0 From caff857d8a21e14ae9f9532914d8fd73221d747b Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 19:11:37 +0200 Subject: [PATCH 057/228] docs/CBOR.md: sync field names/types with the serializer Bring the protocol doc in line with CBORStream2Serializer/Deserializer: - reflection dist_ewald -> rp (dist_ewald is the spot field, not the reflection field) - az_int_std -> az_int_profile_std, az_int_count -> az_int_profile_count - threshold_energy typed as Map(string -> float), not float - end_date typed as string (it is serialized as a plain text string, with no datetime tag, unlike arm_date) - add missing reflection fields: phi, zeta, image_scale_corr - add missing spot fields: image, and indexed-only h, k, l, dist_ewald - add image indexing_lattice_count and end indexed_lattice_count - drop the duplicated image_scale_factor row in the End table Co-Authored-By: Claude Opus 4.8 --- docs/CBOR.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/CBOR.md b/docs/CBOR.md index 58339cc6..a82af4bc 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -68,7 +68,7 @@ There are minor differences at the moment: | max_extra_lattices | uint64 | Maximum number of extra lattices | | | storage_cell_number | uint64 (optional) | Number of storage cells used by JUNGFRAU | | | storage_cell_delay | Rational | Delay of storage cells in JUNGFRAU | | -| threshold_energy | float | Threshold energy for EIGER detector \[eV\] | | +| threshold_energy | Map(string -> float) | Per-channel threshold energy \[eV\] (map of channel name to value) | | | image_dtype | string | Pixel bit type (e.g. uint16) | X | | unit_cell | object (optional) | Unit cell of the system: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\] | | | az_int_q_bin_count | uint64 | Number of azimuthal integration bins in the radial direction | | @@ -139,6 +139,11 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | - ice_ring | bool | spot in resolution range for ice rings | | | | - indexed | bool | indexed solution | | | | - latt | int64 | Lattice to which the peak belongs (negative number = not indexed) | | | +| - image | int64 | image number the spot belongs to | | | +| - h | int64 | Miller index (indexed spots only) | | | +| - k | int64 | Miller index (indexed spots only) | | | +| - l | int64 | Miller index (indexed spots only) | | | +| - dist_ewald | float | distance to Ewald sphere \[Angstrom^-1\] (indexed spots only) | | | | reflections | Array(object) | Reflections: | | | | - h | int64 | Miller index | | | | - k | int64 | Miller index | | | @@ -152,18 +157,22 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | - bkg | float | mean background value (photons) | | | | - sigma | float | standard deviation, estimated from counting statistics (photons) | | | | - image | float | image number (present for each spot) | | | -| - dist_ewald | float | distance to Ewald sphere (present only for indexed spots) | | | +| - rp | float | Distance to Ewald sphere \[Angstrom^-1\] | | | | - rlp | float | Reciprocal Lorentz and polarization corrections | | | | - partiality | float | Partiality of the reflection | | | +| - phi | float | phi angle from XDS: difference from middle of current frame, not absolute \[deg\] | | | +| - zeta | float | Lorentz zeta factor (reciprocal-space geometry term) | | | +| - image_scale_corr | float | Per-image scale correction; I_true = image_scale_corr * I | | | | spot_count | uint64 | Spot count | | | | spot_count_ice_rings | uint64 | Number of spots within identified rings (experimental) | | | | spot_count_low_res | uint64 | Number of spots in low resolution (prior to filtering) | | | | spot_count_indexed | uint64 | Number of spots which fit indexing solution within a given tolerance | | | | az_int_profile | Array(float) | Azimuthal integration results, use az_int_bin_to_q from start message for legend | | | | | | NaN is used for empty bins and has to be taken care by the receiver | | | -| az_int_std | Array(float) | Standard deviation for azimuthal integration. (NaN for less than 2 samples) | | | -| az_int_count | Array(uint64) | Number of pixels contributing to azimuthal bin | | | +| az_int_profile_std | Array(float) | Standard deviation for azimuthal integration. (NaN for less than 2 samples) | | | +| az_int_profile_count | Array(uint64) | Number of pixels contributing to azimuthal bin | | | | indexing_result | bool | Indexing successful | | | +| indexing_lattice_count | int64 | Number of indexing lattices found for this image | | | | indexing_lattice | Array(9 * float) | Indexing result real lattice; present only if indexed | | X | | indexing_extra_lattices | Array(Array(9*float)) | Additional indexed lattices (orientation variants); present only if found | | | | indexing_unit_cell | object | Indexing result unit cell: a, b, c \[angstrom\] and alpha, beta, gamma \[degree\]; present only if indexed | | X | @@ -271,7 +280,7 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | magic_number | uint64 | Number used to describe version of the Jungfraujoch data interface - to allow to detect inconsistency between sender and receiver | | | series_unique_id | string | Unique text ID of the series (run_name parameter) | X | | series_id | uint64 | Unique numeric ID of the series (run_number parameter) | X | -| end_date | date | Approximate end date | | +| end_date | string | Approximate end date | | | max_image_number | uint64 | Number of image with the highest number; counted from 1 to distinguish zero images and one image | | | images_collected | uint64 | Number of images collected | | | images_sent_to_write | uint64 | Number of images sent to writer; if writer queues were full, it is possible this is less than images collected | | @@ -306,9 +315,9 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | error_pixel_count | Array(int32) | Per-image error pixel count | | | image_scale_factor | Array(float) | Per-image scale factor, if scaling/merging was performed | | | integrated_reflections | Array(int32) | Per-image count of integrated reflections | | +| indexed_lattice_count | Array(int32) | Per-image count of indexed lattices | | | niggli_class | Array(uint8) | Per-image Niggli class identifier for indexed images; 0 if unavailable | | | pixel_sum | Array(int64) | Per-image sum of all valid pixels, excluding error/saturated pixels | | -| image_scale_factor | Array(float) | Scaling result: Image scale factor (g) | | | image_scale_mosaicity | Array(float) | Scaling result: Image scale mosaicity \[deg\] | | | image_scale_b_factor | Array(float) | Scaling result: Image scale B factor \[Angstrom^2\] | | | image_scale_cc | Array(float) | Scaling result: Image scale CC | | -- 2.52.0 From 0381d891bc4e8fc65df20d28b0a07ee54988bd20 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 19:36:44 +0200 Subject: [PATCH 058/228] docs: add HDF5.md describing the NeXus layout and JFJoch extensions New docs/HDF5.md documents the on-disk HDF5/NeXus format produced by the writer: a FAIR/derived-metadata rationale (CXI-style per-image spot layout, NXreflections for integration), the master/data-file layout and the three NXmx format variants, the NXmx-standard fields that are populated, and every Jungfraujoch extension group (/entry/MX, /entry/reflections, /entry/azint, /entry/roi, /entry/image, /entry/profiling, /entry/detector, /entry/xfel, detectorSpecific, calibration, fluorescence, user). Content is derived from writer/HDF5NXmx.cpp and writer/HDF5DataFilePlugin*.cpp and cross-checked against the NXmx and NXreflections definitions. JFJOCH_WRITER.md's stale, partial structure table is replaced by a pointer to the new doc; HDF5 is added to the Sphinx toctree. Co-Authored-By: Claude Opus 4.8 --- docs/HDF5.md | 380 ++++++++++++++++++++++++++++++++++++++++++ docs/JFJOCH_WRITER.md | 88 +--------- docs/index.rst | 1 + 3 files changed, 388 insertions(+), 81 deletions(-) create mode 100644 docs/HDF5.md diff --git a/docs/HDF5.md b/docs/HDF5.md new file mode 100644 index 00000000..9824b1cd --- /dev/null +++ b/docs/HDF5.md @@ -0,0 +1,380 @@ +# HDF5 / NeXus data format + +Jungfraujoch stores images and on-the-fly analysis results in HDF5 files that aim to be +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html)-compliant. On top of the +NXmx application definition, Jungfraujoch records a substantial amount of *derived* metadata +(spot finding, indexing, integration, azimuthal integration, per-image statistics, timing). These +extra entries do not exist in NXmx and are documented here so that the layout is unambiguous and +reusable. + +This page documents the **file layout and the data fields**. The operational behaviour of the +writer (running, republishing, file finalisation, CBF/TIFF output) is described in +[jfjoch_writer](JFJOCH_WRITER.md). The wire format that feeds the writer is described in +[CBOR messages](CBOR.md); fields below frequently correspond one-to-one to CBOR message fields, and +that document is a useful companion for their meaning. + +## 1. Motivation: derived metadata and FAIR data + +The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep +them findable, accessible, interoperable and reusable (FAIR). For serial crystallography this is +hard for two practical reasons: + +* **Findability.** Raw diffraction images carry almost no descriptive metadata about *content*. + Quantities such as background level, number of diffraction spots, or indexing outcome let a user + judge the quality and relevance of a dataset *before* inspecting the raw images. +* **Accessibility at scale.** A single experiment can span tens to hundreds of terabytes. Standard + retrieval (e.g. HTTP) makes a dataset *available* but not *inspectable* — users would otherwise + have to download a large fraction of the data just to decide whether it is useful. Compact + derived representations make discovery, assessment and reuse feasible. + +Because Jungfraujoch couples acquisition with real-time analysis used to *steer* experiments, +transparency and reproducibility of that analysis matter. As a minimum the writer therefore +preserves spot-finding and indexing results together with the filters that were applied, and it can +retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse. + +### Why a CXI-style per-image layout for spot finding / indexing + +Spot-finding and indexing results in serial crystallography are inherently *image-centric*: the +natural query is "give me the spots for image *n*". For these products Jungfraujoch adopts a layout +similar to the [Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the +convention understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties +(position, intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by +image number, with each image allocated room for up to a predefined maximum number of spots. These +dense arrays are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are +retrieved without traversing variable-length structures. The cost is some storage overhead for +unused slots (padded with sentinels), which is acceptable for the access pattern. + +We also evaluated the NeXus +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class. +NXreflections models a *dataset-wide* reflection table, which fits integrated rotation data well — +and Jungfraujoch does use it for integration results (see §4.2 below). +But for spot finding/indexing across hundreds of thousands of patterns a single table would force +aggregation over the whole experiment before the spots of one image can be accessed efficiently. +For these intermediate products a per-image representation is more suitable. We encourage the +community to develop standardised NeXus application definitions for image-centric serial +crystallography products that combine NeXus interoperability with the access patterns and scale of +modern experiments. + +## 2. File layout + +A run is written as one **master file** plus, depending on the format, one or more **data files**: + +``` +_master.h5 # NXmx master file (metadata + links / virtual datasets) +_data_000001.h5 # data file: images + per-image analysis +_data_000002.h5 +... +``` + +The master file is produced by `writer/HDF5NXmx.cpp`; data files by `writer/HDF5DataFile.cpp` and +its plugins (`writer/HDF5DataFilePlugin*.cpp`). Files are written to a temporary `*..tmp` +name and renamed on successful close. + +Three master-file variants exist (set via `file_format`): + +| Format | Value | Master ↔ data linking | +|--------|:-----:|------------------------| +| **NXmxLegacy** (default) | 1 | One external link in `/entry/data` per data file (`data_000001`, …). HDF5 1.8 compatible — works with Neggia/Durin XDS plugins and Albula 4.0. | +| **NXmxVDS** | 2 | A single virtual dataset `/entry/data/data` spans all data files; spot finding, azimuthal integration and reflections are linked the same way. Requires HDF5 1.10 / Albula 4.1+. | +| **NXmxIntegrated** | 3 | No separate data files — images and all metadata live in one file. Equivalent in content to the VDS format. | + +In legacy/VDS mode, image-indexed analysis arrays live in the **data files** and are exposed in the +master file through external links or virtual datasets; in integrated mode they are written +directly into the single file. Throughout this document a "✓ in master" column marks entries that +are visible (directly or via link/VDS) from the master file. + +Images are stored chunked (one image per chunk) and compressed with bitshuffle + LZ4 or +bitshuffle + Zstd; signed integer image datasets use `INTx_MIN` as the HDF5 fill value (the +"masked / no-data" sentinel), unsigned use `UINTx_MAX`. + +## 3. NXmx-standard content + +The entries below are part of, or valid base classes for, the +[NXmx](https://manual.nexusformat.org/classes/applications/NXmx.html) application definition. +"NXmx" = listed in the application definition; "base" = a valid field of the relevant NeXus base +class (`NXdetector`, `NXsample`, `NXsource`) but not in the NXmx required/recommended subset. + +### `/entry` (NXentry) + +| Field | Std | Notes | +|-------|:---:|-------| +| `definition` | NXmx | value `"NXmx"` | +| `start_time` | NXmx | arming time | +| `end_time`, `end_time_estimated` | NXmx | approximate end time | + +File-level HDF5 attributes `file_name`, `file_time`, `HDF5_Version` are also set. + +### `/entry/source` (NXsource), `/entry/instrument` (NXinstrument) + +| Field | Std | Units | +|-------|:---:|-------| +| `source/name`, `source/type` | NXmx / base | | +| `source/current` | base | A | +| `instrument/name` | NXmx | | + +### `/entry/instrument/beam` (NXbeam) + +| Field | Std | Units | +|-------|:---:|-------| +| `incident_wavelength` | NXmx | angstrom | +| `incident_wavelength_spread` | NXmx | angstrom (only if polychromatic) | +| `total_flux` | NXmx | Hz | + +### `/entry/instrument/attenuator` (NXattenuator) + +| Field | Std | +|-------|:---:| +| `attenuator_transmission` | NXmx | + +### `/entry/instrument/detector` (NXdetector) + +| Field | Std | Units | +|-------|:---:|-------| +| `depends_on` | NXmx | → `transformations/rot3` | +| `beam_center_x`, `beam_center_y` | NXmx | pixel | +| `distance` | NXmx | m | +| `count_time`, `frame_time` | NXmx | s | +| `sensor_thickness` | NXmx | m | +| `sensor_material` | NXmx | | +| `description` | NXmx | | +| `threshold_energy` | NXmx | eV (EIGER; written only for a single channel) | +| `x_pixel_size`, `y_pixel_size` | base | m | +| `serial_number` | base | | +| `bit_depth_readout` | NXmx | | +| `saturation_value` | NXmx | | +| `flatfield_applied` | NXmx | | +| `pixel_mask`, `pixel_mask_applied` | NXmx | `pixel_mask` is `[y, x]`, hard-linked from `detectorSpecific/pixel_mask` | +| `countrate_correction_applied` | NXmx | | +| `number_of_cycles` | base | frame-summation factor | + +### `/entry/instrument/detector/transformations` (NXtransformations) + +The NXtransformations *mechanism* (the `depends_on` chain, `transformation_type`, `vector`, +`offset` attributes) is standard. The axis **names** follow the PyFAI PONI convention chosen by +Jungfraujoch (see [DETECTOR_GEOMETRY](DETECTOR_GEOMETRY.md)): + +| Axis | Type | Units | Depends on | +|------|------|-------|-----------| +| `translation` | translation | m | `.` | +| `rot1` | rotation | rad | `translation` | +| `rot2` | rotation | rad | `rot1` | +| `rot3` | rotation | rad | `rot2` | + +### `/entry/instrument/detector/module` (NXdetector_module) + +`data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all +NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). + +### `/entry/sample` (NXsample) + +| Field | Std | Units / notes | +|-------|:---:|-------| +| `name` | NXmx | | +| `depends_on` | NXmx | points at the last goniometer / grid-scan axis, or `.` for stills | +| `temperature` | NXmx | K | +| `transformations/` (NXtransformations) | NXmx | rotation axis (e.g. `omega`) or grid-scan translation; hard-linked as `/entry/sample/goniometer` | +| `unit_cell` | base | `[a, b, c, α, β, γ]` | +| `ub_matrix` | base | `[1, 3, 3]`, Angstrom⁻¹ | + +For a rotation scan the goniometer axis is written as a per-image angle array `` plus +`_end`, scalar `_range_average`, `_range_total`, and for helical scans +`_helical_x/_y/_z`. These extra goniometer datasets beyond the bare axis array are Jungfraujoch +conveniences. + +### `/entry/data` (NXdata) + +`data` (3-D image stack, `[n_images, y, x]`) with `image_nr_low` / `image_nr_high` attributes. +In legacy mode this group instead contains one external link `data_000001`, … per data file. + +## 4. Extensions beyond NXmx + +Everything in this section is **outside the NXmx standard**. Each group is declared with +`NX_class = NXcollection` (the NeXus-sanctioned container for non-standardised content) unless noted. +The per-image arrays are indexed by image number, padded to the run length and filled with a +sentinel (`NaN` for floats, `-1`/`0` for integer indices) where a quantity is absent. + +### 4.1 `/entry/MX` — spot finding and indexing (CXI-style) + +The flagship extension. Spot ("peak") properties are stored as fixed-size `[n_images, max_spots]` +arrays (CXI layout, recognised by CrystFEL); scalar-per-image quantities as `[n_images]` vectors. +In legacy/VDS mode these live in the data files and are linked/virtual-stacked into the master. + +**Per-spot arrays `[n_images, max_spots]`:** + +| Dataset | Units | Meaning | Indexing only | +|---------|-------|---------|:---:| +| `peakXPosRaw`, `peakYPosRaw` | pixel | spot position (raw detector frame) | | +| `peakTotalIntensity` | photons | spot intensity | | +| `peakIceRingRes` | | spot lies in an ice-ring resolution band | | +| `peakH`, `peakK`, `peakL` | | Miller indices of the (indexed) spot | ✓ | +| `peakDistEwaldSphere` | Å⁻¹ | distance of the spot from the Ewald sphere | ✓ | +| `peakIndexed` | | spot fits the indexing solution | ✓ | +| `peakLattice` | | lattice the spot belongs to (`-1` = unindexed) | ✓ | + +**Per-image vectors `[n_images]`:** + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `nPeaks` | | number of spots stored for the image (CXI) | +| `strongPixels` | | strong-pixel count (first spot-finding stage) | +| `peakCountUnfiltered` | | spots found before filtering | +| `peakCountLowRes` | | low-resolution spots | +| `peakCountIceRingRes` | | spots inside ice-ring bands | +| `peakCountIndexed` | | spots fitting the indexing solution | +| `imageIndexed` | | image was indexed (0/1) | +| `indexingLatticeCount` | | number of lattices found for the image | +| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A*, Table 3.1.3.1) | +| `bravaisLattice` | | Bravais lattice short code, e.g. `aP`, `mC`, `oF`, `tI`, `hP`, `hR`, `cF` | +| `profileRadius` | Å⁻¹ | crystal profile radius | +| `mosaicity` | deg | mosaicity estimate | +| `bFactor` | Ų | per-image B-factor estimate | +| `resolutionEstimate` | Å | diffraction resolution estimate | +| `integratedReflections` | | number of integrated reflections | +| `bkgEstimate` | photons | mean background in the 3–5 Å resolution band | +| `beam_corr_x`, `beam_corr_y` | pixel | beam-center correction applied during processing | +| `imageScaleFactor` | | on-the-fly per-image scale factor *g* | +| `imageScaleCC` | | on-the-fly scaling correlation coefficient | +| `imageScaleMosaicity` | deg | scaling-model mosaicity | +| `imageScaleBFactor` | Ų | scaling-model B-factor | + +**Per-image lattices:** `latticeIndexed` `[n_images, 9]` (Å) — the real-space lattice (flattened +3×3); `latticeIndexedExtra` `[n_images, max_extra_lattices, 9]` (Å) — additional orientation +variants. + +**Run-level summaries** (written into the master `/entry/MX` at finalisation): + +| Dataset | Units | Meaning | +|---------|-------|---------| +| `indexing_algorithm` | | `FFBIDX` / `FFT (CUDA)` / `FFT (FFTW)` | +| `geom_refinement_algorithm` | | e.g. `beam_center` | +| `rotationLatticeIndexed` | Å | whole-run rotation-indexing lattice (`[9]`) | +| `rotationLatticeIndexedExtra` | Å | additional whole-run lattices (`[m, 9]`) | +| `rotationLatticeNiggliClass` | | Niggli class of the run lattice | +| `imageIndexedMean` | | mean indexing rate over the run | +| `bkgEstimateMean` | photons | mean background over the run | +| `indexedLatticeCount` | | per-image lattice count summary (master). *Note: data files use `indexingLatticeCount`; readers accept either.* | + +CrystFEL can read the spots directly with: + +``` +peak_list = /entry/MX +peak_list_type = cxi +``` + +### 4.2 `/entry/reflections` — integrated reflections (NXreflections) + +Integrated reflections are stored **per image** as +`/entry/reflections/image_NNNNNN` groups, each declared `NX_class = NXreflections`. The columns map +mostly onto the standard +[NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class: + +| Dataset | Units | NXreflections | Meaning | +|---------|-------|:-------------:|---------| +| `h`, `k`, `l` | | standard | Miller indices | +| `d` | Å | standard | resolution | +| `int_sum` | photons | standard | integrated intensity (summation) | +| `int_err` | photons | non-standard name | σ of the intensity (standard equivalent: `int_sum_errors`) | +| `background_mean` | photons | standard | mean background under the peak | +| `predicted_x`, `predicted_y` | pixel | name standard, units differ | predicted position. NXreflections `predicted_x/_y` are *physical* lengths; the pixel datasets are `predicted_px_x/_y` | +| `observed_x`, `observed_y` | pixel | name standard, units differ | observed centroid (pixels; standard pixel form is `observed_px_x/_y`) | +| `observed_frame` | | standard | image number of the reflection | +| `lp` | | standard | Lorentz–polarization factor (stored as `1/rlp`) | +| `partiality` | | standard | recorded fraction of the reflection | +| `delta_phi` | deg | **extension** | XDS Δφ: offset from the centre of the current frame | +| `zeta` | | **extension** | Lorentz ζ factor (reciprocal-space geometry term) | +| `image_scale_corr` | | **extension** | per-image scale correction; `I_true = image_scale_corr · int_sum` | + +In the master file these per-image groups are exposed through `/entry/reflections` external links +(VDS/integrated formats). + +### 4.3 `/entry/azint` — azimuthal integration + +| Dataset | Shape | Units | Meaning | +|---------|-------|-------|---------| +| `bin_to_q` | `[φ_bins, q_bins]` | Å⁻¹ | q value of each bin | +| `bin_to_two_theta` | `[φ_bins, q_bins]` | deg | 2θ of each bin | +| `bin_to_phi` | `[φ_bins, q_bins]` | deg | azimuthal angle of each bin | +| `image` | `[n_images, φ_bins, q_bins]` | | per-image integrated profile (NaN for empty bins) | +| `image_std` | `[n_images, φ_bins, q_bins]` | | per-bin standard deviation | +| `image_count` | `[n_images, φ_bins, q_bins]` | | pixels contributing per bin | +| `map` | `[y, x]` | | pixel→bin mapping (master file only) | + +### 4.4 `/entry/roi/` — regions of interest + +One sub-group per configured ROI, each with `[n_images]` vectors: + +| Dataset | Meaning | +|---------|---------| +| `max` | maximum pixel value in the ROI | +| `sum` | sum of pixel values | +| `sum_sq` | sum of squared pixel values | +| `npixel` | number of valid pixels | +| `x`, `y` | intensity-weighted centroid | + +### 4.5 `/entry/image` — per-image pixel statistics + +`[n_images]` vectors: `max_value`, `min_value` (viable min/max, excluding error/saturated pixels), +`error_pixels`, `saturated_pixels`, `pixel_sum`. Surfaced in the master file under `/entry/image`. + +### 4.6 `/entry/profiling` — per-image timing + +`[n_images]` vectors in seconds: `spotFindingTime`, `indexingTime`, `integrationTime`, +`refinementTime`, `processingTime`, `braggPredictionTime`, `preprocessingTime`, `compressionTime`, +`azIntTime`, `indexAnalysisTime`, `imageScaleTime`. + +### 4.7 `/entry/detector` — acquisition diagnostics (data file) + +A convenience NXcollection in the data file (note: distinct from the standard +`/entry/instrument/detector`). In **integrated** format these datasets are written under +`/entry/instrument/detector/detectorSpecific` instead. + +| Dataset | Meaning | +|---------|---------| +| `timestamp`, `exptime` | per-image timestamp and exposure time | +| `number` | image number (original number if image rejection was used) | +| `det_info` | JUNGFRAU debug field | +| `storage_cell_image` | storage-cell number | +| `rcv_delay`, `rcv_free_send_buffers` | receiver internal diagnostics | +| `packets_expected`, `packets_received` | UDP packets per image | +| `data_collection_efficiency_image` | received / expected packet ratio | + +### 4.8 `/entry/xfel` — pulsed-source metadata + +`[n_images]` vectors `pulseID` and `eventCode`, written for pulsed sources (e.g. SwissFEL). + +### 4.9 Other collections + +| Path | Class | Content | +|------|-------|---------| +| `/entry/instrument/detector/detectorSpecific` | NXcollection | Dectris-style detector metadata + Jungfraujoch fields: `x_pixels_in_detector`, `y_pixels_in_detector`, `nimages`, `ntrigger`, `nimages_collected`, `nimages_written`, `data_collection_efficiency`, `max_receiver_delay`, `storage_cell_number`, `storage_cell_delay` [ns], `software_git_commit`, `software_git_date`, `jfjoch_release`, `jfjoch_writer_release`, `summation_mode`, `detect_ice_rings`, `gain_file_names`, `data_reduction_factor_serialmx`, `adu_histogram/`, `data_collection_efficiency_image` | +| `/entry/instrument/detector/calibration` | NXcollection | per-channel pedestal / calibration images (bitshuffle-compressed) | +| `/entry/instrument/fluorescence` | NXcollection | XRF spectrum: `energy` [eV], `data` | +| `/entry/user` | NXcollection | scalar values supplied under `header_appendix.hdf5` | + +### 4.10 Non-standard fields inside the NXmx detector group + +A few extension scalars are written *inside* the otherwise-standard `/entry/instrument/detector` +group for compatibility with existing tooling: + +| Field | Units | Meaning | +|-------|-------|---------| +| `detector_distance` | m | duplicate of `distance` (Dectris/Neggia compatibility) | +| `detector_number` | | detector identifier (Dectris convention) | +| `error_value` | | masked/error pixel sentinel (NXmx standard would be `underload_value`) | +| `bit_depth_image` | | stored image bit depth (NXmx standard is `bit_depth_readout`) | +| `acquisition_type` | | always `triggered` (Dectris convention) | +| `jungfrau_conversion_applied` | | JUNGFRAU photon/keV conversion applied | +| `jungfrau_conversion_factor` | eV | conversion factor | +| `geometry_transformation_applied` | | module→full-detector geometry applied | + +## 5. Notes + +* **Units** are written as the HDF5 `units` attribute on the dataset (e.g. `m`, `eV`, `deg`, + `Angstrom`, `Angstrom^-1`, `Angstrom^2`, `pixel`, `s`). +* **Sentinels.** Missing per-image values are `NaN` (floats) or `-1`/`0` (integer indices); image + pixels use `INTx_MIN` / `UINTx_MAX`. +* **Master vs data file.** In legacy/VDS formats the analysis arrays physically live in the data + files; the master file links to them (external links in legacy, virtual datasets in VDS). In the + integrated format there are no data files and everything is in one place. +* **CXI / CrystFEL.** `/entry/MX` follows the CXI peak-list convention; see + [CXI file format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf). diff --git a/docs/JFJOCH_WRITER.md b/docs/JFJOCH_WRITER.md index 9fbbb384..4cba4066 100644 --- a/docs/JFJOCH_WRITER.md +++ b/docs/JFJOCH_WRITER.md @@ -121,89 +121,15 @@ For example `header_appendix` of `{"param1": "test1", "param2": ["test1", "test2 Notifications for finalized files are optional, if notification port number is omitted this functionality is not enabled. ## HDF5 file structure -Jungfraujoch aims to generate files compliant with NXmx format. -### Master file +Jungfraujoch writes NXmx-compliant HDF5, with substantial derived metadata (spot finding, indexing, +integration, azimuthal integration, per-image statistics and timing) stored *beyond* the NXmx +standard. The complete file layout — master vs data files, the three format variants +(`NXmxLegacy`, `NXmxVDS`, `NXmxIntegrated`), every NXmx field that is populated and every +Jungfraujoch extension — is documented in [HDF5 / NeXus data format](HDF5.md). -There are custom extension to NXmx format. These will be documented in the future. - -Specifically, if data collection was configured with `header_appendix` having key equal to `hdf5` and value as JSON -object with number and string values. These will be added to `/entry/user`. - -There are three versions of master file possible. - -#### Legacy version (NXmxLegacy) -By default, *legacy version* is used. This version is compatible with DECTRIS file writer version 1.0 format. -This ensures the file compatibility of Neggia and Durin XDS plugins, as well as DECTRIS Albula viewer version 4.0. -Distinct feature is that if images are split into data files, there will be multiple links in `/entry/data`, -each corresponding to a data file. -Yet, certain new HDF5 features, like virtual datasets, are not possible in this format since it has to be compatible with HDF5 1.8 features. - -#### VDS format (NXmxVDS) -Therefore, we have enabled format *VDS version*. This will link to all data files via a single virtual dataset `/entry/data/data`. -The same way spot finding, azimuthal integration and others, will be linked between master and data files. -This format allows to display processing results in currently developed Jungfraujoch Viewer. -For the time being it only works with Durin XDS plugin, and require DECTRIS Albula viewer version 4.1+. - -#### Integrated format (NXmxIntegrated) -This is format, where no data files are created, but both images and metadata are stored in the same master file. -This is generally equivalent to VDS format described above. - -### Data file - -Data file has the following structure: - -| Location | Description | Optional | Linked in master file v. 2 | -|--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------:|:--------------------------:| -| /entry/data/data | Images | | X | -| /entry/detector/timestamp | Timestamp of the image | | | -| /entry/detector/exptime | Exposure time of the image | | | -| /entry/detector/number | Image number; if image rejection was used this will be the original image number | | | -| /entry/detector/det_info | Debug field of the JF detector | X | | -| /entry/detector/storage_cell_image | Storage cell number | X | X * | -| /entry/detector/rcv_delay | Receiver delay for the image (Jungfraujoch debugging) | X | | -| /entry/detector/rcv_free_send_buffers | Receiver number of free send buffers at the time of sending the image (Jungfraujoch debugging) | X | | -| /entry/detector/data_collection_efficiency_image | Ratio of received and expected UDP packets | X | X * | -| /entry/detector/packets_expected | Number of UDP packets expected for the image | X | | -| /entry/detector/packets_received | Number of UDP packets received for the image | X | | -| /entry/image/max_value | Max viable value of the image (excl. overloads, etc.) | X | | -| /entry/azint/bin_to_q | Azimuthal integration - bin-to-Q mapping | X | | -| /entry/azint/image | Azimuthal integration - per image | X | X | -| /entry/MX/peakXPosRaw | Peak position X (see [CXI format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf)) | X | X | -| /entry/MX/peakYPosRaw | Peak position Y (see [CXI format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf)) | X | X | -| /entry/MX/peakTotalIntensity | Peak total intensity (see [CXI format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf)) | X | X | -| /entry/MX/peakH | Miller index h for each detected (indexed) peak | X | X | -| /entry/MX/peakK | Miller index k for each detected (indexed) peak | X | X | -| /entry/MX/peakL | Miller index l for each detected (indexed) peak | X | X | -| /entry/MX/peakDistEwaldSphere | Distance of the peak from the Ewald sphere (prediction) | X | X | -| /entry/MX/nPeaks | Number of peaks per image (see [CXI format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf)) | X | X | -| /entry/MX/strongPixels | Number of strong pixel per image | X | X | -| /entry/MX/nPeaksRingFiltered | Number of peaks not belonging to rings | X | X | -| /entry/MX/imageIndexed | Image is successfully indexed | X | X | -| /entry/MX/profileRadius | Crystal profile radius for indexed images | X | X | -| /entry/MX/latticeIndexed | Crystal lattice for the image, assuming it is indexed | X | X | -| /entry/MX/bkgEstimate | Mean value of pixels in the radius of 3-5 A | X | X | -| /entry/MX/resolutionEstimate | Resolution estimate based on on-the-fly integration | X | X | -| /entry/MX/beam_corr_x | Beam center correction applied during processing (X) [pixel] | X | X | -| /entry/MX/beam_corr_y | Beam center correction applied during processing (Y) [pixel] | X | X | -| /entry/MX/niggli_class | Niggli class identifier of the indexed Bravais lattice (per image) - see [International Tables for Crystallography A (2016). Vol. A, Table 3.1.3.1](https://onlinelibrary.wiley.com/iucr/itc/Ac/ch3o1v0001/table3o1o3o1.pdf) | X | X | -| /entry/MX/bravais_lattice | Bravais lattice short code (per image), e.g., aP, mC, oF, tI, hP, hR, cF | X | X | -| /entry/roi/{roi_name}/max | Max pixel value for roi named {roi_name} | X | X | -| /entry/roi/{roi_name}/sum | Sum pixel value for roi named {roi_name} | X | X | -| /entry/roi/{roi_name}/sum_sq | Sum pixel values squared for roi named {roi_name} | X | X | -| /entry/roi/{roi_name}/npixel | Number of valid pixel for roi named {roi_name} | X | X | -| /entry/roi/{roi_name}/x | Weighted X-coordinate for roi named {roi_name} | X | X | -| /entry/roi/{roi_name}/y | Weighted Y-coordinate for roi named {roi_name} | X | X | -| /entry/xfel/pulseID | Pulse ID (for XFEL only) | X | X | -| /entry/xfel/eventCode | Event code (for XFEL only) | X | X | - -\* - Datasets from `/entry/detector` in data file are mapped to `/entry/instrument/detector/detectorSpecific` in master file. - -If spot finding is enabled, spots are written in the [CXI format](https://raw.githubusercontent.com/cxidb/CXI/master/cxi_file_format.pdf) and are recognized by CrystFEL. The following has to be added to the CrystFEL geometry file: -``` -peak_list = /opt/MX -peak_list_type = cxi -``` +If data collection was configured with a `header_appendix` containing a key `hdf5` whose value is a +JSON object of numbers and strings, those entries are written to `/entry/user`. ## Other formats (CBF and TIFF) In addition to HDF5 format, Jungfraujoch allows to save images in the Crystallographic Binary File (CBF) format. diff --git a/docs/index.rst b/docs/index.rst index 2e7a0350..b8755c9c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -54,6 +54,7 @@ Jungfraujoch is distributed under the GPLv3 license. OPENAPI OPENAPI_SPECS CBOR + HDF5 IMAGE_STREAM PIXEL_MASK WEB_FRONTEND -- 2.52.0 From e8d3eb1b08542ca035b9223965899fa2a6d7391e Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 19:47:36 +0200 Subject: [PATCH 059/228] docs/HDF5.md: generalize to rotation+serial, add process.h5, user_data, ITC link - Motivation no longer frames the format as serial-only: Jungfraujoch serves both rotation and serial MX, and the two-layout rationale (per-image CXI spots vs. dataset-wide NXreflections) is presented for both. - Document the _process.h5 reprocessing output of jfjoch_process: an integrated-format master whose /entry/data/data is a VDS linking back to the original images (all results, no image copies). - Add a section on the header_appendix / image_appendix (user_data) mechanism: how the free-form JSON flows through the start/image CBOR messages and how an "hdf5" sub-object is persisted to /entry/user, with an example. - Link the niggliClass row to International Tables for Crystallography A, Table 3.1.3.1. Co-Authored-By: Claude Opus 4.8 --- docs/HDF5.md | 107 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 22 deletions(-) diff --git a/docs/HDF5.md b/docs/HDF5.md index 9824b1cd..49e72cbd 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -16,8 +16,9 @@ that document is a useful companion for their meaning. ## 1. Motivation: derived metadata and FAIR data The goal of Jungfraujoch is not only to store high-throughput datasets efficiently, but to keep -them findable, accessible, interoperable and reusable (FAIR). For serial crystallography this is -hard for two practical reasons: +them findable, accessible, interoperable and reusable (FAIR). Jungfraujoch is used for both +**rotation** macromolecular crystallography (single- and multi-crystal, including fine-sliced and +helical scans) and **serial** crystallography (stills, grid scans); the same concerns apply to both: * **Findability.** Raw diffraction images carry almost no descriptive metadata about *content*. Quantities such as background level, number of diffraction spots, or indexing outcome let a user @@ -32,28 +33,31 @@ transparency and reproducibility of that analysis matter. As a minimum the write preserves spot-finding and indexing results together with the filters that were applied, and it can retain an unbiased, down-sampled reference set of unfiltered images for validation and reuse. -### Why a CXI-style per-image layout for spot finding / indexing +### Two complementary layouts: per-image spots vs. a reflection table -Spot-finding and indexing results in serial crystallography are inherently *image-centric*: the -natural query is "give me the spots for image *n*". For these products Jungfraujoch adopts a layout -similar to the [Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the -convention understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties -(position, intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by -image number, with each image allocated room for up to a predefined maximum number of spots. These -dense arrays are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are -retrieved without traversing variable-length structures. The cost is some storage overhead for -unused slots (padded with sentinels), which is acceptable for the access pattern. +Jungfraujoch stores analysis products in two shapes, matching how each is accessed. -We also evaluated the NeXus +**Per-image spot finding / indexing.** Spot finding and indexing are inherently *image-centric* — +the natural query is "give me the spots for image *n*" — and this holds for serial stills and for +rotation frames alike. For these products Jungfraujoch adopts a layout similar to the +[Coherent X-ray Imaging (CXI) data bank](https://www.cxidb.org) (Maia, 2012) and the convention +understood by [CrystFEL](https://www.desy.de/~twhite/crystfel/): spot properties (position, +intensity, Miller index, …) are stored in fixed-size two-dimensional arrays indexed by image number, +with each image allocated room for up to a predefined maximum number of spots. These dense arrays +are addressed with ordinary HDF5 hyperslab reads, so the spots of a single image are retrieved +without traversing variable-length structures. The cost is some storage overhead for unused slots +(padded with sentinels), which is acceptable for the access pattern. + +**Integrated reflections.** Integrated intensities are naturally a *dataset-wide* table, which is +exactly the model of the NeXus [NXreflections](https://manual.nexusformat.org/classes/base_classes/NXreflections.html) base class. -NXreflections models a *dataset-wide* reflection table, which fits integrated rotation data well — -and Jungfraujoch does use it for integration results (see §4.2 below). -But for spot finding/indexing across hundreds of thousands of patterns a single table would force -aggregation over the whole experiment before the spots of one image can be accessed efficiently. -For these intermediate products a per-image representation is more suitable. We encourage the -community to develop standardised NeXus application definitions for image-centric serial -crystallography products that combine NeXus interoperability with the access patterns and scale of -modern experiments. +This fits rotation crystallography well, and Jungfraujoch uses NXreflections for its integration +results (see §4.2 below). We deliberately do *not* force spot finding/indexing into a single +experiment-wide table: across the hundreds of thousands of patterns typical of serial — or +fine-sliced rotation — experiments, that would require aggregating the whole experiment before the +spots of one image can be read. We encourage the community to develop standardised NeXus application +definitions for image-centric crystallography products that combine NeXus interoperability with the +access patterns and scale of modern high-throughput experiments. ## 2. File layout @@ -87,6 +91,22 @@ Images are stored chunked (one image per chunk) and compressed with bitshuffle + bitshuffle + Zstd; signed integer image datasets use `INTx_MIN` as the HDF5 fill value (the "masked / no-data" sentinel), unsigned use `UINTx_MAX`. +### Reprocessing output: `_process.h5` + +The offline reprocessing tool [`jfjoch_process`](TOOLS.md) (`tools/jfjoch_process.cpp`) re-runs the +full analysis pipeline (spot finding, indexing, refinement, integration, scaling) on an existing +dataset and writes its results to a master file named **`_process.h5`**. This file uses the +**integrated** format, but instead of copying the images its `/entry/data/data` is a *virtual +dataset that links back to the original image files* (`hdf5_source_data` → +`NXmx::LinkToData_ProcessingVDS`). The result is a compact, self-describing companion file that +holds *all* the derived analysis (everything in §4) plus a virtual view +of the raw images — without duplicating terabytes of data. + +This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead +of) the raw data to convey what is in a dataset and how it processed, while the `/entry/data/data` +VDS still resolves to the original images when they are available. `jfjoch_process` can also process +an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. + ## 3. NXmx-standard content The entries below are part of, or valid base classes for, the @@ -223,7 +243,7 @@ In legacy/VDS mode these live in the data files and are linked/virtual-stacked i | `peakCountIndexed` | | spots fitting the indexing solution | | `imageIndexed` | | image was indexed (0/1) | | `indexingLatticeCount` | | number of lattices found for the image | -| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A*, Table 3.1.3.1) | +| `niggliClass` | | Niggli class of the indexed Bravais lattice (see *International Tables for Crystallography A* (2016), Vol. A, [Table 3.1.3.1](https://onlinelibrary.wiley.com/iucr/itc/Ac/ch3o1v0001/table3o1o3o1.pdf)) | | `bravaisLattice` | | Bravais lattice short code, e.g. `aP`, `mC`, `oF`, `tI`, `hP`, `hR`, `cF` | | `profileRadius` | Å⁻¹ | crystal profile radius | | `mosaicity` | deg | mosaicity estimate | @@ -367,6 +387,49 @@ group for compatibility with existing tooling: | `jungfrau_conversion_factor` | eV | conversion factor | | `geometry_transformation_applied` | | module→full-detector geometry applied | +### 4.11 User-supplied metadata: `header_appendix` and `image_appendix` + +Facilities frequently need to attach metadata that Jungfraujoch does not model explicitly. Two +free-form JSON fields in the `/start` request (`broker/jfjoch_api.yaml`) provide this without any +schema change; both accept *any valid JSON*: + +| Field | Carried in | Persisted to HDF5? | +|-------|-----------|--------------------| +| `header_appendix` | the **start** message, under `user_data.user` (see [CBOR](CBOR.md)) | no — except the `hdf5` sub-object (below) | +| `image_appendix` | **every image** message, as `user_data` | no | + +Both are forwarded verbatim through the ZeroMQ/CBOR stream to every downstream consumer (writer, +republished analysis, viewers), so they are the recommended channel for facility- or +beamline-specific provenance (proposal, operator, optics state, per-image trigger info, …) that has +no dedicated API field. + +**Persisting selected values to HDF5.** `header_appendix` is normally *not* written to the master +file. As an exception, if it contains a key `hdf5` whose value is a JSON object of scalars (strings +and numbers — no arrays or nested objects), the writer stores each entry under `/entry/user/`. + +For example, a `/start` request containing: + +```json +{ + "header_appendix": { + "proposal": "p20001", + "operator": "jdoe", + "hdf5": { "beamline": "X06SA", "ring_mode": "top-up", "attenuator_foils": 2 } + }, + "image_appendix": { "trigger_source": "external" } +} +``` + +forwards the whole `header_appendix` as `user_data.user` on the start message and +`{"trigger_source": "external"}` as `user_data` on every image message, and writes three scalars +into the master file: + +``` +/entry/user/beamline = "X06SA" +/entry/user/ring_mode = "top-up" +/entry/user/attenuator_foils = 2 +``` + ## 5. Notes * **Units** are written as the HDF5 `units` attribute on the dataset (e.g. `m`, `eV`, `deg`, -- 2.52.0 From 5143ba0b1f6be539351dacc816c8947541289a50 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 20:00:40 +0200 Subject: [PATCH 060/228] jfjoch_api.yaml: header_appendix hdf5 values go to /entry/user, not /entry/data The description said the "hdf5" sub-object is written under /entry/data, but the writer (NXmx::UserData) and the example in the same description both use /entry/user. Correct the path. Co-Authored-By: Claude Opus 4.8 --- broker/jfjoch_api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 7834785d..67e80aa8 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -455,7 +455,7 @@ components: Header appendix, added as user_data/user to start ZeroMQ message (can be any valid JSON) In general, it is not saved in HDF5 file. - However, if values are placed in "hdf5" object, `jfjoch_writer` will write them in /entry/data of the HDF5 file. + However, if values are placed in "hdf5" object, `jfjoch_writer` will write them in /entry/user of the HDF5 file. This applies solely to string and number (double floating-point). No arrays/sub-objects is allowed. For example {"hdf5": {"val1":1, "val2":"xyz"}}, will write /entry/user/val1 and /entry/user/val2. image_appendix: -- 2.52.0 From 704e4b4eb3291f8cf123e96c1544dd5cf83b40fd Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 20:24:01 +0200 Subject: [PATCH 061/228] Documentation: Data analysis description --- docs/CPU_DATA_ANALYSIS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS.md b/docs/CPU_DATA_ANALYSIS.md index 2d443b67..17f7b166 100644 --- a/docs/CPU_DATA_ANALYSIS.md +++ b/docs/CPU_DATA_ANALYSIS.md @@ -11,9 +11,10 @@ This document describes the crystallographic algorithms implemented in Jungfrauj 5. Bravais lattice / centering inference, 6. geometry and lattice refinement, 7. reflection prediction (still and rotation), -8. 2D summation integration, +8. Bragg integration by either 2D summation or reference-driven profile fitting, 9. scaling and merging, -10. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, French–Wilson). +10. merge-level error modelling and outlier rejection, +11. auxiliary statistics (Wilson plot, ⟨I/σ(I)⟩, CC1/2, CCref). ## References @@ -346,7 +347,7 @@ Refinement is performed in stages with decreasing acceptance tolerance for inclu --- -## 8. Reflection prediction +## 9. Bragg integration Jungfraujoch predicts reflection positions for integration by enumerating Miller indices within a resolution cutoff and accepting those that satisfy a diffraction condition model. -- 2.52.0 From 638aa379f4cb5c498e5629b172701aeb56972482 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 20:28:25 +0200 Subject: [PATCH 062/228] Fix --- docs/CPU_DATA_ANALYSIS.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS.md b/docs/CPU_DATA_ANALYSIS.md index 17f7b166..620a5339 100644 --- a/docs/CPU_DATA_ANALYSIS.md +++ b/docs/CPU_DATA_ANALYSIS.md @@ -382,13 +382,9 @@ $ which also appears in XDS as the Lorentz component linked to the rotation axis. A Gaussian mosaicity model yields a partiality fraction over an oscillation width $\Delta\phi$: -$ -P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[ -\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) -- -\mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) -\right], -$ + +$ P(\phi;\sigma_M,\zeta,\Delta\phi) = \frac{1}{2}\left[\mathrm{erf}\!\left(\frac{\phi+\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right) - \mathrm{erf}\!\left(\frac{\phi-\Delta\phi/2}{\sqrt{2}\,\sigma_M/\zeta}\right)\right], $ + with mosaicity $\sigma_M$ in radians. Reflections are predicted if they meet minimum $\zeta$ and mosaicity-window criteria, and their predicted detector coordinates fall on the active detector area. @@ -489,8 +485,7 @@ Jungfraujoch supports several partiality choices: 1. **Rotation partiality** (XDS-like; see §8.3): $ P_{ij} = \frac{1}{2}\left[ - \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - - + \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}+\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) - \mathrm{erf}\!\left(\frac{\Delta\phi_{ij}-\Delta\phi/2}{\sqrt{2}\,\sigma_{M,i}/\zeta_{ij}}\right) \right]. $ -- 2.52.0 From 3334e88d3e2d89dba2c3c2a5ac35c264421b8331 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Tue, 16 Jun 2026 20:51:45 +0200 Subject: [PATCH 063/228] frontend: fix state bugs, slim Plotly bundle, drop CRA leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - App: make getValues an arrow field so its `this` survives being passed as the `update` prop (the post-edit statistics refresh was broken). - DataProcessingSettings: guard prop sync with last_downloaded_s + _.isEqual so the 1 s statistics poll no longer overwrites a slider mid-edit (reference `!=` on a freshly-parsed object was always true → snap-back). - DataCollection: collapse nested setState in grid-scan callbacks. - PreviewImage: build an immutable settings copy instead of mutating state in place; fix stale comment. - MultiLinePlotWrapper: drop per-render console.log; render via a custom Plot built on plotly.js-cartesian-dist-min (scatter+heatmap only), cutting the main bundle 6.0->2.6 MB (1.8->0.8 MB gzip). - package.json: remove dead react-scripts test/eject scripts and CRA eslintConfig, move @redocly/cli to devDependencies. - Delete unused CRA scaffolding (serviceWorker.js, react-app-env.d.ts, setupTests.js). Co-Authored-By: Claude Opus 4.8 --- frontend/package-lock.json | 738 ++++++++++++++---- frontend/package.json | 9 +- frontend/src/App.tsx | 2 +- frontend/src/components/DataCollection.tsx | 18 +- .../src/components/DataProcessingSettings.tsx | 43 +- .../src/components/MultiLinePlotWrapper.jsx | 3 +- frontend/src/components/Plot.jsx | 12 + frontend/src/components/PreviewImage.tsx | 23 +- frontend/src/react-app-env.d.ts | 1 - frontend/src/serviceWorker.js | 141 ---- frontend/src/setupTests.js | 5 - 11 files changed, 657 insertions(+), 338 deletions(-) create mode 100644 frontend/src/components/Plot.jsx delete mode 100644 frontend/src/react-app-env.d.ts delete mode 100644 frontend/src/serviceWorker.js delete mode 100644 frontend/src/setupTests.js diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 09f3ba89..8217a526 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14,13 +14,12 @@ "@mui/icons-material": "^6.1.2", "@mui/material": "^6.1.2", "@mui/x-data-grid": "^7.19.0", - "@redocly/cli": "^2.28.1", "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react-swc": "^4.2.3", "lodash": "^4.17.23", - "plotly.js": "^3.3.1", + "plotly.js-cartesian-dist-min": "^3.3.1", "react": "^19.2.4", "react-dom": "^19.2.4", "react-plotly.js": "^2.6.0", @@ -31,6 +30,7 @@ "vite-tsconfig-paths": "^6.1.1" }, "devDependencies": { + "@redocly/cli": "^2.28.1", "@types/lodash": "^4.17.10", "@types/react-plotly.js": "^2.6.3", "esbuild-style-plugin": "^1.6.3", @@ -143,6 +143,7 @@ "version": "7.27.3", "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.27.3.tgz", "integrity": "sha512-fXSwMQqitTGeHLBC08Eq5yXz2m37E4pJX1qAU1+2cNedz/ifv/bVXft90VeSav5nFO61EcNgwr0aJxbyPaWBPg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.27.3" @@ -206,6 +207,7 @@ "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -267,6 +269,7 @@ "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.28.6" @@ -336,6 +339,7 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/@choojs/findup/-/findup-0.2.1.tgz", "integrity": "sha512-YstAqNb0MCN8PjdLCDfRsBcGVRN41f3vgLvaI0IrIcBp4AqILRSS0DeWNGkicC+f/zRIPJLc+9RURVSepwvfBw==", + "peer": true, "dependencies": { "commander": "^2.15.1" }, @@ -346,7 +350,8 @@ "node_modules/@choojs/findup/node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "peer": true }, "node_modules/@emotion/babel-plugin": { "version": "11.13.5", @@ -463,6 +468,7 @@ "version": "0.8.5", "resolved": "https://registry.npmjs.org/@emotion/stylis/-/stylis-0.8.5.tgz", "integrity": "sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ==", + "dev": true, "license": "MIT" }, "node_modules/@emotion/unitless": { @@ -908,12 +914,14 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/@exodus/schemasafe/-/schemasafe-1.3.0.tgz", "integrity": "sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==", + "dev": true, "license": "MIT" }, "node_modules/@faker-js/faker": { "version": "7.6.0", "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-7.6.0.tgz", "integrity": "sha512-XK6BTq1NDMo9Xqw/YkYyGjSsg44fbNwYRx7QK2CuoQgyy+f1rrTDHoExVM5PsyXCtfl2vs2vVJ0MN0yN6LppRw==", + "dev": true, "license": "MIT", "engines": { "node": ">=14.0.0", @@ -924,6 +932,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/@humanwhocodes/momoa/-/momoa-2.0.4.tgz", "integrity": "sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=10.10.0" @@ -1089,6 +1098,7 @@ "version": "0.5.2", "resolved": "https://registry.npmjs.org/@mapbox/geojson-rewind/-/geojson-rewind-0.5.2.tgz", "integrity": "sha512-tJaT+RbYGJYStt7wI3cq4Nl4SXxG8W7JDG5DMJu97V25RnbNg3QtQtf+KD+VLjNpWKYsRvXDNmNrBgEETr1ifA==", + "peer": true, "dependencies": { "get-stream": "^6.0.1", "minimist": "^1.2.6" @@ -1100,12 +1110,14 @@ "node_modules/@mapbox/geojson-types": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@mapbox/geojson-types/-/geojson-types-1.0.2.tgz", - "integrity": "sha512-e9EBqHHv3EORHrSfbR9DqecPNn+AmuAoQxV6aL8Xu30bJMJR1o8PZLZzpk1Wq7/NfCbuhmakHTPYRhoqLsXRnw==" + "integrity": "sha512-e9EBqHHv3EORHrSfbR9DqecPNn+AmuAoQxV6aL8Xu30bJMJR1o8PZLZzpk1Wq7/NfCbuhmakHTPYRhoqLsXRnw==", + "peer": true }, "node_modules/@mapbox/jsonlint-lines-primitives": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@mapbox/jsonlint-lines-primitives/-/jsonlint-lines-primitives-2.0.2.tgz", "integrity": "sha512-rY0o9A5ECsTQRVhv7tL/OyDpGAoUB4tTvLiW1DSzQGq4bvTPhNw1VpSNjDJc5GFZ2XuyOtSWSVN05qOtcD71qQ==", + "peer": true, "engines": { "node": ">= 0.6" } @@ -1114,6 +1126,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/@mapbox/mapbox-gl-supported/-/mapbox-gl-supported-1.5.0.tgz", "integrity": "sha512-/PT1P6DNf7vjEEiPkVIRJkvibbqWtqnyGaBz3nfRdcxclNSnSdaLU5tfAgcD7I8Yt5i+L19s406YLl1koLnLbg==", + "peer": true, "peerDependencies": { "mapbox-gl": ">=0.32.1 <2.0.0" } @@ -1121,22 +1134,26 @@ "node_modules/@mapbox/point-geometry": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz", - "integrity": "sha512-6j56HdLTwWGO0fJPlrZtdU/B13q8Uwmo18Ck2GnGgN9PCFyKTZ3UbXeEdRFh18i9XQ92eH2VdtpJHpBD3aripQ==" + "integrity": "sha512-6j56HdLTwWGO0fJPlrZtdU/B13q8Uwmo18Ck2GnGgN9PCFyKTZ3UbXeEdRFh18i9XQ92eH2VdtpJHpBD3aripQ==", + "peer": true }, "node_modules/@mapbox/tiny-sdf": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/@mapbox/tiny-sdf/-/tiny-sdf-1.2.5.tgz", - "integrity": "sha512-cD8A/zJlm6fdJOk6DqPUV8mcpyJkRz2x2R+/fYcWDYG3oWbG7/L7Yl/WqQ1VZCjnL9OTIMAn6c+BC5Eru4sQEw==" + "integrity": "sha512-cD8A/zJlm6fdJOk6DqPUV8mcpyJkRz2x2R+/fYcWDYG3oWbG7/L7Yl/WqQ1VZCjnL9OTIMAn6c+BC5Eru4sQEw==", + "peer": true }, "node_modules/@mapbox/unitbezier": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz", - "integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==" + "integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==", + "peer": true }, "node_modules/@mapbox/vector-tile": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/@mapbox/vector-tile/-/vector-tile-1.3.1.tgz", "integrity": "sha512-MCEddb8u44/xfQ3oD+Srl/tNcQoqTw3goGk2oLsrFxOTc3dUp+kAnby3PvAeeBYSMSjSPD1nd1AJA6W49WnoUw==", + "peer": true, "dependencies": { "@mapbox/point-geometry": "~0.1.0" } @@ -1145,6 +1162,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@mapbox/whoots-js/-/whoots-js-3.1.0.tgz", "integrity": "sha512-Es6WcD0nO5l+2BOQS4uLfNPYQaNDfbot3X1XUoloz+x0mPDS3eeORZJl06HXjwBG1fOGwCRnzK88LMdxKRrd6Q==", + "peer": true, "engines": { "node": ">=6.0.0" } @@ -1153,6 +1171,7 @@ "version": "20.4.0", "resolved": "https://registry.npmjs.org/@maplibre/maplibre-gl-style-spec/-/maplibre-gl-style-spec-20.4.0.tgz", "integrity": "sha512-AzBy3095fTFPjDjmWpR2w6HVRAZJ6hQZUCwk5Plz6EyfnfuQW1odeW5i2Ai47Y6TBA2hQnC+azscjBSALpaWgw==", + "peer": true, "dependencies": { "@mapbox/jsonlint-lines-primitives": "~2.0.2", "@mapbox/unitbezier": "^0.0.1", @@ -1171,12 +1190,14 @@ "node_modules/@maplibre/maplibre-gl-style-spec/node_modules/@mapbox/unitbezier": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.1.tgz", - "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==" + "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==", + "peer": true }, "node_modules/@maplibre/maplibre-gl-style-spec/node_modules/tinyqueue": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-3.0.0.tgz", - "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==" + "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==", + "peer": true }, "node_modules/@mui/core-downloads-tracker": { "version": "6.4.11", @@ -1655,6 +1676,7 @@ "version": "1.8.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, "license": "MIT", "engines": { "node": "^14.21.3 || >=16" @@ -1667,6 +1689,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.1.tgz", "integrity": "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==", + "dev": true, "funding": [ { "type": "github", @@ -1679,6 +1702,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8.0.0" @@ -1688,6 +1712,7 @@ "version": "0.202.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.202.0.tgz", "integrity": "sha512-fTBjMqKCfotFWfLzaKyhjLvyEyq5vDKTTFfBmx21btv3gvy8Lq6N5Dh2OzqeuN4DjtpSvNT1uNVfg08eD2Rfxw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.3.0" @@ -1700,6 +1725,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.0.1.tgz", "integrity": "sha512-XuY23lSI3d4PEqKA+7SLtAgwqIfc6E/E9eAQWLN1vlpC53ybO3o6jW4BsXo1xvz9lYyyWItfQDDLzezER01mCw==", + "dev": true, "license": "Apache-2.0", "engines": { "node": "^18.19.0 || >=20.6.0" @@ -1712,6 +1738,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.0.1.tgz", "integrity": "sha512-MaZk9SJIDgo1peKevlbhP6+IwIiNPNmswNL4AF0WaQJLbHXjr9SrZMgS12+iqr9ToV4ZVosCcc0f8Rg67LXjxw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" @@ -1727,6 +1754,7 @@ "version": "0.202.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.202.0.tgz", "integrity": "sha512-/hKE8DaFCJuaQqE1IxpgkcjOolUIwgi3TgHElPVKGdGRBSmJMTmN/cr6vWa55pCJIXPyhKvcMrbrya7DZ3VmzA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "2.0.1", @@ -1746,6 +1774,7 @@ "version": "0.202.0", "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.202.0.tgz", "integrity": "sha512-nMEOzel+pUFYuBJg2znGmHJWbmvMbdX5/RhoKNKowguMbURhz0fwik5tUKplLcUtl8wKPL1y9zPnPxeBn65N0Q==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "2.0.1", @@ -1762,6 +1791,7 @@ "version": "0.202.0", "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.202.0.tgz", "integrity": "sha512-5XO77QFzs9WkexvJQL9ksxL8oVFb/dfi9NWQSq7Sv0Efr9x3N+nb1iklP1TeVgxqJ7m1xWiC/Uv3wupiQGevMw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/api-logs": "0.202.0", @@ -1783,6 +1813,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.0.1.tgz", "integrity": "sha512-dZOB3R6zvBwDKnHDTB4X1xtMArB/d324VsbiPkX/Yu0Q8T2xceRthoIVFhJdvgVM2QhGVUyX9tzwiNxGtoBJUw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "2.0.1", @@ -1799,6 +1830,7 @@ "version": "0.202.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.202.0.tgz", "integrity": "sha512-pv8QiQLQzk4X909YKm0lnW4hpuQg4zHwJ4XBd5bZiXcd9urvrJNoNVKnxGHPiDVX/GiLFvr5DMYsDBQbZCypRQ==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/api-logs": "0.202.0", @@ -1816,6 +1848,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.0.1.tgz", "integrity": "sha512-wf8OaJoSnujMAHWR3g+/hGvNcsC16rf9s1So4JlMiFaFHiE4HpIA3oUh+uWZQ7CNuK8gVW/pQSkgoa5HkkOl0g==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "2.0.1", @@ -1832,6 +1865,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.0.1.tgz", "integrity": "sha512-xYLlvk/xdScGx1aEqvxLwf6sXQLXCjk3/1SQT9X9AoN5rXRhkdvIFShuNNmtTEPRBqcsMbS4p/gJLNI2wXaDuQ==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "2.0.1", @@ -1849,6 +1883,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.0.1.tgz", "integrity": "sha512-UhdbPF19pMpBtCWYP5lHbTogLWx9N0EBxtdagvkn5YtsAnCBZzL7SjktG+ZmupRgifsHMjwUaCCaVmqGfSADmA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@opentelemetry/context-async-hooks": "2.0.1", @@ -1866,6 +1901,7 @@ "version": "1.34.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.34.0.tgz", "integrity": "sha512-aKcOkyrorBGlajjRdVoJWHTxfxO1vCNHLJVlSDaRHDIdjU+pX8IYQPvPDkYiujKLbRnWU+1TBwEt0QRgSm4SGA==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=14" @@ -2180,12 +2216,14 @@ "node_modules/@plotly/d3": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/@plotly/d3/-/d3-3.8.2.tgz", - "integrity": "sha512-wvsNmh1GYjyJfyEBPKJLTMzgf2c2bEbSIL50lmqVUi+o1NHaLPi1Lb4v7VxXXJn043BhNyrxUrWI85Q+zmjOVA==" + "integrity": "sha512-wvsNmh1GYjyJfyEBPKJLTMzgf2c2bEbSIL50lmqVUi+o1NHaLPi1Lb4v7VxXXJn043BhNyrxUrWI85Q+zmjOVA==", + "peer": true }, "node_modules/@plotly/d3-sankey": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/@plotly/d3-sankey/-/d3-sankey-0.7.2.tgz", "integrity": "sha512-2jdVos1N3mMp3QW0k2q1ph7Gd6j5PY1YihBrwpkFnKqO+cqtZq3AdEYUeSGXMeLsBDQYiqTVcihYfk8vr5tqhw==", + "peer": true, "dependencies": { "d3-array": "1", "d3-collection": "1", @@ -2196,6 +2234,7 @@ "version": "0.33.1", "resolved": "https://registry.npmjs.org/@plotly/d3-sankey-circular/-/d3-sankey-circular-0.33.1.tgz", "integrity": "sha512-FgBV1HEvCr3DV7RHhDsPXyryknucxtfnLwPtCKKxdolKyTFYoLX/ibEfX39iFYIL7DYbVeRtP43dbFcrHNE+KQ==", + "peer": true, "dependencies": { "d3-array": "^1.2.1", "d3-collection": "^1.0.4", @@ -2207,6 +2246,7 @@ "version": "1.13.4", "resolved": "https://registry.npmjs.org/@plotly/mapbox-gl/-/mapbox-gl-1.13.4.tgz", "integrity": "sha512-sR3/Pe5LqT/fhYgp4rT4aSFf1rTsxMbGiH6Hojc7PH36ny5Bn17iVFUjpzycafETURuFbLZUfjODO8LvSI+5zQ==", + "peer": true, "dependencies": { "@mapbox/geojson-rewind": "^0.5.2", "@mapbox/geojson-types": "^1.0.2", @@ -2239,6 +2279,7 @@ "version": "3.1.9", "resolved": "https://registry.npmjs.org/@plotly/point-cluster/-/point-cluster-3.1.9.tgz", "integrity": "sha512-MwaI6g9scKf68Orpr1pHZ597pYx9uP8UEFXLPbsCmuw3a84obwz6pnMXGc90VhgDNeNiLEdlmuK7CPo+5PIxXw==", + "peer": true, "dependencies": { "array-bounds": "^1.0.1", "binary-search-bounds": "^2.0.4", @@ -2256,7 +2297,8 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/@plotly/regl/-/regl-2.1.2.tgz", "integrity": "sha512-Mdk+vUACbQvjd0m/1JJjOOafmkp/EpmHjISsopEz5Av44CBq7rPC05HHNbYGKVyNUF2zmEoBS/TT0pd0SPFFyw==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -2271,30 +2313,35 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/codegen": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/eventemitter": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/fetch": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "@protobufjs/aspromise": "^1.1.1" @@ -2304,36 +2351,42 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/inquire": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@protobufjs/utf8": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "dev": true, "license": "BSD-3-Clause" }, "node_modules/@redocly/ajv": { "version": "8.18.3", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.18.3.tgz", "integrity": "sha512-l42u0of3hY98sN2A+M4qTX1O/KrpgGH32Hu9kP2GtHyD5Dfqq86PKFLe5dwaD8DEnNmlOlll2BAmeEtf0DaySg==", + "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -2350,6 +2403,7 @@ "version": "2.28.1", "resolved": "https://registry.npmjs.org/@redocly/cli/-/cli-2.28.1.tgz", "integrity": "sha512-gDi0+vC905YHrtGD3WCP86gW44JdXQb0fu4128tVFpgfh5T65tFZkhs2xoPqLJn7RCtkndQ+rSwyXcALKoao0A==", + "dev": true, "license": "MIT", "dependencies": { "@opentelemetry/exporter-trace-otlp-http": "0.202.0", @@ -2395,6 +2449,7 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/@redocly/cli-otel/-/cli-otel-0.1.2.tgz", "integrity": "sha512-Bg7BoO5t1x3lVK+KhA5aGPmeXpQmdf6WtTYHhelKJCsQ+tRMiJoFAQoKHoBHAoNxXrhlS3K9lKFLHGmtxsFQfA==", + "dev": true, "license": "MIT", "dependencies": { "ulid": "^2.3.0" @@ -2404,6 +2459,7 @@ "version": "2.4.0", "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.4.0.tgz", "integrity": "sha512-fIRiVTJNcSRmXKPZtGzFQv9WRrZ3M9eoptl/teFJvjOzmpU+/K/JH6HZ8deBfb5vMEpicJcLn7JmvdknlMq7Zg==", + "dev": true, "license": "MIT", "bin": { "ulid": "bin/cli.js" @@ -2413,6 +2469,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -2425,6 +2482,7 @@ "version": "0.48.0", "resolved": "https://registry.npmjs.org/@redocly/config/-/config-0.48.0.tgz", "integrity": "sha512-8W3wz+Q7y4e9klJWlYOvQWK5r7P2Mo589vcjtlT5coOxsyAdt53k8Vb8iAqnRiGWExbjBQmSbL2XbuU747Nf6Q==", + "dev": true, "license": "MIT", "dependencies": { "json-schema-to-ts": "2.7.2" @@ -2434,6 +2492,7 @@ "version": "2.28.1", "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-2.28.1.tgz", "integrity": "sha512-PXulQY+lUJzeLWfhtJ8UPBFaMvlPDvW/dkozDhUAlYDotEYNMOaKFbJxKcrPCtRYtZ0TJsh5MohdcDLCBAJbFg==", + "dev": true, "license": "MIT", "dependencies": { "@redocly/ajv": "^8.18.0", @@ -2456,6 +2515,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -2468,6 +2528,7 @@ "version": "2.28.1", "resolved": "https://registry.npmjs.org/@redocly/respect-core/-/respect-core-2.28.1.tgz", "integrity": "sha512-r8sf7damvSviJwVif4hZVP/Qw7ciLgwLvHVy9AsUWxWh6JQtTZpV2/lJB681bqjn+GM9EMzhcNL1rBUo4K6Uyg==", + "dev": true, "license": "MIT", "dependencies": { "@faker-js/faker": "^7.6.0", @@ -2492,12 +2553,14 @@ "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, "license": "MIT" }, "node_modules/@redocly/respect-core/node_modules/picomatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -3314,6 +3377,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@turf/area/-/area-7.2.0.tgz", "integrity": "sha512-zuTTdQ4eoTI9nSSjerIy4QwgvxqwJVciQJ8tOPuMHbXJ9N/dNjI7bU8tasjhxas/Cx3NE9NxVHtNpYHL0FSzoA==", + "peer": true, "dependencies": { "@turf/helpers": "^7.2.0", "@turf/meta": "^7.2.0", @@ -3328,6 +3392,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@turf/bbox/-/bbox-7.2.0.tgz", "integrity": "sha512-wzHEjCXlYZiDludDbXkpBSmv8Zu6tPGLmJ1sXQ6qDwpLE1Ew3mcWqt8AaxfTP5QwDNQa3sf2vvgTEzNbPQkCiA==", + "peer": true, "dependencies": { "@turf/helpers": "^7.2.0", "@turf/meta": "^7.2.0", @@ -3342,6 +3407,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@turf/centroid/-/centroid-7.2.0.tgz", "integrity": "sha512-yJqDSw25T7P48au5KjvYqbDVZ7qVnipziVfZ9aSo7P2/jTE7d4BP21w0/XLi3T/9bry/t9PR1GDDDQljN4KfDw==", + "peer": true, "dependencies": { "@turf/helpers": "^7.2.0", "@turf/meta": "^7.2.0", @@ -3356,6 +3422,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-7.2.0.tgz", "integrity": "sha512-cXo7bKNZoa7aC7ydLmUR02oB3IgDe7MxiPuRz3cCtYQHn+BJ6h1tihmamYDWWUlPHgSNF0i3ATc4WmDECZafKw==", + "peer": true, "dependencies": { "@types/geojson": "^7946.0.10", "tslib": "^2.8.1" @@ -3368,6 +3435,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/@turf/meta/-/meta-7.2.0.tgz", "integrity": "sha512-igzTdHsQc8TV1RhPuOLVo74Px/hyPrVgVOTgjWQZzt3J9BVseCdpfY/0cJBdlSRI4S/yTmmHl7gAqjhpYH5Yaw==", + "peer": true, "dependencies": { "@turf/helpers": "^7.2.0", "@types/geojson": "^7946.0.10" @@ -3385,12 +3453,14 @@ "node_modules/@types/geojson": { "version": "7946.0.16", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", - "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==" + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "peer": true }, "node_modules/@types/geojson-vt": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/@types/geojson-vt/-/geojson-vt-3.2.5.tgz", "integrity": "sha512-qDO7wqtprzlpe8FfQ//ClPV9xiuoh2nkIgiouIptON9w5jvD/fA4szvP9GBlDVdJ5dldAl0kX/sy3URbWwLx0g==", + "peer": true, "dependencies": { "@types/geojson": "*" } @@ -3398,7 +3468,8 @@ "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true }, "node_modules/@types/less": { "version": "3.0.8", @@ -3415,12 +3486,14 @@ "node_modules/@types/mapbox__point-geometry": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/@types/mapbox__point-geometry/-/mapbox__point-geometry-0.1.4.tgz", - "integrity": "sha512-mUWlSxAmYLfwnRBmgYV86tgYmMIICX4kza8YnE/eIlywGe2XoOxlpVnXWwir92xRLjwyarqwpu2EJKD2pk0IUA==" + "integrity": "sha512-mUWlSxAmYLfwnRBmgYV86tgYmMIICX4kza8YnE/eIlywGe2XoOxlpVnXWwir92xRLjwyarqwpu2EJKD2pk0IUA==", + "peer": true }, "node_modules/@types/mapbox__vector-tile": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/@types/mapbox__vector-tile/-/mapbox__vector-tile-1.3.4.tgz", "integrity": "sha512-bpd8dRn9pr6xKvuEBQup8pwQfD4VUyqO/2deGjfpe6AwC8YRlyEipvefyRJUSiCJTZuCb8Pl1ciVV5ekqJ96Bg==", + "peer": true, "dependencies": { "@types/geojson": "*", "@types/mapbox__point-geometry": "*", @@ -3444,7 +3517,8 @@ "node_modules/@types/pbf": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.5.tgz", - "integrity": "sha512-j3pOPiEcWZ34R6a6mN07mUkM4o4Lwf6hPNt8eilOeZhTFbxFXmKhvXl9Y28jotFPaI1bpPDJsbCprUoNke6OrA==" + "integrity": "sha512-j3pOPiEcWZ34R6a6mN07mUkM4o4Lwf6hPNt8eilOeZhTFbxFXmKhvXl9Y28jotFPaI1bpPDJsbCprUoNke6OrA==", + "peer": true }, "node_modules/@types/plotly.js": { "version": "2.35.5", @@ -3516,6 +3590,7 @@ "version": "7.1.3", "resolved": "https://registry.npmjs.org/@types/supercluster/-/supercluster-7.1.3.tgz", "integrity": "sha512-Z0pOY34GDFl3Q6hUFYf3HkTwKEE02e7QgtJppBt+beEAxnyOpJua+voGFvxINBHa06GwLFFym7gRPY2SiKIfIA==", + "peer": true, "dependencies": { "@types/geojson": "*" } @@ -3524,6 +3599,7 @@ "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "dev": true, "license": "MIT", "optional": true }, @@ -3547,6 +3623,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -3558,12 +3635,14 @@ "node_modules/abs-svg-path": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/abs-svg-path/-/abs-svg-path-0.1.1.tgz", - "integrity": "sha512-d8XPSGjfyzlXC3Xx891DJRyZfqk5JU0BJrDQcsWomFIV1/BIzPW5HDH5iDdWpqWaav0YVIEzT1RHTwWr0FFshA==" + "integrity": "sha512-d8XPSGjfyzlXC3Xx891DJRyZfqk5JU0BJrDQcsWomFIV1/BIzPW5HDH5iDdWpqWaav0YVIEzT1RHTwWr0FFshA==", + "peer": true }, "node_modules/acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3575,6 +3654,7 @@ "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -3585,6 +3665,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-F+LMD2IDIXuHxgpLJh3nkLj9+tSaEzoUWd+7fONGq5pe2169FUDjpEkOfEpoGLz1sbZni/69p07OsecNfAOpqA==", + "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -3601,6 +3682,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, "license": "MIT", "dependencies": { "ajv": "^8.0.0" @@ -3617,12 +3699,14 @@ "node_modules/almost-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/almost-equal/-/almost-equal-1.1.0.tgz", - "integrity": "sha512-0V/PkoculFl5+0Lp47JoxUcO0xSxhIBvm+BxHdD/OgXNmdRpRHCFnKVuUoWyS9EzQP+otSGv0m9Lb4yVkQBn2A==" + "integrity": "sha512-0V/PkoculFl5+0Lp47JoxUcO0xSxhIBvm+BxHdD/OgXNmdRpRHCFnKVuUoWyS9EzQP+otSGv0m9Lb4yVkQBn2A==", + "peer": true }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, "engines": { "node": ">=8" } @@ -3631,6 +3715,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -3649,12 +3734,14 @@ "node_modules/array-bounds": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/array-bounds/-/array-bounds-1.0.1.tgz", - "integrity": "sha512-8wdW3ZGk6UjMPJx/glyEt0sLzzwAE1bhToPsO1W2pbpR2gULyxe3BjSiuJFheP50T/GgODVPz2fuMUmIywt8cQ==" + "integrity": "sha512-8wdW3ZGk6UjMPJx/glyEt0sLzzwAE1bhToPsO1W2pbpR2gULyxe3BjSiuJFheP50T/GgODVPz2fuMUmIywt8cQ==", + "peer": true }, "node_modules/array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", "integrity": "sha512-M1HQyIXcBGtVywBt8WVdim+lrNaK7VHp99Qt5pSNziXznKHViIBbXWtfRTpEFpF/c4FdfxNAsCCwPp5phBYJtw==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -3663,6 +3750,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/array-normalize/-/array-normalize-1.1.4.tgz", "integrity": "sha512-fCp0wKFLjvSPmCn4F5Tiw4M3lpMZoHlCjfcs7nNzuj3vqQQ1/a8cgB9DXcpDSn18c+coLnaW7rqfcYCvKbyJXg==", + "peer": true, "dependencies": { "array-bounds": "^1.0.0" } @@ -3670,12 +3758,14 @@ "node_modules/array-range": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/array-range/-/array-range-1.0.1.tgz", - "integrity": "sha512-shdaI1zT3CVNL2hnx9c0JMc0ZogGaxDs5e85akgHWKYa0yVbIyp06Ind3dVkTj/uuFrzaHBOyqFzo+VV6aXgtA==" + "integrity": "sha512-shdaI1zT3CVNL2hnx9c0JMc0ZogGaxDs5e85akgHWKYa0yVbIyp06Ind3dVkTj/uuFrzaHBOyqFzo+VV6aXgtA==", + "peer": true }, "node_modules/array-rearrange": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/array-rearrange/-/array-rearrange-2.2.2.tgz", - "integrity": "sha512-UfobP5N12Qm4Qu4fwLDIi2v6+wZsSf6snYSxAMeKhrh37YGnNWZPRmVEKc/2wfms53TLQnzfpG8wCx2Y/6NG1w==" + "integrity": "sha512-UfobP5N12Qm4Qu4fwLDIi2v6+wZsSf6snYSxAMeKhrh37YGnNWZPRmVEKc/2wfms53TLQnzfpG8wCx2Y/6NG1w==", + "peer": true }, "node_modules/babel-plugin-macros": { "version": "3.1.0", @@ -3695,6 +3785,7 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/babel-plugin-styled-components/-/babel-plugin-styled-components-2.1.4.tgz", "integrity": "sha512-Xgp9g+A/cG47sUyRwwYxGM4bR/jDRg5N6it/8+HxCnbT5XNKSKDT9xm4oag/osgqjC2It/vH0yXsomOG6k558g==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-annotate-as-pure": "^7.22.5", @@ -3710,12 +3801,14 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true }, "node_modules/base64-arraybuffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==", + "peer": true, "engines": { "node": ">= 0.6.0" } @@ -3733,6 +3826,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/better-ajv-errors/-/better-ajv-errors-1.2.0.tgz", "integrity": "sha512-UW+IsFycygIo7bclP9h5ugkNH8EjCSgqyFB/yQ4Hqqa1OEYDtb0uFIkYE0b6+CjkgJYVM5UKI/pJPxjYe9EZlA==", + "dev": true, "license": "Apache-2.0", "dependencies": { "@babel/code-frame": "^7.16.0", @@ -3751,22 +3845,26 @@ "node_modules/binary-search-bounds": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.5.tgz", - "integrity": "sha512-H0ea4Fd3lS1+sTEB2TgcLoK21lLhwEJzlQv3IN47pJS976Gx4zoWe0ak3q+uYh60ppQxg9F16Ri4tS1sfD4+jA==" + "integrity": "sha512-H0ea4Fd3lS1+sTEB2TgcLoK21lLhwEJzlQv3IN47pJS976Gx4zoWe0ak3q+uYh60ppQxg9F16Ri4tS1sfD4+jA==", + "peer": true }, "node_modules/bit-twiddle": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bit-twiddle/-/bit-twiddle-1.0.2.tgz", - "integrity": "sha512-B9UhK0DKFZhoTFcfvAzhqsjStvGJp9vYWf3+6SNTtdSQnvIgfkHbgHrg/e4+TH71N2GDu8tpmCVoyfrL1d7ntA==" + "integrity": "sha512-B9UhK0DKFZhoTFcfvAzhqsjStvGJp9vYWf3+6SNTtdSQnvIgfkHbgHrg/e4+TH71N2GDu8tpmCVoyfrL1d7ntA==", + "peer": true }, "node_modules/bitmap-sdf": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/bitmap-sdf/-/bitmap-sdf-1.0.4.tgz", - "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==" + "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==", + "peer": true }, "node_modules/bl": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", + "peer": true, "dependencies": { "readable-stream": "^2.3.5", "safe-buffer": "^5.1.1" @@ -3775,12 +3873,14 @@ "node_modules/bl/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "peer": true }, "node_modules/bl/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -3794,12 +3894,14 @@ "node_modules/bl/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "peer": true }, "node_modules/bl/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -3808,6 +3910,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -3862,12 +3965,14 @@ "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "peer": true }, "node_modules/call-me-maybe": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.2.tgz", - "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==" + "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==", + "dev": true }, "node_modules/callsites": { "version": "3.1.0", @@ -3892,6 +3997,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.1.tgz", "integrity": "sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -3921,6 +4027,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/canvas-fit/-/canvas-fit-1.5.0.tgz", "integrity": "sha512-onIcjRpz69/Hx5bB5HGbYKUF2uC6QT6Gp+pfpGm3A7mPfcluSLV5v4Zu+oflDUwLdUw0rLIBhUbi0v8hM4FJQQ==", + "peer": true, "dependencies": { "element-size": "^1.1.1" } @@ -3929,6 +4036,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -3944,18 +4052,21 @@ "node_modules/clamp": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/clamp/-/clamp-1.0.1.tgz", - "integrity": "sha512-kgMuFyE78OC6Dyu3Dy7vcx4uy97EIbVxJB/B0eJ3bUNAkwdNcxYzgKltnyADiYwsR7SEqkkUPsEUT//OVS6XMA==" + "integrity": "sha512-kgMuFyE78OC6Dyu3Dy7vcx4uy97EIbVxJB/B0eJ3bUNAkwdNcxYzgKltnyADiYwsR7SEqkkUPsEUT//OVS6XMA==", + "peer": true }, "node_modules/classnames": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==", + "dev": true, "license": "MIT" }, "node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, "license": "ISC", "dependencies": { "string-width": "^4.2.0", @@ -3975,6 +4086,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/color-alpha/-/color-alpha-1.0.4.tgz", "integrity": "sha512-lr8/t5NPozTSqli+duAN+x+no/2WaKTeWvxhHGN+aXT6AJ8vPlzLa7UriyjWak0pSC2jHol9JgjBYnnHsGha9A==", + "peer": true, "dependencies": { "color-parse": "^1.3.8" } @@ -3983,6 +4095,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/color-parse/-/color-parse-1.4.3.tgz", "integrity": "sha512-BADfVl/FHkQkyo8sRBwMYBqemqsgnu7JZAwUgvBvuwwuNUZAhSvLTbsEErS5bQXzOjDR0dWzJ4vXN2Q+QoPx0A==", + "peer": true, "dependencies": { "color-name": "^1.0.0" } @@ -3991,6 +4104,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -4002,6 +4116,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/color-id/-/color-id-1.1.0.tgz", "integrity": "sha512-2iRtAn6dC/6/G7bBIo0uupVrIne1NsQJvJxZOBCzQOfk7jRq97feaDZ3RdzuHakRXXnHGNwglto3pqtRx1sX0g==", + "peer": true, "dependencies": { "clamp": "^1.0.1" } @@ -4015,6 +4130,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/color-normalize/-/color-normalize-1.5.0.tgz", "integrity": "sha512-rUT/HDXMr6RFffrR53oX3HGWkDOP9goSAQGBkUaAYKjOE2JxozccdGyufageWDlInRAjm/jYPrf/Y38oa+7obw==", + "peer": true, "dependencies": { "clamp": "^1.0.1", "color-rgba": "^2.1.1", @@ -4025,6 +4141,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/color-parse/-/color-parse-2.0.0.tgz", "integrity": "sha512-g2Z+QnWsdHLppAbrpcFWo629kLOnOPtpxYV69GCqm92gqSgyXbzlfyN3MXs0412fPBkFmiuS+rXposgBgBa6Kg==", + "peer": true, "dependencies": { "color-name": "^1.0.0" } @@ -4033,6 +4150,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/color-rgba/-/color-rgba-2.1.1.tgz", "integrity": "sha512-VaX97wsqrMwLSOR6H7rU1Doa2zyVdmShabKrPEIFywLlHoibgD3QW9Dw6fSqM4+H/LfjprDNAUUW31qEQcGzNw==", + "peer": true, "dependencies": { "clamp": "^1.0.1", "color-parse": "^1.3.8", @@ -4043,6 +4161,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/color-parse/-/color-parse-1.4.3.tgz", "integrity": "sha512-BADfVl/FHkQkyo8sRBwMYBqemqsgnu7JZAwUgvBvuwwuNUZAhSvLTbsEErS5bQXzOjDR0dWzJ4vXN2Q+QoPx0A==", + "peer": true, "dependencies": { "color-name": "^1.0.0" } @@ -4051,6 +4170,7 @@ "version": "1.16.0", "resolved": "https://registry.npmjs.org/color-space/-/color-space-1.16.0.tgz", "integrity": "sha512-A6WMiFzunQ8KEPFmj02OnnoUnqhmSaHaZ/0LVFcPTdlvm8+3aMJ5x1HRHy3bDHPkovkf4sS0f4wsVvwk71fKkg==", + "peer": true, "dependencies": { "hsluv": "^0.0.3", "mumath": "^3.3.4" @@ -4060,6 +4180,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==", + "dev": true, "license": "MIT" }, "node_modules/commander": { @@ -4080,6 +4201,7 @@ "version": "0.7.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -4089,6 +4211,7 @@ "version": "3.48.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.48.0.tgz", "integrity": "sha512-zpEHTy1fjTMZCKLHUZoVeylt9XrzaIN2rbPXEt0k+q7JE5CkCZdo6bNq55bn24a69CH7ErAVLKijxJja4fw+UQ==", + "dev": true, "hasInstallScript": true, "license": "MIT", "peer": true, @@ -4100,7 +4223,8 @@ "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "peer": true }, "node_modules/cosmiconfig": { "version": "7.1.0", @@ -4129,7 +4253,8 @@ "node_modules/country-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/country-regex/-/country-regex-1.1.0.tgz", - "integrity": "sha512-iSPlClZP8vX7MC3/u6s3lrDuoQyhQukh5LyABJ3hvfzbQ3Yyayd4fp04zjLnfi267B/B2FkumcWWgrbban7sSA==" + "integrity": "sha512-iSPlClZP8vX7MC3/u6s3lrDuoQyhQukh5LyABJ3hvfzbQ3Yyayd4fp04zjLnfi267B/B2FkumcWWgrbban7sSA==", + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -4170,6 +4295,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-color-keywords/-/css-color-keywords-1.0.0.tgz", "integrity": "sha512-FyyrDHZKEjXDpNJYvVsV960FiqQyXc/LlYmsxl2BcdMb2WPx0OGRVgTg55rPSyLSNMqP52R9r8geSp7apN3Ofg==", + "dev": true, "license": "ISC", "engines": { "node": ">=4" @@ -4179,6 +4305,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-font/-/css-font-1.2.0.tgz", "integrity": "sha512-V4U4Wps4dPDACJ4WpgofJ2RT5Yqwe1lEH6wlOOaIxMi0gTjdIijsc5FmxQlZ7ZZyKQkkutqqvULOp07l9c7ssA==", + "peer": true, "dependencies": { "css-font-size-keywords": "^1.0.0", "css-font-stretch-keywords": "^1.0.1", @@ -4194,37 +4321,44 @@ "node_modules/css-font-size-keywords": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-font-size-keywords/-/css-font-size-keywords-1.0.0.tgz", - "integrity": "sha512-Q+svMDbMlelgCfH/RVDKtTDaf5021O486ZThQPIpahnIjUkMUslC+WuOQSWTgGSrNCH08Y7tYNEmmy0hkfMI8Q==" + "integrity": "sha512-Q+svMDbMlelgCfH/RVDKtTDaf5021O486ZThQPIpahnIjUkMUslC+WuOQSWTgGSrNCH08Y7tYNEmmy0hkfMI8Q==", + "peer": true }, "node_modules/css-font-stretch-keywords": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/css-font-stretch-keywords/-/css-font-stretch-keywords-1.0.1.tgz", - "integrity": "sha512-KmugPO2BNqoyp9zmBIUGwt58UQSfyk1X5DbOlkb2pckDXFSAfjsD5wenb88fNrD6fvS+vu90a/tsPpb9vb0SLg==" + "integrity": "sha512-KmugPO2BNqoyp9zmBIUGwt58UQSfyk1X5DbOlkb2pckDXFSAfjsD5wenb88fNrD6fvS+vu90a/tsPpb9vb0SLg==", + "peer": true }, "node_modules/css-font-style-keywords": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/css-font-style-keywords/-/css-font-style-keywords-1.0.1.tgz", - "integrity": "sha512-0Fn0aTpcDktnR1RzaBYorIxQily85M2KXRpzmxQPgh8pxUN9Fcn00I8u9I3grNr1QXVgCl9T5Imx0ZwKU973Vg==" + "integrity": "sha512-0Fn0aTpcDktnR1RzaBYorIxQily85M2KXRpzmxQPgh8pxUN9Fcn00I8u9I3grNr1QXVgCl9T5Imx0ZwKU973Vg==", + "peer": true }, "node_modules/css-font-weight-keywords": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-font-weight-keywords/-/css-font-weight-keywords-1.0.0.tgz", - "integrity": "sha512-5So8/NH+oDD+EzsnF4iaG4ZFHQ3vaViePkL1ZbZ5iC/KrsCY+WHq/lvOgrtmuOQ9pBBZ1ADGpaf+A4lj1Z9eYA==" + "integrity": "sha512-5So8/NH+oDD+EzsnF4iaG4ZFHQ3vaViePkL1ZbZ5iC/KrsCY+WHq/lvOgrtmuOQ9pBBZ1ADGpaf+A4lj1Z9eYA==", + "peer": true }, "node_modules/css-global-keywords": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/css-global-keywords/-/css-global-keywords-1.0.1.tgz", - "integrity": "sha512-X1xgQhkZ9n94WDwntqst5D/FKkmiU0GlJSFZSV3kLvyJ1WC5VeyoXDOuleUD+SIuH9C7W05is++0Woh0CGfKjQ==" + "integrity": "sha512-X1xgQhkZ9n94WDwntqst5D/FKkmiU0GlJSFZSV3kLvyJ1WC5VeyoXDOuleUD+SIuH9C7W05is++0Woh0CGfKjQ==", + "peer": true }, "node_modules/css-system-font-keywords": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-system-font-keywords/-/css-system-font-keywords-1.0.0.tgz", - "integrity": "sha512-1umTtVd/fXS25ftfjB71eASCrYhilmEsvDEI6wG/QplnmlfmVM5HkZ/ZX46DT5K3eblFPgLUHt5BRCb0YXkSFA==" + "integrity": "sha512-1umTtVd/fXS25ftfjB71eASCrYhilmEsvDEI6wG/QplnmlfmVM5HkZ/ZX46DT5K3eblFPgLUHt5BRCb0YXkSFA==", + "peer": true }, "node_modules/css-to-react-native": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/css-to-react-native/-/css-to-react-native-3.2.0.tgz", "integrity": "sha512-e8RKaLXMOFii+02mOlqwjbD00KSEKqblnpO9e++1aXS1fPQOpS1YoqdVHBqPjHNoxeF2mimzVqawm2KCbEdtHQ==", + "dev": true, "license": "MIT", "dependencies": { "camelize": "^1.0.0", @@ -4235,7 +4369,8 @@ "node_modules/csscolorparser": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/csscolorparser/-/csscolorparser-1.0.3.tgz", - "integrity": "sha512-umPSgYwZkdFoUrH5hIq5kf0wPSXiro51nPw0j2K/c83KflkPSTBGMz6NJvMB+07VlL0y7VPo6QJcDjcgKTTm3w==" + "integrity": "sha512-umPSgYwZkdFoUrH5hIq5kf0wPSXiro51nPw0j2K/c83KflkPSTBGMz6NJvMB+07VlL0y7VPo6QJcDjcgKTTm3w==", + "peer": true }, "node_modules/cssesc": { "version": "3.0.0", @@ -4259,6 +4394,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/d/-/d-1.0.2.tgz", "integrity": "sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw==", + "peer": true, "dependencies": { "es5-ext": "^0.10.64", "type": "^2.7.2" @@ -4270,17 +4406,20 @@ "node_modules/d3-array": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-1.2.4.tgz", - "integrity": "sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==" + "integrity": "sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==", + "peer": true }, "node_modules/d3-collection": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/d3-collection/-/d3-collection-1.0.7.tgz", - "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==" + "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==", + "peer": true }, "node_modules/d3-color": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "peer": true, "engines": { "node": ">=12" } @@ -4288,12 +4427,14 @@ "node_modules/d3-dispatch": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-1.0.6.tgz", - "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==" + "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==", + "peer": true }, "node_modules/d3-force": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-1.2.1.tgz", "integrity": "sha512-HHvehyaiUlVo5CxBJ0yF/xny4xoaxFxDnBXNvNcfW9adORGZfyNF1dj6DGLKyk4Yh3brP/1h3rnDzdIAwL08zg==", + "peer": true, "dependencies": { "d3-collection": "1", "d3-dispatch": "1", @@ -4304,12 +4445,14 @@ "node_modules/d3-format": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-1.4.5.tgz", - "integrity": "sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==" + "integrity": "sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==", + "peer": true }, "node_modules/d3-geo": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-1.12.1.tgz", "integrity": "sha512-XG4d1c/UJSEX9NfU02KwBL6BYPj8YKHxgBEw5om2ZnTRSbIcego6dhHwcxuSR3clxh0EpE38os1DVPOmnYtTPg==", + "peer": true, "dependencies": { "d3-array": "1" } @@ -4318,6 +4461,7 @@ "version": "2.9.0", "resolved": "https://registry.npmjs.org/d3-geo-projection/-/d3-geo-projection-2.9.0.tgz", "integrity": "sha512-ZULvK/zBn87of5rWAfFMc9mJOipeSo57O+BBitsKIXmU4rTVAnX1kSsJkE0R+TxY8pGNoM1nbyRRE7GYHhdOEQ==", + "peer": true, "dependencies": { "commander": "2", "d3-array": "1", @@ -4335,17 +4479,20 @@ "node_modules/d3-geo-projection/node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "peer": true }, "node_modules/d3-hierarchy": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz", - "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==" + "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==", + "peer": true }, "node_modules/d3-interpolate": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "peer": true, "dependencies": { "d3-color": "1 - 3" }, @@ -4356,17 +4503,20 @@ "node_modules/d3-path": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", - "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "peer": true }, "node_modules/d3-quadtree": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-1.0.7.tgz", - "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==" + "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==", + "peer": true }, "node_modules/d3-shape": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "peer": true, "dependencies": { "d3-path": "1" } @@ -4374,12 +4524,14 @@ "node_modules/d3-time": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-1.1.0.tgz", - "integrity": "sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==" + "integrity": "sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==", + "peer": true }, "node_modules/d3-time-format": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-2.3.0.tgz", "integrity": "sha512-guv6b2H37s2Uq/GefleCDtbe0XZAuy7Wa49VGkPVPMfLL9qObgBST3lEHJBMUp8S7NdLQAGIvr2KXk8Hc98iKQ==", + "peer": true, "dependencies": { "d3-time": "1" } @@ -4387,7 +4539,8 @@ "node_modules/d3-timer": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-1.0.10.tgz", - "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==" + "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==", + "peer": true }, "node_modules/debug": { "version": "4.4.0", @@ -4408,12 +4561,14 @@ "node_modules/decko": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decko/-/decko-1.2.0.tgz", - "integrity": "sha512-m8FnyHXV1QX+S1cl+KPFDIl6NMkxtKsy6+U/aYyjrOqWMuwAwYWu7ePqrsUHtDR5Y8Yk2pi/KIDSgF+vT4cPOQ==" + "integrity": "sha512-m8FnyHXV1QX+S1cl+KPFDIl6NMkxtKsy6+U/aYyjrOqWMuwAwYWu7ePqrsUHtDR5Y8Yk2pi/KIDSgF+vT4cPOQ==", + "dev": true }, "node_modules/defined": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.1.tgz", "integrity": "sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==", + "peer": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -4421,7 +4576,8 @@ "node_modules/detect-kerning": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-kerning/-/detect-kerning-2.1.2.tgz", - "integrity": "sha512-I3JIbrnKPAntNLl1I6TpSQQdQ4AutYzv/sKMFKbepawV/hlH0GmYKhUoOEMd4xqaUHT+Bm0f4127lh5qs1m1tw==" + "integrity": "sha512-I3JIbrnKPAntNLl1I6TpSQQdQ4AutYzv/sKMFKbepawV/hlH0GmYKhUoOEMd4xqaUHT+Bm0f4127lh5qs1m1tw==", + "peer": true }, "node_modules/detect-libc": { "version": "1.0.3", @@ -4450,6 +4606,7 @@ "version": "3.4.0", "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.0.tgz", "integrity": "sha512-nolgK9JcaUXMSmW+j1yaSvaEaoXYHwWyGJlkoCTghc97KgGDDSnpoU/PlEnw63Ah+TGKFOyY+X5LnxaWbCSfXg==", + "dev": true, "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -4468,6 +4625,7 @@ "version": "16.4.7", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -4480,6 +4638,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/draw-svg-path/-/draw-svg-path-1.0.0.tgz", "integrity": "sha512-P8j3IHxcgRMcY6sDzr0QvJDLzBnJJqpTG33UZ2Pvp8rw0apCHhJCWqYprqrXjrgHnJ6tuhP1iTJSAodPDHxwkg==", + "peer": true, "dependencies": { "abs-svg-path": "~0.1.1", "normalize-svg-path": "~0.1.0" @@ -4489,6 +4648,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/dtype/-/dtype-2.0.0.tgz", "integrity": "sha512-s2YVcLKdFGS0hpFqJaTwscsyt0E8nNFdmo73Ocd81xNPj4URI4rj6D60A+vFMIw7BXWlb4yRkEwfBqcZzPGiZg==", + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -4496,12 +4656,14 @@ "node_modules/dup": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dup/-/dup-1.0.0.tgz", - "integrity": "sha512-Bz5jxMMC0wgp23Zm15ip1x8IhYRqJvF3nFC0UInJUDkN1z4uNPk9jTnfCUJXbOGiQ1JbXLQsiV41Fb+HXcj5BA==" + "integrity": "sha512-Bz5jxMMC0wgp23Zm15ip1x8IhYRqJvF3nFC0UInJUDkN1z4uNPk9jTnfCUJXbOGiQ1JbXLQsiV41Fb+HXcj5BA==", + "peer": true }, "node_modules/duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "peer": true, "dependencies": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -4512,12 +4674,14 @@ "node_modules/duplexify/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "peer": true }, "node_modules/duplexify/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -4531,12 +4695,14 @@ "node_modules/duplexify/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "peer": true }, "node_modules/duplexify/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -4544,7 +4710,8 @@ "node_modules/earcut": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", - "integrity": "sha512-/pjZsA1b4RPHbeWZQn66SWS8nZZWLQQ23oE3Eam7aroEFGEvwKAsJfZ9ytiEMycfzXWpca4FA9QIOehf7PocBQ==" + "integrity": "sha512-/pjZsA1b4RPHbeWZQn66SWS8nZZWLQQ23oE3Eam7aroEFGEvwKAsJfZ9ytiEMycfzXWpca4FA9QIOehf7PocBQ==", + "peer": true }, "node_modules/eastasianwidth": { "version": "0.2.0", @@ -4561,12 +4728,14 @@ "node_modules/element-size": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/element-size/-/element-size-1.1.1.tgz", - "integrity": "sha512-eaN+GMOq/Q+BIWy0ybsgpcYImjGIdNLyjLFJU4XsLHXYQao5jCNb36GyN6C2qwmDDYSfIBmKpPpr4VnBdLCsPQ==" + "integrity": "sha512-eaN+GMOq/Q+BIWy0ybsgpcYImjGIdNLyjLFJU4XsLHXYQao5jCNb36GyN6C2qwmDDYSfIBmKpPpr4VnBdLCsPQ==", + "peer": true }, "node_modules/elementary-circuits-directed-graph": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/elementary-circuits-directed-graph/-/elementary-circuits-directed-graph-1.3.1.tgz", "integrity": "sha512-ZEiB5qkn2adYmpXGnJKkxT8uJHlW/mxmBpmeqawEHzPxh9HkLD4/1mFYX5l0On+f6rcPIt8/EWlRU2Vo3fX6dQ==", + "peer": true, "dependencies": { "strongly-connected-components": "^1.0.1" } @@ -4574,12 +4743,14 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "peer": true, "dependencies": { "once": "^1.4.0" } @@ -4608,6 +4779,7 @@ "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz", "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==", "hasInstallScript": true, + "peer": true, "dependencies": { "es6-iterator": "^2.0.3", "es6-symbol": "^3.1.3", @@ -4622,6 +4794,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "peer": true, "dependencies": { "d": "1", "es5-ext": "^0.10.35", @@ -4632,12 +4805,14 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz", "integrity": "sha512-SOp9Phqvqn7jtEUxPWdWfWoLmyt2VaJ6MpvP9Comy1MceMXqE6bxvaTu4iaxpYYPzhny28Lc+M87/c2cPK6lDg==", + "dev": true, "license": "MIT" }, "node_modules/es6-symbol": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.4.tgz", "integrity": "sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg==", + "peer": true, "dependencies": { "d": "^1.0.2", "ext": "^1.7.0" @@ -4650,6 +4825,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", + "peer": true, "dependencies": { "d": "1", "es5-ext": "^0.10.46", @@ -4772,6 +4948,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "peer": true, "dependencies": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -4793,6 +4970,7 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -4801,6 +4979,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz", "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==", + "peer": true, "dependencies": { "d": "^1.0.1", "es5-ext": "^0.10.62", @@ -4815,6 +4994,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "peer": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -4827,6 +5007,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "peer": true, "engines": { "node": ">=4.0" } @@ -4841,6 +5022,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -4849,6 +5031,7 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "peer": true, "dependencies": { "d": "1", "es5-ext": "~0.10.14" @@ -4858,6 +5041,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4867,12 +5051,14 @@ "version": "5.0.4", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "dev": true, "license": "MIT" }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "peer": true, "engines": { "node": ">=0.8.x" } @@ -4881,6 +5067,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "peer": true, "dependencies": { "type": "^2.7.2" } @@ -4889,6 +5076,7 @@ "version": "2.2.5", "resolved": "https://registry.npmjs.org/falafel/-/falafel-2.2.5.tgz", "integrity": "sha512-HuC1qF9iTnHDnML9YZAdCDQwT0yKl/U55K4XSUXqGAA2GLoafFgWRqdAbhWJxXaYD4pyoVxAJ8wH670jMpI9DQ==", + "peer": true, "dependencies": { "acorn": "^7.1.1", "isarray": "^2.0.1" @@ -4901,12 +5089,14 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, "license": "MIT" }, "node_modules/fast-isnumeric": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/fast-isnumeric/-/fast-isnumeric-1.1.4.tgz", "integrity": "sha512-1mM8qOr2LYz8zGaUdmiqRDiuue00Dxjgcb1NQR7TnhLVh6sQyngP9xvLo7Sl7LZpP/sk5eb+bcyWXw530NTBZw==", + "peer": true, "dependencies": { "is-string-blank": "^1.0.1" } @@ -4915,12 +5105,14 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true, "license": "MIT" }, "node_modules/fast-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "dev": true, "funding": [ { "type": "github", @@ -4937,6 +5129,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "dev": true, "funding": [ { "type": "github", @@ -4953,6 +5146,7 @@ "version": "5.8.0", "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.8.0.tgz", "integrity": "sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==", + "dev": true, "funding": [ { "type": "github", @@ -4993,6 +5187,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/flatten-vertex-data/-/flatten-vertex-data-1.0.2.tgz", "integrity": "sha512-BvCBFK2NZqerFTdMDgqfHBwxYWnxeCkwONsw6PvBMcUXqo8U/KDWwmXhqx1x2kLIg7DqIsJfOaJFOmlua3Lxuw==", + "peer": true, "dependencies": { "dtype": "^2.0.0" } @@ -5001,6 +5196,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/font-atlas/-/font-atlas-2.1.0.tgz", "integrity": "sha512-kP3AmvX+HJpW4w3d+PiPR2X6E1yvsBXt2yhuCw+yReO9F1WYhvZwx3c95DGZGwg9xYzDGrgJYa885xmVA+28Cg==", + "peer": true, "dependencies": { "css-font": "^1.0.0" } @@ -5009,6 +5205,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/font-measure/-/font-measure-1.2.2.tgz", "integrity": "sha512-mRLEpdrWzKe9hbfaF3Qpr06TAjquuBVP5cHy4b3hyeNdjc9i0PO6HniGsX5vjL5OWv7+Bd++NiooNpT/s8BvIA==", + "peer": true, "dependencies": { "css-font": "^1.2.0" } @@ -5017,6 +5214,7 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.6.tgz", "integrity": "sha512-k6GAGDyqLe9JaebCsFCoudPPWfihKu8pylYXRlqP1J7ms39iPoTtk2fviNglIeQEwdh0bQeKJ01ZPyuyQvKzwg==", + "dev": true, "license": "MIT" }, "node_modules/foreground-child": { @@ -5039,6 +5237,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", + "peer": true, "dependencies": { "inherits": "^2.0.1", "readable-stream": "^2.0.0" @@ -5047,12 +5246,14 @@ "node_modules/from2/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "peer": true }, "node_modules/from2/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -5066,12 +5267,14 @@ "node_modules/from2/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "peer": true }, "node_modules/from2/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -5131,12 +5334,14 @@ "node_modules/geojson-vt": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/geojson-vt/-/geojson-vt-3.2.1.tgz", - "integrity": "sha512-EvGQQi/zPrDA6zr6BnJD/YhwAkBP8nnJ9emh3EnHQKVMfg/MRVtPbMYdgVy/IaEmn4UfagD2a6fafPDL5hbtwg==" + "integrity": "sha512-EvGQQi/zPrDA6zr6BnJD/YhwAkBP8nnJ9emh3EnHQKVMfg/MRVtPbMYdgVy/IaEmn4UfagD2a6fafPDL5hbtwg==", + "peer": true }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" @@ -5145,12 +5350,14 @@ "node_modules/get-canvas-context": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/get-canvas-context/-/get-canvas-context-1.0.2.tgz", - "integrity": "sha512-LnpfLf/TNzr9zVOGiIY6aKCz8EKuXmlYNV7CM2pUjBa/B+c2I15tS7KLySep75+FuerJdmArvJLcsAXWEy2H0A==" + "integrity": "sha512-LnpfLf/TNzr9zVOGiIY6aKCz8EKuXmlYNV7CM2pUjBa/B+c2I15tS7KLySep75+FuerJdmArvJLcsAXWEy2H0A==", + "peer": true }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "peer": true, "engines": { "node": ">=10" }, @@ -5161,17 +5368,20 @@ "node_modules/gl-mat4": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gl-mat4/-/gl-mat4-1.2.0.tgz", - "integrity": "sha512-sT5C0pwB1/e9G9AvAoLsoaJtbMGjfd/jfxo8jMCKqYYEnjZuFvqV5rehqar0538EmssjdDeiEWnKyBSTw7quoA==" + "integrity": "sha512-sT5C0pwB1/e9G9AvAoLsoaJtbMGjfd/jfxo8jMCKqYYEnjZuFvqV5rehqar0538EmssjdDeiEWnKyBSTw7quoA==", + "peer": true }, "node_modules/gl-matrix": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.3.tgz", - "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==" + "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==", + "peer": true }, "node_modules/gl-text": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/gl-text/-/gl-text-1.4.0.tgz", "integrity": "sha512-o47+XBqLCj1efmuNyCHt7/UEJmB9l66ql7pnobD6p+sgmBUdzfMZXIF0zD2+KRfpd99DJN+QXdvTFAGCKCVSmQ==", + "peer": true, "dependencies": { "bit-twiddle": "^1.0.2", "color-normalize": "^1.5.0", @@ -5196,6 +5406,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/gl-util/-/gl-util-3.1.3.tgz", "integrity": "sha512-dvRTggw5MSkJnCbh74jZzSoTOGnVYK+Bt+Ckqm39CVcl6+zSsxqWk4lr5NKhkqXHL6qvZAU9h17ZF8mIskY9mA==", + "peer": true, "dependencies": { "is-browser": "^2.0.1", "is-firefox": "^1.0.3", @@ -5210,6 +5421,7 @@ "version": "13.0.6", "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "minimatch": "^10.2.2", @@ -5227,6 +5439,7 @@ "version": "11.3.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -5236,6 +5449,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^11.0.0", @@ -5252,6 +5466,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-4.0.0.tgz", "integrity": "sha512-w0Uf9Y9/nyHinEk5vMJKRie+wa4kR5hmDbEhGGds/kG1PwGLLHKRoNMeJOyCQjjBkANlnScqgzcFwGHgmgLkVA==", + "peer": true, "dependencies": { "ini": "^4.1.3", "kind-of": "^6.0.3", @@ -5278,6 +5493,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/glsl-inject-defines/-/glsl-inject-defines-1.0.3.tgz", "integrity": "sha512-W49jIhuDtF6w+7wCMcClk27a2hq8znvHtlGnrYkSWEr8tHe9eA2dcnohlcAmxLYBSpSSdzOkRdyPTrx9fw49+A==", + "peer": true, "dependencies": { "glsl-token-inject-block": "^1.0.0", "glsl-token-string": "^1.0.1", @@ -5288,6 +5504,7 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/glsl-resolve/-/glsl-resolve-0.0.1.tgz", "integrity": "sha512-xxFNsfnhZTK9NBhzJjSBGX6IOqYpvBHxxmo+4vapiljyGNCY0Bekzn0firQkQrazK59c1hYxMDxYS8MDlhw4gA==", + "peer": true, "dependencies": { "resolve": "^0.6.1", "xtend": "^2.1.2" @@ -5296,12 +5513,14 @@ "node_modules/glsl-resolve/node_modules/resolve": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/resolve/-/resolve-0.6.3.tgz", - "integrity": "sha512-UHBY3viPlJKf85YijDUcikKX6tmF4SokIDp518ZDVT92JNDcG5uKIthaT/owt3Sar0lwtOafsQuwrg22/v2Dwg==" + "integrity": "sha512-UHBY3viPlJKf85YijDUcikKX6tmF4SokIDp518ZDVT92JNDcG5uKIthaT/owt3Sar0lwtOafsQuwrg22/v2Dwg==", + "peer": true }, "node_modules/glsl-resolve/node_modules/xtend": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xtend/-/xtend-2.2.0.tgz", "integrity": "sha512-SLt5uylT+4aoXxXuwtQp5ZnMMzhDb1Xkg4pEqc00WUJCQifPfV9Ub1VrNhp9kXkrjZD2I2Hl8WnjP37jzZLPZw==", + "peer": true, "engines": { "node": ">=0.4" } @@ -5309,12 +5528,14 @@ "node_modules/glsl-token-assignments": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/glsl-token-assignments/-/glsl-token-assignments-2.0.2.tgz", - "integrity": "sha512-OwXrxixCyHzzA0U2g4btSNAyB2Dx8XrztY5aVUCjRSh4/D0WoJn8Qdps7Xub3sz6zE73W3szLrmWtQ7QMpeHEQ==" + "integrity": "sha512-OwXrxixCyHzzA0U2g4btSNAyB2Dx8XrztY5aVUCjRSh4/D0WoJn8Qdps7Xub3sz6zE73W3szLrmWtQ7QMpeHEQ==", + "peer": true }, "node_modules/glsl-token-defines": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/glsl-token-defines/-/glsl-token-defines-1.0.0.tgz", "integrity": "sha512-Vb5QMVeLjmOwvvOJuPNg3vnRlffscq2/qvIuTpMzuO/7s5kT+63iL6Dfo2FYLWbzuiycWpbC0/KV0biqFwHxaQ==", + "peer": true, "dependencies": { "glsl-tokenizer": "^2.0.0" } @@ -5322,12 +5543,14 @@ "node_modules/glsl-token-depth": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/glsl-token-depth/-/glsl-token-depth-1.1.2.tgz", - "integrity": "sha512-eQnIBLc7vFf8axF9aoi/xW37LSWd2hCQr/3sZui8aBJnksq9C7zMeUYHVJWMhFzXrBU7fgIqni4EhXVW4/krpg==" + "integrity": "sha512-eQnIBLc7vFf8axF9aoi/xW37LSWd2hCQr/3sZui8aBJnksq9C7zMeUYHVJWMhFzXrBU7fgIqni4EhXVW4/krpg==", + "peer": true }, "node_modules/glsl-token-descope": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/glsl-token-descope/-/glsl-token-descope-1.0.2.tgz", "integrity": "sha512-kS2PTWkvi/YOeicVjXGgX5j7+8N7e56srNDEHDTVZ1dcESmbmpmgrnpjPcjxJjMxh56mSXYoFdZqb90gXkGjQw==", + "peer": true, "dependencies": { "glsl-token-assignments": "^2.0.0", "glsl-token-depth": "^1.1.0", @@ -5338,32 +5561,38 @@ "node_modules/glsl-token-inject-block": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/glsl-token-inject-block/-/glsl-token-inject-block-1.1.0.tgz", - "integrity": "sha512-q/m+ukdUBuHCOtLhSr0uFb/qYQr4/oKrPSdIK2C4TD+qLaJvqM9wfXIF/OOBjuSA3pUoYHurVRNao6LTVVUPWA==" + "integrity": "sha512-q/m+ukdUBuHCOtLhSr0uFb/qYQr4/oKrPSdIK2C4TD+qLaJvqM9wfXIF/OOBjuSA3pUoYHurVRNao6LTVVUPWA==", + "peer": true }, "node_modules/glsl-token-properties": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/glsl-token-properties/-/glsl-token-properties-1.0.1.tgz", - "integrity": "sha512-dSeW1cOIzbuUoYH0y+nxzwK9S9O3wsjttkq5ij9ZGw0OS41BirKJzzH48VLm8qLg+au6b0sINxGC0IrGwtQUcA==" + "integrity": "sha512-dSeW1cOIzbuUoYH0y+nxzwK9S9O3wsjttkq5ij9ZGw0OS41BirKJzzH48VLm8qLg+au6b0sINxGC0IrGwtQUcA==", + "peer": true }, "node_modules/glsl-token-scope": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/glsl-token-scope/-/glsl-token-scope-1.1.2.tgz", - "integrity": "sha512-YKyOMk1B/tz9BwYUdfDoHvMIYTGtVv2vbDSLh94PT4+f87z21FVdou1KNKgF+nECBTo0fJ20dpm0B1vZB1Q03A==" + "integrity": "sha512-YKyOMk1B/tz9BwYUdfDoHvMIYTGtVv2vbDSLh94PT4+f87z21FVdou1KNKgF+nECBTo0fJ20dpm0B1vZB1Q03A==", + "peer": true }, "node_modules/glsl-token-string": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/glsl-token-string/-/glsl-token-string-1.0.1.tgz", - "integrity": "sha512-1mtQ47Uxd47wrovl+T6RshKGkRRCYWhnELmkEcUAPALWGTFe2XZpH3r45XAwL2B6v+l0KNsCnoaZCSnhzKEksg==" + "integrity": "sha512-1mtQ47Uxd47wrovl+T6RshKGkRRCYWhnELmkEcUAPALWGTFe2XZpH3r45XAwL2B6v+l0KNsCnoaZCSnhzKEksg==", + "peer": true }, "node_modules/glsl-token-whitespace-trim": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/glsl-token-whitespace-trim/-/glsl-token-whitespace-trim-1.0.0.tgz", - "integrity": "sha512-ZJtsPut/aDaUdLUNtmBYhaCmhIjpKNg7IgZSfX5wFReMc2vnj8zok+gB/3Quqs0TsBSX/fGnqUUYZDqyuc2xLQ==" + "integrity": "sha512-ZJtsPut/aDaUdLUNtmBYhaCmhIjpKNg7IgZSfX5wFReMc2vnj8zok+gB/3Quqs0TsBSX/fGnqUUYZDqyuc2xLQ==", + "peer": true }, "node_modules/glsl-tokenizer": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/glsl-tokenizer/-/glsl-tokenizer-2.1.5.tgz", "integrity": "sha512-XSZEJ/i4dmz3Pmbnpsy3cKh7cotvFlBiZnDOwnj/05EwNp2XrhQ4XKJxT7/pDt4kp4YcpRSKz8eTV7S+mwV6MA==", + "peer": true, "dependencies": { "through2": "^0.6.3" } @@ -5371,12 +5600,14 @@ "node_modules/glsl-tokenizer/node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "peer": true }, "node_modules/glsl-tokenizer/node_modules/readable-stream": { "version": "1.0.34", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.1", @@ -5387,12 +5618,14 @@ "node_modules/glsl-tokenizer/node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", + "peer": true }, "node_modules/glsl-tokenizer/node_modules/through2": { "version": "0.6.5", "resolved": "https://registry.npmjs.org/through2/-/through2-0.6.5.tgz", "integrity": "sha512-RkK/CCESdTKQZHdmKICijdKKsCRVHs5KsLZ6pACAmF/1GPUQhonHSXWNERctxEp7RmvjdNbZTL5z9V7nSCXKcg==", + "peer": true, "dependencies": { "readable-stream": ">=1.0.33-1 <1.1.0-0", "xtend": ">=4.0.0 <4.1.0-0" @@ -5402,6 +5635,7 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/glslify/-/glslify-7.1.1.tgz", "integrity": "sha512-bud98CJ6kGZcP9Yxcsi7Iz647wuDz3oN+IZsjCRi5X1PI7t/xPKeL0mOwXJjo+CRZMqvq0CkSJiywCcY7kVYog==", + "peer": true, "dependencies": { "bl": "^2.2.1", "concat-stream": "^1.5.2", @@ -5427,6 +5661,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/glslify-bundle/-/glslify-bundle-5.1.1.tgz", "integrity": "sha512-plaAOQPv62M1r3OsWf2UbjN0hUYAB7Aph5bfH58VxJZJhloRNbxOL9tl/7H71K7OLJoSJ2ZqWOKk3ttQ6wy24A==", + "peer": true, "dependencies": { "glsl-inject-defines": "^1.0.1", "glsl-token-defines": "^1.0.0", @@ -5444,6 +5679,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/glslify-deps/-/glslify-deps-1.3.2.tgz", "integrity": "sha512-7S7IkHWygJRjcawveXQjRXLO2FTjijPDYC7QfZyAQanY+yGLCFHYnPtsGT9bdyHiwPTw/5a1m1M9hamT2aBpag==", + "peer": true, "dependencies": { "@choojs/findup": "^0.2.0", "events": "^3.2.0", @@ -5462,6 +5698,7 @@ "engines": [ "node >= 0.8" ], + "peer": true, "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -5472,12 +5709,14 @@ "node_modules/glslify/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "peer": true }, "node_modules/glslify/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -5491,12 +5730,14 @@ "node_modules/glslify/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "peer": true }, "node_modules/glslify/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -5509,12 +5750,14 @@ "node_modules/grid-index": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/grid-index/-/grid-index-1.1.0.tgz", - "integrity": "sha512-HZRwumpOGUrHyxO5bqKZL0B0GlUpwtCAzZ42sgxUPniu33R1LSFH5yrIcBCHjkctCAh3mtWKcKd9J4vDDdeVHA==" + "integrity": "sha512-HZRwumpOGUrHyxO5bqKZL0B0GlUpwtCAzZ42sgxUPniu33R1LSFH5yrIcBCHjkctCAh3mtWKcKd9J4vDDdeVHA==", + "peer": true }, "node_modules/handlebars": { "version": "4.7.9", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", + "dev": true, "license": "MIT", "dependencies": { "minimist": "^1.2.5", @@ -5536,6 +5779,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -5544,6 +5788,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5553,6 +5798,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-hover/-/has-hover-1.0.1.tgz", "integrity": "sha512-0G6w7LnlcpyDzpeGUTuT0CEw05+QlMuGVk1IHNAlHrGJITGodjZu3x8BNDUMfKJSZXNB2ZAclqc1bvrd+uUpfg==", + "peer": true, "dependencies": { "is-browser": "^2.0.1" } @@ -5561,6 +5807,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-passive-events/-/has-passive-events-1.0.0.tgz", "integrity": "sha512-2vSj6IeIsgvsRMyeQ0JaCX5Q3lX4zMn5HpoVc7MEhQ6pv8Iq9rsXjsp+E5ZwaT7T0xhMT0KmU8gtt1EFVdbJiw==", + "peer": true, "dependencies": { "is-browser": "^2.0.1" } @@ -5592,18 +5839,21 @@ "node_modules/hsluv": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/hsluv/-/hsluv-0.0.3.tgz", - "integrity": "sha512-08iL2VyCRbkQKBySkSh6m8zMUa3sADAxGVWs3Z1aPcUkTJeK0ETG4Fc27tEmQBGUAXZjIsXOZqBvacuVNSC/fQ==" + "integrity": "sha512-08iL2VyCRbkQKBySkSh6m8zMUa3sADAxGVWs3Z1aPcUkTJeK0ETG4Fc27tEmQBGUAXZjIsXOZqBvacuVNSC/fQ==", + "peer": true }, "node_modules/http2-client": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/http2-client/-/http2-client-1.3.5.tgz", "integrity": "sha512-EC2utToWl4RKfs5zd36Mxq7nzHHBuomZboI0yYL6Y0RmBgT7Sgkq4rQ0ezFTYoIsSs7Tm9SJe+o2FcAg6GBhGA==", + "dev": true, "license": "MIT" }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -5617,6 +5867,7 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -5653,7 +5904,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "peer": true }, "node_modules/immutable": { "version": "5.1.5", @@ -5686,6 +5938,7 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/ini/-/ini-4.1.3.tgz", "integrity": "sha512-X7rqawQBvfdjS10YU1y1YVreA3SsLrW9dX2CewP2EbBJM4ypVNLDkO5y04gejPwKIY9lR+7r9gn3rFPt/kmWFg==", + "peer": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -5698,7 +5951,8 @@ "node_modules/is-browser": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-browser/-/is-browser-2.1.0.tgz", - "integrity": "sha512-F5rTJxDQ2sW81fcfOR1GnCXT6sVJC104fCyfj+mjpwNEwaPYSn5fte5jiHmBg3DHsIoL/l8Kvw5VN5SsTRcRFQ==" + "integrity": "sha512-F5rTJxDQ2sW81fcfOR1GnCXT6sVJC104fCyfj+mjpwNEwaPYSn5fte5jiHmBg3DHsIoL/l8Kvw5VN5SsTRcRFQ==", + "peer": true }, "node_modules/is-core-module": { "version": "2.16.1", @@ -5728,6 +5982,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.1.0.tgz", "integrity": "sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==", + "peer": true, "engines": { "node": ">=0.10.0" }, @@ -5739,6 +5994,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-firefox/-/is-firefox-1.0.3.tgz", "integrity": "sha512-6Q9ITjvWIm0Xdqv+5U12wgOKEM2KoBw4Y926m0OFkvlCxnbG94HKAsVz8w3fWcfAS5YA2fJORXX1dLrkprCCxA==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5747,6 +6003,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, "engines": { "node": ">=8" } @@ -5768,6 +6025,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-iexplorer/-/is-iexplorer-1.0.0.tgz", "integrity": "sha512-YeLzceuwg3K6O0MLM3UyUUjKAlyULetwryFp1mHy1I5PfArK0AEqlfa+MR4gkJjcbuJXoDJCvXbyqZVf5CR2Sg==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5775,7 +6033,8 @@ "node_modules/is-mobile": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-mobile/-/is-mobile-4.0.0.tgz", - "integrity": "sha512-mlcHZA84t1qLSuWkt2v0I2l61PYdyQDt4aG1mLIXF5FDMm4+haBCxCPYSr/uwqQNRk1MiTizn0ypEuRAOLRAew==" + "integrity": "sha512-mlcHZA84t1qLSuWkt2v0I2l61PYdyQDt4aG1mLIXF5FDMm4+haBCxCPYSr/uwqQNRk1MiTizn0ypEuRAOLRAew==", + "peer": true }, "node_modules/is-number": { "version": "7.0.0", @@ -5791,6 +6050,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5799,6 +6059,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5806,22 +6067,26 @@ "node_modules/is-string-blank": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-string-blank/-/is-string-blank-1.0.1.tgz", - "integrity": "sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==" + "integrity": "sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==", + "peer": true }, "node_modules/is-svg-path": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-svg-path/-/is-svg-path-1.0.2.tgz", - "integrity": "sha512-Lj4vePmqpPR1ZnRctHv8ltSh1OrSxHkhUkd7wi+VQdcdP15/KvQFyk7LhNuM7ZW0EVbJz8kZLVmL9quLrfq4Kg==" + "integrity": "sha512-Lj4vePmqpPR1ZnRctHv8ltSh1OrSxHkhUkd7wi+VQdcdP15/KvQFyk7LhNuM7ZW0EVbJz8kZLVmL9quLrfq4Kg==", + "peer": true }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==" + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "peer": true }, "node_modules/isexe": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", + "peer": true, "engines": { "node": ">=16" } @@ -5845,6 +6110,7 @@ "version": "1.1.6", "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5887,6 +6153,7 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.2.tgz", "integrity": "sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==", + "dev": true, "license": "MIT", "dependencies": { "foreach": "^2.0.4" @@ -5909,6 +6176,7 @@ "version": "2.7.2", "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-2.7.2.tgz", "integrity": "sha512-R1JfqKqbBR4qE8UyBR56Ms30LL62/nlhoz+1UkfI/VE7p54Awu919FZ6ZUPG8zIa3XB65usPJgr1ONVncUGSaQ==", + "dev": true, "license": "MIT", "dependencies": { "@babel/runtime": "^7.18.3", @@ -5923,12 +6191,14 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, "license": "MIT" }, "node_modules/json-stringify-pretty-compact": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-4.0.0.tgz", - "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==" + "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==", + "peer": true }, "node_modules/json5": { "version": "2.2.3", @@ -5957,6 +6227,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/jsonpath-rfc9535/-/jsonpath-rfc9535-1.3.0.tgz", "integrity": "sha512-3jFHya7oZ45aDxIIdx+/zQARahHXxFSMWBkcBUldfXpLS9VCXDJyTKt35kQfEXLqh0K3Ixw/9xFnvcDStaxh7Q==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=20" @@ -5966,6 +6237,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5974,12 +6246,14 @@ "node_modules/kdbush": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", - "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==" + "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==", + "peer": true }, "node_modules/kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5988,6 +6262,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -6022,12 +6297,14 @@ "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "peer": true }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, "license": "Apache-2.0" }, "node_modules/loose-envify": { @@ -6061,12 +6338,14 @@ "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", + "dev": true, "license": "MIT" }, "node_modules/map-limit": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/map-limit/-/map-limit-0.0.1.tgz", "integrity": "sha512-pJpcfLPnIF/Sk3taPW21G/RQsEEirGaFpCW3oXRwH9dnFHPHNGjNyvh++rdmC2fNqEaTw2MhYJraoJWAHx8kEg==", + "peer": true, "dependencies": { "once": "~1.3.0" } @@ -6075,6 +6354,7 @@ "version": "1.3.3", "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", "integrity": "sha512-6vaNInhu+CHxtONf3zw3vq4SP2DOQhjBvIa3rNcG0+P7eKWlYH6Peu7rHizSloRU2EwMz6GraLieis9Ac9+p1w==", + "peer": true, "dependencies": { "wrappy": "1" } @@ -6116,6 +6396,7 @@ "version": "4.7.1", "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-4.7.1.tgz", "integrity": "sha512-lgL7XpIwsgICiL82ITplfS7IGwrB1OJIw/pCvprDp2dhmSSEBgmPzYRvwYYYvJGJD7fxUv1Tvpih4nZ6VrLuaA==", + "peer": true, "dependencies": { "@mapbox/geojson-rewind": "^0.5.2", "@mapbox/jsonlint-lines-primitives": "^2.0.2", @@ -6155,37 +6436,44 @@ "node_modules/maplibre-gl/node_modules/@mapbox/tiny-sdf": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@mapbox/tiny-sdf/-/tiny-sdf-2.0.6.tgz", - "integrity": "sha512-qMqa27TLw+ZQz5Jk+RcwZGH7BQf5G/TrutJhspsca/3SHwmgKQ1iq+d3Jxz5oysPVYTGP6aXxCo5Lk9Er6YBAA==" + "integrity": "sha512-qMqa27TLw+ZQz5Jk+RcwZGH7BQf5G/TrutJhspsca/3SHwmgKQ1iq+d3Jxz5oysPVYTGP6aXxCo5Lk9Er6YBAA==", + "peer": true }, "node_modules/maplibre-gl/node_modules/@mapbox/unitbezier": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.1.tgz", - "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==" + "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==", + "peer": true }, "node_modules/maplibre-gl/node_modules/earcut": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.1.tgz", - "integrity": "sha512-0l1/0gOjESMeQyYaK5IDiPNvFeu93Z/cO0TjZh9eZ1vyCtZnA7KMZ8rQggpsJHIbGSdrqYq9OhuveadOVHCshw==" + "integrity": "sha512-0l1/0gOjESMeQyYaK5IDiPNvFeu93Z/cO0TjZh9eZ1vyCtZnA7KMZ8rQggpsJHIbGSdrqYq9OhuveadOVHCshw==", + "peer": true }, "node_modules/maplibre-gl/node_modules/geojson-vt": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/geojson-vt/-/geojson-vt-4.0.2.tgz", - "integrity": "sha512-AV9ROqlNqoZEIJGfm1ncNjEXfkz2hdFlZf0qkVfmkwdKa8vj7H16YUOT81rJw1rdFhyEDlN2Tds91p/glzbl5A==" + "integrity": "sha512-AV9ROqlNqoZEIJGfm1ncNjEXfkz2hdFlZf0qkVfmkwdKa8vj7H16YUOT81rJw1rdFhyEDlN2Tds91p/glzbl5A==", + "peer": true }, "node_modules/maplibre-gl/node_modules/potpack": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz", - "integrity": "sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw==" + "integrity": "sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw==", + "peer": true }, "node_modules/maplibre-gl/node_modules/quickselect": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", - "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==" + "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", + "peer": true }, "node_modules/maplibre-gl/node_modules/supercluster": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-8.0.1.tgz", "integrity": "sha512-IiOea5kJ9iqzD2t7QJq/cREyLHTtSmUT6gQsweojg9WH2sYJqZK9SswTu6jrscO6D1G5v5vYZ9ru/eq85lXeZQ==", + "peer": true, "dependencies": { "kdbush": "^4.0.2" } @@ -6193,18 +6481,21 @@ "node_modules/maplibre-gl/node_modules/tinyqueue": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-3.0.0.tgz", - "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==" + "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==", + "peer": true }, "node_modules/mark.js": { "version": "8.11.1", "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", + "dev": true, "license": "MIT" }, "node_modules/marked": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "dev": true, "license": "MIT", "bin": { "marked": "bin/marked.js" @@ -6217,6 +6508,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/math-log2/-/math-log2-1.0.1.tgz", "integrity": "sha512-9W0yGtkaMAkf74XGYVy4Dqw3YUMnTNB2eeiw9aQbUl4A3KmuCEHTt2DgAB07ENzOYAjsYSAYufkAq0Zd+jU7zA==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -6239,6 +6531,7 @@ "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.5" @@ -6254,6 +6547,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, "license": "MIT", "engines": { "node": "18 || 20 || >=22" @@ -6263,6 +6557,7 @@ "version": "5.0.6", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -6283,6 +6578,7 @@ "version": "7.1.3", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" @@ -6292,6 +6588,7 @@ "version": "6.15.0", "resolved": "https://registry.npmjs.org/mobx/-/mobx-6.15.0.tgz", "integrity": "sha512-UczzB+0nnwGotYSgllfARAqWCJ5e/skuV2K/l+Zyck/H6pJIhLXuBnz+6vn2i211o7DtbE78HQtsYEKICHGI+g==", + "dev": true, "license": "MIT", "funding": { "type": "opencollective", @@ -6302,6 +6599,7 @@ "version": "9.2.0", "resolved": "https://registry.npmjs.org/mobx-react/-/mobx-react-9.2.0.tgz", "integrity": "sha512-dkGWCx+S0/1mfiuFfHRH8D9cplmwhxOV5CkXMp38u6rQGG2Pv3FWYztS0M7ncR6TyPRQKaTG/pnitInoYE9Vrw==", + "dev": true, "license": "MIT", "dependencies": { "mobx-react-lite": "^4.1.0" @@ -6327,6 +6625,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/mobx-react-lite/-/mobx-react-lite-4.1.1.tgz", "integrity": "sha512-iUxiMpsvNraCKXU+yPotsOncNNmyeS2B5DKL+TL6Tar/xm+wwNJAubJmtRSeAoYawdZqwv8Z/+5nPRHeQxTiXg==", + "dev": true, "license": "MIT", "dependencies": { "use-sync-external-store": "^1.4.0" @@ -6352,6 +6651,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/mouse-change/-/mouse-change-1.4.0.tgz", "integrity": "sha512-vpN0s+zLL2ykyyUDh+fayu9Xkor5v/zRD9jhSqjRS1cJTGS0+oakVZzNm5n19JvvEj0you+MXlYTpNxUDQUjkQ==", + "peer": true, "dependencies": { "mouse-event": "^1.0.0" } @@ -6359,17 +6659,20 @@ "node_modules/mouse-event": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/mouse-event/-/mouse-event-1.0.5.tgz", - "integrity": "sha512-ItUxtL2IkeSKSp9cyaX2JLUuKk2uMoxBg4bbOWVd29+CskYJR9BGsUqtXenNzKbnDshvupjUewDIYVrOB6NmGw==" + "integrity": "sha512-ItUxtL2IkeSKSp9cyaX2JLUuKk2uMoxBg4bbOWVd29+CskYJR9BGsUqtXenNzKbnDshvupjUewDIYVrOB6NmGw==", + "peer": true }, "node_modules/mouse-event-offset": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/mouse-event-offset/-/mouse-event-offset-3.0.2.tgz", - "integrity": "sha512-s9sqOs5B1Ykox3Xo8b3Ss2IQju4UwlW6LSR+Q5FXWpprJ5fzMLefIIItr3PH8RwzfGy6gxs/4GAmiNuZScE25w==" + "integrity": "sha512-s9sqOs5B1Ykox3Xo8b3Ss2IQju4UwlW6LSR+Q5FXWpprJ5fzMLefIIItr3PH8RwzfGy6gxs/4GAmiNuZScE25w==", + "peer": true }, "node_modules/mouse-wheel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mouse-wheel/-/mouse-wheel-1.2.0.tgz", "integrity": "sha512-+OfYBiUOCTWcTECES49neZwL5AoGkXE+lFjIvzwNCnYRlso+EnfvovcBxGoyQ0yQt806eSPjS675K0EwWknXmw==", + "peer": true, "dependencies": { "right-now": "^1.0.0", "signum": "^1.0.0", @@ -6386,6 +6689,7 @@ "resolved": "https://registry.npmjs.org/mumath/-/mumath-3.3.4.tgz", "integrity": "sha512-VAFIOG6rsxoc7q/IaY3jdjmrsuX9f15KlRLYTHmixASBZkZEKC1IFqE2BC5CdhXmK6WLM1Re33z//AGmeRI6FA==", "deprecated": "Redundant dependency in your project.", + "peer": true, "dependencies": { "almost-equal": "^1.1.0" } @@ -6393,7 +6697,8 @@ "node_modules/murmurhash-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz", - "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==" + "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==", + "peer": true }, "node_modules/nanoid": { "version": "3.3.12", @@ -6416,12 +6721,14 @@ "node_modules/native-promise-only": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz", - "integrity": "sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==" + "integrity": "sha512-zkVhZUA3y8mbz652WrL5x0fB0ehrBkulWT3TomAQ9iDtyXZvzKeEA6GPxAItBYeNYl5yngKRX612qHOhvMkDeg==", + "peer": true }, "node_modules/needle": { "version": "2.9.1", "resolved": "https://registry.npmjs.org/needle/-/needle-2.9.1.tgz", "integrity": "sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==", + "peer": true, "dependencies": { "debug": "^3.2.6", "iconv-lite": "^0.4.4", @@ -6438,6 +6745,7 @@ "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "peer": true, "dependencies": { "ms": "^2.1.1" } @@ -6445,12 +6753,14 @@ "node_modules/neo-async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true }, "node_modules/next-tick": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", - "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "peer": true }, "node_modules/no-case": { "version": "3.0.4", @@ -6472,6 +6782,7 @@ "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -6492,6 +6803,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/node-fetch-h2/-/node-fetch-h2-2.3.0.tgz", "integrity": "sha512-ofRW94Ab0T4AOh5Fk8t0h8OBWrmjb0SSB20xh1H8YnPV9EJ+f5AMoYSUQ2zgJ4Iq2HAK0I2l5/Nequ8YzFS3Hg==", + "dev": true, "license": "MIT", "dependencies": { "http2-client": "^1.2.5" @@ -6504,6 +6816,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/node-readfiles/-/node-readfiles-0.2.0.tgz", "integrity": "sha512-SU00ZarexNlE4Rjdm83vglt5Y9yiQ+XI1XpflWlb7q7UTN1JUItm69xMeiQCTxtTfnzt+83T8Cx+vI2ED++VDA==", + "dev": true, "license": "MIT", "dependencies": { "es6-promise": "^3.2.1" @@ -6518,12 +6831,14 @@ "node_modules/normalize-svg-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/normalize-svg-path/-/normalize-svg-path-0.1.0.tgz", - "integrity": "sha512-1/kmYej2iedi5+ROxkRESL/pI02pkg0OBnaR4hJkSIX6+ORzepwbuUXfrdZaPjysTsJInj0Rj5NuX027+dMBvA==" + "integrity": "sha512-1/kmYej2iedi5+ROxkRESL/pI02pkg0OBnaR4hJkSIX6+ORzepwbuUXfrdZaPjysTsJInj0Rj5NuX027+dMBvA==", + "peer": true }, "node_modules/number-is-integer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-integer/-/number-is-integer-1.0.1.tgz", "integrity": "sha512-Dq3iuiFBkrbmuQjGFFF3zckXNCQoSD37/SdSbgcBailUx6knDvDwb5CympBgcoWHy36sfS12u74MHYkXyHq6bg==", + "peer": true, "dependencies": { "is-finite": "^1.0.1" }, @@ -6535,6 +6850,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/oas-kit-common/-/oas-kit-common-1.0.8.tgz", "integrity": "sha512-pJTS2+T0oGIwgjGpw7sIRU8RQMcUoKCDWFLdBqKB2BNmGpbBMH2sdqAaOXUg8OzonZHU0L7vfJu1mJFEiYDWOQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "fast-safe-stringify": "^2.0.7" @@ -6544,6 +6860,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/oas-linter/-/oas-linter-3.2.2.tgz", "integrity": "sha512-KEGjPDVoU5K6swgo9hJVA/qYGlwfbFx+Kg2QB/kd7rzV5N8N5Mg6PlsoCMohVnQmo+pzJap/F610qTodKzecGQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "@exodus/schemasafe": "^1.0.0-rc.2", @@ -6558,6 +6875,7 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "dev": true, "license": "ISC", "engines": { "node": ">= 6" @@ -6567,6 +6885,7 @@ "version": "2.5.6", "resolved": "https://registry.npmjs.org/oas-resolver/-/oas-resolver-2.5.6.tgz", "integrity": "sha512-Yx5PWQNZomfEhPPOphFbZKi9W93CocQj18NlD2Pa4GWZzdZpSJvYwoiuurRI7m3SpcChrnO08hkuQDL3FGsVFQ==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "node-fetch-h2": "^2.3.0", @@ -6586,6 +6905,7 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "dev": true, "license": "ISC", "engines": { "node": ">= 6" @@ -6595,6 +6915,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/oas-schema-walker/-/oas-schema-walker-1.1.5.tgz", "integrity": "sha512-2yucenq1a9YPmeNExoUa9Qwrt9RFkjqaMAA1X+U7sbb0AqBeTIdMHky9SQQ6iN94bO5NW0W4TRYXerG+BdAvAQ==", + "dev": true, "license": "BSD-3-Clause", "funding": { "url": "https://github.com/Mermade/oas-kit?sponsor=1" @@ -6604,6 +6925,7 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/oas-validator/-/oas-validator-5.0.8.tgz", "integrity": "sha512-cu20/HE5N5HKqVygs3dt94eYJfBi0TsZvPVXDhbXQHiEityDN+RROTleefoKRKKJ9dFAF2JBkDHgvWj0sjKGmw==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "call-me-maybe": "^1.0.1", @@ -6623,6 +6945,7 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "dev": true, "license": "ISC", "engines": { "node": ">= 6" @@ -6640,6 +6963,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "peer": true, "dependencies": { "wrappy": "1" } @@ -6648,6 +6972,7 @@ "version": "1.7.2", "resolved": "https://registry.npmjs.org/openapi-sampler/-/openapi-sampler-1.7.2.tgz", "integrity": "sha512-OKytvqB5XIaTgA9xtw8W8UTar+uymW2xPVpFN0NihMtuHPdPTGxBEhGnfFnJW5g/gOSIvkP+H0Xh3XhVI9/n7g==", + "dev": true, "license": "MIT", "dependencies": { "@types/json-schema": "^7.0.7", @@ -6675,6 +7000,7 @@ "version": "0.8.0", "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.8.0.tgz", "integrity": "sha512-KiOAIsdpUTcAXuykya5fnVVT+/5uS0Q1mrkRHcF89tpieSmY33O/tmc54CqwA+bfhbtEfZUNLHaPUiB9X3jt1A==", + "dev": true, "license": "MIT" }, "node_modules/package-json-from-dist": { @@ -6697,7 +7023,8 @@ "node_modules/parenthesis": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/parenthesis/-/parenthesis-3.1.8.tgz", - "integrity": "sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==" + "integrity": "sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==", + "peer": true }, "node_modules/parse-json": { "version": "5.2.0", @@ -6720,6 +7047,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/parse-rect/-/parse-rect-1.2.0.tgz", "integrity": "sha512-4QZ6KYbnE6RTwg9E0HpLchUM9EZt6DnDxajFZZDSV4p/12ZJEvPO702DZpGvRYEPo00yKDys7jASi+/w7aO8LA==", + "peer": true, "dependencies": { "pick-by-alias": "^1.2.0" } @@ -6727,23 +7055,27 @@ "node_modules/parse-svg-path": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/parse-svg-path/-/parse-svg-path-0.1.2.tgz", - "integrity": "sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ==" + "integrity": "sha512-JyPSBnkTJ0AI8GGJLfMXvKq42cj5c006fnLz6fXy6zfoVjJizi8BNTpu8on8ziI1cKy9d9DGNuY17Ce7wuejpQ==", + "peer": true }, "node_modules/parse-unit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parse-unit/-/parse-unit-1.0.1.tgz", - "integrity": "sha512-hrqldJHokR3Qj88EIlV/kAyAi/G5R2+R56TBANxNMy0uPlYcttx0jnMW6Yx5KsKPSbC3KddM/7qQm3+0wEXKxg==" + "integrity": "sha512-hrqldJHokR3Qj88EIlV/kAyAi/G5R2+R56TBANxNMy0uPlYcttx0jnMW6Yx5KsKPSbC3KddM/7qQm3+0wEXKxg==", + "peer": true }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true, "license": "MIT" }, "node_modules/path-expression-matcher": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "dev": true, "funding": [ { "type": "github", @@ -6803,6 +7135,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.3.0.tgz", "integrity": "sha512-XDF38WCH3z5OV/OVa8GKUNtLAyneuzbCisx7QUCF8Q6Nutx0WnJrQe5O+kOtBlLfRNUws98Y58Lblp+NJG5T4Q==", + "peer": true, "dependencies": { "ieee754": "^1.1.12", "resolve-protobuf-schema": "^2.1.0" @@ -6815,17 +7148,20 @@ "version": "1.5.6", "resolved": "https://registry.npmjs.org/perfect-scrollbar/-/perfect-scrollbar-1.5.6.tgz", "integrity": "sha512-rixgxw3SxyJbCaSpo1n35A/fwI1r2rdwMKOTCg/AcG+xOEyZcE8UHVjpZMFCVImzsFoCZeJTT+M/rdEIQYO2nw==", + "dev": true, "license": "MIT" }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" + "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", + "peer": true }, "node_modules/pick-by-alias": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pick-by-alias/-/pick-by-alias-1.2.0.tgz", - "integrity": "sha512-ESj2+eBxhGrcA1azgHs7lARG5+5iLakc/6nlfbpjcLl00HuuUOIuORhYXN4D1HfvMSKuVtFQjAlnwi1JHEeDIw==" + "integrity": "sha512-ESj2+eBxhGrcA1azgHs7lARG5+5iLakc/6nlfbpjcLl00HuuUOIuORhYXN4D1HfvMSKuVtFQjAlnwi1JHEeDIw==", + "peer": true }, "node_modules/picocolors": { "version": "1.1.1", @@ -6836,6 +7172,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -6849,6 +7186,7 @@ "resolved": "https://registry.npmjs.org/plotly.js/-/plotly.js-3.3.1.tgz", "integrity": "sha512-SrGSZ02HvCWQIYsbQX4sgjgGo7k4T+Oz8a+RQwE5Caz+yu1vputBM1UThmiOPY51B5HzO9ajym8Rl4pmLj+i9Q==", "license": "MIT", + "peer": true, "dependencies": { "@plotly/d3": "3.8.2", "@plotly/d3-sankey": "0.7.2", @@ -6905,10 +7243,17 @@ "node": ">=18.0.0" } }, + "node_modules/plotly.js-cartesian-dist-min": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/plotly.js-cartesian-dist-min/-/plotly.js-cartesian-dist-min-3.6.0.tgz", + "integrity": "sha512-Ti4l+8EGRWL6kPdpgeDbHJ7lGN8VvKXhpjdfUUstnUu0vdryermQzeVYAVnyvZa5D0B1EBCtNqOZ0WT42Y2rkw==", + "license": "MIT" + }, "node_modules/plotly.js/node_modules/color-rgba": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/color-rgba/-/color-rgba-3.0.0.tgz", "integrity": "sha512-PPwZYkEY3M2THEHHV6Y95sGUie77S7X8v+h1r6LSAPF3/LL2xJ8duUXSrkic31Nzc4odPwHgUbiX/XuTYzQHQg==", + "peer": true, "dependencies": { "color-parse": "^2.0.0", "color-space": "^2.0.0" @@ -6917,12 +7262,14 @@ "node_modules/plotly.js/node_modules/color-space": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/color-space/-/color-space-2.3.1.tgz", - "integrity": "sha512-5DJdKYwoDji3ik/i0xSn+SiwXsfwr+1FEcCMUz2GS5speGCfGSbBMOLd84SDUBOuX8y4CvdFJmOBBJuC4wp7sQ==" + "integrity": "sha512-5DJdKYwoDji3ik/i0xSn+SiwXsfwr+1FEcCMUz2GS5speGCfGSbBMOLd84SDUBOuX8y4CvdFJmOBBJuC4wp7sQ==", + "peer": true }, "node_modules/pluralize": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz", "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -6931,12 +7278,14 @@ "node_modules/point-in-polygon": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/point-in-polygon/-/point-in-polygon-1.1.0.tgz", - "integrity": "sha512-3ojrFwjnnw8Q9242TzgXuTD+eKiutbzyslcq1ydfu82Db2y+Ogbmyrkpv0Hgj31qwT3lbS9+QAAO/pIQM35XRw==" + "integrity": "sha512-3ojrFwjnnw8Q9242TzgXuTD+eKiutbzyslcq1ydfu82Db2y+Ogbmyrkpv0Hgj31qwT3lbS9+QAAO/pIQM35XRw==", + "peer": true }, "node_modules/polished": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/polished/-/polished-4.3.1.tgz", "integrity": "sha512-OBatVyC/N7SCW/FaDHrSd+vn0o5cS855TOmYi4OkdWUMSJCET/xip//ch8xGUvtr3i44X9LVyWwQlRMTN3pwSA==", + "dev": true, "license": "MIT", "dependencies": { "@babel/runtime": "^7.17.8" @@ -6948,7 +7297,8 @@ "node_modules/polybooljs": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/polybooljs/-/polybooljs-1.2.2.tgz", - "integrity": "sha512-ziHW/02J0XuNuUtmidBc6GXE8YohYydp3DWPWXYsd7O721TjcmN+k6ezjdwkDqep+gnWnFY+yqZHvzElra2oCg==" + "integrity": "sha512-ziHW/02J0XuNuUtmidBc6GXE8YohYydp3DWPWXYsd7O721TjcmN+k6ezjdwkDqep+gnWnFY+yqZHvzElra2oCg==", + "peer": true }, "node_modules/postcss": { "version": "8.5.15", @@ -7072,17 +7422,20 @@ "node_modules/postcss-value-parser": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==" + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true }, "node_modules/potpack": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz", - "integrity": "sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==" + "integrity": "sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==", + "peer": true }, "node_modules/prismjs": { "version": "1.30.0", "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -7092,6 +7445,7 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/probe-image-size/-/probe-image-size-7.2.3.tgz", "integrity": "sha512-HubhG4Rb2UH8YtV4ba0Vp5bQ7L78RTONYu/ujmCu5nBI8wGv24s4E9xSKBi0N1MowRpxk76pFCpJtW0KPzOK0w==", + "peer": true, "dependencies": { "lodash.merge": "^4.6.2", "needle": "^2.5.2", @@ -7101,7 +7455,8 @@ "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "peer": true }, "node_modules/prop-types": { "version": "15.8.1", @@ -7122,6 +7477,7 @@ "version": "7.6.2", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.2.tgz", "integrity": "sha512-N9EiLovGEQOJSPF26Ij7qUGvahfEnq0eeYZ02aigIedkmz1qZSwjnP9SBITHJuF/6MYbIW4HDN8zdYjsjqJKXQ==", + "dev": true, "hasInstallScript": true, "license": "BSD-3-Clause", "dependencies": { @@ -7146,12 +7502,14 @@ "version": "3.6.1", "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.1.tgz", "integrity": "sha512-VG2K63Igkiv9p76tk1lilczEK1cT+kCjKtkdhw1dQZV3k3IXJbd3o6Ho8b9zJZaHSnT2hKe4I+ObmX9w6m5SmQ==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, "funding": [ { "type": "github", @@ -7171,12 +7529,14 @@ "node_modules/quickselect": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-2.0.0.tgz", - "integrity": "sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==" + "integrity": "sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw==", + "peer": true }, "node_modules/raf": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", "integrity": "sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA==", + "peer": true, "dependencies": { "performance-now": "^2.1.0" } @@ -7185,6 +7545,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "^5.1.0" @@ -7233,6 +7594,7 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/react-tabs/-/react-tabs-6.1.0.tgz", "integrity": "sha512-6QtbTRDKM+jA/MZTTefvigNxo0zz+gnBTVFw2CFVvq+f2BuH0nF0vDLNClL045nuTAdOoK/IL1vTP0ZLX0DAyQ==", + "dev": true, "license": "MIT", "dependencies": { "clsx": "^2.0.0", @@ -7275,6 +7637,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -7289,6 +7652,7 @@ "version": "2.5.1", "resolved": "https://registry.npmjs.org/redoc/-/redoc-2.5.1.tgz", "integrity": "sha512-LmqA+4A3CmhTllGG197F0arUpmChukAj9klfSdxNRemT9Hr07xXr7OGKu4PHzBs359sgrJ+4JwmOlM7nxLPGMg==", + "dev": true, "license": "MIT", "dependencies": { "@redocly/openapi-core": "^1.4.0", @@ -7329,12 +7693,14 @@ "version": "0.22.2", "resolved": "https://registry.npmjs.org/@redocly/config/-/config-0.22.2.tgz", "integrity": "sha512-roRDai8/zr2S9YfmzUfNhKjOF0NdcOIqF7bhf4MVC5UxpjIysDjyudvlAiVbpPHp3eDRWbdzUgtkK1a7YiDNyQ==", + "dev": true, "license": "MIT" }, "node_modules/redoc/node_modules/@redocly/openapi-core": { "version": "1.34.6", "resolved": "https://registry.npmjs.org/@redocly/openapi-core/-/openapi-core-1.34.6.tgz", "integrity": "sha512-2+O+riuIUgVSuLl3Lyh5AplWZyVMNuG2F98/o6NrutKJfW4/GTZdPpZlIphS0HGgcOHgmWcCSHj+dWFlZaGSHw==", + "dev": true, "license": "MIT", "dependencies": { "@redocly/ajv": "^8.11.2", @@ -7356,6 +7722,7 @@ "version": "5.1.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -7368,6 +7735,7 @@ "version": "1.1.9", "resolved": "https://registry.npmjs.org/reftools/-/reftools-1.1.9.tgz", "integrity": "sha512-OVede/NQE13xBQ+ob5CKd5KyeJYU2YInb1bmV4nRoOfquZPkAkxuOXicSe1PvqIuZZ4kD13sPKBbR7UFDmli6w==", + "dev": true, "license": "BSD-3-Clause", "funding": { "url": "https://github.com/Mermade/oas-kit?sponsor=1" @@ -7382,12 +7750,14 @@ "name": "@plotly/regl", "version": "2.1.2", "resolved": "https://registry.npmjs.org/@plotly/regl/-/regl-2.1.2.tgz", - "integrity": "sha512-Mdk+vUACbQvjd0m/1JJjOOafmkp/EpmHjISsopEz5Av44CBq7rPC05HHNbYGKVyNUF2zmEoBS/TT0pd0SPFFyw==" + "integrity": "sha512-Mdk+vUACbQvjd0m/1JJjOOafmkp/EpmHjISsopEz5Av44CBq7rPC05HHNbYGKVyNUF2zmEoBS/TT0pd0SPFFyw==", + "peer": true }, "node_modules/regl-error2d": { "version": "2.0.12", "resolved": "https://registry.npmjs.org/regl-error2d/-/regl-error2d-2.0.12.tgz", "integrity": "sha512-r7BUprZoPO9AbyqM5qlJesrSRkl+hZnVKWKsVp7YhOl/3RIpi4UDGASGJY0puQ96u5fBYw/OlqV24IGcgJ0McA==", + "peer": true, "dependencies": { "array-bounds": "^1.0.1", "color-normalize": "^1.5.0", @@ -7402,6 +7772,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/regl-line2d/-/regl-line2d-3.1.3.tgz", "integrity": "sha512-fkgzW+tTn4QUQLpFKsUIE0sgWdCmXAM3ctXcCgoGBZTSX5FE2A0M7aynz7nrZT5baaftLrk9te54B+MEq4QcSA==", + "peer": true, "dependencies": { "array-bounds": "^1.0.1", "array-find-index": "^1.0.2", @@ -7420,6 +7791,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/regl-scatter2d/-/regl-scatter2d-3.3.1.tgz", "integrity": "sha512-seOmMIVwaCwemSYz/y4WE0dbSO9svNFSqtTh5RE57I7PjGo3tcUYKtH0MTSoshcAsreoqN8HoCtnn8wfHXXfKQ==", + "peer": true, "dependencies": { "@plotly/point-cluster": "^3.1.9", "array-range": "^1.0.1", @@ -7442,6 +7814,7 @@ "version": "1.0.14", "resolved": "https://registry.npmjs.org/regl-splom/-/regl-splom-1.0.14.tgz", "integrity": "sha512-OiLqjmPRYbd7kDlHC6/zDf6L8lxgDC65BhC8JirhP4ykrK4x22ZyS+BnY8EUinXKDeMgmpRwCvUmk7BK4Nweuw==", + "peer": true, "dependencies": { "array-bounds": "^1.0.1", "array-range": "^1.0.1", @@ -7457,6 +7830,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7466,6 +7840,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7507,6 +7882,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "peer": true, "dependencies": { "protocol-buffers-schema": "^3.3.1" } @@ -7514,7 +7890,8 @@ "node_modules/right-now": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/right-now/-/right-now-1.0.0.tgz", - "integrity": "sha512-DA8+YS+sMIVpbsuKgy+Z67L9Lxb1p05mNxRpDPNksPDEFir4vmBlUtuN9jkTGn9YMMdlBuK7XQgFiz6ws+yhSg==" + "integrity": "sha512-DA8+YS+sMIVpbsuKgy+Z67L9Lxb1p05mNxRpDPNksPDEFir4vmBlUtuN9jkTGn9YMMdlBuK7XQgFiz6ws+yhSg==", + "peer": true }, "node_modules/rollup": { "version": "4.60.0", @@ -7563,12 +7940,14 @@ "node_modules/rw": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", - "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "peer": true }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, "funding": [ { "type": "github", @@ -7588,7 +7967,8 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "peer": true }, "node_modules/sass": { "version": "1.87.0", @@ -7641,7 +8021,8 @@ "node_modules/sax": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", - "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==" + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "peer": true }, "node_modules/scheduler": { "version": "0.27.0", @@ -7653,6 +8034,7 @@ "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -7665,17 +8047,20 @@ "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "dev": true, "license": "MIT" }, "node_modules/shallow-copy": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/shallow-copy/-/shallow-copy-0.0.1.tgz", - "integrity": "sha512-b6i4ZpVuUxB9h5gfCxPiusKYkqTMOjEbBs4wMaFbkfia4yFv92UKZ6Df8WXcKbn08JNL/abvg3FnMAOfakDvUw==" + "integrity": "sha512-b6i4ZpVuUxB9h5gfCxPiusKYkqTMOjEbBs4wMaFbkfia4yFv92UKZ6Df8WXcKbn08JNL/abvg3FnMAOfakDvUw==", + "peer": true }, "node_modules/shallowequal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz", "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==", + "dev": true, "license": "MIT" }, "node_modules/shebang-command": { @@ -7703,6 +8088,7 @@ "version": "13.2.3", "resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz", "integrity": "sha512-ggLesLtu2xp+ZxI+ysJTmNjh2U0TsC+rQ/pfED9bUZZ4DKefP27D+7YJVVTvKsmjLpIi9jAa7itwDGkDDmt1GQ==", + "dev": true, "license": "MIT", "dependencies": { "should-equal": "^2.0.0", @@ -7716,6 +8102,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/should-equal/-/should-equal-2.0.0.tgz", "integrity": "sha512-ZP36TMrK9euEuWQYBig9W55WPC7uo37qzAEmbjHz4gfyuXrEUgF8cUvQVO+w+d3OMfPvSRQJ22lSm8MQJ43LTA==", + "dev": true, "license": "MIT", "dependencies": { "should-type": "^1.4.0" @@ -7725,6 +8112,7 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/should-format/-/should-format-3.0.3.tgz", "integrity": "sha512-hZ58adtulAk0gKtua7QxevgUaXTTXxIi8t41L3zo9AHvjXO1/7sdLECuHeIN2SRtYXpNkmhoUP2pdeWgricQ+Q==", + "dev": true, "license": "MIT", "dependencies": { "should-type": "^1.3.0", @@ -7735,12 +8123,14 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/should-type/-/should-type-1.4.0.tgz", "integrity": "sha512-MdAsTu3n25yDbIe1NeN69G4n6mUnJGtSJHygX3+oN0ZbO3DTiATnf7XnYJdGT42JCXurTb1JI0qOBR65shvhPQ==", + "dev": true, "license": "MIT" }, "node_modules/should-type-adaptors": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz", "integrity": "sha512-JA4hdoLnN+kebEp2Vs8eBe9g7uy0zbRo+RMcU0EsNy+R+k049Ki+N5tT5Jagst2g7EAja+euFuoXFCa8vIklfA==", + "dev": true, "license": "MIT", "dependencies": { "should-type": "^1.3.0", @@ -7751,6 +8141,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/should-util/-/should-util-1.0.1.tgz", "integrity": "sha512-oXF8tfxx5cDk8r2kYqlkUJzZpDBqVY/II2WhvU0n9Y3XYvAYRmeaf1PvvIvTgPnv4KJ+ES5M0PyDq5Jp+Ygy2g==", + "dev": true, "license": "MIT" }, "node_modules/signal-exit": { @@ -7768,12 +8159,14 @@ "node_modules/signum": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/signum/-/signum-1.0.0.tgz", - "integrity": "sha512-yodFGwcyt59XRh7w5W3jPcIQb3Bwi21suEfT7MAWnBX3iCdklJpgDgvGT9o04UonglZN5SNMfJFkHIR/jO8GHw==" + "integrity": "sha512-yodFGwcyt59XRh7w5W3jPcIQb3Bwi21suEfT7MAWnBX3iCdklJpgDgvGT9o04UonglZN5SNMfJFkHIR/jO8GHw==", + "peer": true }, "node_modules/simple-websocket": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/simple-websocket/-/simple-websocket-9.1.0.tgz", "integrity": "sha512-8MJPnjRN6A8UCp1I+H/dSFyjwJhp6wta4hsVRhjf8w9qBHRzxYt14RaOcjvQnhD1N4yKOddEjflwMnQM4VtXjQ==", + "dev": true, "funding": [ { "type": "github", @@ -7801,6 +8194,7 @@ "version": "1.4.7", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.4.7.tgz", "integrity": "sha512-tf+h5W1IrjNm/9rKKj0JU2MDMruiopx0jjVA5zCdBtcGjfp0+c5rHw/zADLC3IeKlGHtVbHtpfzvYA0OYT+HKg==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.0.0" @@ -7856,6 +8250,7 @@ "version": "0.0.9", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", "integrity": "sha512-vjUc6sfgtgY0dxCdnc40mK6Oftjo9+2K8H/NG81TMhgL392FtiPA9tn9RLyTxXmTLPJPjF3VyzFp6bsWFLisMQ==", + "peer": true, "engines": { "node": "*" } @@ -7864,6 +8259,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.1.1.tgz", "integrity": "sha512-MgWpQ/ZjGieSVB3eOJVs4OA2LT/q1vx98KPCTTQPzq/aLr0YUXTsgryTXr4SLfR0ZfUUCiedM9n/ABeDIyy4mA==", + "peer": true, "dependencies": { "escodegen": "^2.1.0" } @@ -7871,12 +8267,14 @@ "node_modules/stickyfill": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stickyfill/-/stickyfill-1.1.1.tgz", - "integrity": "sha512-GCp7vHAfpao+Qh/3Flh9DXEJ/qSi0KJwJw6zYlZOtRYXWUIpMM6mC2rIep/dK8RQqwW0KxGJIllmjPIBOGN8AA==" + "integrity": "sha512-GCp7vHAfpao+Qh/3Flh9DXEJ/qSi0KJwJw6zYlZOtRYXWUIpMM6mC2rIep/dK8RQqwW0KxGJIllmjPIBOGN8AA==", + "dev": true }, "node_modules/stream-parser": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/stream-parser/-/stream-parser-0.3.1.tgz", "integrity": "sha512-bJ/HgKq41nlKvlhccD5kaCr/P+Hu0wPNKPJOH7en+YrJu/9EgqUF+88w5Jb6KNcjOFMhfX4B2asfeAtIGuHObQ==", + "peer": true, "dependencies": { "debug": "2" } @@ -7885,6 +8283,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "peer": true, "dependencies": { "ms": "2.0.0" } @@ -7892,17 +8291,20 @@ "node_modules/stream-parser/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "peer": true }, "node_modules/stream-shift": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "peer": true }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -7918,6 +8320,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/string-split-by/-/string-split-by-1.0.0.tgz", "integrity": "sha512-KaJKY+hfpzNyet/emP81PJA9hTVSfxNLS9SFTWxdCnnW1/zOOwiV248+EfoX7IQFcBaOp4G5YE6xTJMF+pLg6A==", + "peer": true, "dependencies": { "parenthesis": "^3.1.5" } @@ -7926,6 +8329,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -7954,6 +8358,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -7978,6 +8383,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz", "integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==", + "dev": true, "funding": [ { "type": "github", @@ -7989,12 +8395,14 @@ "node_modules/strongly-connected-components": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/strongly-connected-components/-/strongly-connected-components-1.0.1.tgz", - "integrity": "sha512-i0TFx4wPcO0FwX+4RkLJi1MxmcTv90jNZgxMu9XRnMXMeFUY1VJlIoXpZunPUvUUqbCT1pg5PEkFqqpcaElNaA==" + "integrity": "sha512-i0TFx4wPcO0FwX+4RkLJi1MxmcTv90jNZgxMu9XRnMXMeFUY1VJlIoXpZunPUvUUqbCT1pg5PEkFqqpcaElNaA==", + "peer": true }, "node_modules/styled-components": { "version": "5.3.11", "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-5.3.11.tgz", "integrity": "sha512-uuzIIfnVkagcVHv9nE0VPlHPSCmXIUGKfJ42LNjxCCTDTL5sgnJ8Z7GZBq0EnLYGln77tPpEpExt2+qa+cZqSw==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.0.0", @@ -8025,12 +8433,14 @@ "version": "0.7.5", "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.7.5.tgz", "integrity": "sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg==", + "dev": true, "license": "MIT" }, "node_modules/styled-components/node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -8040,6 +8450,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^3.0.0" @@ -8057,6 +8468,7 @@ "version": "7.1.5", "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-7.1.5.tgz", "integrity": "sha512-EulshI3pGUM66o6ZdH3ReiFcvHpM3vAigyK+vcxdjpJyEbIIrtbmBdY23mGgnI24uXiGFvrGq9Gkum/8U7vJWg==", + "peer": true, "dependencies": { "kdbush": "^3.0.0" } @@ -8064,17 +8476,20 @@ "node_modules/supercluster/node_modules/kdbush": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-3.0.0.tgz", - "integrity": "sha512-hRkd6/XW4HTsA9vjVpY9tuXJYLSlelnkTmVFu4M9/7MIYQtFcHpbugAU7UbOfjOiVSVYl2fqgBuJ32JUmRo5Ew==" + "integrity": "sha512-hRkd6/XW4HTsA9vjVpY9tuXJYLSlelnkTmVFu4M9/7MIYQtFcHpbugAU7UbOfjOiVSVYl2fqgBuJ32JUmRo5Ew==", + "peer": true }, "node_modules/superscript-text": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/superscript-text/-/superscript-text-1.0.0.tgz", - "integrity": "sha512-gwu8l5MtRZ6koO0icVTlmN5pm7Dhh1+Xpe9O4x6ObMAsW+3jPbW14d1DsBq1F4wiI+WOFjXF35pslgec/G8yCQ==" + "integrity": "sha512-gwu8l5MtRZ6koO0icVTlmN5pm7Dhh1+Xpe9O4x6ObMAsW+3jPbW14d1DsBq1F4wiI+WOFjXF35pslgec/G8yCQ==", + "peer": true }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -8097,7 +8512,8 @@ "node_modules/svg-arc-to-cubic-bezier": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/svg-arc-to-cubic-bezier/-/svg-arc-to-cubic-bezier-3.2.0.tgz", - "integrity": "sha512-djbJ/vZKZO+gPoSDThGNpKDO+o+bAeA4XQKovvkNCqnIS2t+S4qnLAGQhyyrulhCFRl1WWzAp0wUDV8PpTVU3g==" + "integrity": "sha512-djbJ/vZKZO+gPoSDThGNpKDO+o+bAeA4XQKovvkNCqnIS2t+S4qnLAGQhyyrulhCFRl1WWzAp0wUDV8PpTVU3g==", + "peer": true }, "node_modules/svg-parser": { "version": "2.0.4", @@ -8108,6 +8524,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/svg-path-bounds/-/svg-path-bounds-1.0.2.tgz", "integrity": "sha512-H4/uAgLWrppIC0kHsb2/dWUYSmb4GE5UqH06uqWBcg6LBjX2fu0A8+JrO2/FJPZiSsNOKZAhyFFgsLTdYUvSqQ==", + "peer": true, "dependencies": { "abs-svg-path": "^0.1.1", "is-svg-path": "^1.0.1", @@ -8119,6 +8536,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/normalize-svg-path/-/normalize-svg-path-1.1.0.tgz", "integrity": "sha512-r9KHKG2UUeB5LoTouwDzBy2VxXlHsiM6fyLQvnJa0S5hrhzqElH/CH7TUGhT1fVvIYBIKf3OpY4YJ4CK+iaqHg==", + "peer": true, "dependencies": { "svg-arc-to-cubic-bezier": "^3.0.0" } @@ -8127,6 +8545,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/svg-path-sdf/-/svg-path-sdf-1.1.3.tgz", "integrity": "sha512-vJJjVq/R5lSr2KLfVXVAStktfcfa1pNFjFOgyJnzZFXlO/fDZ5DmM8FpnSKKzLPfEYTVeXuVBTHF296TpxuJVg==", + "peer": true, "dependencies": { "bitmap-sdf": "^1.0.0", "draw-svg-path": "^1.0.0", @@ -8139,6 +8558,7 @@ "version": "7.0.8", "resolved": "https://registry.npmjs.org/swagger2openapi/-/swagger2openapi-7.0.8.tgz", "integrity": "sha512-upi/0ZGkYgEcLeGieoz8gT74oWHA0E7JivX7aN9mAf+Tc7BQoRBvnIGHoPDw+f9TXTW4s6kGYCZJtauP6OYp7g==", + "dev": true, "license": "BSD-3-Clause", "dependencies": { "call-me-maybe": "^1.0.1", @@ -8166,6 +8586,7 @@ "version": "1.10.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.3.tgz", "integrity": "sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA==", + "dev": true, "license": "ISC", "engines": { "node": ">= 6" @@ -8214,6 +8635,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "peer": true, "dependencies": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" @@ -8222,12 +8644,14 @@ "node_modules/through2/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "peer": true }, "node_modules/through2/node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "peer": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -8241,12 +8665,14 @@ "node_modules/through2/node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "peer": true }, "node_modules/through2/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "peer": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -8254,7 +8680,8 @@ "node_modules/tinycolor2": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", - "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==" + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "peer": true }, "node_modules/tinyglobby": { "version": "0.2.15", @@ -8304,17 +8731,20 @@ "node_modules/tinyqueue": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-2.0.3.tgz", - "integrity": "sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA==" + "integrity": "sha512-ppJZNDuKGgxzkHihX8v9v9G5f+18gzaTfrukGrq6ueg0lmH4nqVnA2IPG0AEH3jKEk2GRJCUhDoqpoiw3PHLBA==", + "peer": true }, "node_modules/to-float32": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/to-float32/-/to-float32-1.1.0.tgz", - "integrity": "sha512-keDnAusn/vc+R3iEiSDw8TOF7gPiTLdK1ArvWtYbJQiVfmRg6i/CAvbKq3uIS0vWroAC7ZecN3DjQKw3aSklUg==" + "integrity": "sha512-keDnAusn/vc+R3iEiSDw8TOF7gPiTLdK1ArvWtYbJQiVfmRg6i/CAvbKq3uIS0vWroAC7ZecN3DjQKw3aSklUg==", + "peer": true }, "node_modules/to-px": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-px/-/to-px-1.0.1.tgz", "integrity": "sha512-2y3LjBeIZYL19e5gczp14/uRWFDtDUErJPVN3VU9a7SJO+RjGRtYR47aMN2bZgGlxvW4ZcEz2ddUPVHXcMfuXw==", + "peer": true, "dependencies": { "parse-unit": "^1.0.1" } @@ -8336,6 +8766,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", + "peer": true, "dependencies": { "commander": "2" }, @@ -8348,18 +8779,21 @@ "node_modules/topojson-client/node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "peer": true }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, "license": "MIT" }, "node_modules/ts-algebra": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz", "integrity": "sha512-kloPhf1hq3JbCPOTYoOWDKxebWjNb2o/LKnNfkWhxVVisFFmMJPPdJeGoGmM+iRLyoXAR61e08Pb+vUXINg8aA==", + "dev": true, "license": "MIT" }, "node_modules/tsconfck": { @@ -8389,17 +8823,20 @@ "node_modules/type": { "version": "2.7.3", "resolved": "https://registry.npmjs.org/type/-/type-2.7.3.tgz", - "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==" + "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==", + "peer": true }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "peer": true }, "node_modules/typedarray-pool": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/typedarray-pool/-/typedarray-pool-1.2.0.tgz", "integrity": "sha512-YTSQbzX43yvtpfRtIDAYygoYtgT+Rpjuxy9iOpczrjpXLgGoyG7aS5USJXV2d3nn8uHTeb9rXDvzS27zUg5KYQ==", + "peer": true, "dependencies": { "bit-twiddle": "^1.0.0", "dup": "^1.0.0" @@ -8422,6 +8859,7 @@ "version": "3.19.3", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "dev": true, "optional": true, "bin": { "uglifyjs": "bin/uglifyjs" @@ -8434,6 +8872,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/ulid/-/ulid-3.0.2.tgz", "integrity": "sha512-yu26mwteFYzBAot7KVMqFGCVpsF6g8wXfJzQUHvu1no3+rRRSFcSV2nKeYvNPLD2J4b08jYBDhHUjeH0ygIl9w==", + "dev": true, "license": "MIT", "bin": { "ulid": "dist/cli.js" @@ -8443,6 +8882,7 @@ "version": "6.24.0", "resolved": "https://registry.npmjs.org/undici/-/undici-6.24.0.tgz", "integrity": "sha512-lVLNosgqo5EkGqh5XUDhGfsMSoO8K0BAN0TyJLvwNRSl4xWGZlCVYsAIpa/OpA3TvmnM01GWcoKmc3ZWo5wKKA==", + "dev": true, "license": "MIT", "engines": { "node": ">=18.17" @@ -8466,7 +8906,8 @@ "node_modules/unquote": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unquote/-/unquote-1.1.1.tgz", - "integrity": "sha512-vRCqFv6UhXpWxZPyGDh/F3ZpNv8/qo7w6iufLpQg9aKnQ71qM4B5KiI7Mia9COcjEhrO9LueHpMYjYzsWH3OIg==" + "integrity": "sha512-vRCqFv6UhXpWxZPyGDh/F3ZpNv8/qo7w6iufLpQg9aKnQ71qM4B5KiI7Mia9COcjEhrO9LueHpMYjYzsWH3OIg==", + "peer": true }, "node_modules/update-browserslist-db": { "version": "1.2.3", @@ -8501,12 +8942,14 @@ "node_modules/update-diff": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-diff/-/update-diff-1.1.0.tgz", - "integrity": "sha512-rCiBPiHxZwT4+sBhEbChzpO5hYHjm91kScWgdHf4Qeafs6Ba7MBl+d9GlGv72bcTZQO0sLmtQS1pHSWoCLtN/A==" + "integrity": "sha512-rCiBPiHxZwT4+sBhEbChzpO5hYHjm91kScWgdHf4Qeafs6Ba7MBl+d9GlGv72bcTZQO0sLmtQS1pHSWoCLtN/A==", + "peer": true }, "node_modules/url-template": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==", + "dev": true, "license": "BSD" }, "node_modules/use-sync-external-store": { @@ -8657,6 +9100,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/vt-pbf/-/vt-pbf-3.1.3.tgz", "integrity": "sha512-2LzDFzt0mZKZ9IpVF2r69G9bXaP2Q2sArJCmcCgvfTdCCZzSyz4aCLoQyUilu37Ll56tCblIZrXFIjNUpGIlmA==", + "peer": true, "dependencies": { "@mapbox/point-geometry": "0.1.0", "@mapbox/vector-tile": "^1.3.1", @@ -8666,12 +9110,14 @@ "node_modules/weak-map": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/weak-map/-/weak-map-1.0.8.tgz", - "integrity": "sha512-lNR9aAefbGPpHO7AEnY0hCFjz1eTkWCXYvkTRrTHs9qv8zJp+SkVYpzfLIFXQQiG3tVvbNFQgVg2bQS8YGgxyw==" + "integrity": "sha512-lNR9aAefbGPpHO7AEnY0hCFjz1eTkWCXYvkTRrTHs9qv8zJp+SkVYpzfLIFXQQiG3tVvbNFQgVg2bQS8YGgxyw==", + "peer": true }, "node_modules/webgl-context": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/webgl-context/-/webgl-context-2.2.0.tgz", "integrity": "sha512-q/fGIivtqTT7PEoF07axFIlHNk/XCPaYpq64btnepopSWvKNFkoORlQYgqDigBIuGA1ExnFd/GnSUnBNEPQY7Q==", + "peer": true, "dependencies": { "get-canvas-context": "^1.0.1" } @@ -8680,12 +9126,14 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, "license": "BSD-2-Clause" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", @@ -8696,6 +9144,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", + "peer": true, "dependencies": { "isexe": "^3.1.1" }, @@ -8709,13 +9158,15 @@ "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==" + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true }, "node_modules/world-calendars": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/world-calendars/-/world-calendars-1.0.4.tgz", "integrity": "sha512-VGRnLJS+xJmGDPodgJRnGIDwGu0s+Cr9V2HB3EzlDZ5n0qb8h5SJtGUEkjrphZYAglEiXZ6kiXdmk0H/h/uu/w==", "license": "MIT", + "peer": true, "dependencies": { "object-assign": "^4.1.0" } @@ -8724,6 +9175,7 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -8758,12 +9210,14 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "peer": true }, "node_modules/ws": { "version": "7.5.10", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=8.3.0" @@ -8785,6 +9239,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "dev": true, "funding": [ { "type": "github", @@ -8800,6 +9255,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "peer": true, "engines": { "node": ">=0.4" } @@ -8808,6 +9264,7 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -8839,12 +9296,14 @@ "version": "0.0.43", "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz", "integrity": "sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==", + "dev": true, "license": "Apache-2.0" }, "node_modules/yargs": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.0.1.tgz", "integrity": "sha512-xBBulfCc8Y6gLFcrPvtqKz9hz8SO0l1Ni8GgDekvBX2ro0HRQImDGnikfc33cgzcYUSncapnNcZDjVFIH3f6KQ==", + "dev": true, "license": "MIT", "dependencies": { "cliui": "^7.0.2", @@ -8863,6 +9322,7 @@ "version": "20.2.9", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, "license": "ISC", "engines": { "node": ">=10" diff --git a/frontend/package.json b/frontend/package.json index 99d5c9d6..531a5de2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,13 +11,12 @@ "@mui/icons-material": "^6.1.2", "@mui/material": "^6.1.2", "@mui/x-data-grid": "^7.19.0", - "@redocly/cli": "^2.28.1", "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react-swc": "^4.2.3", "lodash": "^4.17.23", - "plotly.js": "^3.3.1", + "plotly.js-cartesian-dist-min": "^3.3.1", "react": "^19.2.4", "react-dom": "^19.2.4", "react-plotly.js": "^2.6.0", @@ -33,16 +32,11 @@ "serve": "vite preview", "redocly": "redocly build-docs ../broker/jfjoch_api.yaml --output=dist/openapi.html", "redocly4broker": "redocly build-docs ../broker/jfjoch_api.yaml --output=../broker/redoc-static.html", - "test": "react-scripts test", - "eject": "react-scripts eject", "openapi": "./node_modules/openapi-typescript-codegen/bin/index.js -i ../broker/jfjoch_api.yaml --output ./src/openapi" }, "overrides": { "styled-components": "5.3.11" }, - "eslintConfig": { - "extends": "react-app" - }, "browserslist": { "production": [ ">0.2%", @@ -56,6 +50,7 @@ ] }, "devDependencies": { + "@redocly/cli": "^2.28.1", "@types/lodash": "^4.17.10", "@types/react-plotly.js": "^2.6.3", "esbuild-style-plugin": "^1.6.3", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3db6ea81..b3570596 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -64,7 +64,7 @@ class App extends Component { s: {} } - getValues() { + getValues = () => { DefaultService.getStatistics() .then(data => this.setState({s: data, connection_error: false})) .catch(_ => { diff --git a/frontend/src/components/DataCollection.tsx b/frontend/src/components/DataCollection.tsx index 28a72311..de02f731 100644 --- a/frontend/src/components/DataCollection.tsx +++ b/frontend/src/components/DataCollection.tsx @@ -264,29 +264,23 @@ class DataCollection extends React.Component { this.setState((e) => this.setState( - prevState => ( - {grid : {...prevState.grid, n_fast: val}} - )))} + callback={(val: number) => this.setState( + prevState => ({grid : {...prevState.grid, n_fast: val}}))} min={1} default={this.state.grid.n_fast} /> this.setState((e) => this.setState( - prevState => ( - {grid : {...prevState.grid, step_x_um: val}} - )))} + callback={(val: number) => this.setState( + prevState => ({grid : {...prevState.grid, step_x_um: val}}))} units="μm" float={true} default={this.state.grid.step_x_um} /> this.setState((e) => this.setState( - prevState => ( - {grid : {...prevState.grid, step_y_um: val}} - )))} + callback={(val: number) => this.setState( + prevState => ({grid : {...prevState.grid, step_y_um: val}}))} units="μm" float={true} default={this.state.grid.step_y_um} diff --git a/frontend/src/components/DataProcessingSettings.tsx b/frontend/src/components/DataProcessingSettings.tsx index 52b322ef..c6352e96 100644 --- a/frontend/src/components/DataProcessingSettings.tsx +++ b/frontend/src/components/DataProcessingSettings.tsx @@ -3,6 +3,7 @@ import React, {Component} from 'react'; import Paper from '@mui/material/Paper'; import {Grid, Slider, Switch, Typography} from "@mui/material"; import {DefaultService, spot_finding_settings} from "../openapi"; +import _ from "lodash"; type MyProps = { s?: spot_finding_settings, @@ -11,25 +12,29 @@ type MyProps = { type MyState = { s: spot_finding_settings, + last_downloaded_s: spot_finding_settings, high_res_gap_Q_recipA: number } +const default_spot_finding_settings: spot_finding_settings = { + enable: true, + indexing: true, + photon_count_threshold: 8, + signal_to_noise_threshold: 3.0, + min_pix_per_spot: 2, + max_pix_per_spot: 50, + high_resolution_limit: 2.5, + low_resolution_limit: 50.0, + quick_integration: false, + high_resolution_limit_for_spot_count_low_res: 5.0, + ice_ring_width_q_recipA: 0.02, + high_res_gap_Q_recipA: 1.5 +}; + class DataProcessingSettings extends Component { state : MyState = { - s: { - enable: true, - indexing: true, - photon_count_threshold: 8, - signal_to_noise_threshold: 3.0, - min_pix_per_spot: 2, - max_pix_per_spot: 50, - high_resolution_limit: 2.5, - low_resolution_limit: 50.0, - quick_integration: false, - high_resolution_limit_for_spot_count_low_res: 5.0, - ice_ring_width_q_recipA: 0.02, - high_res_gap_Q_recipA: 1.5 - }, + s: default_spot_finding_settings, + last_downloaded_s: default_spot_finding_settings, high_res_gap_Q_recipA: 1.5 } @@ -41,9 +46,12 @@ class DataProcessingSettings extends Component { getValues() { const incoming = this.props.s; - if (incoming !== undefined) + // Only adopt the server copy when it actually changed, otherwise the + // 1 s statistics poll would overwrite edits the user is making. + if ((incoming !== undefined) && !_.isEqual(incoming, this.state.last_downloaded_s)) this.setState(prevState => ({ s: incoming, + last_downloaded_s: incoming, high_res_gap_Q_recipA: incoming.high_res_gap_Q_recipA ?? prevState.high_res_gap_Q_recipA })); } @@ -52,9 +60,8 @@ class DataProcessingSettings extends Component { this.getValues(); } - componentDidUpdate(prevProps: Readonly) { - if ((this.props.s !== undefined) && (prevProps.s != this.props.s)) - this.setState({s: this.props.s}); + componentDidUpdate() { + this.getValues(); } setPhotonCountThreshold = (event: Event, newValue: number | number[]) => { diff --git a/frontend/src/components/MultiLinePlotWrapper.jsx b/frontend/src/components/MultiLinePlotWrapper.jsx index 28b645c2..0d4a3667 100644 --- a/frontend/src/components/MultiLinePlotWrapper.jsx +++ b/frontend/src/components/MultiLinePlotWrapper.jsx @@ -1,6 +1,6 @@ import React, {Component} from 'react'; -import Plot from "react-plotly.js"; +import Plot from "./Plot"; // Not using TypeScript, as plotly is not TypeScript :( @@ -41,7 +41,6 @@ class MultiLinePlotWrapper extends Component { } } - console.log(layout); return v === value ) as color_scale; - // If no match is found, default to file_writer_format.NONE + // If no match is found, default to the indigo color scale return enumValue || color_scale.INDIGO; } @@ -89,25 +89,24 @@ class PreviewImage extends Component { } update_image_id_mode = (event: React.ChangeEvent) => { - let s : preview_settings = this.state.settings; - + let image_id : number; switch (event.target.value) { case "last": - s.image_id = -1; - this.setState({settings: s, image_id_mode: event.target.value}); - this.getValues(s); + image_id = -1; break; case "last_indexed": - s.image_id = -2; - this.setState({settings: s, image_id_mode: event.target.value}); - this.getValues(s); + image_id = -2; break; case "select": - s.image_id = this.state.image_id; - this.setState({settings: s, image_id_mode: event.target.value}); - this.getValues(s); + image_id = this.state.image_id; break; + default: + return; } + + let s : preview_settings = {...this.state.settings, image_id}; + this.setState({settings: s, image_id_mode: event.target.value}); + this.getValues(s); } updateToggle = (event: React.ChangeEvent) => { diff --git a/frontend/src/react-app-env.d.ts b/frontend/src/react-app-env.d.ts deleted file mode 100644 index 6431bc5f..00000000 --- a/frontend/src/react-app-env.d.ts +++ /dev/null @@ -1 +0,0 @@ -/// diff --git a/frontend/src/serviceWorker.js b/frontend/src/serviceWorker.js deleted file mode 100644 index b04b771a..00000000 --- a/frontend/src/serviceWorker.js +++ /dev/null @@ -1,141 +0,0 @@ -// This optional code is used to register a service worker. -// register() is not called by default. - -// This lets the app load faster on subsequent visits in production, and gives -// it offline capabilities. However, it also means that developers (and users) -// will only see deployed updates on subsequent visits to a page, after all the -// existing tabs open on the page have been closed, since previously cached -// resources are updated in the background. - -// To learn more about the benefits of this model and instructions on how to -// opt-in, read https://bit.ly/CRA-PWA - -const isLocalhost = Boolean( - window.location.hostname === 'localhost' || - // [::1] is the IPv6 localhost address. - window.location.hostname === '[::1]' || - // 127.0.0.0/8 are considered localhost for IPv4. - window.location.hostname.match( - /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/ - ) -); - -export function register(config) { - if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) { - // The URL constructor is available in all browsers that support SW. - const publicUrl = new URL(process.env.PUBLIC_URL, window.location.href); - if (publicUrl.origin !== window.location.origin) { - // Our service worker won't work if PUBLIC_URL is on a different origin - // from what our page is served on. This might happen if a CDN is used to - // serve assets; see https://github.com/facebook/create-react-app/issues/2374 - return; - } - - window.addEventListener('load', () => { - const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`; - - if (isLocalhost) { - // This is running on localhost. Let's check if a service worker still exists or not. - checkValidServiceWorker(swUrl, config); - - // Add some additional logging to localhost, pointing developers to the - // service worker/PWA documentation. - navigator.serviceWorker.ready.then(() => { - console.log( - 'This web app is being served cache-first by a service ' + - 'worker. To learn more, visit https://bit.ly/CRA-PWA' - ); - }); - } else { - // Is not localhost. Just register service worker - registerValidSW(swUrl, config); - } - }); - } -} - -function registerValidSW(swUrl, config) { - navigator.serviceWorker - .register(swUrl) - .then(registration => { - registration.onupdatefound = () => { - const installingWorker = registration.installing; - if (installingWorker == null) { - return; - } - installingWorker.onstatechange = () => { - if (installingWorker.state === 'installed') { - if (navigator.serviceWorker.controller) { - // At this point, the updated precached content has been fetched, - // but the previous service worker will still serve the older - // content until all client tabs are closed. - console.log( - 'New content is available and will be used when all ' + - 'tabs for this page are closed. See https://bit.ly/CRA-PWA.' - ); - - // Execute callback - if (config && config.onUpdate) { - config.onUpdate(registration); - } - } else { - // At this point, everything has been precached. - // It's the perfect time to display a - // "Content is cached for offline use." message. - console.log('Content is cached for offline use.'); - - // Execute callback - if (config && config.onSuccess) { - config.onSuccess(registration); - } - } - } - }; - }; - }) - .catch(error => { - console.error('Error during service worker registration:', error); - }); -} - -function checkValidServiceWorker(swUrl, config) { - // Check if the service worker can be found. If it can't reload the page. - fetch(swUrl, { - headers: { 'Service-Worker': 'script' }, - }) - .then(response => { - // Ensure service worker exists, and that we really are getting a JS file. - const contentType = response.headers.get('content-type'); - if ( - response.status === 404 || - (contentType != null && contentType.indexOf('javascript') === -1) - ) { - // No service worker found. Probably a different app. Reload the page. - navigator.serviceWorker.ready.then(registration => { - registration.unregister().then(() => { - window.location.reload(); - }); - }); - } else { - // Service worker found. Proceed as normal. - registerValidSW(swUrl, config); - } - }) - .catch(() => { - console.log( - 'No internet connection found. App is running in offline mode.' - ); - }); -} - -export function unregister() { - if ('serviceWorker' in navigator) { - navigator.serviceWorker.ready - .then(registration => { - registration.unregister(); - }) - .catch(error => { - console.error(error.message); - }); - } -} diff --git a/frontend/src/setupTests.js b/frontend/src/setupTests.js deleted file mode 100644 index 74b1a275..00000000 --- a/frontend/src/setupTests.js +++ /dev/null @@ -1,5 +0,0 @@ -// jest-dom adds custom jest matchers for asserting on DOM nodes. -// allows you to do things like: -// expect(element).toHaveTextContent(/react/i) -// learn more: https://github.com/testing-library/jest-dom -import '@testing-library/jest-dom/extend-expect'; -- 2.52.0 From c69b5297d5793f3a14591d5a98313528e2ff6dfb Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 07:38:36 +0200 Subject: [PATCH 064/228] docs: add jfjoch_process page, refresh viewer/tools docs, unify CLI naming - Add docs/JFJOCH_PROCESS.md describing the offline analysis tool, its options, output files, and the broker/viewer/process distinction; mention jfjoch_scale for re-scaling/merging. - Rewrite docs/JFJOCH_VIEWER.md for consistency: functionality, HTTP env vars (JUNGFRAUJOCH_HTTP_HOST/PORT), command line, and the real D-Bus API. - Refresh docs/TOOLS.md to the current set of tools; add both pages to index.rst. - jfjoch_process: fix stale self-name (jfjoch_analysis -> jfjoch_process) in usage/license/logger. - jfjoch_scale: unify --scaling-high-resolution with jfjoch_process (drop -D short flag, make it long-only) and remove dead p/q/i short options. Co-Authored-By: Claude Opus 4.8 --- docs/JFJOCH_PROCESS.md | 159 +++++++++++++++++++++++++++++++++++++++ docs/JFJOCH_VIEWER.md | 66 ++++++++++++---- docs/TOOLS.md | 76 +++++++++++++------ docs/index.rst | 2 + tools/jfjoch_process.cpp | 6 +- tools/jfjoch_scale.cpp | 11 +-- 6 files changed, 274 insertions(+), 46 deletions(-) create mode 100644 docs/JFJOCH_PROCESS.md diff --git a/docs/JFJOCH_PROCESS.md b/docs/JFJOCH_PROCESS.md new file mode 100644 index 00000000..ac8886a7 --- /dev/null +++ b/docs/JFJOCH_PROCESS.md @@ -0,0 +1,159 @@ +# jfjoch_process + +`jfjoch_process` is the **offline** crystallographic data-analysis tool of Jungfraujoch. +It takes an existing HDF5 dataset, runs the full analysis pipeline — spot finding, indexing, +geometry refinement, Bragg integration and (optionally) scaling and merging — and writes the +results to a `_process.h5` file, plus reflection files (`.mtz`/`.cif`/`.hkl`) when merging is +requested. + +It runs the *same* analysis code as the online and interactive tools, just driven from the +command line over a file rather than a live detector stream. + +> **Note.** `jfjoch_process` is under very active development. This page describes the tool and +> its options at a high level; the authoritative, always-current list of options is the program's +> own usage message — run `jfjoch_process` with no arguments. + +## Where it fits among the three analysis tools + +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| [`jfjoch_viewer`](JFJOCH_VIEWER.md) | Interactive, on-screen exploration | Qt desktop application | Displayed on screen (results not saved to disk) | +| **`jfjoch_process`** | **Offline batch processing of a stored dataset** | **Command-line interface** | **`_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging** | + +Use `jfjoch_process` to re-analyse data after acquisition, to experiment with processing +parameters, or to produce merged intensities for downstream structure solution. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. The CUDA build +provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`fft`); without CUDA +only the CPU `fftw` indexer is available. Spot finding, integration and scaling run on the CPU and +scale with the thread count (`-N`). + +## Input and output + +**Input** is a single Jungfraujoch HDF5 master file (NXmx-based). If the dataset already contains +stored spot lists, two-pass rotation indexing can reuse them instead of re-running spot finding on +the first pass. + +**Output** (controlled by `-o, --output-prefix`, default `output`): + +- `_process.h5` — NXmx-compliant HDF5 with derived metadata (spots, indexing, + integration, azimuthal integration, per-image statistics). See + [HDF5 / NeXus data format](HDF5.md) for the layout. +- When merging (`-M`, or whenever a `--reference-mtz` is supplied), the merged reflections are + written as `.mtz` (default), or `.cif` / `.hkl` depending on + `--scaling-output`. No-reference scaling additionally emits per-iteration `_iterN_scale.dat`. + +Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the +console. + +## Re-scaling and re-merging (`jfjoch_scale`) + +The companion tool `jfjoch_scale` re-scales and merges the *already-integrated* reflections stored +in one or more `_process.h5` files, without re-running spot finding or integration. Use it to +re-merge quickly with a different space group, partiality model, resolution limit or reference MTZ, +or to combine several processed runs into one set of merged intensities. + +## Quick start + +### Rotation data + +Two-pass rotation indexing, rotation partiality, scale and merge in space group 96: + +``` +jfjoch_process rotation_master.h5 \ + -o lyso_rot -N 16 \ + -R -S 96 \ + -M -P rot +``` + +`-R` runs the two-pass rotation indexer (index the sweep once, then process every frame against +that lattice); `-P rot` selects the rotation partiality model; `-M` scales and merges. For strong +rotation data the de-novo FFT indexer often indexes more frames — add `-X fft` (and drop `-C` to +let it find the cell from scratch). + +### Still / serial data + +Known-cell indexing of independent stills with the GPU fast-feedback indexer, then merge against a +reference structure: + +``` +jfjoch_process serial_master.h5 \ + -o lyso_serial -N 16 \ + -X ffbidx -C 79,79,38,90,90,90 -S 96 \ + --spot-sigma 4 \ + -M -z reference.mtz -r pixelrefine \ + --scaling-high-resolution 1.8 +``` + +`ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills. +`-r pixelrefine` selects the experimental reference-driven still integrator (needs +`--reference-mtz`). For weak serial data, tightening spot finding with `--spot-sigma 4` typically +raises the indexing rate substantially. + +## Command-line options + +General: + +| Option | Description | +| --- | --- | +| `-o, --output-prefix ` | Output file prefix (default: `output`) | +| `-N, --threads ` | Number of worker threads (default: 1) | +| `-s, --start-image ` | First image to process (default: 0) | +| `-e, --end-image ` | Last image to process (default: all) | +| `-t, --stride ` | Process every *n*-th image (default: 1) | +| `-v, --verbose` | Verbose output | + +Spot finding: + +| Option | Description | +| --- | --- | +| `--spot-sigma ` | Noise sigma level for spot finding (default: 3.0) | +| `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | +| `--spot-high-resolution ` | High-resolution limit for spot finding, Å (default: 1.5) | +| `--max-spots ` | Maximum spot count (default: 250) | + +Indexing: + +| Option | Description | +| --- | --- | +| `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`) | +| `-S, --space-group ` | Space group number (used for indexing and scaling) | +| `-r, --refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `pixelrefine` | +| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (optional image count, default 30) | +| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) | +| `--redo-rotation-spots` | Redo spot finding for the two-pass rotation first pass | +| `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass | + +Indexer choice in brief: `ffbidx` (GPU) refines toward a **known cell** and is best for sparse +serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the +[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS.md) for the algorithms. + +Scaling and merging: + +| Option | Description | +| --- | --- | +| `-M, --scale-merge` | Scale and merge | +| `-P, --partiality ` | Partiality model: `fixed` (default) \| `rot` \| `unity` | +| `-A, --anomalous` | Anomalous mode (keep Friedel pairs separate) | +| `-B, --refine-bfactor` | Refine a per-image B-factor | +| `-w, --wedge[=num]` | Refine the per-image rotation wedge (optional starting value) | +| `--scaling-high-resolution ` | High-resolution limit for scaling, Å (default: no limit) | +| `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02) | +| `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: off) | +| `--reject-delta-cchalf ` | Drop images with ΔCC1/2 below mean − N·stddev (default: off) | +| `--min-image-cc ` | Per-image CC limit, percent (default: no limit) | +| `--scaling-iterations ` | Scaling iterations with no reference data (default: 3) | +| `--scaling-output ` | Reflection output format: `mtz` (default) \| `cif` \| `txt` | +| `-z, --reference-mtz ` | Reference MTZ (enables reference-driven scaling) | + +Pixel refinement (experimental; select with `-r pixelrefine`, requires `--reference-mtz`): + +| Option | Description | +| --- | --- | +| `--bandwidth ` | Relative X-ray bandwidth FWHM (e.g. `0.01` for a 1% DMM); default from file or 0 (monochromatic) | +| `--integration-radius ` | Signal-box radius `r1`, or `r1,r2,r3` (px). One value ⇒ `r2=r1+2`, `r3=r1+4` | +| `--profile-multiplier ` | Scale the measured tangential profile width (default: 6; XDS-style generous aperture) | diff --git a/docs/JFJOCH_VIEWER.md b/docs/JFJOCH_VIEWER.md index 9e160be8..5a7ff0a8 100644 --- a/docs/JFJOCH_VIEWER.md +++ b/docs/JFJOCH_VIEWER.md @@ -1,19 +1,57 @@ # jfjoch_viewer -Jungfraujoch diffraction viewer is distributed as a standalone application. -It uses Qt library version 6, and allows to open HDF5 files generated by [`jfjoch_writer`](JFJOCH_WRITER.md). -It as well allows to open NXmx files written by DECTRIS detectors, though testing is very limited. -It can be downloaded pre-built from Gitea release page, or from Jungfraujoch RPM/APT repositories. -See [Deployment](DEPLOYMENT.md) section for more information. -This viewer can be also sync online with [`jfjoch_broker`](JFJOCH_BROKER.md) HTTP interface -to visualize data during data collection. +`jfjoch_viewer` is the **interactive** desktop application of Jungfraujoch. It opens diffraction +datasets, displays each image together with the analysis overlay (spots, predictions, azimuthal +integration, per-image statistics), and can follow a live data collection by syncing with a +running [`jfjoch_broker`](JFJOCH_BROKER.md) over its HTTP interface. -# Data processing pipeline -Viewer contains embedded data processing pipeline, which is the same as the one used -in Jungfraujoch installation. On systems equipped with GPU, it is preferable to use version -compiled with CUDA support (proper RPM/APT repositories). +It is a standalone Qt 6 application, distributed pre-built on the Gitea release page and in the +Jungfraujoch RPM/APT repositories (see [Deployment](DEPLOYMENT.md)). -At the moment data processing results are not saved to disk. +## Where it fits among the three analysis tools -# DBus interface -Viewer can be controlled externally with DBus interface, allowing to open file, open detector via HTTP or load image. \ No newline at end of file +| Tool | Mode | Driven by | Output | +| --- | --- | --- | --- | +| [`jfjoch_broker`](JFJOCH_BROKER.md) | Online, real-time streaming analysis on FPGA + GPU | HTTP/REST + ZeroMQ | Live results and statistics, images streamed to [`jfjoch_writer`](JFJOCH_WRITER.md) | +| **`jfjoch_viewer`** | **Interactive, on-screen exploration** | **Qt desktop application** | **Displayed on screen (results not saved to disk)** | +| [`jfjoch_process`](JFJOCH_PROCESS.md) | Offline batch processing of a stored dataset | Command-line interface | `_process.h5`, and `.mtz`/`.cif`/`.hkl` when merging | + +## Functionality + +- Opens HDF5 files written by [`jfjoch_writer`](JFJOCH_WRITER.md) (`*_master.h5`) and the + `*_process.h5` files produced by [`jfjoch_process`](JFJOCH_PROCESS.md). It also opens NXmx files + written by DECTRIS detectors, though that path has had only limited testing. +- Runs an **embedded data-processing pipeline** — the same analysis code as the rest of + Jungfraujoch — performing spot finding, indexing and integration on the displayed images. + Results are shown on screen but are **not** saved to disk. +- Auxiliary windows and panels: image list, image metadata, spot list, reflection list, + per-region-of-interest statistics, the azimuthal-integration profile, and dataset-info charts. +- User-mask editing: build a user mask interactively, clear it, save it as TIFF, or upload it to a + connected server. + +## Hardware + +As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. On systems with a +GPU, use the CUDA build (provided as separate RPM/APT repositories) for the embedded indexing and +integration; the non-CUDA build runs the same pipeline on the CPU at much lower throughput. + +## Opening data + +- **File ▸ Open** (`Ctrl+O`) — open a local HDF5 file. +- **File ▸ Open HTTP** (`Ctrl+H`) — connect to a `jfjoch_broker` HTTP endpoint to follow a live + collection. The dialog defaults to host `localhost` and port `8080`; these defaults can be + overridden with the environment variables `JUNGFRAUJOCH_HTTP_HOST` and `JUNGFRAUJOCH_HTTP_PORT`. +- **Command line** — `jfjoch_viewer ` opens a file (or an `http://host:port` URL) on + start-up. `--dbus ` (`-d`) enables or disables the D-Bus interface (default: enabled); + `--help` and `--version` behave as usual. + +## D-Bus interface + +When enabled, the viewer registers the D-Bus interface `ch.psi.jfjoch_viewer`, so other processes +can drive it: + +- `LoadFile(filename, image_number=0, summation=1)` — open a file (or an `http://host:port` URL) + and display the given image. +- `LoadImage(image_number, summation=1)` — navigate to an image in the already-open dataset. + +`summation` sums that many consecutive images before display. diff --git a/docs/TOOLS.md b/docs/TOOLS.md index d567125f..e723d240 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -1,47 +1,75 @@ # Tools -## jfjoch_pcie_status -Prints detailed status information about the card. Execute by adding device path, e.g.: +Besides the main services ([`jfjoch_broker`](JFJOCH_BROKER.md), +[`jfjoch_writer`](JFJOCH_WRITER.md), [`jfjoch_viewer`](JFJOCH_VIEWER.md)), the repository ships a +number of command-line tools. Each prints its own usage when run with `-h` or without arguments. + +## Data analysis + +### jfjoch_process +Offline CLI tool that runs the full crystallographic analysis pipeline (spot finding, indexing, +integration, scaling/merging) on a stored HDF5 dataset, producing a `_process.h5` file and, when +merging, reflection files. See [jfjoch_process](JFJOCH_PROCESS.md). + +### jfjoch_scale +Re-scales and merges the already-integrated reflections from one or more `_process.h5` files +(no re-integration). Useful to re-merge with a different space group, partiality, resolution limit +or reference MTZ, or to combine several runs. See [jfjoch_process](JFJOCH_PROCESS.md). + +### jfjoch_extract_hkl +Extracts reflections (HKL list) from a Jungfraujoch master file; can sum the same HKL across +neighbouring images and compare against an XDS `INTEGRATE.HKL` reference. + +## FPGA / PCIe card management + +### jfjoch_pcie_status +Prints detailed status information about the card. Safe to run during data collection: ``` ./jfjoch_pcie_status /dev/jfjoch0 ``` -The program is safe to execute during a running data collection. - -## jfjoch_pcie_clear_net_counters -Network counters in the card give information about Ethernet, UDP and ICMP packets encountered by the network stack prior to Jungfraujoch logic. -These counters are running from the moment card is powered on. They can be reset by running the program with device name, e.g.: -``` -./jfjoch_pcie_clear_net_counters /dev/jfjoch0 -``` - -## jfjoch_pcie_net_cfg -Network configuration can be retrieved and modified with `jfjoch_pcie_net_cfg` tool. Usage: +### jfjoch_pcie_net_cfg +Reads and modifies the network configuration of the card's interfaces: ``` jfjoch_pcie_net_cfg - Read configuration for all network interface of a device + Read configuration for all network interfaces of a device jfjoch_pcie_net_cfg |fgen - Read configuration for a particular network interface / internal frame generator" + Read configuration for a particular network interface / internal frame generator jfjoch_pcie_net_cfg |fgen ipv4 Set IPv4 address for a particular network interface / internal frame generator jfjoch_pcie_net_cfg |fgen direct 0|1 Set direct mode for a particular network interface / internal frame generator jfjoch_pcie_net_cfg |fgen clear - Clear Ethernet counter for a particular network interface / internal frame generator + Clear Ethernet counters for a particular network interface / internal frame generator ``` -## jfjoch_hdf5_tools +### jfjoch_pcie_clear_net_counters +Resets the card's Ethernet, UDP and ICMP packet counters (which otherwise run from power-on): +``` +./jfjoch_pcie_clear_net_counters /dev/jfjoch0 +``` -Tool to test single threaded HDF5 writer performance +### jfjoch_pcie_read_register +Reads raw Jungfraujoch FPGA registers. -## jfjoch_offline_process +## Testing, benchmarking and simulation -Tool to run offline processing on existing HDF5 dataset +### jfjoch_udp_simulator +UDP packet simulator used to test the Jungfraujoch FPGA receiver. -## jfjoch_udp_simulator +### jfjoch_fpga_test +Exercises and benchmarks the FPGA data path and receiver. With `-H` it runs the high-level +synthesis C model on the CPU, so no FPGA device is required. -UDP simulator to test Jungfraujoch FPGA +### jfjoch_lite_perf_test +Performance test of the lite (CPU/GPU) analysis path — indexing, integration and optional file +writing. -## jfjoch_pcie_read_register +### jfjoch_hdf5_test +Tests single-threaded HDF5 writer performance. -Tool to read verbatim Jungfraujoch FPGA registers \ No newline at end of file +### jfjoch_azint_test +Tests the azimuthal integration code on synthetic data. + +### jfjoch_simplon_test +Minimal test client for a DECTRIS SIMPLON detector API. diff --git a/docs/index.rst b/docs/index.rst index b8755c9c..c6016839 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,8 @@ Jungfraujoch is distributed under the GPLv3 license. JFJOCH_BROKER JFJOCH_WRITER + JFJOCH_PROCESS + JFJOCH_VIEWER SOFTWARE_INTEGRATION TOOLS diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index acadd7ba..74b422b9 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -36,7 +36,7 @@ #include "../image_analysis/UpdateReflectionResolution.h" void print_usage() { - std::cout << "Usage ./jfjoch_analysis {} " << std::endl; + std::cout << "Usage jfjoch_process {} " << std::endl; std::cout << "Options:" << std::endl; std::cout << " -o, --output-prefix Output file prefix (default: output)" << std::endl; std::cout << " -N, --threads Number of threads (default: 1)" << std::endl; @@ -281,9 +281,9 @@ int main(int argc, char **argv) { RegisterHDF5Filter(); - print_license("jfjoch_analysis"); + print_license("jfjoch_process"); - Logger logger("jfjoch_analysis"); + Logger logger("jfjoch_process"); std::string input_file; std::string output_prefix = "output"; diff --git a/tools/jfjoch_scale.cpp b/tools/jfjoch_scale.cpp index e6da83f2..bb04f64a 100644 --- a/tools/jfjoch_scale.cpp +++ b/tools/jfjoch_scale.cpp @@ -36,7 +36,7 @@ void print_usage() { std::cout << "" << std::endl; std::cout << " Scaling and merging" << std::endl; std::cout << " -S, --space-group Space group number" << std::endl; - std::cout << " -D, --scaling-high-resolution High resolution limit for scaling/merging (default: 0.0; no limit)" + std::cout << " --scaling-high-resolution High resolution limit for scaling/merging (default: 0.0; no limit)" << std::endl; std::cout << " -P, --partiality Partiality refinement fixed|rot|unity (default: fixed)" << std::endl; @@ -57,7 +57,8 @@ enum { OPT_MIN_PARTIALITY = 1000, OPT_MIN_IMAGE_CC, OPT_SCALING_ITERATIONS, - OPT_SCALING_OUTPUT + OPT_SCALING_OUTPUT, + OPT_SCALING_HIGH_RESOLUTION }; @@ -73,7 +74,7 @@ static option long_options[] = { {"anomalous", no_argument, nullptr, 'A'}, {"refine-bfactor", no_argument, nullptr, 'B'}, {"wedge", optional_argument, nullptr, 'w'}, - {"scaling-high-resolution", required_argument, nullptr, 'D'}, + {"scaling-high-resolution", required_argument, nullptr, OPT_SCALING_HIGH_RESOLUTION}, {"min-partiality", required_argument, nullptr, OPT_MIN_PARTIALITY}, {"min-image-cc", required_argument, nullptr, OPT_MIN_IMAGE_CC}, {"scaling-iterations", required_argument, nullptr, OPT_SCALING_ITERATIONS}, @@ -116,7 +117,7 @@ int main(int argc, char **argv) { int opt; int option_index = 0; - const char *short_opts = "vo:N:s:e:z:S:P:ABw::D:p:q:i:"; + const char *short_opts = "vo:N:s:e:z:S:P:ABw::"; while ((opt = getopt_long(argc, argv, short_opts, long_options, &option_index)) != -1) { switch (opt) { @@ -165,7 +166,7 @@ int main(int argc, char **argv) { if (optarg) wedge_for_scaling = std::stod(optarg); break; - case 'D': + case OPT_SCALING_HIGH_RESOLUTION: d_min_scale_merge = atof(optarg); logger.Info("High resolution limit for scaling/merging set to {:.2f} A", d_min_scale_merge.value()); break; -- 2.52.0 From 39c808776f0668ba103eee544066ef2adad0faca Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 08:37:21 +0200 Subject: [PATCH 065/228] build: portability groundwork toward a Windows/MSVC viewer - CMakeLists.txt: fetch libzmq at the top level (zeromq/libzmq v4.3.5) before slsDetectorPackage, so this project controls the ZeroMQ version instead of sls's bundled archive. sls reuses it via its if(NOT libzmq_POPULATED) guard, so a single libzmq-static target is built (no duplicate-target/double-symbol clash). Verified the full Linux build still links (JFJochZMQ -> JFJochReceiver -> jfjoch_process). - common/NetworkAddressConvert.cpp: guard the network includes for _WIN32 (winsock2/ws2tcpip vs arpa/inet). - common/ImageBuffer.cpp: use std::malloc/std::free for the non-NUMA path instead of an anonymous mmap (the mapping had no huge-page/mbind flags, so it was equivalent) - portable and removes the POSIX-only dependency. The NUMA-interleave path is unchanged. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 20 ++++++++++++++++++++ common/ImageBuffer.cpp | 19 +++++++++---------- common/NetworkAddressConvert.cpp | 5 +++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 02668e08..d675e610 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,26 @@ FetchContent_Declare( EXCLUDE_FROM_ALL ) +# ZeroMQ (libzmq): fetch it here, at the top level, so that THIS project - not +# slsDetectorPackage - controls the version. slsDetectorPackage bundles its own libzmq +# archive, but only populates it behind `if(NOT libzmq_POPULATED)`; by making libzmq +# available before sls below, sls reuses this copy and a single libzmq-static target is +# built (no duplicate target / double-symbol clash). A future viewer-only / Windows build +# (which does not build sls) fetches the same libzmq standalone. +SET(BUILD_SHARED OFF CACHE BOOL "" FORCE) # libzmq: static only (matches sls) +SET(BUILD_TESTS OFF CACHE BOOL "" FORCE) # libzmq: no test build +SET(WITH_PERF_TOOL OFF CACHE BOOL "" FORCE) # libzmq: no perf tools +SET(WITH_DOCS OFF CACHE BOOL "" FORCE) # libzmq: no docs +SET(ENABLE_CPACK OFF CACHE BOOL "" FORCE) # libzmq: no CPack injection +FetchContent_Declare( + libzmq + GIT_REPOSITORY https://github.com/zeromq/libzmq.git + GIT_TAG v4.3.5 + EXCLUDE_FROM_ALL +) + +# libzmq must be made available BEFORE sls_detector_package for the override above to take effect. +FetchContent_MakeAvailable(libzmq) FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib) ADD_SUBDIRECTORY(jungfrau) diff --git a/common/ImageBuffer.cpp b/common/ImageBuffer.cpp index e3ced97f..3145998b 100644 --- a/common/ImageBuffer.cpp +++ b/common/ImageBuffer.cpp @@ -6,28 +6,27 @@ #include "JFJochException.h" #include "ZeroCopyReturnValue.h" +#include + #ifdef JFJOCH_USE_NUMA #include #endif -#include ImageBuffer::ImageBuffer(size_t buffer_size_bytes) : buffer_size(buffer_size_bytes) { #ifdef JFJOCH_USE_NUMA + // Interleave the buffer across NUMA nodes - throughput-critical path (receiver), Linux-only. buffer = static_cast(numa_alloc_interleaved(buffer_size)); +#else + // The non-NUMA mapping carried no huge-page / mbind flags, so a plain heap allocation is + // equivalent (and portable). The buffer is zeroed below. + buffer = static_cast(std::malloc(buffer_size)); +#endif if (buffer == nullptr) - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, - "Failed to allocate image buffer"); - - #else - buffer = (uint8_t *) mmap (nullptr, buffer_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) ; - if (buffer == MAP_FAILED) throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Failed to allocate image buffer"); -#endif memset(buffer, 0, buffer_size); } @@ -39,7 +38,7 @@ ImageBuffer::~ImageBuffer() { #ifdef JFJOCH_USE_NUMA numa_free(buffer, buffer_size); #else - munmap(buffer, buffer_size); + std::free(buffer); #endif } diff --git a/common/NetworkAddressConvert.cpp b/common/NetworkAddressConvert.cpp index 9d58b1d0..baa920db 100644 --- a/common/NetworkAddressConvert.cpp +++ b/common/NetworkAddressConvert.cpp @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only +#ifdef _WIN32 +#include +#include +#else #include +#endif #include #include -- 2.52.0 From 1056acc3a6110d0cc8fa8f067be7f95c88861419 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 08:40:09 +0200 Subject: [PATCH 066/228] build: guard the librt link to Linux only librt is Linux/glibc-only and does not exist on macOS/Windows. Link it via a $<$:rt> generator expression (both the main JFJochCommon link and the CUDA block) so -lrt is still passed on Linux - where some distros need it for clock_gettime/timer_*/pthreads - and dropped elsewhere. Verified -lrt is still on the jfjoch_process link line on Linux. Co-Authored-By: Claude Opus 4.8 --- common/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 53dd9333..f6273dc4 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -139,14 +139,17 @@ ADD_LIBRARY(JFJochCommon STATIC CorrelationCoefficient.h ) -TARGET_LINK_LIBRARIES(JFJochCommon JFJochLogger Compression JFCalibration gemmi Threads::Threads -lrt ) +# librt is Linux/glibc-only (older glibc keeps clock_gettime/timer_*/sem_* there, and some +# distros are picky about it for pthreads); it does not exist on macOS/Windows. +TARGET_LINK_LIBRARIES(JFJochCommon JFJochLogger Compression JFCalibration gemmi Threads::Threads + $<$:rt>) TARGET_LINK_LIBRARIES(JFJochZMQ "$") IF (JFJOCH_CUDA_AVAILABLE) TARGET_SOURCES(JFJochCommon PRIVATE CUDAWrapper.cu ) TARGET_LINK_LIBRARIES(JFJochCommon CUDA::cudart_static - ${CMAKE_DL_LIBS} rt) + ${CMAKE_DL_LIBS} $<$:rt>) ENDIF() IF(HAS_NUMAIF AND HAS_NUMA_H AND NUMA_LIBRARY) -- 2.52.0 From 02fa15c2b95cc94745b20394efebcb8477654b29 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 15:29:52 +0200 Subject: [PATCH 067/228] jfjoch_process: spread per-image GPU work across all visible GPUs The offline worker threads built MXAnalysisWithoutFPGA without selecting a CUDA device, so all per-image preprocessing/spot-finding/azimuthal integration ran on GPU 0 (only the indexer pool was distributed). Add pin_gpu() to CUDAWrapper - a process-wide round-robin counter (counter++ % get_gpu_count(), no thread id, no-op without a GPU, honours CUDA_VISIBLE_DEVICES) - and call it once per worker before building the analysis resources so their CUDA streams/engines land on distinct devices. Also add NUMA_GPU_REVIEW.md: a working note mapping ImageBuffer/NUMAHWPolicy/GPU dispatch with goals and a staged plan (multi-broker GPU isolation via CUDA_VISIBLE_DEVICES, dropping libnuma, reassessing NUMA pinning for the FPGA path). Co-Authored-By: Claude Opus 4.8 --- NUMA_GPU_REVIEW.md | 115 +++++++++++++++++++++++++++++++++++++++ common/CUDAWrapper.cpp | 2 + common/CUDAWrapper.cu | 8 +++ common/CUDAWrapper.h | 5 ++ tools/jfjoch_process.cpp | 6 ++ 5 files changed, 136 insertions(+) create mode 100644 NUMA_GPU_REVIEW.md diff --git a/NUMA_GPU_REVIEW.md b/NUMA_GPU_REVIEW.md new file mode 100644 index 00000000..5132cca7 --- /dev/null +++ b/NUMA_GPU_REVIEW.md @@ -0,0 +1,115 @@ +# NUMA / GPU usage — current state, goals, proposed changes (for review) + +Working note to agree on direction **before** modifying `ImageBuffer` / `NUMAHWPolicy` and the +GPU-dispatch logic. Nothing here is implemented yet. File:line anchors are from branch +`2606-pixel-refine`. + +## 1. Current state (the mind map) + +### 1a. `ImageBuffer` — the big RAM ring buffer +- **One instance**, a member of the receiver: `JFJochReceiverService::image_buffer` + (`receiver/JFJochReceiverService.h:21`), sized `image_buffer_MiB` from broker config + (`broker/jfjoch_broker.cpp:104` → `JFJochReceiverService` ctor + `receiver/JFJochReceiverService.cpp:15`, `image_buffer(send_buffer_size_MiB*1024*1024)`). + This is the 150–200 GB allocation. +- **Allocated + zeroed in the ctor** (`common/ImageBuffer.cpp`): `numa_alloc_interleaved` if libnuma, + else `std::malloc`; then a **single-threaded `memset`** to pre-fault every page (deliberate — kills + first-use page-fault latency in the hot path). Happens once at broker startup. +- **Producers/consumers**: receiver/decompression threads write frames into slots; consumers are + preview/TIFF/JPEG/HTTP retrieval (`GetImage`) and the ZMQ/file sender. Access is random and + unpinned (any thread → any slot). +- **Not used by `jfjoch_process` or `jfjoch_viewer`** — they read HDF5 through the reader, never + instantiate `ImageBuffer`. So this buffer is **broker/receiver-only** (it only needs to *compile* + for the viewer). + +### 1b. `NUMAHWPolicy` — bundles three concerns per worker thread +Built from the broker config `numa_policy` string (e.g. `n2g2`, `n8g4`, `n8g4_hbm`) into a table of +`NUMABinding{cpu_node, mem_node, gpu}`; `GetBinding(thread) = bindings[thread % nbindings]` +(round-robin, `common/NUMAHWPolicy.cpp:54`). `Bind(thread)` does **all three** at once +(`common/NUMAHWPolicy.cpp:61`): +1. **CPU pin** — `RunOnNode` / `numa_run_on_node` +2. **Memory bind** — `MemOnNode` / `numa_set_membind` (on `n8g4_hbm`, `mem_node = i+8` → binds to HBM nodes) +3. **GPU select** — `SelectGPU` / `set_gpu` (= `cudaSetDevice`, **not** libnuma) + +Call sites: +- `receiver/JFJochReceiverFPGA.cpp:180/217/264` — data-stream threads `RunOnNode(FPGA's NUMA node)` + (device-locality pin, the NIC-era idea applied to the FPGA card). +- `receiver/JFJochReceiverFPGA.cpp:299`, `receiver/JFJochReceiverLite.cpp:234` — analysis worker + threads `numa_policy.Bind(threadid)` → cpu + mem + **GPU** per the policy table. +- `image_analysis/indexing/IndexerThreadPool.cpp:34` — each indexer thread + `SelectGPUAndItsNUMA(threadid % gpu_count)` (GPU round-robin + that GPU's own NUMA node; + independent of the `numa_policy` table). + +libnuma is used in exactly three files: `NUMAHWPolicy.cpp` (the above), `ImageBuffer.cpp` +(`numa_alloc_interleaved`/`numa_free`), and `CUDAWrapper.cpp` (one `numa_node` lookup). + +### 1c. GPU dispatch — and why `jfjoch_process` underuses GPUs +- `get_gpu_count()` = `cudaGetDeviceCount()` (`common/CUDAWrapper.*`), so it already honours + **`CUDA_VISIBLE_DEVICES`**. All dispatch is `% gpu_count`. +- **Broker/receiver**: worker threads spread over GPUs via `numa_policy.Bind` (the `gpu` field) + + the indexer pool via `threadid % gpu_count`. → uses all visible GPUs. +- **`jfjoch_process`**: its worker lambda (`tools/jfjoch_process.cpp:849`, launched `nthreads` times) + constructs `MXAnalysisWithoutFPGA` but **never calls `Bind`/`SelectGPU`**, and + `MXAnalysisWithoutFPGA` itself does not select a device (`image_analysis/MXAnalysisWithoutFPGA.cpp:38` + just builds GPU engines on the *current* device). → all per-image preprocessing / spot-finding / + azimuthal integration run on **GPU 0**. Only the indexer pool spreads. **This is the root cause of + "`jfjoch_process` doesn't use all GPUs."** + +## 2. Goals + +- **G1 — multiple brokers, disjoint GPUs.** Run >1 `jfjoch_broker` on one machine, each confined to a + subset of GPUs, with the code transparently using "all it can see" (no hard-coded indices). Pure + workload control, no security requirement. +- **G2 — `jfjoch_process` should use all visible GPUs**, not just GPU 0. +- **G3 — drop the libnuma dependency** if it doesn't cost real performance (annoying dep; also a + blocker for the long-term Windows/MSVC viewer). +- **G4 — reassess whether NUMA CPU/mem pinning is still worth it** given the FPGA pipeline (DMA into + kernel-mmap'd buffers, negligible IRQ traffic) rather than the old network-RX model. + +## 3. Proposed changes + +- **G1 (zero code):** launch each broker under `CUDA_DEVICE_ORDER=PCI_BUS_ID + CUDA_VISIBLE_DEVICES= jfjoch_broker …`. `get_gpu_count()`/`% gpu_count` already do the rest. + Action item: **document this** (deployment note) and set `CUDA_DEVICE_ORDER=PCI_BUS_ID` so indices + are stable across boots. + +- **G2 (DONE):** added `pin_gpu()` to `CUDAWrapper` — a process-wide round-robin counter + (`counter++ % get_gpu_count()`, no thread id needed, no-op when no GPU). The `jfjoch_process` worker + calls it once before building `MXAnalysisWithoutFPGA`, so each worker's CUDA streams/engines land on + a distinct device. Caller-agnostic and reusable by other thread pools later. + +- **G3 / `ImageBuffer`:** replace `numa_alloc_interleaved` + single-threaded `memset` with **plain + `malloc` + a parallel first-touch `memset`** (N threads, unpinned). Threads spread by the scheduler + → balanced placement ≈ interleave for random access, *and* faster startup, *and* no libnuma. Safe + here because the buffer is 30–40 % of RAM (first-touch spills to the other node if one fills; no OOM; + just check `vm.zone_reclaim_mode == 0`). Deterministic per-page interleave (if ever needed under + tight RAM) is a raw `mbind(MPOL_INTERLEAVE)` syscall — still libnuma-free. + +- **G3 / G4 / `NUMAHWPolicy`:** split the bundled concerns: + - **CPU pin** (`RunOnNode`) — likely **drop** for the FPGA path (G4). If ever wanted back, use + `sched_setaffinity` (no libnuma). + - **GPU select** (`SelectGPU`/`SelectGPUAndItsNUMA`) — **keep**; already `cudaSetDevice`, no libnuma. + - **Memory bind** (`MemOnNode`) — **drop.** (HBM is out of scope: the only Xeon MAX box didn't pay + off and isn't worth special-casing, so no `mbind` path is needed — treat every host as plain + multi-socket.) + - **`CUDAWrapper` `numa_node`** — read the GPU PCIe device's `/sys/.../numa_node` instead of libnuma. + - Result: `NUMA_LIBRARY` leaves the CMake entirely. + +## 4. Open questions / to validate before deleting anything + +- **G4 is empirical.** A/B at production frame rate (pinning on vs off): sustained throughput, dropped + frames, latency jitter. The reasoning predicts "no regression," but measure on one real box first — + production systems are currently tuned around this. +- Confirm `vm.zone_reclaim_mode` is `0` on the broker hosts (else first-touch reclaims locally before + spilling → latency stalls). +- Parallel first-touch placement is *approximate* (depends on the scheduler spreading the zeroing + threads); fine with RAM headroom, but note it's not the guaranteed 50/50 of `mbind` interleave. + +## 5. Suggested order (low-risk first) + +1. **G1** — document `CUDA_VISIBLE_DEVICES` launch (no code). +2. ~~**G2** — per-worker GPU pin in `jfjoch_process`.~~ **DONE** (`pin_gpu()`). +3. **ImageBuffer** parallel first-touch (drops one libnuma user, helps startup; stands alone). +4. **G4 A/B** on a real broker; if clean, drop CPU pinning. +5. **NUMAHWPolicy** simplify (keep GPU select, drop CPU pin + mem bind) + `CUDAWrapper` sysfs + → remove `NUMA_LIBRARY` from CMake. diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index e2cf8733..d73e3760 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -11,6 +11,8 @@ int32_t get_gpu_count() { void set_gpu(int32_t dev_id) {} +void pin_gpu() {} + int get_gpu_numa_node(int dev_id) { return -1; } diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 54e06740..684f8b82 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-only #include +#include #include "CUDAWrapper.h" #include "JFJochException.h" @@ -38,6 +39,13 @@ void set_gpu(int32_t dev_id) { } } +void pin_gpu() { + static std::atomic counter{0}; + auto dev_count = get_gpu_count(); + if (dev_count > 0) + set_gpu(counter.fetch_add(1) % dev_count); +} + // Return CUDA device PCI Bus ID as "domain:bus:device.function", e.g., "0000:65:00.0" static std::string get_cuda_device_pci_bus_id(int dev_id) { // CUDA API provides cudaDeviceGetPCIBusId diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index dce83bb0..dcc97f3c 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -8,3 +8,8 @@ int32_t get_gpu_count(); void set_gpu(int32_t dev_id); int get_gpu_numa_node(int dev_id); + +// Pin the calling thread to the next GPU in round-robin order, using a process-wide counter +// (counter++ % get_gpu_count()). Call once per thread; no thread id needed. No-op when no GPU +// is visible. Honours CUDA_VISIBLE_DEVICES via get_gpu_count(). +void pin_gpu(); diff --git a/tools/jfjoch_process.cpp b/tools/jfjoch_process.cpp index 74b422b9..ffb707bc 100644 --- a/tools/jfjoch_process.cpp +++ b/tools/jfjoch_process.cpp @@ -25,6 +25,7 @@ #include "../common/print_license.h" #include "../image_analysis/MXAnalysisWithoutFPGA.h" #include "../image_analysis/indexing/IndexerFactory.h" +#include "../common/CUDAWrapper.h" #include "../writer/FileWriter.h" #include "../image_analysis/IndexAndRefine.h" #include "../receiver/JFJochReceiverPlots.h" @@ -847,6 +848,11 @@ int main(int argc, char **argv) { std::atomic finished_count = 0; auto worker = [&](int thread_id) { + // Spread the per-image GPU pipeline across all visible GPUs (round-robin). + // Must run before building the analysis resources so their CUDA streams/engines are + // created on this thread's device. + pin_gpu(); + // Thread-local analysis resources MXAnalysisWithoutFPGA analysis(experiment, mapping, pixel_mask, indexer); -- 2.52.0 From d373ba049017d655ea3657dd8699e90756b2ecac Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 17:03:13 +0200 Subject: [PATCH 068/228] ImageBuffer: replace NUMA interleave with parallel first-touch Allocate the ring buffer with plain malloc and zero it across hardware_concurrency() threads, so each page is first-touched - and thus NUMA-placed - by whichever node the scheduler ran the zeroing thread on. For the random-access buffer this approximates the previous numa_alloc_interleaved placement, speeds up the one-time fault-in of the 150-200 GB allocation, and drops the libnuma dependency from this file. Co-Authored-By: Claude Opus 4.8 --- common/ImageBuffer.cpp | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/common/ImageBuffer.cpp b/common/ImageBuffer.cpp index 3145998b..5b2809b3 100644 --- a/common/ImageBuffer.cpp +++ b/common/ImageBuffer.cpp @@ -6,28 +6,39 @@ #include "JFJochException.h" #include "ZeroCopyReturnValue.h" +#include #include +#include +#include +#include -#ifdef JFJOCH_USE_NUMA -#include -#endif +namespace { + // Zero the buffer in parallel so that each page is first-touched by whichever NUMA node the + // scheduler placed the zeroing thread on. With RAM headroom this approximates an interleaved + // placement for the random-access ring buffer, while also slashing the one-time cost of + // faulting in a 150-200 GB allocation. Replaces numa_alloc_interleaved + a single-threaded + // memset, so this file no longer needs libnuma. + void parallel_first_touch(uint8_t *buffer, size_t buffer_size) { + const unsigned n = std::max(1u, std::thread::hardware_concurrency()); + const size_t chunk = (buffer_size + n - 1) / n; + std::vector threads; + for (size_t begin = 0; begin < buffer_size; begin += chunk) { + size_t len = std::min(chunk, buffer_size - begin); + threads.emplace_back([=] { memset(buffer + begin, 0, len); }); + } + for (auto &t : threads) + t.join(); + } +} ImageBuffer::ImageBuffer(size_t buffer_size_bytes) : buffer_size(buffer_size_bytes) { - -#ifdef JFJOCH_USE_NUMA - // Interleave the buffer across NUMA nodes - throughput-critical path (receiver), Linux-only. - buffer = static_cast(numa_alloc_interleaved(buffer_size)); -#else - // The non-NUMA mapping carried no huge-page / mbind flags, so a plain heap allocation is - // equivalent (and portable). The buffer is zeroed below. buffer = static_cast(std::malloc(buffer_size)); -#endif if (buffer == nullptr) throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Failed to allocate image buffer"); - memset(buffer, 0, buffer_size); + parallel_first_touch(buffer, buffer_size); } ImageBuffer::~ImageBuffer() { @@ -35,11 +46,7 @@ ImageBuffer::~ImageBuffer() { std::unique_lock ul(m); FinalizeInternal(ul); -#ifdef JFJOCH_USE_NUMA - numa_free(buffer, buffer_size); -#else std::free(buffer); -#endif } void ImageBuffer::StartMeasurement(size_t in_location_size) { -- 2.52.0 From 040cdeacf1d0048ffaa5b3d90cb7c8e011214eae Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 17:09:28 +0200 Subject: [PATCH 069/228] acquisition_device: give each device sole ownership of its buffers The base AcquisitionDevice no longer allocates or frees frame buffers; buffer_device is now just a non-owning view of addresses. Each subclass owns its backing memory and the matching lifecycle: - PCIExpressDevice mmap's the kernel DMA buffers and munmap's them in its own destructor (and on ctor failure), symmetric with MapKernelBuffer. - HLSSimulatedDevice owns plain zeroed heap buffers it points buffer_device into, declared before the HLSDevice so they outlive the action thread that writes them. The buffers are page-aligned to match the real device's kernel DMA buffers - the modelled AXI datamover and FPGAIntegrationTest require aligned output buffers. This drops the NUMA/mmap dance from the simulated path (not performance-critical) - removing libnuma from acquisition_device - and replaces the base-class cleanup that had to guess the allocation strategy with a single clear owner per device. Co-Authored-By: Claude Opus 4.8 --- acquisition_device/AcquisitionDevice.cpp | 38 ----------------------- acquisition_device/AcquisitionDevice.h | 7 +++-- acquisition_device/HLSSimulatedDevice.cpp | 8 +++-- acquisition_device/HLSSimulatedDevice.h | 9 +++++- acquisition_device/PCIExpressDevice.cpp | 9 ++++++ acquisition_device/PCIExpressDevice.h | 2 ++ 6 files changed, 29 insertions(+), 44 deletions(-) diff --git a/acquisition_device/AcquisitionDevice.cpp b/acquisition_device/AcquisitionDevice.cpp index 452338fa..fc31af1e 100644 --- a/acquisition_device/AcquisitionDevice.cpp +++ b/acquisition_device/AcquisitionDevice.cpp @@ -1,11 +1,6 @@ // SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only -#ifdef JFJOCH_USE_NUMA -#include -#endif - -#include #include #include #include @@ -14,24 +9,6 @@ #include "AcquisitionDevice.h" #include "../common/NetworkAddressConvert.h" -void *mmap_acquisition_buffer(size_t size, int16_t numa_node) { - void *ret = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ret == MAP_FAILED) { - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "frame_buffer"); - } -#ifdef JFJOCH_USE_NUMA - if (numa_node >= 0) { - unsigned long nodemask = 1L << numa_node;; - if (numa_node > sizeof(nodemask)*8) - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Mask too small for NUMA node"); - if (mbind(ret, size, MPOL_BIND, &nodemask, sizeof(nodemask)*8, MPOL_MF_STRICT) == -1) - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Cannot apply NUMA policy"); - } -#endif - memset(ret, 0, size); - return ret; -} - AcquisitionDevice::AcquisitionDevice(uint16_t in_data_stream) { logger = nullptr; data_stream = in_data_stream; @@ -238,21 +215,6 @@ void AcquisitionDevice::InitializePixelMask(const DiffractionExperiment &experim } } -void AcquisitionDevice::MapBuffersStandard(size_t c2h_buffer_count, int16_t numa_node) { - try { - for (int i = 0; i < c2h_buffer_count; i++) - buffer_device.emplace_back((DeviceOutput *) mmap_acquisition_buffer(FPGA_BUFFER_LOCATION_SIZE, numa_node)); - } catch (const JFJochException &e) { - UnmapBuffers(); - throw; - } -} - -void AcquisitionDevice::UnmapBuffers() { - for (auto &i: buffer_device) - if (i != nullptr) munmap(i, FPGA_BUFFER_LOCATION_SIZE); -} - void AcquisitionDevice::FrameBufferRelease(size_t frame_number, uint16_t module_number) { auto handle = counters.GetBufferHandleAndClear(frame_number, module_number); if (handle != AcquisitionCounters::HandleNotFound) diff --git a/acquisition_device/AcquisitionDevice.h b/acquisition_device/AcquisitionDevice.h index 71e60ca3..7b9d59e1 100644 --- a/acquisition_device/AcquisitionDevice.h +++ b/acquisition_device/AcquisitionDevice.h @@ -49,6 +49,9 @@ protected: ThreadSafeFIFO work_completion_queue; ThreadSafeFIFO work_request_queue; + // Non-owning view of the per-buffer addresses. Each device subclass owns the backing memory + // and its lifecycle: PCIExpressDevice mmap's/munmap's kernel DMA buffers, HLSSimulatedDevice + // points these at plain heap buffers it owns. std::vector buffer_device; Logger *logger; @@ -58,8 +61,6 @@ protected: uint32_t ipv4_addr; explicit AcquisitionDevice(uint16_t data_stream); - void UnmapBuffers(); - void MapBuffersStandard(size_t c2h_buffer_count, int16_t numa_node); const DeviceOutput *GetDeviceOutput(size_t handle) const; DeviceOutput *GetDeviceOutput(size_t handle); virtual void HW_RunInternalGenerator(const FrameGeneratorConfig& config) = 0; @@ -70,7 +71,7 @@ public: static constexpr const uint64_t HandleNotValid = UINT64_MAX; - virtual ~AcquisitionDevice() { UnmapBuffers(); }; + virtual ~AcquisitionDevice() = default; void StartAction(const DiffractionExperiment &experiment, uint32_t optional_flags = 0); void PrepareAction(const DiffractionExperiment &experiment); diff --git a/acquisition_device/HLSSimulatedDevice.cpp b/acquisition_device/HLSSimulatedDevice.cpp index 24fd222b..54e8c5b3 100644 --- a/acquisition_device/HLSSimulatedDevice.cpp +++ b/acquisition_device/HLSSimulatedDevice.cpp @@ -3,14 +3,18 @@ #include "HLSSimulatedDevice.h" -HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node) +HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules) : FPGAAcquisitionDevice(data_stream) { mac_addr = 0xCCAA11223344; ipv4_addr = 0x0132010A; max_modules = MAX_MODULES_FPGA; - MapBuffersStandard(in_frame_buffer_size_modules, numa_node); + buffers.reserve(in_frame_buffer_size_modules); + for (size_t i = 0; i < in_frame_buffer_size_modules; i++) { + buffers.push_back(std::make_unique()); // zero-initialised, 64-byte aligned + buffer_device.push_back(reinterpret_cast(buffers.back().get())); + } device = std::make_unique(buffer_device); } diff --git a/acquisition_device/HLSSimulatedDevice.h b/acquisition_device/HLSSimulatedDevice.h index 4fee45bc..83e40ef9 100644 --- a/acquisition_device/HLSSimulatedDevice.h +++ b/acquisition_device/HLSSimulatedDevice.h @@ -11,6 +11,13 @@ #include "FPGAAcquisitionDevice.h" class HLSSimulatedDevice : public FPGAAcquisitionDevice { + // Owns the simulated frame buffers. Plain heap (this path is not performance-critical, so no + // NUMA placement and no mmap), but page-aligned (4 KiB) to match the real device's kernel DMA + // buffers - more than enough for the data path's alignment needs (AXI datamover 64 B, + // FPGAIntegrationTest 128 B). Declared before `device` so the buffers outlive the HLSDevice + // action thread that writes into them; buffer_device points into these. + struct alignas(4096) FrameBuffer { uint8_t data[FPGA_BUFFER_LOCATION_SIZE]; }; + std::vector> buffers; std::unique_ptr device; void HW_ReadActionRegister(DataCollectionConfig *job) override; @@ -25,7 +32,7 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) override; void HW_RunInternalGenerator(const FrameGeneratorConfig &config) override; public: - HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1); + HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules); ~HLSSimulatedDevice() override = default; void CreateJFPacket(const DiffractionExperiment& experiment, uint64_t frame_number, uint32_t eth_packet, uint32_t module_number, const uint16_t *data, int8_t adjust_axis = 0, uint8_t user = 0); diff --git a/acquisition_device/PCIExpressDevice.cpp b/acquisition_device/PCIExpressDevice.cpp index 1ddb6f9b..3fa146fd 100644 --- a/acquisition_device/PCIExpressDevice.cpp +++ b/acquisition_device/PCIExpressDevice.cpp @@ -33,6 +33,15 @@ PCIExpressDevice::PCIExpressDevice(uint16_t data_stream, const std::string &devi } } +PCIExpressDevice::~PCIExpressDevice() { + UnmapBuffers(); +} + +void PCIExpressDevice::UnmapBuffers() { + for (auto &buf: buffer_device) + if (buf != nullptr) dev.UnmapKernelBuffer(buf); +} + bool PCIExpressDevice::HW_ReadMailbox(uint32_t *values) { PCI_EXCEPT(return dev.ReadWorkCompletion(values);) } diff --git a/acquisition_device/PCIExpressDevice.h b/acquisition_device/PCIExpressDevice.h index 2362a1fe..ac1f1b3c 100644 --- a/acquisition_device/PCIExpressDevice.h +++ b/acquisition_device/PCIExpressDevice.h @@ -20,10 +20,12 @@ class PCIExpressDevice : public FPGAAcquisitionDevice { void FPGA_EndAction() override; uint32_t GetNumKernelBuffers() const; void HW_RunInternalGenerator(const FrameGeneratorConfig &config) override; + void UnmapBuffers(); public: explicit PCIExpressDevice(uint16_t data_stream); PCIExpressDevice(uint16_t data_stream, uint16_t pci_slot); PCIExpressDevice(uint16_t data_stream, const std::string &device_name); + ~PCIExpressDevice() override; void Cancel() override; int32_t GetNUMANode() const override; -- 2.52.0 From 17732ea774369abb9df7e967d66a7b7201a66ce6 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 19:22:00 +0200 Subject: [PATCH 070/228] docs: refresh NUMA_GPU_REVIEW to match implemented state Several items in the note had landed or were inaccurate. Update it to: - mark DONE: G2 pin_gpu, ImageBuffer first-touch, acquisition_device de-NUMA, CUDAWrapper sysfs node lookup; - add the previously-missing FPGA DMA buffer section - placement is a kernel concern (dma_alloc_coherent, device-local), not the userspace mbind, which was only the simulator; - record that libnuma is now down to a single file (NUMAHWPolicy.cpp); - note that dependency removal (G3) and pinning behaviour (G4) are separable axes. Co-Authored-By: Claude Opus 4.8 --- NUMA_GPU_REVIEW.md | 186 ++++++++++++++++++++++++++------------------- 1 file changed, 108 insertions(+), 78 deletions(-) diff --git a/NUMA_GPU_REVIEW.md b/NUMA_GPU_REVIEW.md index 5132cca7..840d86ac 100644 --- a/NUMA_GPU_REVIEW.md +++ b/NUMA_GPU_REVIEW.md @@ -1,115 +1,145 @@ -# NUMA / GPU usage — current state, goals, proposed changes (for review) +# NUMA / GPU usage — current state, goals, remaining work -Working note to agree on direction **before** modifying `ImageBuffer` / `NUMAHWPolicy` and the -GPU-dispatch logic. Nothing here is implemented yet. File:line anchors are from branch +Working note on the NUMA/GPU direction. Several items are now **implemented and committed** on +branch `2606-pixel-refine` (see §3/§5); this note tracks what landed, what's left, and — importantly +— the corrected mental model of *where NUMA actually matters*. File:line anchors are from `2606-pixel-refine`. +**Headline:** after the committed work, **libnuma is used in exactly one file** +(`common/NUMAHWPolicy.cpp`). Everything else (GPU node lookup, the big RAM buffer, the FPGA DMA +buffers, the simulator) is libnuma-free. + ## 1. Current state (the mind map) ### 1a. `ImageBuffer` — the big RAM ring buffer - **One instance**, a member of the receiver: `JFJochReceiverService::image_buffer` (`receiver/JFJochReceiverService.h:21`), sized `image_buffer_MiB` from broker config - (`broker/jfjoch_broker.cpp:104` → `JFJochReceiverService` ctor - `receiver/JFJochReceiverService.cpp:15`, `image_buffer(send_buffer_size_MiB*1024*1024)`). - This is the 150–200 GB allocation. -- **Allocated + zeroed in the ctor** (`common/ImageBuffer.cpp`): `numa_alloc_interleaved` if libnuma, - else `std::malloc`; then a **single-threaded `memset`** to pre-fault every page (deliberate — kills - first-use page-fault latency in the hot path). Happens once at broker startup. + (`broker/jfjoch_broker.cpp:104` → ctor `receiver/JFJochReceiverService.cpp:15`). The 150–200 GB + allocation. +- **Allocation (DONE, commit `d373ba04`):** plain `std::malloc` + a **parallel first-touch `memset`** + (`hardware_concurrency()` threads, unpinned) in `common/ImageBuffer.cpp`. Each page is + first-touched — and thus NUMA-placed — by whichever node the scheduler ran the zeroing thread on: + approximates the old `numa_alloc_interleaved` placement for the random-access buffer, *and* + pre-faults every page (no first-use fault in the hot path), *and* speeds up startup, *and* drops + libnuma here. (Was `numa_alloc_interleaved` + single-threaded `memset`.) - **Producers/consumers**: receiver/decompression threads write frames into slots; consumers are preview/TIFF/JPEG/HTTP retrieval (`GetImage`) and the ZMQ/file sender. Access is random and - unpinned (any thread → any slot). + unpinned (any thread → any slot) — which is *why* interleave/first-touch placement (not per-node + binding) is the right model. - **Not used by `jfjoch_process` or `jfjoch_viewer`** — they read HDF5 through the reader, never - instantiate `ImageBuffer`. So this buffer is **broker/receiver-only** (it only needs to *compile* - for the viewer). + instantiate `ImageBuffer`. Broker/receiver-only (it only needs to *compile* for the viewer). -### 1b. `NUMAHWPolicy` — bundles three concerns per worker thread +### 1b. FPGA DMA buffers — placement is a *kernel* concern, not libnuma *(this section was missing)* +The real per-frame DMA buffers are **allocated and NUMA-placed by the kernel driver**, with zero +userspace/libnuma involvement. The earlier draft framed the userspace `mbind` as "the real +hardware-locality win" — that was wrong; the win is in the kernel: +- `fpga/pcie_driver/jfjoch_memory.c:28` — `dma_alloc_coherent(&pdev->dev, FPGA_BUFFER_LOCATION_SIZE, + …)` × `nbuffer` (512). Gives **physically contiguous, DMA-coherent** pages (required: each buffer's + bus address is written into the FPGA address table at `:37-38`). Placement is **device-local by + construction** — the kernel DMA/page allocator uses `dev_to_node(&pdev->dev)`. This is exactly why + it can't be a userspace `malloc`+`mbind`: userspace virtual memory is neither physically contiguous + nor a valid DMA target. +- `jfjoch_memory.c:99` — `dma_mmap_coherent` maps those *same physical pages* into userspace; + `JungfraujochDevice::MapKernelBuffer` (`fpga/host_library/JungfraujochDevice.cpp:166`) is a plain + `mmap` of the char dev. No second allocation, no first-touch, no migration. +- `IOCTL_JFJOCH_NUMA` (`fpga/pcie_driver/jfjoch_ioctl.c:133`) just **reports** + `drvdata->pdev->dev.numa_node`. Userspace's only NUMA action on the real path is to pin the + *acquire thread's CPU* to that node (the `RunOnNode` calls in §1c). +- **Simulated path (DONE, commit `042df308`):** `HLSSimulatedDevice` used to *emulate* device locality + with a userspace `mmap`+`mbind` (``) — the only libnuma user in `acquisition_device/`. It + now uses **plain zeroed heap buffers** (`std::vector`, not performance-critical). Buffer ownership + was refactored so each device owns its lifecycle: `PCIExpressDevice` `mmap`s/`munmap`s the kernel + DMA buffers; `HLSSimulatedDevice` owns the heap buffers; the base `AcquisitionDevice` is just a + non-owning address view. No libnuma, no mmap on the sim path. + +### 1c. `NUMAHWPolicy` — bundles three concerns per worker thread (the last libnuma user) Built from the broker config `numa_policy` string (e.g. `n2g2`, `n8g4`, `n8g4_hbm`) into a table of `NUMABinding{cpu_node, mem_node, gpu}`; `GetBinding(thread) = bindings[thread % nbindings]` -(round-robin, `common/NUMAHWPolicy.cpp:54`). `Bind(thread)` does **all three** at once -(`common/NUMAHWPolicy.cpp:61`): -1. **CPU pin** — `RunOnNode` / `numa_run_on_node` -2. **Memory bind** — `MemOnNode` / `numa_set_membind` (on `n8g4_hbm`, `mem_node = i+8` → binds to HBM nodes) +(round-robin, `common/NUMAHWPolicy.cpp:50`). `Bind(thread)` does **all three** at once: +1. **CPU pin** — `RunOnNode` / `numa_run_on_node` ← libnuma +2. **Memory bind** — `MemOnNode` / `numa_set_membind` (`n8g4_hbm` → HBM nodes) ← libnuma 3. **GPU select** — `SelectGPU` / `set_gpu` (= `cudaSetDevice`, **not** libnuma) Call sites: -- `receiver/JFJochReceiverFPGA.cpp:180/217/264` — data-stream threads `RunOnNode(FPGA's NUMA node)` - (device-locality pin, the NIC-era idea applied to the FPGA card). +- `receiver/JFJochReceiverFPGA.cpp:180/217/264` — acquire (data-stream) threads + `RunOnNode(acquisition_device.GetNUMANode())` → the kernel-reported device node (§1b). CPU pin only. - `receiver/JFJochReceiverFPGA.cpp:299`, `receiver/JFJochReceiverLite.cpp:234` — analysis worker - threads `numa_policy.Bind(threadid)` → cpu + mem + **GPU** per the policy table. + threads `numa_policy.Bind(threadid)` → cpu + mem + GPU per the policy table. - `image_analysis/indexing/IndexerThreadPool.cpp:34` — each indexer thread - `SelectGPUAndItsNUMA(threadid % gpu_count)` (GPU round-robin + that GPU's own NUMA node; + `SelectGPUAndItsNUMA(threadid % gpu_count)` (GPU round-robin + that GPU's NUMA node via sysfs; independent of the `numa_policy` table). -libnuma is used in exactly three files: `NUMAHWPolicy.cpp` (the above), `ImageBuffer.cpp` -(`numa_alloc_interleaved`/`numa_free`), and `CUDAWrapper.cpp` (one `numa_node` lookup). +The GPU NUMA node is now read from sysfs (`/sys/bus/pci/devices//numa_node`, +`common/CUDAWrapper.cu:75`), **not** libnuma. So the only remaining libnuma calls are `RunOnNode` and +`MemOnNode` in this file. -### 1c. GPU dispatch — and why `jfjoch_process` underuses GPUs +### 1d. GPU dispatch — `jfjoch_process` now uses all visible GPUs (was the root cause) - `get_gpu_count()` = `cudaGetDeviceCount()` (`common/CUDAWrapper.*`), so it already honours **`CUDA_VISIBLE_DEVICES`**. All dispatch is `% gpu_count`. - **Broker/receiver**: worker threads spread over GPUs via `numa_policy.Bind` (the `gpu` field) + - the indexer pool via `threadid % gpu_count`. → uses all visible GPUs. -- **`jfjoch_process`**: its worker lambda (`tools/jfjoch_process.cpp:849`, launched `nthreads` times) - constructs `MXAnalysisWithoutFPGA` but **never calls `Bind`/`SelectGPU`**, and - `MXAnalysisWithoutFPGA` itself does not select a device (`image_analysis/MXAnalysisWithoutFPGA.cpp:38` - just builds GPU engines on the *current* device). → all per-image preprocessing / spot-finding / - azimuthal integration run on **GPU 0**. Only the indexer pool spreads. **This is the root cause of - "`jfjoch_process` doesn't use all GPUs."** + indexer pool via `threadid % gpu_count`. → uses all visible GPUs. +- **`jfjoch_process` (DONE, G2):** its worker (`tools/jfjoch_process.cpp:854`) now calls `pin_gpu()` + before building `MXAnalysisWithoutFPGA`, so each worker's CUDA streams/engines land on a distinct + device. Previously everything ran on GPU 0 (only the indexer pool spread). ## 2. Goals - **G1 — multiple brokers, disjoint GPUs.** Run >1 `jfjoch_broker` on one machine, each confined to a - subset of GPUs, with the code transparently using "all it can see" (no hard-coded indices). Pure - workload control, no security requirement. -- **G2 — `jfjoch_process` should use all visible GPUs**, not just GPU 0. -- **G3 — drop the libnuma dependency** if it doesn't cost real performance (annoying dep; also a - blocker for the long-term Windows/MSVC viewer). -- **G4 — reassess whether NUMA CPU/mem pinning is still worth it** given the FPGA pipeline (DMA into - kernel-mmap'd buffers, negligible IRQ traffic) rather than the old network-RX model. + subset of GPUs, code transparently using "all it can see" (no hard-coded indices). Workload + control, no security requirement. +- **G2 — `jfjoch_process` should use all visible GPUs**, not just GPU 0. **DONE** (§1d). +- **G3 — drop the libnuma dependency.** Annoying dep; also a blocker for the long-term Windows/MSVC + viewer. Mostly done — one file (`NUMAHWPolicy.cpp`) left. +- **G4 — reassess whether NUMA CPU/mem pinning is still worth it.** Note the corrected model: FPGA DMA + buffer *placement* is the kernel's job (§1b), so the only behavioural NUMA op left on real data flow + is the **CPU pin** of the acquire/analysis threads to the kernel-reported device node. That's the + thing to A/B. -## 3. Proposed changes +**Key separability insight:** dependency removal (G3) and behaviour removal (G4) are *independent +axes*. Dropping libnuma does **not** require dropping any NUMA behaviour — `numa_run_on_node` → +`sched_setaffinity`, `mbind`/`set_mempolicy` are raw syscalls. So G3 can complete regardless of how +the G4 measurement turns out. -- **G1 (zero code):** launch each broker under `CUDA_DEVICE_ORDER=PCI_BUS_ID - CUDA_VISIBLE_DEVICES= jfjoch_broker …`. `get_gpu_count()`/`% gpu_count` already do the rest. - Action item: **document this** (deployment note) and set `CUDA_DEVICE_ORDER=PCI_BUS_ID` so indices - are stable across boots. +## 3. Status of changes -- **G2 (DONE):** added `pin_gpu()` to `CUDAWrapper` — a process-wide round-robin counter - (`counter++ % get_gpu_count()`, no thread id needed, no-op when no GPU). The `jfjoch_process` worker - calls it once before building `MXAnalysisWithoutFPGA`, so each worker's CUDA streams/engines land on - a distinct device. Caller-agnostic and reusable by other thread pools later. +- **G1 (zero code) — TODO (docs).** Launch each broker under `CUDA_DEVICE_ORDER=PCI_BUS_ID + CUDA_VISIBLE_DEVICES= jfjoch_broker …`. `get_gpu_count()`/`% gpu_count` do the rest. Set + `CUDA_DEVICE_ORDER=PCI_BUS_ID` so indices are stable across boots. Action: write the deployment note. +- **G2 — DONE.** `pin_gpu()` in `CUDAWrapper` (process-wide round-robin `counter++ % get_gpu_count()`, + no-op when no GPU); `jfjoch_process` worker calls it once. Caller-agnostic, reusable. +- **`ImageBuffer` — DONE (`d373ba04`).** malloc + parallel first-touch (§1a). +- **`acquisition_device` — DONE (`042df308`).** Simulator off NUMA/mmap → plain heap; per-device + buffer ownership; libnuma gone from `acquisition_device/` (§1b). +- **`CUDAWrapper` `numa_node` — DONE.** sysfs lookup, not libnuma (§1c). +- **`NUMAHWPolicy` — REMAINING.** Split the bundled concerns: + - **Memory bind** (`MemOnNode`) — **drop.** HBM out of scope (the one Xeon MAX box didn't pay off; + treat every host as plain multi-socket). + - **CPU pin** (`RunOnNode`) — gated on **G4**. If kept, reimplement with `sched_setaffinity` + + node→cpulist from `/sys/devices/system/node/nodeN/cpulist` (no libnuma). Note it pins to the + *kernel-reported device node* (§1b) — well-founded if any pinning is kept. + - **GPU select** (`SelectGPU`) — **keep** (`cudaSetDevice`). `SelectGPUAndItsNUMA` then collapses to + `set_gpu` (+ optional CPU pin); `Bind` collapses to `SelectGPU` (+ optional CPU pin). + - Result: `NUMA_LIBRARY` leaves the CMake (`CMakeLists.txt:78-80`, + `common/CMakeLists.txt:155-161`). -- **G3 / `ImageBuffer`:** replace `numa_alloc_interleaved` + single-threaded `memset` with **plain - `malloc` + a parallel first-touch `memset`** (N threads, unpinned). Threads spread by the scheduler - → balanced placement ≈ interleave for random access, *and* faster startup, *and* no libnuma. Safe - here because the buffer is 30–40 % of RAM (first-touch spills to the other node if one fills; no OOM; - just check `vm.zone_reclaim_mode == 0`). Deterministic per-page interleave (if ever needed under - tight RAM) is a raw `mbind(MPOL_INTERLEAVE)` syscall — still libnuma-free. +## 4. Open questions / to validate before deleting `NUMAHWPolicy` pinning -- **G3 / G4 / `NUMAHWPolicy`:** split the bundled concerns: - - **CPU pin** (`RunOnNode`) — likely **drop** for the FPGA path (G4). If ever wanted back, use - `sched_setaffinity` (no libnuma). - - **GPU select** (`SelectGPU`/`SelectGPUAndItsNUMA`) — **keep**; already `cudaSetDevice`, no libnuma. - - **Memory bind** (`MemOnNode`) — **drop.** (HBM is out of scope: the only Xeon MAX box didn't pay - off and isn't worth special-casing, so no `mbind` path is needed — treat every host as plain - multi-socket.) - - **`CUDAWrapper` `numa_node`** — read the GPU PCIe device's `/sys/.../numa_node` instead of libnuma. - - Result: `NUMA_LIBRARY` leaves the CMake entirely. - -## 4. Open questions / to validate before deleting anything - -- **G4 is empirical.** A/B at production frame rate (pinning on vs off): sustained throughput, dropped - frames, latency jitter. The reasoning predicts "no regression," but measure on one real box first — +- **G4 is empirical.** A/B at production frame rate (CPU pin on vs off): sustained throughput, dropped + frames, latency jitter. Reasoning predicts "no regression," but measure on one real box first — production systems are currently tuned around this. -- Confirm `vm.zone_reclaim_mode` is `0` on the broker hosts (else first-touch reclaims locally before - spilling → latency stalls). -- Parallel first-touch placement is *approximate* (depends on the scheduler spreading the zeroing - threads); fine with RAM headroom, but note it's not the guaranteed 50/50 of `mbind` interleave. +- Confirm `vm.zone_reclaim_mode` is `0` on the broker hosts (`cat /proc/sys/vm/zone_reclaim_mode`). + Non-zero turns `ImageBuffer`'s first-touch into synchronous local reclaim (page-cache eviction / + writeback) → startup latency stalls; `0` just spills the page to a remote node. +- `ImageBuffer` first-touch placement is *approximate* (depends on the scheduler spreading the zeroing + threads); fine with RAM headroom, but not the guaranteed 50/50 of `mbind` interleave. Deterministic + per-page interleave, if ever needed, is a raw `mbind(MPOL_INTERLEAVE)` syscall — still libnuma-free. -## 5. Suggested order (low-risk first) +## 5. Remaining order (low-risk first) -1. **G1** — document `CUDA_VISIBLE_DEVICES` launch (no code). -2. ~~**G2** — per-worker GPU pin in `jfjoch_process`.~~ **DONE** (`pin_gpu()`). -3. **ImageBuffer** parallel first-touch (drops one libnuma user, helps startup; stands alone). -4. **G4 A/B** on a real broker; if clean, drop CPU pinning. -5. **NUMAHWPolicy** simplify (keep GPU select, drop CPU pin + mem bind) + `CUDAWrapper` sysfs - → remove `NUMA_LIBRARY` from CMake. +1. **G1** — document the `CUDA_VISIBLE_DEVICES` launch (no code). +2. **G4 A/B** on a real broker — the gate for dropping the `RunOnNode` CPU pin. +3. **`NUMAHWPolicy`** — drop `MemOnNode`; drop/replace `RunOnNode` per G4; keep `SelectGPU`. Then + remove `NUMA_LIBRARY` from CMake → G3 complete. + +*(DONE so far: G2 `pin_gpu`, `ImageBuffer` first-touch, `acquisition_device` de-NUMA, `CUDAWrapper` +sysfs node lookup.)* -- 2.52.0 From cc925b26683526ad18890f1f5ba4ec33fa5d91e4 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 20:25:36 +0200 Subject: [PATCH 071/228] Remove NUMAHWPolicy and the libnuma dependency NUMA CPU/memory pinning is no longer worthwhile: the FPGA DMA buffers are placed device-local by the kernel (dma_alloc_coherent), the big RAM ring buffer is random-access (first-touch handles placement), and GPU work is already spread across all visible devices. So drop the pinning entirely and with it libnuma. - Delete NUMAHWPolicy; the only concern worth keeping - GPU selection - is done directly via pin_gpu() (round-robin over visible GPUs) in the indexer pool and the Lite analysis threads. CPU-only threads (FPGA acquire/pedestal/summation/frame-transform) no longer bind anything. - Drop get_gpu_numa_node() (sysfs lookup) - only SelectGPUAndItsNUMA used it. - numa_policy broker setting is deprecated and ignored (kept in the API for backward compatibility; warns once on startup). - Remove NUMA_LIBRARY / numa.h / numaif.h detection from CMake. - Docs: drop the NUMA dependency, remove the numa_policy config example, and document running multiple brokers on disjoint GPUs via CUDA_VISIBLE_DEVICES. - Remove NUMA_GPU_REVIEW.md (the planning note; this work is now done). Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 5 - NUMA_GPU_REVIEW.md | 145 ------------------ broker/JFJochBrokerParser.cpp | 4 - broker/jfjoch_api.yaml | 3 +- broker/jfjoch_broker.cpp | 2 + common/CMakeLists.txt | 9 -- common/CUDAWrapper.cpp | 4 - common/CUDAWrapper.cu | 54 ------- common/CUDAWrapper.h | 1 - common/NUMAHWPolicy.cpp | 125 --------------- common/NUMAHWPolicy.h | 33 ---- docs/DEPLOYMENT.md | 8 +- docs/JFJOCH_BROKER.md | 1 - docs/SOFTWARE.md | 1 - image_analysis/indexing/IndexerThreadPool.cpp | 10 +- image_analysis/indexing/IndexerThreadPool.h | 1 - receiver/JFJochReceiver.cpp | 2 - receiver/JFJochReceiver.h | 4 - receiver/JFJochReceiverFPGA.cpp | 24 --- receiver/JFJochReceiverFPGA.h | 2 - receiver/JFJochReceiverLite.cpp | 11 +- receiver/JFJochReceiverLite.h | 2 - receiver/JFJochReceiverService.cpp | 12 -- receiver/JFJochReceiverService.h | 4 - receiver/JFJochReceiverTest.cpp | 8 +- receiver/JFJochReceiverTest.h | 2 - tools/jfjoch_fpga_test.cpp | 6 +- tools/jfjoch_lite_perf_test.cpp | 4 +- 28 files changed, 24 insertions(+), 463 deletions(-) delete mode 100644 NUMA_GPU_REVIEW.md delete mode 100644 common/NUMAHWPolicy.cpp delete mode 100644 common/NUMAHWPolicy.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d675e610..f81c5ea0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,11 +73,6 @@ IF(HAS_FFTW3_H AND FFTWF_LIBRARY) ENDIF() INCLUDE_DIRECTORIES(include) -INCLUDE(CheckIncludeFile) - -FIND_LIBRARY(NUMA_LIBRARY NAMES numa DOC "NUMA Library") -CHECK_INCLUDE_FILE(numaif.h HAS_NUMAIF) -CHECK_INCLUDE_FILE(numa.h HAS_NUMA_H) include(FetchContent) diff --git a/NUMA_GPU_REVIEW.md b/NUMA_GPU_REVIEW.md deleted file mode 100644 index 840d86ac..00000000 --- a/NUMA_GPU_REVIEW.md +++ /dev/null @@ -1,145 +0,0 @@ -# NUMA / GPU usage — current state, goals, remaining work - -Working note on the NUMA/GPU direction. Several items are now **implemented and committed** on -branch `2606-pixel-refine` (see §3/§5); this note tracks what landed, what's left, and — importantly -— the corrected mental model of *where NUMA actually matters*. File:line anchors are from -`2606-pixel-refine`. - -**Headline:** after the committed work, **libnuma is used in exactly one file** -(`common/NUMAHWPolicy.cpp`). Everything else (GPU node lookup, the big RAM buffer, the FPGA DMA -buffers, the simulator) is libnuma-free. - -## 1. Current state (the mind map) - -### 1a. `ImageBuffer` — the big RAM ring buffer -- **One instance**, a member of the receiver: `JFJochReceiverService::image_buffer` - (`receiver/JFJochReceiverService.h:21`), sized `image_buffer_MiB` from broker config - (`broker/jfjoch_broker.cpp:104` → ctor `receiver/JFJochReceiverService.cpp:15`). The 150–200 GB - allocation. -- **Allocation (DONE, commit `d373ba04`):** plain `std::malloc` + a **parallel first-touch `memset`** - (`hardware_concurrency()` threads, unpinned) in `common/ImageBuffer.cpp`. Each page is - first-touched — and thus NUMA-placed — by whichever node the scheduler ran the zeroing thread on: - approximates the old `numa_alloc_interleaved` placement for the random-access buffer, *and* - pre-faults every page (no first-use fault in the hot path), *and* speeds up startup, *and* drops - libnuma here. (Was `numa_alloc_interleaved` + single-threaded `memset`.) -- **Producers/consumers**: receiver/decompression threads write frames into slots; consumers are - preview/TIFF/JPEG/HTTP retrieval (`GetImage`) and the ZMQ/file sender. Access is random and - unpinned (any thread → any slot) — which is *why* interleave/first-touch placement (not per-node - binding) is the right model. -- **Not used by `jfjoch_process` or `jfjoch_viewer`** — they read HDF5 through the reader, never - instantiate `ImageBuffer`. Broker/receiver-only (it only needs to *compile* for the viewer). - -### 1b. FPGA DMA buffers — placement is a *kernel* concern, not libnuma *(this section was missing)* -The real per-frame DMA buffers are **allocated and NUMA-placed by the kernel driver**, with zero -userspace/libnuma involvement. The earlier draft framed the userspace `mbind` as "the real -hardware-locality win" — that was wrong; the win is in the kernel: -- `fpga/pcie_driver/jfjoch_memory.c:28` — `dma_alloc_coherent(&pdev->dev, FPGA_BUFFER_LOCATION_SIZE, - …)` × `nbuffer` (512). Gives **physically contiguous, DMA-coherent** pages (required: each buffer's - bus address is written into the FPGA address table at `:37-38`). Placement is **device-local by - construction** — the kernel DMA/page allocator uses `dev_to_node(&pdev->dev)`. This is exactly why - it can't be a userspace `malloc`+`mbind`: userspace virtual memory is neither physically contiguous - nor a valid DMA target. -- `jfjoch_memory.c:99` — `dma_mmap_coherent` maps those *same physical pages* into userspace; - `JungfraujochDevice::MapKernelBuffer` (`fpga/host_library/JungfraujochDevice.cpp:166`) is a plain - `mmap` of the char dev. No second allocation, no first-touch, no migration. -- `IOCTL_JFJOCH_NUMA` (`fpga/pcie_driver/jfjoch_ioctl.c:133`) just **reports** - `drvdata->pdev->dev.numa_node`. Userspace's only NUMA action on the real path is to pin the - *acquire thread's CPU* to that node (the `RunOnNode` calls in §1c). -- **Simulated path (DONE, commit `042df308`):** `HLSSimulatedDevice` used to *emulate* device locality - with a userspace `mmap`+`mbind` (``) — the only libnuma user in `acquisition_device/`. It - now uses **plain zeroed heap buffers** (`std::vector`, not performance-critical). Buffer ownership - was refactored so each device owns its lifecycle: `PCIExpressDevice` `mmap`s/`munmap`s the kernel - DMA buffers; `HLSSimulatedDevice` owns the heap buffers; the base `AcquisitionDevice` is just a - non-owning address view. No libnuma, no mmap on the sim path. - -### 1c. `NUMAHWPolicy` — bundles three concerns per worker thread (the last libnuma user) -Built from the broker config `numa_policy` string (e.g. `n2g2`, `n8g4`, `n8g4_hbm`) into a table of -`NUMABinding{cpu_node, mem_node, gpu}`; `GetBinding(thread) = bindings[thread % nbindings]` -(round-robin, `common/NUMAHWPolicy.cpp:50`). `Bind(thread)` does **all three** at once: -1. **CPU pin** — `RunOnNode` / `numa_run_on_node` ← libnuma -2. **Memory bind** — `MemOnNode` / `numa_set_membind` (`n8g4_hbm` → HBM nodes) ← libnuma -3. **GPU select** — `SelectGPU` / `set_gpu` (= `cudaSetDevice`, **not** libnuma) - -Call sites: -- `receiver/JFJochReceiverFPGA.cpp:180/217/264` — acquire (data-stream) threads - `RunOnNode(acquisition_device.GetNUMANode())` → the kernel-reported device node (§1b). CPU pin only. -- `receiver/JFJochReceiverFPGA.cpp:299`, `receiver/JFJochReceiverLite.cpp:234` — analysis worker - threads `numa_policy.Bind(threadid)` → cpu + mem + GPU per the policy table. -- `image_analysis/indexing/IndexerThreadPool.cpp:34` — each indexer thread - `SelectGPUAndItsNUMA(threadid % gpu_count)` (GPU round-robin + that GPU's NUMA node via sysfs; - independent of the `numa_policy` table). - -The GPU NUMA node is now read from sysfs (`/sys/bus/pci/devices//numa_node`, -`common/CUDAWrapper.cu:75`), **not** libnuma. So the only remaining libnuma calls are `RunOnNode` and -`MemOnNode` in this file. - -### 1d. GPU dispatch — `jfjoch_process` now uses all visible GPUs (was the root cause) -- `get_gpu_count()` = `cudaGetDeviceCount()` (`common/CUDAWrapper.*`), so it already honours - **`CUDA_VISIBLE_DEVICES`**. All dispatch is `% gpu_count`. -- **Broker/receiver**: worker threads spread over GPUs via `numa_policy.Bind` (the `gpu` field) + - indexer pool via `threadid % gpu_count`. → uses all visible GPUs. -- **`jfjoch_process` (DONE, G2):** its worker (`tools/jfjoch_process.cpp:854`) now calls `pin_gpu()` - before building `MXAnalysisWithoutFPGA`, so each worker's CUDA streams/engines land on a distinct - device. Previously everything ran on GPU 0 (only the indexer pool spread). - -## 2. Goals - -- **G1 — multiple brokers, disjoint GPUs.** Run >1 `jfjoch_broker` on one machine, each confined to a - subset of GPUs, code transparently using "all it can see" (no hard-coded indices). Workload - control, no security requirement. -- **G2 — `jfjoch_process` should use all visible GPUs**, not just GPU 0. **DONE** (§1d). -- **G3 — drop the libnuma dependency.** Annoying dep; also a blocker for the long-term Windows/MSVC - viewer. Mostly done — one file (`NUMAHWPolicy.cpp`) left. -- **G4 — reassess whether NUMA CPU/mem pinning is still worth it.** Note the corrected model: FPGA DMA - buffer *placement* is the kernel's job (§1b), so the only behavioural NUMA op left on real data flow - is the **CPU pin** of the acquire/analysis threads to the kernel-reported device node. That's the - thing to A/B. - -**Key separability insight:** dependency removal (G3) and behaviour removal (G4) are *independent -axes*. Dropping libnuma does **not** require dropping any NUMA behaviour — `numa_run_on_node` → -`sched_setaffinity`, `mbind`/`set_mempolicy` are raw syscalls. So G3 can complete regardless of how -the G4 measurement turns out. - -## 3. Status of changes - -- **G1 (zero code) — TODO (docs).** Launch each broker under `CUDA_DEVICE_ORDER=PCI_BUS_ID - CUDA_VISIBLE_DEVICES= jfjoch_broker …`. `get_gpu_count()`/`% gpu_count` do the rest. Set - `CUDA_DEVICE_ORDER=PCI_BUS_ID` so indices are stable across boots. Action: write the deployment note. -- **G2 — DONE.** `pin_gpu()` in `CUDAWrapper` (process-wide round-robin `counter++ % get_gpu_count()`, - no-op when no GPU); `jfjoch_process` worker calls it once. Caller-agnostic, reusable. -- **`ImageBuffer` — DONE (`d373ba04`).** malloc + parallel first-touch (§1a). -- **`acquisition_device` — DONE (`042df308`).** Simulator off NUMA/mmap → plain heap; per-device - buffer ownership; libnuma gone from `acquisition_device/` (§1b). -- **`CUDAWrapper` `numa_node` — DONE.** sysfs lookup, not libnuma (§1c). -- **`NUMAHWPolicy` — REMAINING.** Split the bundled concerns: - - **Memory bind** (`MemOnNode`) — **drop.** HBM out of scope (the one Xeon MAX box didn't pay off; - treat every host as plain multi-socket). - - **CPU pin** (`RunOnNode`) — gated on **G4**. If kept, reimplement with `sched_setaffinity` + - node→cpulist from `/sys/devices/system/node/nodeN/cpulist` (no libnuma). Note it pins to the - *kernel-reported device node* (§1b) — well-founded if any pinning is kept. - - **GPU select** (`SelectGPU`) — **keep** (`cudaSetDevice`). `SelectGPUAndItsNUMA` then collapses to - `set_gpu` (+ optional CPU pin); `Bind` collapses to `SelectGPU` (+ optional CPU pin). - - Result: `NUMA_LIBRARY` leaves the CMake (`CMakeLists.txt:78-80`, - `common/CMakeLists.txt:155-161`). - -## 4. Open questions / to validate before deleting `NUMAHWPolicy` pinning - -- **G4 is empirical.** A/B at production frame rate (CPU pin on vs off): sustained throughput, dropped - frames, latency jitter. Reasoning predicts "no regression," but measure on one real box first — - production systems are currently tuned around this. -- Confirm `vm.zone_reclaim_mode` is `0` on the broker hosts (`cat /proc/sys/vm/zone_reclaim_mode`). - Non-zero turns `ImageBuffer`'s first-touch into synchronous local reclaim (page-cache eviction / - writeback) → startup latency stalls; `0` just spills the page to a remote node. -- `ImageBuffer` first-touch placement is *approximate* (depends on the scheduler spreading the zeroing - threads); fine with RAM headroom, but not the guaranteed 50/50 of `mbind` interleave. Deterministic - per-page interleave, if ever needed, is a raw `mbind(MPOL_INTERLEAVE)` syscall — still libnuma-free. - -## 5. Remaining order (low-risk first) - -1. **G1** — document the `CUDA_VISIBLE_DEVICES` launch (no code). -2. **G4 A/B** on a real broker — the gate for dropping the `RunOnNode` CPU pin. -3. **`NUMAHWPolicy`** — drop `MemOnNode`; drop/replace `RunOnNode` per G4; keep `SelectGPU`. Then - remove `NUMA_LIBRARY` from CMake → G3 complete. - -*(DONE so far: G2 `pin_gpu`, `ImageBuffer` first-touch, `acquisition_device` de-NUMA, `CUDAWrapper` -sysfs node lookup.)* diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index 68b000ae..ffb3d092 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -234,10 +234,6 @@ void ParseAcquisitionDeviceGroup(const org::openapitools::server::model::Jfjoch_ } void ParseReceiverSettings(const org::openapitools::server::model::Jfjoch_settings &input, JFJochReceiverService &service) { - std::string numa_policy = input.getNumaPolicy(); - if (!numa_policy.empty()) - service.NUMAPolicy(numa_policy); - // Using default in case service.NumThreads(input.getReceiverThreads()); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 67e80aa8..6e2b42db 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -2264,7 +2264,8 @@ components: description: Number of threads used by the receiver numa_policy: type: string - description: NUMA policy to bind CPUs + deprecated: true + description: Ignored value frontend_directory: type: string description: Location of built JavaScript web frontend diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index dde40cbe..e8997992 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -74,6 +74,8 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } + if (settings.numaPolicyIsSet()) + logger.Warning("NUMA policy is deprecated and ignored - processed are split over all the CPUs/GPUs in the system"); if (settings.verboseIsSet() && settings.isVerbose()) logger.Verbose(true); else diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f6273dc4..06a365c2 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -55,7 +55,6 @@ ADD_LIBRARY(JFJochCommon STATIC DetectorModuleGeometry.cpp DetectorModuleGeometry.h DetectorSetup.h DetectorSetup.cpp ZeroCopyReturnValue.h Histogram.h DiffractionGeometry.h CUDAWrapper.cpp CUDAWrapper.h - NUMAHWPolicy.cpp NUMAHWPolicy.h ADUHistogram.cpp ADUHistogram.h RawToConvertedGeometryCore.h Plot.h @@ -151,11 +150,3 @@ IF (JFJOCH_CUDA_AVAILABLE) TARGET_LINK_LIBRARIES(JFJochCommon CUDA::cudart_static ${CMAKE_DL_LIBS} $<$:rt>) ENDIF() - -IF(HAS_NUMAIF AND HAS_NUMA_H AND NUMA_LIBRARY) - TARGET_COMPILE_DEFINITIONS(JFJochCommon PUBLIC JFJOCH_USE_NUMA) - TARGET_LINK_LIBRARIES(JFJochCommon ${NUMA_LIBRARY}) - MESSAGE(STATUS "NUMA memory/CPU pinning enabled") -ELSE() - MESSAGE(WARNING "NUMA memory/CPU pinning disabled") -ENDIF() diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index d73e3760..ab3e2e6c 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -13,8 +13,4 @@ void set_gpu(int32_t dev_id) {} void pin_gpu() {} -int get_gpu_numa_node(int dev_id) { - return -1; -} - #endif diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 684f8b82..554c54c6 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -45,57 +45,3 @@ void pin_gpu() { if (dev_count > 0) set_gpu(counter.fetch_add(1) % dev_count); } - -// Return CUDA device PCI Bus ID as "domain:bus:device.function", e.g., "0000:65:00.0" -static std::string get_cuda_device_pci_bus_id(int dev_id) { - // CUDA API provides cudaDeviceGetPCIBusId - char buf[64] = {0}; - cudaDeviceProp prop; - cudaError_t st = cudaGetDeviceProperties(&prop, dev_id); - if (st != cudaSuccess) { - throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(st)); - } - // Prefer cudaDeviceGetPCIBusId for full id including domain and function - cudaError_t st2 = cudaDeviceGetPCIBusId(buf, static_cast(sizeof(buf)), dev_id); - if (st2 == cudaSuccess) { - return std::string(buf); - } - // Fallback: synthesize from properties (domain may be missing on very old drivers) - // Note: function is typically ".0" - char alt[64]; - std::snprintf(alt, sizeof(alt), "%04x:%02x:%02x.%u", - prop.pciDomainID, prop.pciBusID, prop.pciDeviceID, 0u); - return std::string(alt); -} - -// Resolve NUMA node from PCI address using Linux sysfs -// Returns: -// >=0 NUMA node index -// -1 if NUMA node is not available/unknown -int get_gpu_numa_node(int dev_id) { - auto dev_count = get_gpu_count(); - if (dev_count <= 0) return -1; - if (dev_id < 0 || dev_id >= dev_count) { - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Invalid CUDA device ID"); - } - - // We don't need to call cudaSetDevice here; querying by id is sufficient. - const std::string pci_bus_id = get_cuda_device_pci_bus_id(dev_id); // "dddd:bb:dd.f" - - // sysfs path for PCI device. Examples: - // - /sys/bus/pci/devices/0000:65:00.0/numa_node - const std::string sysfs_path = std::string("/sys/bus/pci/devices/") + pci_bus_id + "/numa_node"; - - std::ifstream f(sysfs_path); - if (!f.is_open()) { - // On some systems, the symlink may be via /sys/class/drm or nvidia, but primary path should exist. - return -1; - } - - int numa = -1; - f >> numa; - if (!f.good()) { - return -1; - } - return numa; -} diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index dcc97f3c..4b288ee0 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -7,7 +7,6 @@ int32_t get_gpu_count(); void set_gpu(int32_t dev_id); -int get_gpu_numa_node(int dev_id); // Pin the calling thread to the next GPU in round-robin order, using a process-wide counter // (counter++ % get_gpu_count()). Call once per thread; no thread id needed. No-op when no GPU diff --git a/common/NUMAHWPolicy.cpp b/common/NUMAHWPolicy.cpp deleted file mode 100644 index ac6621ae..00000000 --- a/common/NUMAHWPolicy.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "NUMAHWPolicy.h" - -#include "../common/CUDAWrapper.h" -#include "JFJochException.h" - -#ifdef JFJOCH_USE_NUMA -#include -#endif - -NUMAHWPolicy::NUMAHWPolicy(const std::string &policy) : name(policy) { - if ((policy.empty()) || (policy == "none")) { - name = "none"; - } else if (policy == "n2g2") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 1}); - } else if (policy == "n2g4") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 3}); - } else if (policy == "n2g2_hbm") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 1}); - } else if (policy == "n2g4_hbm") { - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 3}); - } else if (policy == "n8g4") { - for (int32_t i = 0; i < 8; i++) - bindings.emplace_back(NUMABinding{.cpu_node = i, .mem_node = i, .gpu = i/2}); - } else if (policy == "n8g4_hbm") { - for (int32_t i = 0; i < 8; i++) - bindings.emplace_back(NUMABinding{.cpu_node = i, .mem_node = i + 8, .gpu = i / 2}); - } else if (policy == "g2") { - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 1}); - } else if (policy == "g4") { - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 0}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 1}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 2}); - bindings.emplace_back(NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = 3}); - } else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Unknown NUMA policy"); -} - -NUMABinding NUMAHWPolicy::GetBinding(uint32_t thread) const { - if (bindings.empty()) - return NUMABinding{.cpu_node = -1, .mem_node = -1, .gpu = -1}; - else - return bindings.at(thread % bindings.size()); -} - -void NUMAHWPolicy::Bind(uint32_t thread) const { - Bind(GetBinding(thread)); -} - -void NUMAHWPolicy::Bind(const NUMABinding &binding) { - RunOnNode(binding.cpu_node); - MemOnNode(binding.mem_node); - SelectGPU(binding.gpu); -} - -void NUMAHWPolicy::RunOnNode(int32_t cpu_node) { -#ifdef JFJOCH_USE_NUMA - if (numa_available() != -1) { - auto max_nodes = numa_num_configured_nodes(); - - if (cpu_node >= 0) { - if (cpu_node < max_nodes) - numa_run_on_node(cpu_node); - else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "CPU NUMA node out of bounds"); - } - } -#endif -} - -void NUMAHWPolicy::MemOnNode(int32_t mem_node) { -#ifdef JFJOCH_USE_NUMA - if (numa_available() != -1) { - auto max_nodes = numa_num_configured_nodes(); - - if (mem_node >= 0) { - if (mem_node < max_nodes) { - struct bitmask *mask = numa_allocate_nodemask(); - numa_bitmask_setbit(mask, mem_node); - numa_set_membind(mask); - numa_bitmask_free(mask); - } else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Memory NUMA node out of bounds"); - } - } -#endif -} - -void NUMAHWPolicy::SelectGPU(int32_t gpu) { - auto gpu_count = get_gpu_count(); - - if ((gpu_count > 0) && (gpu >= 0)) { - if (gpu < gpu_count) - set_gpu(gpu); - else - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "GPU device out of bounds"); - } -} - -void NUMAHWPolicy::SelectGPUAndItsNUMA(int32_t gpu) { - int numa = get_gpu_numa_node(gpu); - if (numa >= 0) { - RunOnNode(numa); - MemOnNode(numa); - } - set_gpu(gpu); -} - - -const std::string &NUMAHWPolicy::GetName() const { - return name; -} - - diff --git a/common/NUMAHWPolicy.h b/common/NUMAHWPolicy.h deleted file mode 100644 index 6c1375dd..00000000 --- a/common/NUMAHWPolicy.h +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include -#include -#include -#include - -struct NUMABinding { - int32_t cpu_node; - int32_t mem_node; - int32_t gpu; -}; - -class NUMAHWPolicy { - std::string name; - std::vector bindings; -public: - NUMAHWPolicy() = default; - explicit NUMAHWPolicy(const std::string& policy); - NUMABinding GetBinding(uint32_t thread) const; - - const std::string &GetName() const; - - void Bind(uint32_t thread) const; - static void Bind(const NUMABinding &binding); - static void RunOnNode(int32_t cpu_node); - static void MemOnNode(int32_t mem_node); - static void SelectGPU(int32_t gpu); - static void SelectGPUAndItsNUMA(int32_t gpu); -}; diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index fb91982c..1a3e757b 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -110,6 +110,12 @@ Example JSON files are placed in `etc/` folder. JSON file format is also explain When running the service can be accessed via HTTP interface from a web browser for configuration and monitoring. +Jungfraujoch automatically uses every GPU visible to the process and spreads the per-image work across all of them. To run more than one `jfjoch_broker` on a single machine, each confined to a disjoint subset of GPUs, set `CUDA_VISIBLE_DEVICES`; setting `CUDA_DEVICE_ORDER=PCI_BUS_ID` keeps the GPU indices stable across reboots. For example, two brokers on a 4-GPU host: +``` +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=0,1 jfjoch_broker broker_a.json 5232 +CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=2,3 jfjoch_broker broker_b.json 5233 +``` + To prepare the configuration file one also needs to reference calibration files: gain files for PSI JUNGFRAU and trim-bit files for PSI EIGER. These need to be obtained from the PSI Detector Group. @@ -118,7 +124,7 @@ These need to be obtained from the PSI Detector Group. To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_fpga_test` tool. For example to simulate 10M pixel system with 4 FPGA cards and 200k images on a 2 CPU system with 2 GPUs: ``` -jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 -Pn2g2 +jfjoch_fpga_test ~/nextgendcu/ -m20 -s4 -i 200000 ``` Or 1M pixel system with one FPGA card: ``` diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index d7e58f0b..76e3f2b7 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -129,7 +129,6 @@ Example with all fields: }, "image_buffer_MiB": 2048, "receiver_threads": 64, - "numa_policy": "n2g2", "frontend_directory": "/usr/share/jfjoch/frontend", "image_pusher": "ZeroMQ", "zeromq_metadata": { diff --git a/docs/SOFTWARE.md b/docs/SOFTWARE.md index d64671e8..8b144dbe 100644 --- a/docs/SOFTWARE.md +++ b/docs/SOFTWARE.md @@ -21,7 +21,6 @@ Required: Optional: * CUDA compiler version 12.7 or newer - required for MX fast feedback indexer * FFTW library - for indexing if GPU/CUDA is absent -* NUMA library - to pin threads to nodes/CPUs * Node.js - to make frontend * Qt version 6 (for jfjoch_viewer) diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 86900f12..51913d9a 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -28,15 +28,11 @@ IndexerThread::IndexerThread(const IndexingSettings &settings, int threadid) { void IndexerThread::Worker(const IndexingSettings &settings, int threadid) { try { -#ifdef JFJOCH_USE_CUDA - auto gpu_count = get_gpu_count(); - if (gpu_count > 0) - NUMAHWPolicy::SelectGPUAndItsNUMA(threadid % gpu_count); -#endif + pin_gpu(); } catch (const std::exception &e) { - spdlog::error("Failed to bind thread to NUMA node: {}", e.what()); + spdlog::error("Failed to pin to GPU {}", e.what()); } catch (...) { - // NUMA policy errors are not critical and should be ignored for the time being. + // GPU pinning errors are not critical and should be ignored for the time being. } std::unique_ptr fft_indexer, ffbidx_indexer, fftw_indexer; diff --git a/image_analysis/indexing/IndexerThreadPool.h b/image_analysis/indexing/IndexerThreadPool.h index 224e2943..fd7e0a04 100644 --- a/image_analysis/indexing/IndexerThreadPool.h +++ b/image_analysis/indexing/IndexerThreadPool.h @@ -17,7 +17,6 @@ #include "../common/JFJochMessages.h" #include "../common/DiffractionSpot.h" #include "../common/DiffractionExperiment.h" -#include "../common/NUMAHWPolicy.h" #include "Indexer.h" class IndexerThread { diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 64e82e33..752013ff 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -13,7 +13,6 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, JFJochReceiverPlots &in_plots, const SpotFindingSettings &spot_finding_settings, Logger &logger, - const NUMAHWPolicy &in_numa_policy, const PixelMask &in_pixel_mask, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, @@ -29,7 +28,6 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment &in_experiment, plots(in_plots), scan_result(in_experiment), serialmx_filter(in_experiment), - numa_policy(in_numa_policy), pixel_mask(in_pixel_mask), indexer(experiment, indexing_thread_pool) { logger.Info("Initializing receiver"); diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 83918987..8c2a1362 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -19,7 +19,6 @@ #include "JFJochReceiverPlots.h" #include "LossyFilter.h" #include "../common/MovingAverage.h" -#include "../common/NUMAHWPolicy.h" #include "../common/ScanResultGenerator.h" #include "../image_analysis/indexing/IndexerThreadPool.h" #include "../image_analysis/IndexAndRefine.h" @@ -75,8 +74,6 @@ protected: std::optional images_written; - NUMAHWPolicy numa_policy; - std::vector> adu_histogram_module; PixelMask pixel_mask; @@ -104,7 +101,6 @@ public: JFJochReceiverPlots &plots, const SpotFindingSettings &spot_finding_settings, Logger &logger, - const NUMAHWPolicy &numa_policy, const PixelMask &pixel_mask, ZMQPreviewSocket *zmq_preview_socket = nullptr, ZMQMetadataSocket *zmq_metadata_socket = nullptr, diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 802b7faf..3b2cbbb8 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -6,7 +6,6 @@ #include #include "ImageMetadata.h" -#include "../common/CUDAWrapper.h" JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_pixel_mask, @@ -14,7 +13,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen AcquisitionDeviceGroup &in_aq_device, ImagePusher &in_image_sender, Logger &in_logger, int64_t in_forward_and_sum_nthreads, - const NUMAHWPolicy &in_numa_policy, const SpotFindingSettings &in_spot_finding_settings, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, @@ -29,7 +27,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen in_plots, in_spot_finding_settings, in_logger, - in_numa_policy, in_pixel_mask, in_zmq_preview_socket, in_zmq_metadata_socket, @@ -60,8 +57,6 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Number of threads must be more than zero"); - logger.Info("NUMA policy: {}", numa_policy.GetName()); - expected_packets_per_image = 0; for (int d = 0; d < ndatastreams; d++) { acquisition_device[d].PrepareAction(experiment); @@ -176,12 +171,6 @@ void JFJochReceiverFPGA::SendCalibration() { } void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Warning("NUMA bind error {} for device thread {} - continuing without binding", e.what(), data_stream); - } - try { LoadCalibrationToFPGA(data_stream); frame_transformation_ready.wait(); @@ -213,12 +202,6 @@ void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) { void JFJochReceiverFPGA::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell, uint32_t threadid, bool ignore) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Error("HW bind error {}", e.what()); - } - JFPedestalCalc pedestal_calc(experiment); uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber(); @@ -260,12 +243,6 @@ int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream, uint16_t module_number, uint32_t threadid, ModuleSummation &summation) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Error("HW bind error {}", e.what()); - } - ModuleSummation local_summation(experiment); int64_t starting_frame = image_number * experiment.GetSummation(); @@ -296,7 +273,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { std::unique_ptr analyzer; try { - numa_policy.Bind(threadid); analyzer = std::make_unique(experiment, indexer); } catch (const JFJochException &e) { frame_transformation_ready.count_down(); diff --git a/receiver/JFJochReceiverFPGA.h b/receiver/JFJochReceiverFPGA.h index c6146c91..a34af2c0 100644 --- a/receiver/JFJochReceiverFPGA.h +++ b/receiver/JFJochReceiverFPGA.h @@ -18,7 +18,6 @@ #include "../image_pusher/ImagePusher.h" #include "../common/Logger.h" #include "../common/ThreadSafeFIFO.h" -#include "../common/NUMAHWPolicy.h" #include "../common/ImageBuffer.h" #include "../common/PixelMask.h" @@ -97,7 +96,6 @@ public: AcquisitionDeviceGroup &acquisition_devices, ImagePusher &image_pusher, Logger &logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &numa_policy, const SpotFindingSettings &spot_finding_settings, JFJochReceiverCurrentStatus ¤t_status, JFJochReceiverPlots &plots, diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index e986956e..cc8fbcad 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -3,6 +3,7 @@ #include "JFJochReceiverLite.h" #include "../image_analysis/indexing/IndexerFactory.h" +#include "../common/CUDAWrapper.h" using namespace std::chrono_literals; @@ -34,7 +35,6 @@ JFJochReceiverLite::JFJochReceiverLite(const DiffractionExperiment &in_experimen ImagePusher &in_image_pusher, Logger &in_logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &in_numa_policy, const SpotFindingSettings &in_spot_finding_settings, JFJochReceiverCurrentStatus &in_current_status, JFJochReceiverPlots &in_plots, @@ -49,7 +49,6 @@ JFJochReceiverLite::JFJochReceiverLite(const DiffractionExperiment &in_experimen in_plots, in_spot_finding_settings, in_logger, - in_numa_policy, in_pixel_mask, in_zmq_preview_socket, in_zmq_metadata_socket, @@ -231,14 +230,12 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { logger.Debug("Thread {} started", id); try { - numa_policy.Bind(id); - data_analysis_started.count_down(); + pin_gpu(); } catch (const JFJochException &e) { - data_analysis_started.count_down(); - Cancel(e); - return; + logger.Warning("Error pinning GPU {}", e.what()); } + data_analysis_started.count_down(); measurement_started.wait(); try { diff --git a/receiver/JFJochReceiverLite.h b/receiver/JFJochReceiverLite.h index b8c1b546..33a0d0aa 100644 --- a/receiver/JFJochReceiverLite.h +++ b/receiver/JFJochReceiverLite.h @@ -12,7 +12,6 @@ #include "../common/ImageBuffer.h" #include "../common/Logger.h" #include "../image_pusher/ImagePusher.h" -#include "../common/NUMAHWPolicy.h" #include "../preview/PreviewImage.h" #include "../preview/ZMQPreviewSocket.h" #include "../preview/ZMQMetadataSocket.h" @@ -53,7 +52,6 @@ public: ImagePuller &image_puller, ImagePusher &image_pusher, Logger &logger, int64_t forward_and_sum_nthreads, - const NUMAHWPolicy &numa_policy, const SpotFindingSettings &spot_finding_settings, JFJochReceiverCurrentStatus ¤t_status, JFJochReceiverPlots &plots, diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index a6e92f49..63b7b5b4 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -24,16 +24,6 @@ JFJochReceiverService &JFJochReceiverService::NumThreads(int64_t input) { return *this; } -JFJochReceiverService &JFJochReceiverService::NUMAPolicy(const NUMAHWPolicy &policy) { - numa_policy = policy; - return *this; -} - -JFJochReceiverService &JFJochReceiverService::NUMAPolicy(const std::string &policy) { - numa_policy = NUMAHWPolicy(policy); - return *this; -} - void JFJochReceiverService::FinalizeMeasurementChangeState() { std::unique_lock ul(state_mutex); state = ReceiverState::Idle; @@ -80,7 +70,6 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, aq_devices, image_pusher, logger, nthreads_local, - numa_policy, spot_finding_settings, receiver_status, plots, @@ -102,7 +91,6 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, image_pusher, logger, nthreads_local, - numa_policy, spot_finding_settings, receiver_status, plots, diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index 1edec952..5c4748fd 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -9,12 +9,10 @@ #include "JFJochReceiver.h" #include "../acquisition_device/AcquisitionDeviceGroup.h" -#include "../common/NUMAHWPolicy.h" #include "../preview/ZMQMetadataSocket.h" #include "../preview/ZMQPreviewSocket.h" class JFJochReceiverService { - NUMAHWPolicy numa_policy; std::unique_ptr receiver; AcquisitionDeviceGroup &aq_devices; Logger &logger; @@ -45,8 +43,6 @@ public: ImagePusher &pusher, size_t send_buffer_size_MiB = 512); JFJochReceiverService& NumThreads(int64_t input); - JFJochReceiverService& NUMAPolicy(const NUMAHWPolicy& policy); - JFJochReceiverService& NUMAPolicy(const std::string& policy); JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); JFJochReceiverService& MetadataSocket(const std::string &addr); diff --git a/receiver/JFJochReceiverTest.cpp b/receiver/JFJochReceiverTest.cpp index 1e00e7ab..3bd87105 100644 --- a/receiver/JFJochReceiverTest.cpp +++ b/receiver/JFJochReceiverTest.cpp @@ -38,7 +38,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const DiffractionExperiment &x, const PixelMask &pixel_mask, uint16_t nthreads, - const std::string &numa_policy, size_t send_buf_size_MiB, bool quick_integrate) { @@ -47,8 +46,8 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, for (int i = 0; i < raw_expected_image.size(); i++) raw_expected_image[i] = i % 65536; - return JFJochReceiverTest(output, logger, aq_devices, x, pixel_mask, raw_expected_image, nthreads, numa_policy, send_buf_size_MiB, - quick_integrate); + return JFJochReceiverTest(output, logger, aq_devices, x, pixel_mask, raw_expected_image, nthreads, + send_buf_size_MiB, quick_integrate); } bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, @@ -57,7 +56,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const PixelMask &pixel_mask, const std::vector &raw_expected_image, uint16_t nthreads, - const std::string &numa_policy, size_t send_buf_size_MiB, bool quick_integrate) { std::vector raw_expected_image_with_mask = raw_expected_image; @@ -79,7 +77,7 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, TestImagePusher pusher(image_number); JFJochReceiverService service(aq_devices, logger, pusher, send_buf_size_MiB); - service.NumThreads(nthreads).NUMAPolicy(numa_policy); + service.NumThreads(nthreads); service.LoadInternalGeneratorImage(x, raw_expected_image, 0); service.Indexing(x.GetIndexingSettings()); diff --git a/receiver/JFJochReceiverTest.h b/receiver/JFJochReceiverTest.h index 89281494..2cc03ee0 100644 --- a/receiver/JFJochReceiverTest.h +++ b/receiver/JFJochReceiverTest.h @@ -10,7 +10,6 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const DiffractionExperiment &x, const PixelMask &pixel_mask, uint16_t nthreads, - const std::string &numa_policy = "", size_t send_buf_size_MiB = 1024, bool quick_integrate = false); @@ -20,6 +19,5 @@ bool JFJochReceiverTest(JFJochReceiverOutput &output, Logger &logger, const PixelMask &pixel_mask, const std::vector &data, uint16_t nthreads, - const std::string &numa_policy = "", size_t send_buf_size_MiB = 1024, bool quick_integrate = false); diff --git a/tools/jfjoch_fpga_test.cpp b/tools/jfjoch_fpga_test.cpp index 9cd98e3a..b4359252 100644 --- a/tools/jfjoch_fpga_test.cpp +++ b/tools/jfjoch_fpga_test.cpp @@ -23,7 +23,6 @@ void print_usage(Logger &logger) { logger.Info(" -m Number of modules"); logger.Info(" -i Number of images"); logger.Info(" -N Number of image processing threads"); - logger.Info(" -P NUMA Policy (none|n2g2|n8g4|n8g4_hbm), none is default"); logger.Info(" -B Size of send buffer in MiB (default 2048)"); logger.Info(" -q Use Poisson lossy compression, with square root of counts"); logger.Info(" -T Use thresholding for low counts"); @@ -53,7 +52,6 @@ int main(int argc, char **argv) { std::optional fft_num_vectors; bool verbose = false; - std::string numa_policy_name; bool raw_data = false; bool force_32bit = false; bool force_8bit = false; @@ -100,7 +98,7 @@ int main(int argc, char **argv) { verbose = true; break; case 'P': - numa_policy_name = std::string(optarg); + logger.Warning("NUMA policy is deprecated and ignored"); break; case 'R': raw_data = true; @@ -247,7 +245,7 @@ int main(int argc, char **argv) { bool ret; std::thread run_thread([&] { try { - ret = JFJochReceiverTest(output, logger, aq_devices, x, mask, input, nthreads, numa_policy_name, + ret = JFJochReceiverTest(output, logger, aq_devices, x, mask, input, nthreads, send_buffer_size_MiB, quick_integrate); } catch (std::exception &e) { logger.Error(e.what()); diff --git a/tools/jfjoch_lite_perf_test.cpp b/tools/jfjoch_lite_perf_test.cpp index 5840d25f..13ba0103 100644 --- a/tools/jfjoch_lite_perf_test.cpp +++ b/tools/jfjoch_lite_perf_test.cpp @@ -18,7 +18,6 @@ void print_usage(Logger &logger) { logger.Info("Options:"); logger.Info(" -i Number of images"); logger.Info(" -N Number of image processing threads (default: 8)"); - logger.Info(" -P NUMA policy: none|n2g2|n8g4|n8g4_hbm (default: none)"); logger.Info(" -F{} Write file, optional parameter is name (default: lyso_lite_perf_test)"); logger.Info(" -X Indexing (none|fft|fftw|ffbidx), ffbidx is default"); logger.Info(" -t Indexing thread pool size (default: 4)"); @@ -58,7 +57,7 @@ int main(int argc, char **argv) { nthreads = atol(optarg); break; case 'P': - numa_policy_name = std::string(optarg); + logger.Warning("NUMA policy is deprecated and ignored"); break; case 'i': nimages = atol(optarg); @@ -166,7 +165,6 @@ int main(int argc, char **argv) { AcquisitionDeviceGroup group; JFJochReceiverService service(group, logger, pusher); - service.NUMAPolicy(numa_policy_name); service.NumThreads(nthreads); IndexingSettings i_settings; -- 2.52.0 From 58910274bf257e8f53bd5ecaaab66a5f7d363073 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 21:15:54 +0200 Subject: [PATCH 072/228] image_analysis: compute ROI statistics in the non-FPGA path MXAnalysisWithoutFPGA never filled DataMessage.roi, so ROI integrals were only available on the FPGA path. Add a software ROI engine that mirrors the FPGA roi_calc kernel: per-ROI sum, sum of squares, good-pixel count, max and intensity-weighted centre of mass, with each pixel carrying a 16-bit mask so it can contribute to any subset of up to 16 ROIs. New image_analysis/roi/ library (JFJochROIIntegration), structured like azint: a base that precomputes the per-pixel mask and names, a templated CPU engine (generic over pixel type for a future 16-bit path), and a GPU kernel using per-block shared-memory atomics for the STXM case (half-detector ROIs). Masked pixels are excluded entirely; saturated pixels are excluded from the sums but still count towards the max, matching roi_calc exactly. The engine is only constructed when at least one ROI is defined. Downstream CBOR/HDF5 already consume message.roi, so no further changes are needed. Co-Authored-By: Claude Opus 4.8 --- image_analysis/CMakeLists.txt | 3 +- image_analysis/MXAnalysisWithoutFPGA.cpp | 9 ++ image_analysis/MXAnalysisWithoutFPGA.h | 2 + image_analysis/roi/CMakeLists.txt | 5 + image_analysis/roi/ROIIntegration.cpp | 34 +++++ image_analysis/roi/ROIIntegration.h | 46 +++++++ image_analysis/roi/ROIIntegrationCPU.cpp | 11 ++ image_analysis/roi/ROIIntegrationCPU.h | 71 +++++++++++ image_analysis/roi/ROIIntegrationGPU.cu | 152 +++++++++++++++++++++++ image_analysis/roi/ROIIntegrationGPU.h | 37 ++++++ tests/CMakeLists.txt | 1 + tests/ROIIntegrationCPUTest.cpp | 95 ++++++++++++++ 12 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 image_analysis/roi/CMakeLists.txt create mode 100644 image_analysis/roi/ROIIntegration.cpp create mode 100644 image_analysis/roi/ROIIntegration.h create mode 100644 image_analysis/roi/ROIIntegrationCPU.cpp create mode 100644 image_analysis/roi/ROIIntegrationCPU.h create mode 100644 image_analysis/roi/ROIIntegrationGPU.cu create mode 100644 image_analysis/roi/ROIIntegrationGPU.h create mode 100644 tests/ROIIntegrationCPUTest.cpp diff --git a/image_analysis/CMakeLists.txt b/image_analysis/CMakeLists.txt index 1bf6e448..00996187 100644 --- a/image_analysis/CMakeLists.txt +++ b/image_analysis/CMakeLists.txt @@ -50,6 +50,7 @@ ADD_SUBDIRECTORY(lattice_search) ADD_SUBDIRECTORY(scale_merge) ADD_SUBDIRECTORY(image_preprocessing) ADD_SUBDIRECTORY(azint) +ADD_SUBDIRECTORY(roi) ADD_SUBDIRECTORY(pixel_refinement) -TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochPixelRefine gemmi) +TARGET_LINK_LIBRARIES(JFJochImageAnalysis JFJochAzIntEngine JFJochROIIntegration JFJochImagePreprocessing JFJochBraggPrediction JFJochBraggIntegration JFJochLatticeSearch JFJochIndexing JFJochSpotFinding JFJochCommon JFJochGeomRefinement JFJochScaleMerge JFJochPixelRefine gemmi) diff --git a/image_analysis/MXAnalysisWithoutFPGA.cpp b/image_analysis/MXAnalysisWithoutFPGA.cpp index 7bade492..cbe26e6a 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.cpp +++ b/image_analysis/MXAnalysisWithoutFPGA.cpp @@ -11,9 +11,11 @@ #include "image_preprocessing/ImagePreprocessorCPU.h" #include "azint/AzIntEngineCPU.h" +#include "roi/ROIIntegrationCPU.h" #include "spot_finding/ImageSpotFinderCPU.h" #ifdef JFJOCH_USE_CUDA #include "azint/AzIntEngineGPU.h" +#include "roi/ROIIntegrationGPU.h" #include "spot_finding/ImageSpotFinderGPU.h" #include "image_preprocessing/ImagePreprocessorGPU.h" #include "image_preprocessing/ImagePreprocessorBufferGPU.h" @@ -42,6 +44,8 @@ MXAnalysisWithoutFPGA::MXAnalysisWithoutFPGA(const DiffractionExperiment &in_exp spotFinder = std::make_unique(experiment.GetXPixelsNum(), experiment.GetYPixelsNum()); azint = std::make_unique(integration); preprocessor = std::make_unique(in_experiment, in_mask); + if (experiment.ROI().size() >= 1) + roi = std::make_unique(experiment); #ifdef JFJOCH_USE_CUDA } else { auto stream = std::make_shared(); @@ -49,6 +53,8 @@ MXAnalysisWithoutFPGA::MXAnalysisWithoutFPGA(const DiffractionExperiment &in_exp preprocessor = std::make_unique(in_experiment, in_mask, stream); spotFinder = std::make_unique(experiment.GetXPixelsNum(), experiment.GetYPixelsNum(), stream); azint = std::make_unique(integration, stream); + if (experiment.ROI().size() >= 1) + roi = std::make_unique(experiment, stream); } #endif } @@ -77,6 +83,9 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output, const auto azint_end_time = std::chrono::steady_clock::now(); output.azint_time_s = std::chrono::duration(azint_end_time - azint_start_time).count(); + if (roi) + roi->Run(*preprocessor_buffer, output.roi); + if (spot_finding_settings.enable) { // Update resolution mask if (mask_high_res != spot_finding_settings.high_resolution_limit diff --git a/image_analysis/MXAnalysisWithoutFPGA.h b/image_analysis/MXAnalysisWithoutFPGA.h index e5864d41..9d6bd0a9 100644 --- a/image_analysis/MXAnalysisWithoutFPGA.h +++ b/image_analysis/MXAnalysisWithoutFPGA.h @@ -14,6 +14,7 @@ #include "spot_finding/ImageSpotFinder.h" #include "indexing/IndexerThreadPool.h" #include "azint/AzIntEngine.h" +#include "roi/ROIIntegration.h" #include "IndexAndRefine.h" #include "image_preprocessing/ImagePreprocessor.h" #include "image_preprocessing/ImagePreprocessorBuffer.h" @@ -31,6 +32,7 @@ class MXAnalysisWithoutFPGA { size_t xpixels; std::unique_ptr azint; + std::unique_ptr roi; std::unique_ptr spotFinder; IndexAndRefine &indexer; std::unique_ptr prediction; diff --git a/image_analysis/roi/CMakeLists.txt b/image_analysis/roi/CMakeLists.txt new file mode 100644 index 00000000..afa1daa6 --- /dev/null +++ b/image_analysis/roi/CMakeLists.txt @@ -0,0 +1,5 @@ +ADD_LIBRARY(JFJochROIIntegration STATIC ROIIntegration.cpp ROIIntegration.h ROIIntegrationCPU.cpp ROIIntegrationCPU.h) +TARGET_LINK_LIBRARIES(JFJochROIIntegration JFJochCommon) +IF (JFJOCH_CUDA_AVAILABLE) + TARGET_SOURCES(JFJochROIIntegration PRIVATE ../indexing/CUDAMemHelpers.h ROIIntegrationGPU.cu ROIIntegrationGPU.h) +ENDIF() diff --git a/image_analysis/roi/ROIIntegration.cpp b/image_analysis/roi/ROIIntegration.cpp new file mode 100644 index 00000000..0cef8cca --- /dev/null +++ b/image_analysis/roi/ROIIntegration.cpp @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "ROIIntegration.h" +#include "../../common/DiffractionExperiment.h" + +ROIIntegration::ROIIntegration(const DiffractionExperiment &experiment) + : roi_map(experiment.ExportROIMap()), + width(experiment.GetXPixelsNumConv()), + npixel(roi_map.size()), + roi_count(experiment.ROI().size()), + roi_name(roi_count), + roi_sum(roi_count), + roi_sum2(roi_count), + roi_pixels(roi_count), + roi_x_weighted(roi_count), + roi_y_weighted(roi_count), + roi_max(roi_count) { + for (const auto &[name, id] : experiment.ROI().GetROINameMap()) + roi_name[id] = name; +} + +void ROIIntegration::Export(std::map &out) const { + for (uint16_t r = 0; r < roi_count; r++) + out[roi_name[r]] = ROIMessage{ + .sum = roi_sum[r], + .sum_square = roi_sum2[r], + .max_count = roi_max[r], + .pixels = roi_pixels[r], + .pixels_masked = 0, + .x_weighted = roi_x_weighted[r], + .y_weighted = roi_y_weighted[r], + }; +} diff --git a/image_analysis/roi/ROIIntegration.h b/image_analysis/roi/ROIIntegration.h new file mode 100644 index 00000000..2d0ddd26 --- /dev/null +++ b/image_analysis/roi/ROIIntegration.h @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include + +#include "../../common/JFJochMessages.h" +#include "../image_preprocessing/ImagePreprocessorBuffer.h" + +class DiffractionExperiment; + +// Computes per-ROI statistics over an assembled image: sum, sum of squares, +// good-pixel count, maximum and intensity-weighted centre of mass. Each pixel +// carries a 16-bit mask selecting which of up to 16 ROIs it belongs to, so a +// single pixel can contribute to none, one, or several ROIs at once. This is +// the software counterpart of the FPGA roi_calc kernel. +class ROIIntegration { +protected: + std::vector roi_map; // per-pixel ROI bitmask, assembled (converted) geometry + size_t width; // assembled image width, to recover x/y from a pixel index + size_t npixel; + uint16_t roi_count; + std::vector roi_name; // ROI name indexed by bit position + + // Result accumulators, filled by Run() in the derived class + std::vector roi_sum; + std::vector roi_sum2; + std::vector roi_pixels; + std::vector roi_x_weighted; + std::vector roi_y_weighted; + std::vector roi_max; + + void Export(std::map &out) const; +public: + explicit ROIIntegration(const DiffractionExperiment &experiment); + virtual ~ROIIntegration() = default; + + [[nodiscard]] uint16_t Count() const { return roi_count; } + [[nodiscard]] bool empty() const { return roi_count == 0; } + + virtual void Run(const ImagePreprocessorBuffer &image, std::map &out) = 0; +}; diff --git a/image_analysis/roi/ROIIntegrationCPU.cpp b/image_analysis/roi/ROIIntegrationCPU.cpp new file mode 100644 index 00000000..432b2ca4 --- /dev/null +++ b/image_analysis/roi/ROIIntegrationCPU.cpp @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "ROIIntegrationCPU.h" + +ROIIntegrationCPU::ROIIntegrationCPU(const DiffractionExperiment &experiment) + : ROIIntegration(experiment) {} + +void ROIIntegrationCPU::Run(const ImagePreprocessorBuffer &image, std::map &out) { + RunROI(image, out); +} diff --git a/image_analysis/roi/ROIIntegrationCPU.h b/image_analysis/roi/ROIIntegrationCPU.h new file mode 100644 index 00000000..0e5fb4eb --- /dev/null +++ b/image_analysis/roi/ROIIntegrationCPU.h @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +#include "ROIIntegration.h" +#include "../../common/JFJochException.h" + +class ROIIntegrationCPU : public ROIIntegration { +public: + explicit ROIIntegrationCPU(const DiffractionExperiment &experiment); + + // image is anything indexable with operator[] and size(). Templated on the + // pixel type so a future narrow-integer (e.g. 16-bit) path works as well. + template + void RunROI(const T &image, std::map &out) { + if (image.size() != npixel) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "ROIIntegration: mismatch in image size"); + + for (uint16_t r = 0; r < roi_count; r++) { + roi_sum[r] = 0; + roi_sum2[r] = 0; + roi_pixels[r] = 0; + roi_x_weighted[r] = 0; + roi_y_weighted[r] = 0; + roi_max[r] = INT64_MIN; + } + + using pixel_t = std::remove_cv_t>; + + for (size_t i = 0; i < npixel; i++) { + const uint16_t mask = roi_map[i]; + if (mask == 0) + continue; + + const pixel_t v = image[i]; + // masked/bad pixels (signed types only) are excluded entirely + if constexpr (std::is_signed_v) { + if (v == std::numeric_limits::min()) + continue; + } + // saturated pixels still count towards the max, but not the sums + const bool saturated = (v == std::numeric_limits::max()); + + const int64_t val = static_cast(v); + const int64_t x = static_cast(i % width); + const int64_t y = static_cast(i / width); + + for (uint16_t r = 0; r < roi_count; r++) { + if (!(mask & (1u << r))) + continue; + if (!saturated) { + roi_sum[r] += val; + roi_sum2[r] += static_cast(val * val); + roi_pixels[r] += 1; + roi_x_weighted[r] += val * x; + roi_y_weighted[r] += val * y; + } + if (val > roi_max[r]) + roi_max[r] = val; + } + } + Export(out); + } + + void Run(const ImagePreprocessorBuffer &image, std::map &out) override; +}; diff --git a/image_analysis/roi/ROIIntegrationGPU.cu b/image_analysis/roi/ROIIntegrationGPU.cu new file mode 100644 index 00000000..66701cfa --- /dev/null +++ b/image_analysis/roi/ROIIntegrationGPU.cu @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include "ROIIntegrationGPU.h" +#include "../../common/DiffractionExperiment.h" + +inline void cuda_err(cudaError_t val) { + if (val != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); +} + +// One pixel carries a 16-bit mask, so it can feed any subset of the ROIs. +// Each block reduces into shared memory first to keep global atomics low. +__global__ +void gpu_roi( + const uint16_t *__restrict__ roi_map, + const int32_t *__restrict__ input_buffer, + size_t num_pixels, + size_t width, + int roi_count, + unsigned long long *__restrict__ roi_sum, + unsigned long long *__restrict__ roi_sum2, + unsigned long long *__restrict__ roi_pixels, + unsigned long long *__restrict__ roi_x_weighted, + unsigned long long *__restrict__ roi_y_weighted, + int *__restrict__ roi_max) { + extern __shared__ unsigned long long shared[]; + unsigned long long *s_sum = shared; + unsigned long long *s_sum2 = &s_sum[roi_count]; + unsigned long long *s_pixels = &s_sum2[roi_count]; + unsigned long long *s_xw = &s_pixels[roi_count]; + unsigned long long *s_yw = &s_xw[roi_count]; + int *s_max = (int *) &s_yw[roi_count]; + + for (int r = threadIdx.x; r < roi_count; r += blockDim.x) { + s_sum[r] = 0; + s_sum2[r] = 0; + s_pixels[r] = 0; + s_xw[r] = 0; + s_yw[r] = 0; + s_max[r] = INT_MIN; + } + __syncthreads(); + + for (size_t idx = blockIdx.x * blockDim.x + threadIdx.x; + idx < num_pixels; + idx += blockDim.x * gridDim.x) { + const uint16_t mask = roi_map[idx]; + if (mask == 0) + continue; + + const int32_t v = input_buffer[idx]; + if (v == INT32_MIN) // masked/bad pixel + continue; + const bool saturated = (v == INT32_MAX); + + const long long val = v; + const long long x = idx % width; + const long long y = idx / width; + const unsigned long long val_u = (unsigned long long) val; + const unsigned long long val2_u = (unsigned long long) (val * val); + const unsigned long long vx_u = (unsigned long long) (val * x); + const unsigned long long vy_u = (unsigned long long) (val * y); + + for (int r = 0; r < roi_count; r++) { + if (!(mask & (1u << r))) + continue; + if (!saturated) { + atomicAdd(&s_sum[r], val_u); + atomicAdd(&s_sum2[r], val2_u); + atomicAdd(&s_pixels[r], 1ULL); + atomicAdd(&s_xw[r], vx_u); + atomicAdd(&s_yw[r], vy_u); + } + atomicMax(&s_max[r], v); + } + } + __syncthreads(); + + for (int r = threadIdx.x; r < roi_count; r += blockDim.x) { + atomicAdd(&roi_sum[r], s_sum[r]); + atomicAdd(&roi_sum2[r], s_sum2[r]); + atomicAdd(&roi_pixels[r], s_pixels[r]); + atomicAdd(&roi_x_weighted[r], s_xw[r]); + atomicAdd(&roi_y_weighted[r], s_yw[r]); + atomicMax(&roi_max[r], s_max[r]); + } +} + +ROIIntegrationGPU::ROIIntegrationGPU(const DiffractionExperiment &experiment, std::shared_ptr stream) + : ROIIntegration(experiment), + stream(stream), + gpu_roi_map(npixel), + gpu_sum(roi_count), + gpu_sum2(roi_count), + gpu_pixels(roi_count), + gpu_x_weighted(roi_count), + gpu_y_weighted(roi_count), + gpu_max(roi_count), + host_sum(roi_count), + host_sum2(roi_count), + host_pixels(roi_count), + host_x_weighted(roi_count), + host_y_weighted(roi_count), + host_max(roi_count), + max_init(roi_count, INT_MIN) { + + cudaDeviceProp prop{}; + cuda_err(cudaGetDeviceProperties(&prop, 0)); + threads = 128; + blocks = 4 * prop.multiProcessorCount; + shared_needed = roi_count * (5 * sizeof(unsigned long long) + sizeof(int)); + + cuda_err(cudaMemcpy(gpu_roi_map, roi_map.data(), sizeof(uint16_t) * npixel, cudaMemcpyHostToDevice)); +} + +void ROIIntegrationGPU::Run(const ImagePreprocessorBuffer &image, std::map &out) { + if (image.size() != npixel) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "ROIIntegration: mismatch in image size"); + + cuda_err(cudaMemsetAsync(gpu_sum, 0, sizeof(unsigned long long) * roi_count, *stream)); + cuda_err(cudaMemsetAsync(gpu_sum2, 0, sizeof(unsigned long long) * roi_count, *stream)); + cuda_err(cudaMemsetAsync(gpu_pixels, 0, sizeof(unsigned long long) * roi_count, *stream)); + cuda_err(cudaMemsetAsync(gpu_x_weighted, 0, sizeof(unsigned long long) * roi_count, *stream)); + cuda_err(cudaMemsetAsync(gpu_y_weighted, 0, sizeof(unsigned long long) * roi_count, *stream)); + cuda_err(cudaMemcpyAsync(gpu_max, max_init.data(), sizeof(int) * roi_count, cudaMemcpyHostToDevice, *stream)); + + gpu_roi<<>>( + gpu_roi_map, image.getGPUBuffer(), npixel, width, roi_count, + gpu_sum, gpu_sum2, gpu_pixels, gpu_x_weighted, gpu_y_weighted, gpu_max); + + cudaMemcpyAsync(host_sum.data(), gpu_sum, sizeof(unsigned long long) * roi_count, cudaMemcpyDeviceToHost, *stream); + cudaMemcpyAsync(host_sum2.data(), gpu_sum2, sizeof(unsigned long long) * roi_count, cudaMemcpyDeviceToHost, *stream); + cudaMemcpyAsync(host_pixels.data(), gpu_pixels, sizeof(unsigned long long) * roi_count, cudaMemcpyDeviceToHost, *stream); + cudaMemcpyAsync(host_x_weighted.data(), gpu_x_weighted, sizeof(unsigned long long) * roi_count, cudaMemcpyDeviceToHost, *stream); + cudaMemcpyAsync(host_y_weighted.data(), gpu_y_weighted, sizeof(unsigned long long) * roi_count, cudaMemcpyDeviceToHost, *stream); + cudaMemcpyAsync(host_max.data(), gpu_max, sizeof(int) * roi_count, cudaMemcpyDeviceToHost, *stream); + cuda_err(cudaStreamSynchronize(*stream)); + + for (uint16_t r = 0; r < roi_count; r++) { + roi_sum[r] = static_cast(host_sum[r]); + roi_sum2[r] = host_sum2[r]; + roi_pixels[r] = host_pixels[r]; + roi_x_weighted[r] = static_cast(host_x_weighted[r]); + roi_y_weighted[r] = static_cast(host_y_weighted[r]); + roi_max[r] = (host_max[r] == INT_MIN) ? INT64_MIN : static_cast(host_max[r]); + } + Export(out); +} diff --git a/image_analysis/roi/ROIIntegrationGPU.h b/image_analysis/roi/ROIIntegrationGPU.h new file mode 100644 index 00000000..5528b37b --- /dev/null +++ b/image_analysis/roi/ROIIntegrationGPU.h @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "ROIIntegration.h" +#include "../indexing/CUDAMemHelpers.h" + +class ROIIntegrationGPU : public ROIIntegration { + std::shared_ptr stream; + int threads; + int blocks; + size_t shared_needed; + + CudaDevicePtr gpu_roi_map; + // 64-bit sums are accumulated as unsigned long long (two's-complement bit + // pattern) because CUDA atomicAdd has no signed 64-bit overload. + CudaDevicePtr gpu_sum; + CudaDevicePtr gpu_sum2; + CudaDevicePtr gpu_pixels; + CudaDevicePtr gpu_x_weighted; + CudaDevicePtr gpu_y_weighted; + CudaDevicePtr gpu_max; + + std::vector host_sum; + std::vector host_sum2; + std::vector host_pixels; + std::vector host_x_weighted; + std::vector host_y_weighted; + std::vector host_max; + std::vector max_init; // INT_MIN seed copied into gpu_max each frame +public: + ROIIntegrationGPU(const DiffractionExperiment &experiment, std::shared_ptr stream); + void Run(const ImagePreprocessorBuffer &image, std::map &out) override; +}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 751e8f64..59af6e96 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,6 +32,7 @@ ADD_EXECUTABLE(jfjoch_test JPEGTest.cpp HistogramTest.cpp ROIMapTest.cpp + ROIIntegrationCPUTest.cpp LossyFilterTest.cpp ImageBufferTest.cpp PixelMaskTest.cpp diff --git a/tests/ROIIntegrationCPUTest.cpp b/tests/ROIIntegrationCPUTest.cpp new file mode 100644 index 00000000..747d9b16 --- /dev/null +++ b/tests/ROIIntegrationCPUTest.cpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include "../image_analysis/roi/ROIIntegrationCPU.h" +#include "../common/DiffractionExperiment.h" + +TEST_CASE("ROIIntegrationCPU") { + DiffractionExperiment experiment(DetJF(1)); + experiment.ROI().SetROI(ROIDefinition{.boxes = { + ROIBox("roiA", 10, 20, 30, 40), + ROIBox("roiB", 15, 25, 35, 45) + }}); + + ROIIntegrationCPU roi(experiment); + REQUIRE(roi.Count() == 2); + REQUIRE_FALSE(roi.empty()); + + const auto roi_map = experiment.ExportROIMap(); + const size_t width = experiment.GetXPixelsNumConv(); + const size_t npixel = roi_map.size(); + + ImagePreprocessorBuffer image(npixel); + for (size_t i = 0; i < npixel; i++) + image[i] = static_cast((i * 7 + 3) % 251); // deterministic, well below INT32_MAX + + // Inject one saturated pixel into roiA (bit 0) and one masked pixel into roiB (bit 1) + int64_t saturated_index = -1; + int64_t masked_index = -1; + for (size_t i = 0; i < npixel && (saturated_index < 0 || masked_index < 0); i++) { + if (saturated_index < 0 && (roi_map[i] & (1 << 0))) + saturated_index = static_cast(i); + else if (masked_index < 0 && (roi_map[i] & (1 << 1))) + masked_index = static_cast(i); + } + REQUIRE(saturated_index >= 0); + REQUIRE(masked_index >= 0); + image[saturated_index] = INT32_MAX; + image[masked_index] = INT32_MIN; + + // Independent reference matching the documented semantics (masked excluded + // entirely; saturated excluded from sums but counted towards the max) + struct Ref { int64_t sum = 0; uint64_t sum2 = 0; uint64_t pixels = 0; + int64_t xw = 0; int64_t yw = 0; int64_t max = INT64_MIN; }; + Ref ref[2]; + for (size_t i = 0; i < npixel; i++) { + const uint16_t mask = roi_map[i]; + if (mask == 0) + continue; + const int32_t v = image[i]; + if (v == INT32_MIN) + continue; + const bool saturated = (v == INT32_MAX); + const int64_t val = v; + const int64_t x = static_cast(i % width); + const int64_t y = static_cast(i / width); + for (int r = 0; r < 2; r++) { + if (!(mask & (1 << r))) + continue; + if (!saturated) { + ref[r].sum += val; + ref[r].sum2 += static_cast(val * val); + ref[r].pixels += 1; + ref[r].xw += val * x; + ref[r].yw += val * y; + } + if (val > ref[r].max) + ref[r].max = val; + } + } + + std::map out; + roi.Run(image, out); + + REQUIRE(out.size() == 2); + REQUIRE(out.contains("roiA")); + REQUIRE(out.contains("roiB")); + + const std::pair roi_ids[2] = {{"roiA", 0}, {"roiB", 1}}; + for (const auto &[name, r] : roi_ids) { + const auto &msg = out.at(name); + CHECK(msg.sum == ref[r].sum); + CHECK(msg.sum_square == ref[r].sum2); + CHECK(msg.pixels == ref[r].pixels); + CHECK(msg.x_weighted == ref[r].xw); + CHECK(msg.y_weighted == ref[r].yw); + CHECK(msg.max_count == ref[r].max); + CHECK(msg.pixels_masked == 0); + } + + // Targeted checks: saturated pixel sets the max for roiA but is not summed + CHECK(out.at("roiA").max_count == INT32_MAX); + CHECK(ref[0].pixels > 0); +} -- 2.52.0 From c0a4801bc47c998c6ad771a14c5b43edae2d9065 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 22:02:09 +0200 Subject: [PATCH 073/228] TCPStreamPusher: fix zero-copy buffer reuse and make send/END timeouts progress-based Three robustness fixes for the writer-facing TCP stream, addressing the spurious "wrong number of images" / connection failures seen under load. 1. Never MSG_ZEROCOPY a transient buffer. The synchronous SendImage path passes a caller-owned buffer with z == nullptr and reuses it for the next frame immediately. With MSG_ZEROCOPY the kernel still references that buffer after send() returns, so the peer could receive corrupted frames and drop the connection mid-stream, truncating the run. Zero-copy is now gated on a ZeroCopyReturnValue that keeps the buffer alive until completion. 2. Make the SendAll watchdog measure lack of progress, not total wall-clock. The previous absolute deadline tore down a healthy but back-pressured connection (slow/starved writer) after a fixed window; the watchdog now resets on every byte actually sent, so only a genuinely stuck socket is closed. Dead peers are still caught by OS keepalive and POLLHUP/POLLERR. 3. Make the END-ack wait progress-based (WaitForEndAck). The writer may still be draining a backlog of DATA frames when END is sent; each DATA ACK is progress, so the timeout only arms once the writer falls silent rather than firing while images are legitimately still being drained. Co-Authored-By: Claude Opus 4.8 --- image_pusher/TCPStreamPusher.cpp | 52 +++++++++++++++++++++++++++++--- image_pusher/TCPStreamPusher.h | 1 + 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/image_pusher/TCPStreamPusher.cpp b/image_pusher/TCPStreamPusher.cpp index d3184d2c..d4a4295b 100644 --- a/image_pusher/TCPStreamPusher.cpp +++ b/image_pusher/TCPStreamPusher.cpp @@ -197,7 +197,10 @@ bool TCPStreamPusher::SendAll(Connection& c, const void* buf, size_t len, bool a bool zc_used = false; uint32_t zc_first = 0; uint32_t zc_last = 0; - const auto deadline = std::chrono::steady_clock::now() + send_total_timeout; + // Watchdog on *lack of progress*, not total time: a slow but advancing transfer + // (a writer applying back-pressure) must not be torn down, only a socket that + // stops moving bytes altogether. Reset on every byte actually handed to the kernel. + auto last_progress = std::chrono::steady_clock::now(); bool try_zerocopy = false; #if defined(MSG_ZEROCOPY) @@ -213,7 +216,7 @@ bool TCPStreamPusher::SendAll(Connection& c, const void* buf, size_t len, bool a return false; } - if (std::chrono::steady_clock::now() >= deadline) { + if (std::chrono::steady_clock::now() - last_progress >= send_total_timeout) { c.broken = true; CloseFd(c.fd); if (zc_used_out) *zc_used_out = zc_used; @@ -288,6 +291,9 @@ bool TCPStreamPusher::SendAll(Connection& c, const void* buf, size_t len, bool a } #endif + if (rc > 0) + last_progress = std::chrono::steady_clock::now(); + sent += static_cast(rc); } @@ -315,7 +321,12 @@ bool TCPStreamPusher::SendFrame(Connection& c, const uint8_t* data, size_t size, uint32_t zc_last = 0; if (size > 0) { - const bool allow_zerocopy = (type == TCPFrameType::DATA || type == TCPFrameType::CALIBRATION); + // MSG_ZEROCOPY leaves the kernel referencing the caller's buffer after send() + // returns, so it is only safe when a ZeroCopyReturnValue keeps that buffer alive + // until the completion notification. The synchronous path passes a transient + // buffer (z == nullptr) that the caller reuses immediately — never zero-copy it. + const bool allow_zerocopy = (z != nullptr) + && (type == TCPFrameType::DATA || type == TCPFrameType::CALIBRATION); if (!SendAll(c, data, size, allow_zerocopy, &zc_used, &zc_first, &zc_last)) { if (z) { if (zc_used) EnqueueZeroCopyPending(c, z, zc_first, zc_last); @@ -899,6 +910,39 @@ bool TCPStreamPusher::WaitForAck(Connection& c, TCPFrameType ack_for, std::chron return ack_ok; } +bool TCPStreamPusher::WaitForEndAck(Connection& c, std::chrono::milliseconds no_progress_timeout, std::string* error_text) { + std::unique_lock ul(c.ack_mutex); + + // After END is sent the writer may still be draining a backlog of DATA frames. + // Each DATA ACK that arrives is forward progress, so the END timer is effectively + // armed only once the writer falls silent: the timeout fires when neither a DATA + // ACK nor the END ACK is seen for the whole window, not while images still drain. + uint64_t last_acked = c.data_acked_total.load(std::memory_order_relaxed); + + while (!c.end_ack_received && !c.broken) { + const bool progressed = c.ack_cv.wait_for(ul, no_progress_timeout, [&] { + return c.end_ack_received || c.broken.load() + || c.data_acked_total.load(std::memory_order_relaxed) != last_acked; + }); + + if (!progressed) { + if (error_text) *error_text = "END ACK timeout (writer stalled)"; + return false; + } + last_acked = c.data_acked_total.load(std::memory_order_relaxed); + } + + if (c.broken) { + if (error_text) *error_text = c.last_ack_error.empty() ? "Socket broken" : c.last_ack_error; + return false; + } + + if (!c.end_ack_ok && error_text) + *error_text = c.last_ack_error.empty() ? "END ACK rejected" : c.last_ack_error; + + return c.end_ack_ok; +} + void TCPStreamPusher::StartDataCollection(StartMessage& message) { if (message.images_per_file < 1) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Images per file cannot be zero or negative"); @@ -1076,7 +1120,7 @@ bool TCPStreamPusher::EndDataCollection(const EndMessage& message) { } std::string ack_err; - if (!WaitForAck(c, TCPFrameType::END, std::chrono::seconds(10), &ack_err)) + if (!WaitForEndAck(c, std::chrono::seconds(10), &ack_err)) ret = false; } diff --git a/image_pusher/TCPStreamPusher.h b/image_pusher/TCPStreamPusher.h index 1b6eb0ef..bd5dfed6 100644 --- a/image_pusher/TCPStreamPusher.h +++ b/image_pusher/TCPStreamPusher.h @@ -161,6 +161,7 @@ class TCPStreamPusher : public ImagePusher { bool WaitForZeroCopyDrain(Connection& c, std::chrono::milliseconds timeout); bool WaitForAck(Connection& c, TCPFrameType ack_for, std::chrono::milliseconds timeout, std::string* error_text); + bool WaitForEndAck(Connection& c, std::chrono::milliseconds no_progress_timeout, std::string* error_text); public: explicit TCPStreamPusher(const std::string& addr, size_t in_max_connections, -- 2.52.0 From 188cbb659dc190b657583bdd13bae0c34583339d Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Wed, 17 Jun 2026 22:28:42 +0200 Subject: [PATCH 074/228] tests: add GPU/CPU equivalence test for ROI integration Runs ROIIntegrationGPU and ROIIntegrationCPU on identical input and asserts every per-ROI field (sum, sum_square, max, pixels, weighted centre, masked count) matches bit-for-bit. Uses overlapping ROI boxes (multi-bit masks), negative pixel values (signed weighted-sum path), and an injected saturated and masked pixel per ROI to cover the "max only" and "fully excluded" branches. Guarded by JFJOCH_USE_CUDA and skips with a warning when no CUDA GPU is present, mirroring ImageSpotFinderGPUTest. Co-Authored-By: Claude Opus 4.8 --- tests/CMakeLists.txt | 1 + tests/ROIIntegrationGPUTest.cpp | 114 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 tests/ROIIntegrationGPUTest.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 59af6e96..bc6539d3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -33,6 +33,7 @@ ADD_EXECUTABLE(jfjoch_test HistogramTest.cpp ROIMapTest.cpp ROIIntegrationCPUTest.cpp + ROIIntegrationGPUTest.cpp LossyFilterTest.cpp ImageBufferTest.cpp PixelMaskTest.cpp diff --git a/tests/ROIIntegrationGPUTest.cpp b/tests/ROIIntegrationGPUTest.cpp new file mode 100644 index 00000000..dc6d2841 --- /dev/null +++ b/tests/ROIIntegrationGPUTest.cpp @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include "../common/CUDAWrapper.h" + +#ifdef JFJOCH_USE_CUDA + +#include +#include +#include + +#include "../image_analysis/roi/ROIIntegrationCPU.h" +#include "../image_analysis/roi/ROIIntegrationGPU.h" +#include "../image_analysis/image_preprocessing/ImagePreprocessorBufferGPU.h" +#include "../common/DiffractionExperiment.h" + +namespace { +DiffractionExperiment make_roi_experiment() { + DiffractionExperiment experiment(DetJF(1)); + // Overlapping boxes so some pixels belong to several ROIs at once (multi-bit + // mask), exercising the per-bit accumulation that has to match between CPU and GPU. + // ROIBox is (name, x_min, x_max, y_min, y_max), kept within the JF module bounds. + experiment.ROI().SetROI(ROIDefinition{.boxes = { + ROIBox("roiA", 10, 210, 20, 220), + ROIBox("roiB", 100, 300, 100, 300), + ROIBox("roiC", 0, 150, 0, 150), + ROIBox("roiD", 50, 250, 50, 250), + }}); + return experiment; +} + +void compare_results(const std::map &cpu, + const std::map &gpu) { + REQUIRE(cpu.size() == gpu.size()); + for (const auto &[name, c] : cpu) { + INFO("ROI " << name); + REQUIRE(gpu.contains(name)); + const auto &g = gpu.at(name); + CHECK(g.sum == c.sum); + CHECK(g.sum_square == c.sum_square); + CHECK(g.max_count == c.max_count); + CHECK(g.pixels == c.pixels); + CHECK(g.x_weighted == c.x_weighted); + CHECK(g.y_weighted == c.y_weighted); + CHECK(g.pixels_masked == c.pixels_masked); + } +} +} // namespace + +// The GPU kernel reduces with atomics and two's-complement unsigned accumulators, +// while the CPU path is a plain serial loop. On identical input every per-ROI +// statistic must be bit-for-bit identical, so we run both and compare. +TEST_CASE("ROIIntegrationGPU_MatchesCPU") { + if (get_gpu_count() == 0) { + WARN("No CUDA GPU present. Skipping ROIIntegrationGPU_MatchesCPU"); + return; + } + + const DiffractionExperiment experiment = make_roi_experiment(); + const auto roi_map = experiment.ExportROIMap(); + const size_t npixel = roi_map.size(); + const uint16_t roi_count = experiment.ROI().size(); + REQUIRE(roi_count == 4); + + // Deterministic image with both positive and negative values; negatives exercise + // the signed weighted-sum path (val * x can be negative). + std::vector values(npixel); + for (size_t i = 0; i < npixel; i++) + values[i] = static_cast((i * 2654435761u) % 1000) - 500; + + // Inject one saturated (INT32_MAX) and one masked (INT32_MIN) pixel into every ROI + // so both the "max only, not summed" and "fully excluded" branches are covered. + for (uint16_t r = 0; r < roi_count; r++) { + bool injected_sat = false, injected_mask = false; + for (size_t i = 0; i < npixel && !(injected_sat && injected_mask); i++) { + if (!(roi_map[i] & (1u << r))) + continue; + if (!injected_sat) { values[i] = INT32_MAX; injected_sat = true; } + else if (!injected_mask) { values[i] = INT32_MIN; injected_mask = true; } + } + REQUIRE(injected_sat); + REQUIRE(injected_mask); + } + + // CPU reference + ImagePreprocessorBuffer cpu_image(npixel); + for (size_t i = 0; i < npixel; i++) + cpu_image[i] = values[i]; + + ROIIntegrationCPU cpu(experiment); + std::map out_cpu; + cpu.Run(cpu_image, out_cpu); + + // GPU under test — identical input uploaded to the device + auto stream = std::make_shared(); + ImagePreprocessorBufferGPU gpu_image(npixel); + for (size_t i = 0; i < npixel; i++) + gpu_image[i] = values[i]; + + REQUIRE(cudaMemcpyAsync(gpu_image.getGPUBuffer(), + gpu_image.getBuffer().data(), + npixel * sizeof(int32_t), + cudaMemcpyHostToDevice, + *stream) == cudaSuccess); + + ROIIntegrationGPU gpu(experiment, stream); + std::map out_gpu; + gpu.Run(gpu_image, out_gpu); + + compare_results(out_cpu, out_gpu); +} + +#endif -- 2.52.0 From 56ddfaef96bfcf725711385d18c4d6f4c0203a8d Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 08:13:07 +0200 Subject: [PATCH 075/228] viewer: live detector status bar + dataset-follow sync mode Add a status-bar cluster, shown when connected over HTTP, that surfaces the broker state and live acquisition info: - broker state box (QProgressBar) with progress drawn as the bar fill, polled ~1 Hz on its own timer independent of image sync - Live/Disconnected connection badge (host:port in tooltip) - "+N new" badge and effective live-rate (Hz) readout All widgets are fixed-width and only blanked in place, so the bar never reflows when things appear/disappear. Add a third autoload mode (HTTPSyncDataset): manually selecting an image while following live now freezes the displayed image but keeps the dataset, plots and image count updating. Surfaced via a tri-state HTTP Sync button (off / live / data-only). Also fix GetBrokerStatus() dropping the message field and a couple of copy-pasted exception strings in JFJochHttpReader. Co-Authored-By: Claude Opus 4.8 --- reader/JFJochHttpReader.cpp | 32 +++++- reader/JFJochHttpReader.h | 4 + viewer/JFJochImageReadingWorker.cpp | 75 +++++++++++- viewer/JFJochImageReadingWorker.h | 16 ++- viewer/JFJochViewerStatusBar.cpp | 120 +++++++++++++++++++- viewer/JFJochViewerStatusBar.h | 34 +++++- viewer/JFJochViewerWindow.cpp | 12 ++ viewer/toolbar/JFJochViewerToolbarImage.cpp | 33 +++++- 8 files changed, 308 insertions(+), 18 deletions(-) diff --git a/reader/JFJochHttpReader.cpp b/reader/JFJochHttpReader.cpp index 30784ed9..dd383d27 100644 --- a/reader/JFJochHttpReader.cpp +++ b/reader/JFJochHttpReader.cpp @@ -50,7 +50,7 @@ BrokerStatus JFJochHttpReader::GetBrokerStatus() const { auto res = cli_cmd.Get("/status"); if (!res || res->status != httplib::StatusCode::OK_200) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Could not get image buffer status"); + "Could not get broker status"); try { org::openapitools::server::model::Broker_status input = nlohmann::json::parse(res->body); @@ -59,6 +59,8 @@ BrokerStatus JFJochHttpReader::GetBrokerStatus() const { ret.gpu_count = input.getGpuCount(); if (input.progressIsSet()) ret.progress = input.getProgress(); + if (input.messageIsSet()) + ret.message = input.getMessage(); if (input.getState() == "Inactive") ret.state = JFJochState::Inactive; @@ -87,7 +89,7 @@ BrokerStatus JFJochHttpReader::GetBrokerStatus() const { return ret; } catch (std::exception &e) { throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Could not parse image buffer status"); + "Could not parse broker status"); } } @@ -209,6 +211,30 @@ void JFJochHttpReader::ReadURL(const std::string &url) { SetStartMessage(UpdateDataset_i()); } +std::shared_ptr JFJochHttpReader::RefreshDatasetIfChanged(int64_t &num_images_out) { + std::unique_lock ul(http_mutex); + + num_images_out = 0; + if (addr.empty()) + return {}; + + auto status = GetImageBufferStatus(); + num_images_out = status.max_image_number + 1; + + // Re-fetch the dataset (start message + plots) only when the buffer actually changed, + // so the dataset-only follow mode does not hammer the broker with plot requests. + const bool buffer_changed = !last_image_buffer_counter.has_value() + || !status.current_counter.has_value() + || last_image_buffer_counter.value() != status.current_counter.value(); + last_image_buffer_counter = status.current_counter; + + if (!buffer_changed) + return {}; + + SetStartMessage(UpdateDataset_i()); + return GetDataset(); +} + bool JFJochHttpReader::LoadImage_i(std::shared_ptr &dataset, DataMessage &message, std::vector &buffer, @@ -342,7 +368,7 @@ std::vector JFJochHttpReader::GetPlot_i(const std::string &plot_type, flo return {}; } catch (nlohmann::json::parse_error &e) { throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Could not parse image buffer status"); + "Could not parse plot " + plot_type); } } diff --git a/reader/JFJochHttpReader.h b/reader/JFJochHttpReader.h index 13dc6911..e558d98a 100644 --- a/reader/JFJochHttpReader.h +++ b/reader/JFJochHttpReader.h @@ -30,6 +30,10 @@ public: uint64_t GetNumberOfImages() const override; void Close() override; + // Refresh dataset/plots if the image buffer changed since the last poll (returns nullptr if + // unchanged). Always writes the current number of images to num_images_out. Does not load an image. + std::shared_ptr RefreshDatasetIfChanged(int64_t &num_images_out); + void UploadUserMask(const std::vector& mask); BrokerStatus GetBrokerStatus() const; diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index a10ded30..15a441d4 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -68,7 +68,8 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se indexing_settings(experiment.GetIndexingSettings()), azint_settings(experiment.GetAzimuthalIntegrationSettings()) { qRegisterMetaType>("QVector"); - spot_finding_settings = settings;; + qRegisterMetaType("BrokerStatus"); + spot_finding_settings = settings; indexing = std::make_unique(indexing_settings); http_reader.Experiment(experiment); @@ -78,6 +79,10 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se autoload_timer->setInterval(autoload_interval); connect(autoload_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::AutoLoadTimerExpired); + status_timer = new QTimer(this); + status_timer->setInterval(status_interval); + connect(status_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::StatusTimerExpired); + file_open_retry_timer = new QTimer(this); file_open_retry_timer->setSingleShot(true); connect(file_open_retry_timer, &QTimer::timeout, this, &JFJochImageReadingWorker::FileOpenRetryTimerExpired); @@ -171,6 +176,8 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_ http_reader.ReadURL(filename.toStdString()); total_images = http_reader.GetNumberOfImages(); dataset = http_reader.GetDataset(); + SetHttpConnected_i(true, filename); + status_timer->start(); if (image_number < 0) setAutoLoadMode_i(AutoloadMode::HTTPSync); else @@ -178,6 +185,8 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_ } else { http_mode = false; + status_timer->stop(); + SetHttpConnected_i(false, ""); if (retry) { pending_load.filename = filename; @@ -279,6 +288,10 @@ void JFJochImageReadingWorker::CloseFile() { else file_reader.Close(); + status_timer->stop(); + setAutoLoadMode_i(AutoloadMode::None); + SetHttpConnected_i(false, ""); + current_image_ptr.reset(); current_image.reset(); current_summation = 1; @@ -290,7 +303,12 @@ void JFJochImageReadingWorker::CloseFile() { void JFJochImageReadingWorker::LoadImage(int64_t image_number, int64_t summation) { QMutexLocker ul(&m); - setAutoLoadMode_i(AutoloadMode::None); + // Manually selecting an image while following live drops to dataset-only follow: + // the chosen image stays put while plots and image count keep updating. + if (autoload_mode == AutoloadMode::HTTPSync || autoload_mode == AutoloadMode::HTTPSyncDataset) + setAutoLoadMode_i(AutoloadMode::HTTPSyncDataset); + else + setAutoLoadMode_i(AutoloadMode::None); if ((image_number == current_image) && (current_summation == summation)) return; LoadImage_i(image_number, summation); @@ -366,6 +384,9 @@ void JFJochImageReadingWorker::LoadImage_i(int64_t image_number, int64_t summati autoload_timer->setInterval(autoload_interval); } } + + if (autoload_mode == AutoloadMode::HTTPSync) + emit liveRateChanged(autoload_interval > 0 ? 1000.0 / autoload_interval : 0.0); } logger.Info("Loaded image {} in {}/{} ms Autoload timer set to {} ms", image_number, duration_1, duration_2, autoload_interval); @@ -656,6 +677,10 @@ void JFJochImageReadingWorker::AutoLoadTimerExpired() { if (http_mode) LoadImage_i(-1 , 1); break; + case AutoloadMode::HTTPSyncDataset: + if (http_mode) + RefreshDatasetOnly_i(); + break; case AutoloadMode::Movie: { if (total_images == 0 || !current_image) return; @@ -668,12 +693,57 @@ void JFJochImageReadingWorker::AutoLoadTimerExpired() { } } +void JFJochImageReadingWorker::SetHttpConnected_i(bool connected, const QString &addr) { + // Assumes m locked! Only signals on a real change so the status bar does not flicker. + if (connected == http_connected) + return; + http_connected = connected; + emit httpConnectionChanged(connected, addr); +} + +void JFJochImageReadingWorker::RefreshDatasetOnly_i() { + // Assumes m locked! Updates dataset/plots and image count without touching the displayed image. + if (!http_mode) + return; + try { + int64_t num_images = 0; + auto dataset = http_reader.RefreshDatasetIfChanged(num_images); + SetHttpConnected_i(true, current_file); + + if (num_images != total_images) { + total_images = num_images; + if (current_image.has_value()) + emit imageNumberChanged(total_images, current_image.value()); + } + if (dataset) + emit datasetLoaded(dataset); + } catch (std::exception &e) { + logger.Debug("Dataset refresh failed: {}", e.what()); + SetHttpConnected_i(false, current_file); + } +} + +void JFJochImageReadingWorker::StatusTimerExpired() { + QMutexLocker locker(&m); + if (!http_mode) + return; + try { + BrokerStatus status = http_reader.GetBrokerStatus(); + SetHttpConnected_i(true, current_file); + emit brokerStatusUpdated(status); + } catch (std::exception &e) { + SetHttpConnected_i(false, current_file); + } +} + void JFJochImageReadingWorker::setAutoLoadMode_i(AutoloadMode in_mode) { autoload_mode = in_mode; if (autoload_mode == AutoloadMode::None) autoload_timer->stop(); else autoload_timer->start(); + if (autoload_mode != AutoloadMode::HTTPSync) + emit liveRateChanged(0.0); emit autoloadChanged(autoload_mode); } @@ -682,6 +752,7 @@ void JFJochImageReadingWorker::setAutoLoadMode(AutoloadMode mode) { switch (mode) { case AutoloadMode::HTTPSync: + case AutoloadMode::HTTPSyncDataset: if (http_mode) setAutoLoadMode_i(mode); else diff --git a/viewer/JFJochImageReadingWorker.h b/viewer/JFJochImageReadingWorker.h index fcf74d5d..07144f52 100644 --- a/viewer/JFJochImageReadingWorker.h +++ b/viewer/JFJochImageReadingWorker.h @@ -30,12 +30,15 @@ Q_DECLARE_METATYPE(AzimuthalIntegrationSettings) Q_DECLARE_METATYPE(UnitCell) Q_DECLARE_METATYPE(std::shared_ptr) +Q_DECLARE_METATYPE(BrokerStatus) class JFJochImageReadingWorker : public QObject { Q_OBJECT public: - enum class AutoloadMode {HTTPSync, Movie, None}; + // HTTPSync: follow latest image + dataset. HTTPSyncDataset: follow dataset/plots/image count + // but keep the displayed image frozen (entered by manually selecting an image while following). + enum class AutoloadMode {HTTPSync, HTTPSyncDataset, Movie, None}; Q_ENUM(AutoloadMode) private: mutable QMutex m; @@ -76,6 +79,11 @@ private: QTimer *autoload_timer; int autoload_interval = 500; // milliseconds + // Broker status polling (independent of image sync, runs whenever connected over HTTP) + QTimer *status_timer = nullptr; + int status_interval = 1000; // milliseconds + bool http_connected = false; + // Adaptive autoload interval based on recent load+analysis time MovingAverage autoload_ms_ma{8}; // window size (tune as needed) int autoload_interval_min_ms = 50; // 20 Hz is the top performance! @@ -103,6 +111,8 @@ private: void LoadFile_i(const QString &filename, qint64 image_number, qint64 summation, bool retry); void LoadImage_i(int64_t image_number, int64_t summation); + void RefreshDatasetOnly_i(); + void SetHttpConnected_i(bool connected, const QString &addr); void ReanalyzeImage_i(); void UpdateDataset_i(const std::optional& experiment); void UpdateAzint_i(const JFJochReaderDataset *dataset); @@ -120,6 +130,9 @@ signals: void autoloadChanged(AutoloadMode mode); void fileLoadError(QString title, QString message); void fileLoadRetryStatus(bool active, QString message); + void brokerStatusUpdated(BrokerStatus status); + void httpConnectionChanged(bool connected, QString addr); + void liveRateChanged(double hz); public: JFJochImageReadingWorker(const SpotFindingSettings &settings, const DiffractionExperiment& experiment, QObject *parent = nullptr); @@ -127,6 +140,7 @@ public: private slots: void AutoLoadTimerExpired(); + void StatusTimerExpired(); void FileOpenRetryTimerExpired(); public slots: diff --git a/viewer/JFJochViewerStatusBar.cpp b/viewer/JFJochViewerStatusBar.cpp index 445d48f7..680f7ede 100644 --- a/viewer/JFJochViewerStatusBar.cpp +++ b/viewer/JFJochViewerStatusBar.cpp @@ -3,8 +3,126 @@ #include "JFJochViewerStatusBar.h" -JFJochViewerStatusBar::JFJochViewerStatusBar(QWidget *parent) : QStatusBar(parent) {} +#include + +JFJochViewerStatusBar::JFJochViewerStatusBar(QWidget *parent) : QStatusBar(parent) { + // Added left-to-right within the permanent (right-aligned) area. + conn_label = new QLabel(this); + conn_label->setFixedWidth(100); + conn_label->setAlignment(Qt::AlignCenter); + addPermanentWidget(conn_label); + + state_box = new QProgressBar(this); + state_box->setFixedWidth(110); + state_box->setRange(0, 1000); + state_box->setValue(0); + state_box->setTextVisible(true); + state_box->setAlignment(Qt::AlignCenter); + state_box->setFormat(""); + addPermanentWidget(state_box); + + new_label = new QLabel(this); + new_label->setFixedWidth(70); + new_label->setAlignment(Qt::AlignCenter); + addPermanentWidget(new_label); + + rate_label = new QLabel(this); + rate_label->setFixedWidth(60); + rate_label->setAlignment(Qt::AlignCenter); + addPermanentWidget(rate_label); +} void JFJochViewerStatusBar::display(QString input) { showMessage(input, 60000); } + +void JFJochViewerStatusBar::setBrokerStatus(BrokerStatus status) { + QString text; + switch (status.state) { + case JFJochState::Inactive: text = "Inactive"; break; + case JFJochState::Idle: text = "Idle"; break; + case JFJochState::Measuring: text = "Measuring"; break; + case JFJochState::Error: text = "Error"; break; + case JFJochState::Busy: text = "Busy"; break; + case JFJochState::Calibration: text = "Calibration"; break; + } + state_box->setFormat(text); + + if (status.progress.has_value()) + state_box->setValue(std::clamp(static_cast(status.progress.value() * 1000.0f), 0, 1000)); + else + state_box->setValue(0); + + const QString chunk = (status.state == JFJochState::Error) ? "#d9534f" : "#5cb85c"; + state_box->setStyleSheet(QString( + "QProgressBar { border: 1px solid #aaa; border-radius: 2px; background: transparent; }" + "QProgressBar::chunk { background-color: %1; }").arg(chunk)); + + if (status.message.has_value() && !status.message.value().empty()) + state_box->setToolTip(QString::fromStdString(status.message.value())); + else + state_box->setToolTip(text); +} + +void JFJochViewerStatusBar::setHttpConnection(bool connected, QString addr) { + if (addr.isEmpty()) { + // No detector session at all -> blank the whole cluster (but keep the reserved space). + conn_label->setText(""); + conn_label->setToolTip(""); + conn_label->setStyleSheet(""); + state_box->setFormat(""); + state_box->setValue(0); + state_box->setStyleSheet(""); + state_box->setToolTip(""); + rate_label->setText(""); + new_label->setText(""); + new_label->setStyleSheet(""); + return; + } + + conn_label->setToolTip(addr); + if (connected) { + conn_label->setText("Live"); + conn_label->setStyleSheet("color: white; background-color: #5cb85c; border-radius: 3px;"); + } else { + conn_label->setText("Disconnected"); + conn_label->setStyleSheet("color: white; background-color: #d9534f; border-radius: 3px;"); + // Connection lost: clear live readouts but keep the badge so the user sees why. + state_box->setFormat(""); + state_box->setValue(0); + state_box->setToolTip(""); + rate_label->setText(""); + new_label->setText(""); + } +} + +void JFJochViewerStatusBar::setAutoloadMode(JFJochImageReadingWorker::AutoloadMode mode) { + autoload_mode = mode; + UpdateNewLabel(); +} + +void JFJochViewerStatusBar::setImageNumber(int64_t in_total_images, int64_t in_current_image) { + total_images = in_total_images; + current_image = in_current_image; + UpdateNewLabel(); +} + +void JFJochViewerStatusBar::setLiveRate(double hz) { + if (hz > 0.0) + rate_label->setText(QString::number(hz, 'f', 1) + " Hz"); + else + rate_label->setText(""); +} + +void JFJochViewerStatusBar::UpdateNewLabel() { + if (autoload_mode == JFJochImageReadingWorker::AutoloadMode::HTTPSyncDataset) { + const int64_t n = total_images - 1 - current_image; + if (n > 0) { + new_label->setText(QString("+%1 new").arg(n)); + new_label->setStyleSheet("color: #f0ad4e; font-weight: bold;"); + return; + } + } + new_label->setText(""); + new_label->setStyleSheet(""); +} diff --git a/viewer/JFJochViewerStatusBar.h b/viewer/JFJochViewerStatusBar.h index 59d2de0a..7592f523 100644 --- a/viewer/JFJochViewerStatusBar.h +++ b/viewer/JFJochViewerStatusBar.h @@ -4,14 +4,36 @@ #pragma once #include +#include +#include + +#include "../common/BrokerStatus.h" +#include "JFJochImageReadingWorker.h" class JFJochViewerStatusBar : public QStatusBar { + Q_OBJECT + + // Permanent (right-side) widgets. These are created once and never shown/hidden or resized, + // so the status bar never reflows; when not applicable they are just blanked in place. + QLabel *conn_label; // "Live" / "Disconnected" (host:port in tooltip) + QProgressBar *state_box; // Jungfraujoch state text, progress drawn as the bar fill + QLabel *new_label; // "+N new" when following the dataset with a frozen image + QLabel *rate_label; // effective live update rate + + JFJochImageReadingWorker::AutoloadMode autoload_mode = JFJochImageReadingWorker::AutoloadMode::None; + int64_t total_images = 0; + int64_t current_image = 0; + + void UpdateNewLabel(); + +public: + explicit JFJochViewerStatusBar(QWidget *parent = nullptr); + public slots: void display(QString input); -public: - JFJochViewerStatusBar(QWidget * parent = nullptr); - + void setBrokerStatus(BrokerStatus status); + void setHttpConnection(bool connected, QString addr); + void setAutoloadMode(JFJochImageReadingWorker::AutoloadMode mode); + void setImageNumber(int64_t total_images, int64_t current_image); + void setLiveRate(double hz); }; - - - diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index 903b7528..eab463ca 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -294,6 +294,18 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString connect(side_panel, &JFJochViewerSidePanel::writeStatusBar, statusbar, &JFJochViewerStatusBar::display); + // Detector connection / broker state / live readouts in the status bar + connect(reading_worker, &JFJochImageReadingWorker::brokerStatusUpdated, + statusbar, &JFJochViewerStatusBar::setBrokerStatus); + connect(reading_worker, &JFJochImageReadingWorker::httpConnectionChanged, + statusbar, &JFJochViewerStatusBar::setHttpConnection); + connect(reading_worker, &JFJochImageReadingWorker::autoloadChanged, + statusbar, &JFJochViewerStatusBar::setAutoloadMode); + connect(reading_worker, &JFJochImageReadingWorker::imageNumberChanged, + statusbar, &JFJochViewerStatusBar::setImageNumber); + connect(reading_worker, &JFJochImageReadingWorker::liveRateChanged, + statusbar, &JFJochViewerStatusBar::setLiveRate); + connect(metadataWindow, &JFJochViewerMetadataWindow::datasetUpdated, reading_worker, &JFJochImageReadingWorker::UpdateDataset); diff --git a/viewer/toolbar/JFJochViewerToolbarImage.cpp b/viewer/toolbar/JFJochViewerToolbarImage.cpp index ea6f99e9..b570194e 100644 --- a/viewer/toolbar/JFJochViewerToolbarImage.cpp +++ b/viewer/toolbar/JFJochViewerToolbarImage.cpp @@ -62,9 +62,11 @@ JFJochViewerToolbarImage::JFJochViewerToolbarImage(QWidget *parent) : QToolBar(p movie_button->setShortcut(QKeySequence(Qt::CTRL | Qt::Key_M)); addWidget(movie_button); - autoload_button = new QPushButton("HTTP &Sync"); + autoload_button = new QPushButton("HTTP Sync"); autoload_button->setCheckable(true); autoload_button->setChecked(false); + // Fixed width so the tri-state glyph prefix never reflows the toolbar. + autoload_button->setFixedWidth(120); autoload_button->setShortcut(QKeySequence(Qt::CTRL | Qt::Key_S)); addWidget(autoload_button); @@ -195,10 +197,18 @@ void JFJochViewerToolbarImage::setSummation(int val) { } void JFJochViewerToolbarImage::autoloadButtonPressed() { - if (autoload_button->isChecked()) - emit autoLoadButtonPressed(JFJochImageReadingWorker::AutoloadMode::HTTPSync); - else - emit autoLoadButtonPressed(JFJochImageReadingWorker::AutoloadMode::None); + // The button either starts/resumes live following or stops it; the data-only "frozen" state + // is entered automatically by manually selecting an image while following. + switch (autoload_mode) { + case JFJochImageReadingWorker::AutoloadMode::HTTPSync: + emit autoLoadButtonPressed(JFJochImageReadingWorker::AutoloadMode::None); + break; + case JFJochImageReadingWorker::AutoloadMode::HTTPSyncDataset: + case JFJochImageReadingWorker::AutoloadMode::Movie: + case JFJochImageReadingWorker::AutoloadMode::None: + emit autoLoadButtonPressed(JFJochImageReadingWorker::AutoloadMode::HTTPSync); + break; + } } void JFJochViewerToolbarImage::movieButtonPressed() { @@ -213,16 +223,29 @@ void JFJochViewerToolbarImage::reanalyzeButtonPressed() { } void JFJochViewerToolbarImage::setAutoloadMode(JFJochImageReadingWorker::AutoloadMode input) { + autoload_mode = input; switch (input) { case JFJochImageReadingWorker::AutoloadMode::HTTPSync: + autoload_button->setText("▣ HTTP Sync"); + autoload_button->setToolTip("Following live (image + data)"); + autoload_button->setChecked(true); + movie_button->setChecked(false); + break; + case JFJochImageReadingWorker::AutoloadMode::HTTPSyncDataset: + autoload_button->setText("◐ HTTP Sync"); + autoload_button->setToolTip("Following data only — image frozen (click to resume live)"); autoload_button->setChecked(true); movie_button->setChecked(false); break; case JFJochImageReadingWorker::AutoloadMode::Movie: + autoload_button->setText("HTTP Sync"); + autoload_button->setToolTip(""); autoload_button->setChecked(false); movie_button->setChecked(true); break; case JFJochImageReadingWorker::AutoloadMode::None: + autoload_button->setText("HTTP Sync"); + autoload_button->setToolTip(""); autoload_button->setChecked(false); movie_button->setChecked(false); break; -- 2.52.0 From b29b870ef2b672a6d282990e64d1c41170b84294 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 13:45:04 +0200 Subject: [PATCH 076/228] broker/gen: gitignore generated cpp-pistache-server output Only gen/model/ is compiled (into JFJochAPI) and tracked; the REST server is hand-written on cpp-httplib, so the generated Pistache server stubs (api/, impl/, main-api-server.cpp, CMakeLists.txt, README.md, .openapi-generator*) are regeneration debris that was previously left untracked and tended to carry a newer spec version than the committed models. Ignore it so it can't be committed by accident. Co-Authored-By: Claude Opus 4.8 --- broker/gen/.gitignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 broker/gen/.gitignore diff --git a/broker/gen/.gitignore b/broker/gen/.gitignore new file mode 100644 index 00000000..c24e5b6c --- /dev/null +++ b/broker/gen/.gitignore @@ -0,0 +1,14 @@ +# Generated OpenAPI cpp-pistache-server output. +# +# Only model/ is compiled (into the JFJochAPI library) and tracked. The REST +# server is hand-written on cpp-httplib (see broker/JFJochBrokerHttp.cpp), so the +# generated Pistache server stubs below are regeneration debris: never compiled, +# and they tend to carry a newer spec version than the committed models. Keep +# them out of git to avoid accidentally committing mismatched artefacts. +/api/ +/impl/ +/main-api-server.cpp +/CMakeLists.txt +/README.md +/.openapi-generator/ +/.openapi-generator-ignore -- 2.52.0 From ec1308f4a9d0da504171774bd21e0657d988d631 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 13:50:49 +0200 Subject: [PATCH 077/228] viewer: guard Linux-only constructs for Windows/MSVC builds Step toward building jfjoch_viewer on Windows/MSVC. No change to the Linux build: D-Bus stays on by default and the XCB plugin is still used. - CMake: make Qt6::DBus optional via JFJOCH_VIEWER_DBUS (ON on Linux, OFF on Windows/macOS where Qt6::DBus does not exist); compile/link/install the dbus/ adaptor + service file only when enabled. Select the platform integration plugin per-OS (QXcb on Linux, QWindows on Windows, QCocoa on macOS) instead of hard-coding QXcbIntegrationPlugin. - JFJochViewerWindow: wrap the adaptor include and D-Bus registration in #ifdef JFJOCH_VIEWER_DBUS. - JFJochImageReadingWorker: the POSIX open()/fstat()/NFS-errno preflight is now #ifndef _WIN32, with a portable QFileInfo fallback elsewhere; POSIX-only headers are guarded too. - Replace the POSIX M_PI/M_PI_2 extension with C++20 std::numbers::pi across the viewer (not defined by MSVC without _USE_MATH_DEFINES). Co-Authored-By: Claude Opus 4.8 --- viewer/CMakeLists.txt | 42 ++++++++++++++----- viewer/JFJochImageReadingWorker.cpp | 27 ++++++++++-- viewer/JFJochViewerWindow.cpp | 6 +++ viewer/image_viewer/JFJochAzIntImage.cpp | 3 +- .../image_viewer/JFJochDiffractionImage.cpp | 15 +++---- viewer/widgets/PowderCalibrationWidget.cpp | 3 +- viewer/windows/JFJochViewerMetadataWindow.cpp | 9 ++-- 7 files changed, 79 insertions(+), 26 deletions(-) diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index d7f0875c..b8a54837 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -5,7 +5,19 @@ SET(CMAKE_AUTOMOC ON) SET(CMAKE_AUTORCC ON) SET(CMAKE_AUTOUIC ON) -FIND_PACKAGE(Qt6 COMPONENTS Core Gui Widgets Charts DBus Concurrent OpenGL OpenGLWidgets REQUIRED) +# D-Bus (single-instance / remote control) is Linux-only: Qt6::DBus does not +# exist on Windows or macOS. Default ON on Linux, OFF elsewhere. +IF(UNIX AND NOT APPLE) + OPTION(JFJOCH_VIEWER_DBUS "Build viewer with D-Bus support (Linux only)" ON) +ELSE() + OPTION(JFJOCH_VIEWER_DBUS "Build viewer with D-Bus support (Linux only)" OFF) +ENDIF() + +SET(JFJOCH_VIEWER_QT_COMPONENTS Core Gui Widgets Charts Concurrent OpenGL OpenGLWidgets) +IF(JFJOCH_VIEWER_DBUS) + LIST(APPEND JFJOCH_VIEWER_QT_COMPONENTS DBus) +ENDIF() +FIND_PACKAGE(Qt6 COMPONENTS ${JFJOCH_VIEWER_QT_COMPONENTS} REQUIRED) QT_ADD_RESOURCES(APP_RESOURCES resources/resources.qrc) @@ -34,8 +46,6 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView JFJochViewerStatusBar.h windows/JFJochViewerImageListWindow.cpp windows/JFJochViewerImageListWindow.h - dbus/JFJochViewerAdaptor.h - dbus/JFJochViewerAdaptor.cpp widgets/NumericComboBox.cpp widgets/NumericComboBox.h windows/JFJochViewerMetadataWindow.cpp @@ -90,16 +100,21 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView windows/JFJochMagnifierWindow.h ) -TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::DBus Qt6::Concurrent +TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::Concurrent Qt6::OpenGL Qt6::OpenGLWidgets JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis) INSTALL(TARGETS jfjoch_viewer RUNTIME COMPONENT viewer) -INSTALL( - FILES ${CMAKE_CURRENT_SOURCE_DIR}/dbus/ch.psi.jfjoch_viewer.service - DESTINATION share/dbus-1/services - COMPONENT viewer -) +IF(JFJOCH_VIEWER_DBUS) + TARGET_SOURCES(jfjoch_viewer PRIVATE dbus/JFJochViewerAdaptor.cpp dbus/JFJochViewerAdaptor.h) + TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::DBus) + TARGET_COMPILE_DEFINITIONS(jfjoch_viewer PRIVATE JFJOCH_VIEWER_DBUS) + INSTALL( + FILES ${CMAKE_CURRENT_SOURCE_DIR}/dbus/ch.psi.jfjoch_viewer.service + DESTINATION share/dbus-1/services + COMPONENT viewer + ) +ENDIF() INSTALL( FILES resources/jfjoch.png @@ -113,7 +128,14 @@ INSTALL( COMPONENT viewer ) -qt_import_plugins(jfjoch_viewer INCLUDE Qt::QXcbIntegrationPlugin) +# Platform integration plugin: XCB on Linux, native plugin on Windows/macOS. +IF(WIN32) + qt_import_plugins(jfjoch_viewer INCLUDE Qt::QWindowsIntegrationPlugin) +ELSEIF(APPLE) + qt_import_plugins(jfjoch_viewer INCLUDE Qt::QCocoaIntegrationPlugin) +ELSE() + qt_import_plugins(jfjoch_viewer INCLUDE Qt::QXcbIntegrationPlugin) +ENDIF() IF(HAS_FFTW3_H AND FFTWF_LIBRARY) TARGET_LINK_LIBRARIES(jfjoch_viewer ${FFTWF_LIBRARY}) diff --git a/viewer/JFJochImageReadingWorker.cpp b/viewer/JFJochImageReadingWorker.cpp index 15a441d4..67496478 100644 --- a/viewer/JFJochImageReadingWorker.cpp +++ b/viewer/JFJochImageReadingWorker.cpp @@ -1,11 +1,14 @@ // SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only +#include +#include +#include +#ifndef _WIN32 #include #include #include -#include -#include +#endif #include "JFJochImageReadingWorker.h" #include "../reader/JFJochReaderImage.h" // JFJochReaderImage + GAP/ERROR/SATURATED sentinels @@ -31,6 +34,7 @@ namespace { PreflightResult preflight_open_ro(const QString& filename, std::string& reason_out) { reason_out.clear(); +#ifndef _WIN32 const QByteArray path = filename.toLocal8Bit(); errno = 0; const int fd = ::open(path.constData(), O_RDONLY | O_CLOEXEC); @@ -59,6 +63,23 @@ namespace { return PreflightResult::IsDirectory; return PreflightResult::OtherError; +#else + // No NFS-style transient-errno semantics off POSIX; use a portable check. + const QFileInfo info(filename); + if (!info.exists()) { + reason_out = "File not visible yet"; + return PreflightResult::NotYetVisible; + } + if (info.isDir()) { + reason_out = "Path is a directory"; + return PreflightResult::IsDirectory; + } + if (!info.isReadable()) { + reason_out = "Permission denied"; + return PreflightResult::PermissionDenied; + } + return PreflightResult::Ok; +#endif } } @@ -518,7 +539,7 @@ void JFJochImageReadingWorker::FindCenter(const UnitCell& calibrant, bool guess) QVector rings; for (int i = 0; i < 15 && i < ring_Q.size(); i++) { - rings.push_back(2 * M_PI / ring_Q[i]); + rings.push_back(2 * std::numbers::pi / ring_Q[i]); } emit setRings(rings); } diff --git a/viewer/JFJochViewerWindow.cpp b/viewer/JFJochViewerWindow.cpp index eab463ca..7763b9e1 100644 --- a/viewer/JFJochViewerWindow.cpp +++ b/viewer/JFJochViewerWindow.cpp @@ -15,7 +15,9 @@ #include "../common/CUDAWrapper.h" #include "windows/JFJochViewerImageListWindow.h" #include "windows/JFJochViewerMetadataWindow.h" +#ifdef JFJOCH_VIEWER_DBUS #include "dbus/JFJochViewerAdaptor.h" +#endif #include "windows/JFJochViewerProcessingWindow.h" #include "windows/JFJochViewerSpotListWindow.h" #include "windows/JFJochViewerReflectionListWindow.h" @@ -119,6 +121,7 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString menuBar->AddWindowEntry(azintImageWindow, "Azimuthal integration 2D image"); menuBar->AddWindowEntry(magnifierWindow, "Magnifier"); +#ifdef JFJOCH_VIEWER_DBUS if (dbus) { // Create adaptor attached to this window new JFJochViewerAdaptor(this); @@ -132,6 +135,9 @@ JFJochViewerWindow::JFJochViewerWindow(QWidget *parent, bool dbus, const QString } } } +#else + (void) dbus; +#endif connect(this, &JFJochViewerWindow::LoadFileRequest, reading_worker, &JFJochImageReadingWorker::LoadFile); diff --git a/viewer/image_viewer/JFJochAzIntImage.cpp b/viewer/image_viewer/JFJochAzIntImage.cpp index e055f729..74c86733 100644 --- a/viewer/image_viewer/JFJochAzIntImage.cpp +++ b/viewer/image_viewer/JFJochAzIntImage.cpp @@ -5,6 +5,7 @@ #include #include "../../common/JFJochException.h" +#include JFJochAzIntImage::JFJochAzIntImage(QWidget *parent) : JFJochImage(parent) { @@ -109,7 +110,7 @@ void JFJochAzIntImage::mouseDoubleClickEvent(QMouseEvent *event) { float phi = image->Dataset().az_int_bin_to_phi[idx]; auto geom = image->Dataset().experiment.GetDiffractionGeometry(); - auto coord = geom.ResPhiToPxl(2 * M_PI / q, phi / 180.0 * M_PI); + auto coord = geom.ResPhiToPxl(2 * std::numbers::pi / q, phi / 180.0 * std::numbers::pi); emit zoomOnBin(QPointF(coord.first, coord.second)); } } \ No newline at end of file diff --git a/viewer/image_viewer/JFJochDiffractionImage.cpp b/viewer/image_viewer/JFJochDiffractionImage.cpp index b204a624..51dde0f3 100644 --- a/viewer/image_viewer/JFJochDiffractionImage.cpp +++ b/viewer/image_viewer/JFJochDiffractionImage.cpp @@ -3,6 +3,7 @@ #include "JFJochDiffractionImage.h" #include "../../common/DiffractionGeometry.h" +#include #include #include @@ -239,9 +240,9 @@ void JFJochDiffractionImage::DrawResolutionRings() { continue; auto [x1,y1] = geom.ResPhiToPxl(d, 0); - auto [x2,y2] = geom.ResPhiToPxl(d, M_PI_2); - auto [x3,y3] = geom.ResPhiToPxl(d, M_PI); - auto [x4,y4] = geom.ResPhiToPxl(d, 3.0 * M_PI_2); + auto [x2,y2] = geom.ResPhiToPxl(d, std::numbers::pi / 2); + auto [x3,y3] = geom.ResPhiToPxl(d, std::numbers::pi); + auto [x4,y4] = geom.ResPhiToPxl(d, 3.0 * std::numbers::pi / 2); auto x_min = std::min({x1, x2, x3, x4}); auto x_max = std::max({x1, x2, x3, x4}); @@ -252,9 +253,9 @@ void JFJochDiffractionImage::DrawResolutionRings() { addOverlayItem(scene()->addEllipse(boundingRect, pen)); auto [x5,y5] = geom.ResPhiToPxl(d, phi_offset + 0); - auto [x6,y6] = geom.ResPhiToPxl(d, phi_offset + M_PI_2); - auto [x7,y7] = geom.ResPhiToPxl(d, phi_offset + M_PI); - auto [x8,y8] = geom.ResPhiToPxl(d, phi_offset + 3.0 * M_PI_2); + auto [x6,y6] = geom.ResPhiToPxl(d, phi_offset + std::numbers::pi / 2); + auto [x7,y7] = geom.ResPhiToPxl(d, phi_offset + std::numbers::pi); + auto [x8,y8] = geom.ResPhiToPxl(d, phi_offset + 3.0 * std::numbers::pi / 2); QPointF point_1(x5, y5); QPointF point_2(x6, y6); @@ -285,7 +286,7 @@ void JFJochDiffractionImage::DrawResolutionRings() { scene()->addItem(textItem); addOverlayItem(textItem); } - phi_offset += 4.0 / 180.0 * M_PI; + phi_offset += 4.0 / 180.0 * std::numbers::pi; } } diff --git a/viewer/widgets/PowderCalibrationWidget.cpp b/viewer/widgets/PowderCalibrationWidget.cpp index d6d9c31d..b7a78076 100644 --- a/viewer/widgets/PowderCalibrationWidget.cpp +++ b/viewer/widgets/PowderCalibrationWidget.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "../image_analysis/geom_refinement/AssignSpotsToRings.h" @@ -25,7 +26,7 @@ PowderCalibrationWidget::PowderCalibrationWidget(QWidget *parent) : QWidget(pare std::vector rings = CalculateXtalRings(GetCalibrant(), 10); QVector q_rings; for (float ring : rings) { - q_rings.append(2 * M_PI / ring); + q_rings.append(2 * std::numbers::pi / ring); } emit ringsFromCalibration(q_rings); }); diff --git a/viewer/windows/JFJochViewerMetadataWindow.cpp b/viewer/windows/JFJochViewerMetadataWindow.cpp index 85d0e430..97db7e02 100644 --- a/viewer/windows/JFJochViewerMetadataWindow.cpp +++ b/viewer/windows/JFJochViewerMetadataWindow.cpp @@ -10,6 +10,7 @@ #include #include "JFJochViewerMetadataWindow.h" +#include JFJochViewerMetadataWindow::JFJochViewerMetadataWindow(QWidget *parent) : JFJochHelperWindow(parent) { @@ -135,8 +136,8 @@ void JFJochViewerMetadataWindow::datasetUpdate() { tmp.BeamX_pxl(beam_center_x->value()); tmp.BeamY_pxl(beam_center_y->value()); tmp.IncidentEnergy_keV(WVL_1A_IN_KEV / wavelength_A->value()); - tmp.PoniRot1_rad(poni_rot1_deg->value() * M_PI / 180.0); - tmp.PoniRot2_rad(poni_rot2_deg->value() * M_PI / 180.0); + tmp.PoniRot1_rad(poni_rot1_deg->value() * std::numbers::pi / 180.0); + tmp.PoniRot2_rad(poni_rot2_deg->value() * std::numbers::pi / 180.0); tmp.DetectIceRings(detect_ice_rings->isChecked()); if (unit_cell_enabled->isChecked()) { @@ -176,8 +177,8 @@ void JFJochViewerMetadataWindow::datasetLoaded(std::shared_ptrsetValue(dataset->experiment.GetBeamX_pxl()); beam_center_y->setValue(dataset->experiment.GetBeamY_pxl()); wavelength_A->setValue(dataset->experiment.GetWavelength_A()); - poni_rot1_deg->setValue(dataset->experiment.GetPoniRot1_rad() * 180.0 / M_PI); - poni_rot2_deg->setValue(dataset->experiment.GetPoniRot2_rad() * 180.0 / M_PI); + poni_rot1_deg->setValue(dataset->experiment.GetPoniRot1_rad() * 180.0 / std::numbers::pi); + poni_rot2_deg->setValue(dataset->experiment.GetPoniRot2_rad() * 180.0 / std::numbers::pi); detect_ice_rings->setChecked(dataset->experiment.IsDetectIceRings()); auto unit_cell = dataset->experiment.GetUnitCell(); -- 2.52.0 From e1622c3750e5631634f04fd6b0d991fbf0a10364 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 13:52:41 +0200 Subject: [PATCH 078/228] plots: expose compression_ratio over the REST/UI plot path The receiver already computes and stores the compression ratio per image (JFJochReceiverPlots Add/GetPlots/GetPlotRaw all handle PlotType::CompressionRatio), but the type was unreachable from the API: it was missing from ConvertPlotType, the OpenAPI plot_type enum, and the TS client, so requesting it returned 500 and no UI menu entry existed. Wire it through all layers: - broker: add the compression_ratio -> PlotType::CompressionRatio case in ConvertPlotType. - spec: add compression_ratio to the plot_type parameter enum. - frontend: add COMPRESSION_RATIO to the generated plot_type enum (matches what regeneration would produce), a "Compression ratio" menu entry, and a Y-axis label. Co-Authored-By: Claude Opus 4.8 --- broker/OpenAPIConvert.cpp | 1 + broker/jfjoch_api.yaml | 1 + frontend/src/components/DataProcessingPlot.tsx | 2 ++ frontend/src/components/DataProcessingPlots.tsx | 1 + frontend/src/openapi/models/plot_type.ts | 1 + 5 files changed, 6 insertions(+) diff --git a/broker/OpenAPIConvert.cpp b/broker/OpenAPIConvert.cpp index c601f594..ff273c16 100644 --- a/broker/OpenAPIConvert.cpp +++ b/broker/OpenAPIConvert.cpp @@ -886,6 +886,7 @@ PlotType ConvertPlotType(const std::optional& input) { if (input == "image_scale_factor") return PlotType::ImageScaleFactor; if (input == "image_scale_cc") return PlotType::ImageScaleCC; if (input == "image_scale_b") return PlotType::ImageScaleBFactor; + if (input == "compression_ratio") return PlotType::CompressionRatio; if (input == "indexing_lattice_count") return PlotType::IndexingLatticeCount; throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Plot type not recognized"); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 6e2b42db..1a44430a 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -120,6 +120,7 @@ components: - image_scale_factor - image_scale_cc - image_scale_b + - compression_ratio roi: in: query name: roi diff --git a/frontend/src/components/DataProcessingPlot.tsx b/frontend/src/components/DataProcessingPlot.tsx index d2ab2d61..e0faae69 100644 --- a/frontend/src/components/DataProcessingPlot.tsx +++ b/frontend/src/components/DataProcessingPlot.tsx @@ -90,6 +90,8 @@ function AxisTypeY(plot: plot_type) : string | ReactNode { case plot_type.BEAM_CENTER_X: case plot_type.BEAM_CENTER_Y: return "Beam center [pixel]"; + case plot_type.COMPRESSION_RATIO: + return "Compression ratio"; default: return "?"; } diff --git a/frontend/src/components/DataProcessingPlots.tsx b/frontend/src/components/DataProcessingPlots.tsx index cc632a0e..49162e35 100644 --- a/frontend/src/components/DataProcessingPlots.tsx +++ b/frontend/src/components/DataProcessingPlots.tsx @@ -80,6 +80,7 @@ class DataProcessingPlots extends Component { Indexed lattice count Integrated reflections Image collection efficiency + Compression ratio Processing time Receiver delay (internal debugging) Receiver free send buffers (internal debugging) diff --git a/frontend/src/openapi/models/plot_type.ts b/frontend/src/openapi/models/plot_type.ts index 370ee04d..26ac39b2 100644 --- a/frontend/src/openapi/models/plot_type.ts +++ b/frontend/src/openapi/models/plot_type.ts @@ -44,4 +44,5 @@ export enum plot_type { IMAGE_SCALE_FACTOR = 'image_scale_factor', IMAGE_SCALE_CC = 'image_scale_cc', IMAGE_SCALE_B = 'image_scale_b', + COMPRESSION_RATIO = 'compression_ratio', } -- 2.52.0 From 086129f767ef8562c5b2538b1c8e399491097c99 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 14:12:14 +0200 Subject: [PATCH 079/228] receiver: count only images the pusher accepted images_sent was incremented right after image_pusher.SendImage(*loc), but the ZeroCopyReturnValue overload was void and, for the TCP pusher, asynchronous: it silently drops the image (releases the slot and returns) when there is no live connection or the 2 s enqueue deadline expires. So images_sent over-counted on a broken/slow writer connection and disagreed with the ACK-based GetImagesWritten(). Make SendImage(ZeroCopyReturnValue&) return whether the image was accepted (enqueued/handed off) and only increment images_sent on success. The slot is still released on the drop path. The authoritative delivered count remains GetImagesWritten() (total_data_acked_ok for TCP). File/ZMQ pushers return true on accept, preserving their previous always-counted behaviour. Co-Authored-By: Claude Opus 4.8 --- image_pusher/HDF5FilePusher.cpp | 3 ++- image_pusher/HDF5FilePusher.h | 2 +- image_pusher/ImagePusher.cpp | 5 +++-- image_pusher/ImagePusher.h | 7 ++++++- image_pusher/TCPStreamPusher.cpp | 11 ++++++----- image_pusher/TCPStreamPusher.h | 2 +- image_pusher/ZMQStream2Pusher.cpp | 7 +++++-- image_pusher/ZMQStream2Pusher.h | 2 +- receiver/JFJochReceiverFPGA.cpp | 6 ++++-- receiver/JFJochReceiverLite.cpp | 6 ++++-- 10 files changed, 33 insertions(+), 18 deletions(-) diff --git a/image_pusher/HDF5FilePusher.cpp b/image_pusher/HDF5FilePusher.cpp index a5f9138c..5c93f24e 100644 --- a/image_pusher/HDF5FilePusher.cpp +++ b/image_pusher/HDF5FilePusher.cpp @@ -91,12 +91,13 @@ bool HDF5FilePusher::SendImage(const uint8_t *image_data, size_t image_size, int return true; } -void HDF5FilePusher::SendImage(ZeroCopyReturnValue &z) { +bool HDF5FilePusher::SendImage(ZeroCopyReturnValue &z) { writer_queue.PutBlocking(ImagePusherQueueElement{ .image_data = (uint8_t *) z.GetImage(), .z = &z, .end = false }); + return true; } bool HDF5FilePusher::SendCalibration(const CompressedImage &message) { diff --git a/image_pusher/HDF5FilePusher.h b/image_pusher/HDF5FilePusher.h index 4039135e..f7fcbdc3 100644 --- a/image_pusher/HDF5FilePusher.h +++ b/image_pusher/HDF5FilePusher.h @@ -34,7 +34,7 @@ public: void StartDataCollection(StartMessage &message) override; bool EndDataCollection(const EndMessage &message) override; bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; - void SendImage(ZeroCopyReturnValue &z) override; + bool SendImage(ZeroCopyReturnValue &z) override; bool SendCalibration(const CompressedImage &message) override; std::string PrintSetup() const override; diff --git a/image_pusher/ImagePusher.cpp b/image_pusher/ImagePusher.cpp index cca67c54..8c518ba6 100644 --- a/image_pusher/ImagePusher.cpp +++ b/image_pusher/ImagePusher.cpp @@ -25,7 +25,8 @@ std::string ImagePusher::GetWriterNotificationSocketAddress() const { return ""; } -void ImagePusher::SendImage(ZeroCopyReturnValue &z) { - SendImage((uint8_t *) z.GetImage(), z.GetImageSize(), z.GetImageNumber()); +bool ImagePusher::SendImage(ZeroCopyReturnValue &z) { + const bool accepted = SendImage((uint8_t *) z.GetImage(), z.GetImageSize(), z.GetImageNumber()); z.release(); + return accepted; } diff --git a/image_pusher/ImagePusher.h b/image_pusher/ImagePusher.h index 17d543ed..065d4ef1 100644 --- a/image_pusher/ImagePusher.h +++ b/image_pusher/ImagePusher.h @@ -41,7 +41,12 @@ public: virtual void StartDataCollection(StartMessage& message) = 0; virtual bool EndDataCollection(const EndMessage& message) = 0; // Non-blocking virtual bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) = 0; - virtual void SendImage(ZeroCopyReturnValue &z); + // Returns true if the image was accepted (handed to the writer / enqueued for + // sending), false if it was dropped (e.g. no TCP connection, or the enqueue + // deadline expired). Ownership is settled either way: on a false return the + // slot has already been release()d. The authoritative count of images actually + // delivered is GetImagesWritten() (ACK-based for TCP). + virtual bool SendImage(ZeroCopyReturnValue &z); virtual bool SendCalibration(const CompressedImage& message) = 0; virtual std::string Finalize(); // Ensure that all streams are closed, can throw exception virtual std::string GetWriterNotificationSocketAddress() const; diff --git a/image_pusher/TCPStreamPusher.cpp b/image_pusher/TCPStreamPusher.cpp index d4a4295b..892b1bbc 100644 --- a/image_pusher/TCPStreamPusher.cpp +++ b/image_pusher/TCPStreamPusher.cpp @@ -1051,7 +1051,7 @@ bool TCPStreamPusher::SendImage(const uint8_t *image_data, size_t image_size, in return SendFrame(c, image_data, image_size, TCPFrameType::DATA, image_number, nullptr); } -void TCPStreamPusher::SendImage(ZeroCopyReturnValue &z) { +bool TCPStreamPusher::SendImage(ZeroCopyReturnValue &z) { // Look up the target connection while holding the mutex, but do NOT call // PutBlocking while holding it — that can block indefinitely and deadlock // against AcceptorThread/KeepaliveThread. @@ -1061,7 +1061,7 @@ void TCPStreamPusher::SendImage(ZeroCopyReturnValue &z) { const auto& use = (!session_connections.empty() ? session_connections : connections); if (use.empty()) { z.release(); - return; + return false; } auto idx = static_cast((z.GetImageNumber() / images_per_file) % static_cast(use.size())); @@ -1070,14 +1070,14 @@ void TCPStreamPusher::SendImage(ZeroCopyReturnValue &z) { if (!target || target->broken || !target->active) { z.release(); - return; + return false; } const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); while (std::chrono::steady_clock::now() < deadline) { if (target->broken || !target->active) { z.release(); - return; + return false; } if (target->queue.PutTimeout(ImagePusherQueueElement{ @@ -1085,12 +1085,13 @@ void TCPStreamPusher::SendImage(ZeroCopyReturnValue &z) { .z = &z, .end = false }, std::chrono::milliseconds(50))) { - return; + return true; } } target->broken = true; z.release(); + return false; } bool TCPStreamPusher::EndDataCollection(const EndMessage& message) { diff --git a/image_pusher/TCPStreamPusher.h b/image_pusher/TCPStreamPusher.h index bd5dfed6..dfa62c03 100644 --- a/image_pusher/TCPStreamPusher.h +++ b/image_pusher/TCPStreamPusher.h @@ -177,7 +177,7 @@ public: void StartDataCollection(StartMessage& message) override; bool EndDataCollection(const EndMessage& message) override; bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; - void SendImage(ZeroCopyReturnValue &z) override; + bool SendImage(ZeroCopyReturnValue &z) override; bool SendCalibration(const CompressedImage& message) override; std::string Finalize() override; diff --git a/image_pusher/ZMQStream2Pusher.cpp b/image_pusher/ZMQStream2Pusher.cpp index f115ce94..a29ca5fd 100644 --- a/image_pusher/ZMQStream2Pusher.cpp +++ b/image_pusher/ZMQStream2Pusher.cpp @@ -26,12 +26,15 @@ bool ZMQStream2Pusher::SendImage(const uint8_t *image_data, size_t image_size, i } -void ZMQStream2Pusher::SendImage(ZeroCopyReturnValue &z) { +bool ZMQStream2Pusher::SendImage(ZeroCopyReturnValue &z) { if (!socket.empty()) { auto socket_number = (z.GetImageNumber() / images_per_file) % socket.size(); socket[socket_number]->SendImage(z); - } else + return true; + } else { z.release(); + return false; + } } void ZMQStream2Pusher::StartDataCollection(StartMessage& message) { diff --git a/image_pusher/ZMQStream2Pusher.h b/image_pusher/ZMQStream2Pusher.h index 9d58f7a3..3fc17de4 100644 --- a/image_pusher/ZMQStream2Pusher.h +++ b/image_pusher/ZMQStream2Pusher.h @@ -37,7 +37,7 @@ public: bool SendCalibration(const CompressedImage& message) override; // Thread-safe - void SendImage(ZeroCopyReturnValue &z) override; + bool SendImage(ZeroCopyReturnValue &z) override; bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; std::string Finalize() override; diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 3b2cbbb8..63745408 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -481,8 +481,10 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { zmq_metadata_socket->AddDataMessage(message); if (push_images_to_writer) { - image_pusher.SendImage(*loc); - ++images_sent; // Handle case when image not sent properly + // Only count images the pusher accepted; TCP can drop on a + // broken/absent connection or an expired enqueue deadline. + if (image_pusher.SendImage(*loc)) + ++images_sent; } else loc->release(); UpdateMaxImageSent(message.number); diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index cc8fbcad..b88972e2 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -328,8 +328,10 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { zmq_metadata_socket->AddDataMessage(data_msg); if (push_images_to_writer) { - image_pusher.SendImage(*loc); - ++images_sent; // Handle case when image not sent properly + // Only count images the pusher accepted; TCP can drop on a + // broken/absent connection or an expired enqueue deadline. + if (image_pusher.SendImage(*loc)) + ++images_sent; } else loc->release(); UpdateMaxImageSent(data_msg.number); -- 2.52.0 From 52f349b9f151c54c3e7568fd536fb9ae7d3641c8 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 14:13:03 +0200 Subject: [PATCH 080/228] frame_serialize: use reentrant gmtime in CBOR date decoding gmtime() returns a pointer to a shared static buffer, so concurrent deserialization of CborUnixTime_tTag values was a data race. Decode into a local struct tm via gmtime_r (gmtime_s on Windows) instead. Co-Authored-By: Claude Opus 4.8 --- frame_serialize/CBORStream2Deserializer.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 2a598de6..06241be0 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -85,8 +85,14 @@ namespace { return GetCBORString(value); else if (tag == CborUnixTime_tTag) { time_t t = GetCBORUInt(value); + struct tm tm_buf{}; +#ifdef _WIN32 + gmtime_s(&tm_buf, &t); +#else + gmtime_r(&t, &tm_buf); +#endif char buf1[255]; - strftime(buf1, sizeof(buf1), "%FT%T", gmtime(&t)); + strftime(buf1, sizeof(buf1), "%FT%T", &tm_buf); return std::string(buf1) + "Z"; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Time/date tag error"); -- 2.52.0 From 01a97d61feb7d14ae8e00646909b85033bf225d6 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 14:15:22 +0200 Subject: [PATCH 081/228] common: spell the azimuthal d-spacing unit d_A consistently PlotAzintUnit used D_A while MultiLinePlotUnits used d_A for the same d-spacing-in-angstrom unit. Align PlotAzintUnit to the lowercase crystallographic convention already used by MultiLinePlotUnits. Generated Plot_unit_x::D_A is left as-is (controlled by the OpenAPI generator's C++ enum casing). Co-Authored-By: Claude Opus 4.8 --- broker/JFJochBrokerHttp.cpp | 2 +- common/AzimuthalIntegrationProfile.cpp | 2 +- common/Plot.h | 2 +- receiver/JFJochReceiverPlots.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/broker/JFJochBrokerHttp.cpp b/broker/JFJochBrokerHttp.cpp index ec2d90bf..8166fb87 100644 --- a/broker/JFJochBrokerHttp.cpp +++ b/broker/JFJochBrokerHttp.cpp @@ -880,7 +880,7 @@ void JFJochBrokerHttp::preview_plot_get(const std::optional &type, if (azintUnit == "Q_recipA" || azintUnit == "q_recipa") unit = PlotAzintUnit::Q_recipA; else if (azintUnit == "d_A" || azintUnit == "d_a") - unit = PlotAzintUnit::D_A; + unit = PlotAzintUnit::d_A; else if (azintUnit == "two_theta_deg") unit = PlotAzintUnit::TwoTheta_deg; } diff --git a/common/AzimuthalIntegrationProfile.cpp b/common/AzimuthalIntegrationProfile.cpp index 4ccfb609..dc31a79d 100644 --- a/common/AzimuthalIntegrationProfile.cpp +++ b/common/AzimuthalIntegrationProfile.cpp @@ -120,7 +120,7 @@ const std::vector &AzimuthalIntegrationProfile::GetXAxis(PlotAzintUnit un switch (unit) { case PlotAzintUnit::TwoTheta_deg: return bin_to_2theta; - case PlotAzintUnit::D_A: + case PlotAzintUnit::d_A: return bin_to_d; default: case PlotAzintUnit::Q_recipA: diff --git a/common/Plot.h b/common/Plot.h index 1075b5c4..943d0135 100644 --- a/common/Plot.h +++ b/common/Plot.h @@ -19,7 +19,7 @@ enum class PlotType { }; enum class PlotAzintUnit { - Q_recipA, TwoTheta_deg, D_A + Q_recipA, TwoTheta_deg, d_A }; struct PlotRequest { diff --git a/receiver/JFJochReceiverPlots.cpp b/receiver/JFJochReceiverPlots.cpp index 3f7e2599..553b99ce 100644 --- a/receiver/JFJochReceiverPlots.cpp +++ b/receiver/JFJochReceiverPlots.cpp @@ -250,7 +250,7 @@ MultiLinePlot JFJochReceiverPlots::GetPlots(const PlotRequest &request) { case PlotAzintUnit::TwoTheta_deg: units = MultiLinePlotUnits::Angle_deg; break; - case PlotAzintUnit::D_A: + case PlotAzintUnit::d_A: units = MultiLinePlotUnits::d_A; break; } -- 2.52.0 From 1558dddbb8e78ab13edbe44b480627cb8e7c1469 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 14:19:23 +0200 Subject: [PATCH 082/228] indexing: make the resolved-algorithm invariant explicit in the pool IndexerThreadPool dispatches on DiffractionExperiment::GetIndexingAlgorithm(), which already resolves Auto to a concrete algorithm (FFTW/FFT/FFBIDX) or None; the pool has no policy to resolve Auto itself. The worker handled a stray Auto with a dead branch and silently produced no result when the resolved algorithm had no matching indexer built. - Document on GetIndexingAlgorithm() that it never returns Auto. - Throw a clear internal error at the pool boundary if Auto ever arrives. - In the worker, replace the dead Auto branch with a loud failure for any resolved algorithm that has no matching indexer (e.g. a GPU algorithm on a host without a GPU), instead of returning no result silently. Co-Authored-By: Claude Opus 4.8 --- common/DiffractionExperiment.h | 3 +++ image_analysis/indexing/IndexerThreadPool.cpp | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index a8bbb9a8..a542ad03 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -392,6 +392,9 @@ public: void CalcSpotFinderResolutionMap(float *data, size_t module_number) const; CompressedImageMode GetImageMode() const; + // Resolves the configured algorithm to a concrete one from GPU availability and + // unit-cell presence: Auto -> FFTW/FFT/FFBIDX, and FFBIDX without a cell -> None. + // Never returns Auto - the indexer pool relies on this and cannot resolve Auto itself. IndexingAlgorithmEnum GetIndexingAlgorithm() const; GeomRefinementAlgorithmEnum GetGeomRefinementAlgorithm() const; float GetIndexingTolerance() const; diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 51913d9a..262aa4b6 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -118,9 +118,14 @@ void IndexerThread::Worker(const IndexingSettings &settings, int threadid) { indexer = ffbidx_indexer.get(); } else if (algorithm == IndexingAlgorithmEnum::FFTW && fftw_indexer) { indexer = fftw_indexer.get(); - } else if (algorithm == IndexingAlgorithmEnum::Auto) { + } else { + // Algorithm is already resolved here (never Auto/None - see + // IndexerThreadPool::Run). Reaching this means the resolved algorithm + // has no matching indexer in this worker (e.g. a GPU algorithm on a + // host without a GPU) - fail loudly instead of silently not indexing. throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Internal error: Invalid indexing algorithm provided"); + "Internal error: no indexer available for the resolved " + "indexing algorithm"); } if (indexer) { @@ -206,9 +211,17 @@ int IndexerThreadPool::GetFreeWorker() { } IndexerResult IndexerThreadPool::Run(const DiffractionExperiment &experiment, const std::vector &recip) { - if (experiment.GetIndexingAlgorithm() == IndexingAlgorithmEnum::None) + const auto algorithm = experiment.GetIndexingAlgorithm(); + if (algorithm == IndexingAlgorithmEnum::None) return IndexerResult{.lattice = {}, .indexing_time_s = 0, .executed = false}; + // GetIndexingAlgorithm() must already have resolved Auto to a concrete algorithm; + // the pool has no policy to resolve it, so Auto here is an upstream contract bug. + if (algorithm == IndexingAlgorithmEnum::Auto) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Internal error: indexing algorithm must be resolved (not Auto) " + "before reaching the indexer pool"); + // Check if there is available worker const int task = GetFreeWorker(); -- 2.52.0 From d0a1175c65da776f7d5516efad769a6101119508 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Thu, 18 Jun 2026 14:29:53 +0200 Subject: [PATCH 083/228] Clean-up of OpenAPI code --- broker/gen/model/Dataset_settings.h | 2 +- broker/gen/model/Jfjoch_settings.h | 2 +- broker/redoc-static.html | 8 ++++---- docs/python_client/docs/DatasetSettings.md | 2 +- docs/python_client/docs/JfjochSettings.md | 2 +- frontend/src/openapi/models/dataset_settings.ts | 2 +- frontend/src/openapi/models/jfjoch_settings.ts | 3 ++- frontend/src/openapi/services/DefaultService.ts | 4 ++-- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index c9e5d574..8a06b8e0 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -168,7 +168,7 @@ public: bool gridScanIsSet() const; void unsetGrid_scan(); /// - /// Header appendix, added as user_data/user to start ZeroMQ message (can be any valid JSON) In general, it is not saved in HDF5 file. However, if values are placed in \"hdf5\" object, `jfjoch_writer` will write them in /entry/data of the HDF5 file. This applies solely to string and number (double floating-point). No arrays/sub-objects is allowed. For example {\"hdf5\": {\"val1\":1, \"val2\":\"xyz\"}}, will write /entry/user/val1 and /entry/user/val2. + /// Header appendix, added as user_data/user to start ZeroMQ message (can be any valid JSON) In general, it is not saved in HDF5 file. However, if values are placed in \"hdf5\" object, `jfjoch_writer` will write them in /entry/user of the HDF5 file. This applies solely to string and number (double floating-point). No arrays/sub-objects is allowed. For example {\"hdf5\": {\"val1\":1, \"val2\":\"xyz\"}}, will write /entry/user/val1 and /entry/user/val2. /// nlohmann::json getHeaderAppendix() const; void setHeaderAppendix(nlohmann::json const& value); diff --git a/broker/gen/model/Jfjoch_settings.h b/broker/gen/model/Jfjoch_settings.h index e1e2ab26..8661427a 100644 --- a/broker/gen/model/Jfjoch_settings.h +++ b/broker/gen/model/Jfjoch_settings.h @@ -164,7 +164,7 @@ public: bool receiverThreadsIsSet() const; void unsetReceiver_threads(); /// - /// NUMA policy to bind CPUs + /// Ignored value /// std::string getNumaPolicy() const; void setNumaPolicy(std::string const& value); diff --git a/broker/redoc-static.html b/broker/redoc-static.html index 7475bf3e..e6a0dbfc 100644 --- a/broker/redoc-static.html +++ b/broker/redoc-static.html @@ -466,7 +466,7 @@ Transmission of attenuator (filter) [no units]

object (grid_scan)

Definition of a grid scan (mutually exclusive with rotation_axis)

header_appendix
any

Header appendix, added as user_data/user to start ZeroMQ message (can be any valid JSON) In general, it is not saved in HDF5 file.

-

However, if values are placed in "hdf5" object, jfjoch_writer will write them in /entry/data of the HDF5 file. +

However, if values are placed in "hdf5" object, jfjoch_writer will write them in /entry/user of the HDF5 file. This applies solely to string and number (double floating-point). No arrays/sub-objects is allowed. For example {"hdf5": {"val1":1, "val2":"xyz"}}, will write /entry/user/val1 and /entry/user/val2.

image_appendix
any

Image appendix, added as user_data to image ZeroMQ message (can be any valid JSON) @@ -868,7 +868,7 @@ User mask is not automatically applied - i.e. pixels with user mask will have a

Generate 1D plot from Jungfraujoch

query Parameters
binning
integer
Default: 1

Binning of frames for the plot (0 = default binning)

-
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b"

Type of requested plot

+
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b" "compression_ratio"

Type of requested plot

fill
number <float>

Fill value for elements that were missed during data collection

experimental_coord
boolean
Default: false

If measurement has goniometer axis defined, plot X-axis will represent rotation angle If measurement has grid scan defined, plot X-axis and Y-axis will represent grid position, Z will be used as the final value @@ -880,7 +880,7 @@ For still measurement the number is ignored

http://localhost:5232/preview/plot

Response samples

Content type
application/json
{
  • "title": "string",
  • "unit_x": "image_number",
  • "size_x": 0.1,
  • "size_y": 0.1,
  • "plot": [
    ]
}

Generate 1D plot from Jungfraujoch and send in raw binary format. Data are provided as (32-bit) float binary array. This format doesn't transmit information about X-axis, only values, so it is of limited use for azimuthal integration. -

query Parameters
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b"

Type of requested plot

+
query Parameters
type
required
string
Enum: "bkg_estimate" "azint" "azint_1d" "spot_count" "spot_count_low_res" "spot_count_indexed" "spot_count_ice" "indexing_rate" "indexing_lattice_count" "indexing_unit_cell_length" "indexing_unit_cell_angle" "profile_radius" "mosaicity" "b_factor" "error_pixels" "saturated_pixels" "image_collection_efficiency" "receiver_delay" "receiver_free_send_buf" "strong_pixels" "roi_sum" "roi_mean" "roi_max_count" "roi_pixels" "roi_weighted_x" "roi_weighted_y" "packets_received" "max_pixel_value" "resolution_estimate" "pixel_sum" "processing_time" "beam_center_x" "beam_center_y" "integrated_reflections" "image_scale_factor" "image_scale_cc" "image_scale_b" "compression_ratio"

Type of requested plot

roi
string non-empty

Name of ROI for which plot is requested

Responses