From d1d7691da1cabc5c3d09173e11a75f1a031a20f9 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 15 Nov 2026 12:19:29 +0100 Subject: [PATCH] Decay decided late, against the correction surfaces fitted without it Since the decay is fitted about each frame's mean s^2 it fits the full, unattenuated slope and cross-validates on sweeps where it did not before. It is fitted first and the time x detector surface after it takes only what the slope left, though that surface follows a damage trend that is not linear in dose better than the single slope: on a 1440 deg multi-turn sweep the slope took 34 A^2, the surface's held-out gain fell from +0.45 (no decay) to +0.17 and ISa from 20.7 to 17.4, R_meas 7.7 -> 8.1%. Applying the decay is a decision the surfaces' outcome should inform, so where it is adopted the surfaces are fitted a second time without it and the chain with the higher merged CC1/2 per shell (Fisher's z) is kept. On three such sweeps ISa returns to 20.7-21.3 (17.4-18.9), R_meas to 7.3-7.9% (7.5-8.2%). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../scale_merge/RotationScaleMerge.cpp | 102 +++++++++++++----- 1 file changed, 74 insertions(+), 28 deletions(-) diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index e21fde542..bda19955a 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -8078,6 +8078,15 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool "one is judged against fits no surfaces, so this is what the two are compared on)", *cc_half_before_corrections); } + auto fulls_corr = [&]() { + std::vector c(fulls.size()); + for (size_t i = 0; i < fulls.size(); ++i) c[i] = fulls[i].corr; + return c; + }; + auto set_fulls_corr = [&](const std::vector &c) { + for (size_t i = 0; i < fulls.size(); ++i) fulls[i].corr = c[i]; + }; + const std::vector corr_before_decay = refine_surfaces ? fulls_corr() : std::vector{}; sub("decay"); if (refine_surfaces && refine_decay_b) RefineDecay(n_groups); @@ -8086,34 +8095,71 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool sub("relative B"); if (refine_surfaces && along_frames && relative_b_deg > 0.0) RefineRelativeB(n_groups); // per-batch relative-B on top of the single decay slope - // The three surfaces overlap, so the one fitted first gets the first claim and the others are tested - // on what it left - the order decides what is adopted. Detector first: every frame of the sweep - // measures the static detector pattern, so its fine grid is the best-determined of the three. Then - // the time x detector surface, which takes what changes through the sweep. The goniometer-frame - // surface last: its cells collect directions from the whole sweep and the whole resolution range, - // so fitted first it soaks up part of both others and fits it where it is not. Measured with the gate - // below: goniometer frame first, a 5 keV sweep then refused the time surface and fell from ISa 31.8 - // to 20.1 (R_meas 7.8 -> 8.9%, CC to the model 0.8307 -> 0.8282), and a finely sliced sweep with a - // large drift went from R_meas 18.2 to 19.6%; time surface first, the detector surface was refused - // after it on a 16 keV sweep that it takes from ISa 33 to 40. In this order neither happens. - // The two crystal-frame candidates come last, the smooth spherical-harmonic one before the grid: - // the grid is then tested on what the smooth surface left, and where that surface is refused the - // grid sees exactly what it saw before. - sub("modulation"); - if (refine_surfaces && along_frames && modulation_iter > 0) - RefineModulation(modulation_iter, n_groups); - sub("absorption time"); - if (refine_surfaces && along_frames && absorption_iter > 0) - RefineAbsorptionTime(absorption_iter, n_groups); - sub("absorption SH"); - if (refine_surfaces && along_frames && absorption_iter > 0) - RefineAbsorptionSH(absorption_iter, n_groups); - sub("absorption gonio"); - if (refine_surfaces && along_frames && absorption_iter > 0) - RefineAbsorption(absorption_iter, n_groups); - sub("absorption joint"); - if (refine_surfaces && !along_frames && absorption_iter > 0) - RefineAbsorptionJoint(absorption_iter, n_groups); + const bool decay_applied = refine_surfaces && fulls_corr() != corr_before_decay; + auto fit_surfaces = [&]() { + // The three surfaces overlap, so the one fitted first gets the first claim and the others are tested + // on what it left - the order decides what is adopted. Detector first: every frame of the sweep + // measures the static detector pattern, so its fine grid is the best-determined of the three. Then + // the time x detector surface, which takes what changes through the sweep. The goniometer-frame + // surface last: its cells collect directions from the whole sweep and the whole resolution range, + // so fitted first it soaks up part of both others and fits it where it is not. Measured with the gate + // below: goniometer frame first, a 5 keV sweep then refused the time surface and fell from ISa 31.8 + // to 20.1 (R_meas 7.8 -> 8.9%, CC to the model 0.8307 -> 0.8282), and a finely sliced sweep with a + // large drift went from R_meas 18.2 to 19.6%; time surface first, the detector surface was refused + // after it on a 16 keV sweep that it takes from ISa 33 to 40. In this order neither happens. + // The two crystal-frame candidates come last, the smooth spherical-harmonic one before the grid: + // the grid is then tested on what the smooth surface left, and where that surface is refused the + // grid sees exactly what it saw before. + sub("modulation"); + if (refine_surfaces && along_frames && modulation_iter > 0) + RefineModulation(modulation_iter, n_groups); + sub("absorption time"); + if (refine_surfaces && along_frames && absorption_iter > 0) + RefineAbsorptionTime(absorption_iter, n_groups); + sub("absorption SH"); + if (refine_surfaces && along_frames && absorption_iter > 0) + RefineAbsorptionSH(absorption_iter, n_groups); + sub("absorption gonio"); + if (refine_surfaces && along_frames && absorption_iter > 0) + RefineAbsorption(absorption_iter, n_groups); + sub("absorption joint"); + if (refine_surfaces && !along_frames && absorption_iter > 0) + RefineAbsorptionJoint(absorption_iter, n_groups); + }; + fit_surfaces(); + // The decay and the relative B take the first claim on a resolution x time trend, and the time x + // detector surface after them can take only what they left - though it models a trend that is not + // linear in dose better than the single slope does. Both chains are cheap next to the pass, so where + // the decay was adopted the surfaces are fitted a second time without it and the merge decides: the + // chain whose merged CC1/2, per shell on Fisher's z, is higher is kept. + if (decay_applied) { + auto chain_score = [&]() { + const std::vector keep = fulls_corr(); +#ifdef JFJOCH_USE_CUDA + if (combined_on_gpu && scaled_fulls_on_gpu) + upload_fulls_corr(); +#endif + const auto m = MergeAndStats(n_groups, for_search, combined_on_gpu && scaled_fulls_on_gpu, + /*full_stats=*/false).statistics; + set_fulls_corr(keep); + double z = 0.0; + int n = 0; + for (const auto &sh : m.shells) + if (std::isfinite(sh.cc_half) && std::fabs(sh.cc_half) < 1.0) { z += std::atanh(sh.cc_half); ++n; } + return n > 0 ? z / n : std::atanh(std::clamp(m.overall.cc_half, -0.999999, 0.999999)); + }; + const double z_with = chain_score(); + const std::vector corr_with = fulls_corr(); + set_fulls_corr(corr_before_decay); + fit_surfaces(); + const double z_without = chain_score(); + const bool keep_decay = z_with > z_without; + if (keep_decay) + set_fulls_corr(corr_with); + logger->Info("Decay against the surfaces alone: merged CC1/2 per shell on Fisher's z {:+.4f} with the " + "decay, {:+.4f} without => {}", z_with, z_without, + keep_decay ? "decay kept" : "decay dropped, the surfaces fitted without it kept"); + } sub("disposition"); // --- 4c. The disposition. What keeping each batch of the sweep costs the merged intensities