Files
Jungfraujoch/tests/RotationScaleWalkTest.cpp
leonarski_fandFilip Leonarski 19b135186d
Build Packages / Create release (push) Successful in 7s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 4m56s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 9m10s
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 13m51s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 16m56s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 20m59s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 21m25s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 21m17s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 14m14s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 25m59s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 25m48s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 15m52s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 19m33s
Build Packages / Generate python client (push) Successful in 29s
Build Packages / Build documentation (push) Successful in 1m19s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 20m49s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 25m31s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 25m24s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 26m5s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 22m19s
Build Packages / Unit tests (push) Failing after 2h28m24s
Build Packages / Validation Battery (push) Successful in 40m33s
v1.0.0-rc.175 (#85)
* Rugnux: Processes a collection live while jfjoch_broker writes it (`rugnux <master.h5> <broker URL>`).
* Rugnux: Processing starts on the first images and confirms its choices once all are in, so analysis can begin during collection.
* Rugnux: Faster processing, especially on servers with several GPUs and CPU sockets and on computers without a GPU (`--no-gpu`, macOS).
* Rugnux: Uses much less GPU memory, so large data sets stay on the GPU of smaller cards, and a data set too large for the card is processed on the CPU instead.
* Rugnux: On macOS, Bragg integration runs on the Apple GPU.
* Rugnux: Better handling of twinning and pseudo-symmetry, with the twin hypothesis written as `<prefix>_twin_<SG>.mtz`, and a wrong `--model` no longer confirms the data or decides the hand.
* Rugnux: `--model` accepts predicted (AlphaFold) models and falls back to molecular replacement when the model is not placed as a rigid body (experimental `--mr` forces it), settling space group, hand and twinning and writing a different answer as `<prefix>_mr.mtz`.
* Rugnux: Molecular replacement is faster and settles the space group in more cases with several copies in the asymmetric unit.
* Rugnux: Deposition-ready mmCIF, a richer report, and numbered steps on the terminal with the full log in `<prefix>_log.txt`; `_P1.mtz`, `--estimate-beam-center` and `--beam-center-search` are removed.
* Rugnux: The beam-stop shadow is found correctly on tilted detectors.
* Rugnux: Stills files that record a stationary goniometer axis get the default stills geometry refinement.
* jfjoch_viewer: "Analyze dataset" uses the same defaults as a bare `rugnux <file>`, and shows the results of processed CBF, marCCD and SMV sweeps.
* jfjoch_viewer: The Files panel refreshes on F5 and every few seconds, and the plot type and other selections stay put during live collection.
* jfjoch_writer: Memory stays bounded and old data files are closed efficiently on long runs, any failure ends the run in an error state the broker is told about, and the master file appears only after its data files; the ZeroMQ pusher logs when a stalled writer makes it drop images.
* Fixes: The TCP image pusher no longer blocks the broker's receiver, GPU out-of-memory failures stop with a clear message, and the XDS plugin masks module and chip edges.

---------

Co-authored-by: Filip Leonarski <jungfrau@filips-mac-mini.home>
Reviewed-on: #85
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-10-16 15:34:43 +02:00

185 lines
9.5 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <cmath>
#include <catch2/catch_all.hpp>
#include "../rugnux/Rugnux.h"
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
namespace {
// A synthetic sweep whose stage turned `true_scale` times the stored angles. Scored at a scale k,
// the validation spots stay on the lattice as long as the angles track the rotation, and the share
// that does falls off with the relative rate error; a wrong spindle angle keeps half a percent.
struct SyntheticSweep {
double true_scale;
int64_t spots = 35000;
int index_calls = 0;
int refit_calls = 0;
ValidationSpotEvidence IndexAt(float k) {
++index_calls;
const double error = std::fabs(true_scale / k - 1.0);
const double on = 0.9 * std::max(0.0, 1.0 - 30.0 * error);
return ValidationSpotEvidence{spots, std::llround(on * spots), std::llround(0.005 * spots)};
}
// The post-refinement at k, relative to k. It reads only 70 % of the error that is left: it
// sees only the frames the angles at k still track.
std::optional<double> RefitAt(float k) {
++refit_calls;
return 1.0 + 0.7 * (true_scale / k - 1.0);
}
RotationScaleWalk Walk(double first_fit, int64_t sweep_images = 3600) {
return WalkRotationScale(first_fit, [this](float k) { return IndexAt(k); },
[this](float k) { return RefitAt(k); }, 8, sweep_images);
}
};
}
TEST_CASE("ValidationEvidencePrefers", "[RotationScale]") {
const ValidationSpotEvidence base{10000, 3000, 50};
// 1 % more of the spots beyond chance is under the noise of two 30 % shares over 10000 spots
// (sqrt(2 * 0.3 * 0.7 / 10000) = 0.65 %, times 3.29); 5 % is well over it.
CHECK_FALSE(ValidationEvidencePrefers(base, {10000, 3100, 50}));
CHECK(ValidationEvidencePrefers(base, {10000, 3500, 50}));
// A candidate is judged against its own null: more spots on the lattice bought by a null that
// rose just as much is no gain.
CHECK_FALSE(ValidationEvidencePrefers(base, {10000, 3500, 550}));
// Never against itself, and nothing that scored nothing wins.
CHECK_FALSE(ValidationEvidencePrefers(base, base));
CHECK_FALSE(ValidationEvidencePrefers(base, {}));
CHECK(ValidationEvidencePrefers({}, base));
}
TEST_CASE("WalkRotationScale_ReachesTheFixedPoint", "[RotationScale]") {
// A stage 3 % slow. The first fit reads 70 % of that; each refit at the adopted scale reads 70 %
// of what is left, and the walk goes on as long as the validation frames prefer the new scale.
SyntheticSweep sweep{0.97};
const auto walk = sweep.Walk(sweep.RefitAt(1.0f).value());
CHECK(walk.scale == Catch::Approx(0.97).margin(0.001));
CHECK(walk.scale != 1.0f);
CHECK(walk.evidence.on_lattice > sweep.IndexAt(1.0f).on_lattice);
CHECK(sweep.refit_calls > 2);
CHECK_FALSE(walk.trail.empty());
}
TEST_CASE("WalkRotationScale_StopsWhereTheNextStepIsUnderHalfAFrame", "[RotationScale]") {
// Spots enough for the validation frames to prefer every step: what ends the walk is the step the
// contraction predicts, measured against the sweep. Each refit reads 70 % of the error left, so the
// steps contract by 0.3.
SyntheticSweep short_sweep{0.97};
short_sweep.spots = 1000000000;
const auto a = short_sweep.Walk(short_sweep.RefitAt(1.0f).value(), 360);
SyntheticSweep long_sweep{0.97};
long_sweep.spots = 1000000000;
const auto b = long_sweep.Walk(long_sweep.RefitAt(1.0f).value(), 36000);
CHECK(short_sweep.refit_calls < long_sweep.refit_calls);
// Where it stops, what is left of the error moves the end of the sweep by under a frame.
CHECK(std::fabs(a.scale - 0.97) * 360 < 1.0);
CHECK(std::fabs(b.scale - 0.97) * 36000 < 1.0);
}
TEST_CASE("WalkRotationScale_StoredAnglesStand", "[RotationScale]") {
SECTION("A healthy stage: a fit off by noise scores no better than the stored angles") {
SyntheticSweep sweep{1.0};
const auto walk = sweep.Walk(1.0002);
CHECK(walk.scale == 1.0f);
CHECK(sweep.index_calls == 2); // the stored angles and the fit, nothing more
CHECK(sweep.refit_calls == 0);
}
SECTION("A real but small error the spots cannot resolve beyond their noise") {
SyntheticSweep sweep{0.999};
sweep.spots = 400;
const auto walk = sweep.Walk(0.9993);
CHECK(walk.scale == 1.0f);
}
SECTION("A fit that tracks something other than the rotation scores worse, and is refused") {
SyntheticSweep sweep{1.0};
const auto walk = sweep.Walk(0.98);
CHECK(walk.scale == 1.0f);
CHECK(sweep.refit_calls == 0);
}
SECTION("A fit of exactly one asks for no probe at all") {
SyntheticSweep sweep{1.0};
const auto walk = sweep.Walk(1.0);
CHECK(walk.scale == 1.0f);
CHECK(walk.trail.empty());
CHECK(sweep.index_calls == 0);
}
}
TEST_CASE("SmoothLogScale_FollowsInformationBridgesGaps", "[RotationScale]") {
const int n = 60;
// A ramp is no curvature, so any amount of smoothing keeps it exactly.
std::vector<double> ramp(n), J(n, 1.0);
for (int f = 0; f < n; ++f) ramp[f] = -0.1 * f;
auto x = RotationScaleMerge::SmoothLogScale(ramp, J, 1e6);
for (int f = 0; f < n; ++f) CHECK(x[f] == Catch::Approx(ramp[f]).margin(1e-6));
// A stretch with no information is bridged by the straight line through its neighbours.
std::vector<double> Jgap(J);
for (int f = 20; f < 40; ++f) Jgap[f] = 0.0;
x = RotationScaleMerge::SmoothLogScale(ramp, Jgap, 1.0);
CHECK(x[30] == Catch::Approx(-3.0).margin(1e-6));
// A frame with far more information than its neighbours keeps its own value.
std::vector<double> y(n, 0.0), Jone(n, 1.0);
y[30] = 1.0; Jone[30] = 1e6;
x = RotationScaleMerge::SmoothLogScale(y, Jone, 10.0);
CHECK(x[30] == Catch::Approx(1.0).margin(1e-3));
// Fewer than two frames with information: nothing to smooth against.
std::vector<double> Jsingle(n, 0.0);
Jsingle[5] = 1.0;
CHECK(RotationScaleMerge::SmoothLogScale(y, Jsingle, 1.0) == y);
}
TEST_CASE("ChooseLogScaleSmoothing_SmoothsNoiseFollowsSignal", "[RotationScale]") {
const int n = 400;
std::vector<double> J(n, 1.0), noisy(n), step(n);
// Deterministic noise about a flat scale: the chosen curve is close to flat.
for (int f = 0; f < n; ++f) noisy[f] = 0.3 * std::sin(12.9898 * f) * std::cos(78.233 * f);
const double l_noise = RotationScaleMerge::ChooseLogScaleSmoothing(noisy, J, 1);
const auto flat = RotationScaleMerge::SmoothLogScale(noisy, J, l_noise);
double rms = 0.0;
for (double v : flat) rms += v * v;
CHECK(std::sqrt(rms / n) < 0.05);
// A precise slow wave is followed.
for (int f = 0; f < n; ++f) step[f] = 2.0 * std::sin(f / 30.0);
const double l_wave = RotationScaleMerge::ChooseLogScaleSmoothing(step, J, 1);
const auto wave = RotationScaleMerge::SmoothLogScale(step, J, l_wave);
CHECK(wave[47] == Catch::Approx(step[47]).margin(0.01));
CHECK(l_wave < l_noise);
}
TEST_CASE("ValidationEvidenceBeatsChance_ExchangeableArrangements", "[RotationScale]") {
// Four reflections taken at the recorded angles and none at any wrong one: (1/6)^4 < 0.001.
CHECK(ValidationEvidenceBeatsChance({.reflections = 4, .reflections_by_chance = 0}));
CHECK_FALSE(ValidationEvidenceBeatsChance({.reflections = 3, .reflections_by_chance = 0}));
// The same four with two more spread over the wrong angles is what chance does once in ~100.
CHECK_FALSE(ValidationEvidenceBeatsChance({.reflections = 4, .reflections_by_chance = 2}));
CHECK_FALSE(ValidationEvidenceBeatsChance({}));
// A real lattice: thousands against a handful.
CHECK(ValidationEvidenceBeatsChance({.reflections = 3000, .reflections_by_chance = 250}));
// As many at the recorded angles as at an average wrong one is no lattice.
CHECK_FALSE(ValidationEvidenceBeatsChance({.reflections = 100, .reflections_by_chance = 500}));
}
TEST_CASE("ValidationEvidenceBeatsChance_NoCrystalSignificance", "[RotationScale]") {
// A handful of stray reflections passes 0.001 but not the no-crystal test's own level.
CHECK(ValidationEvidenceBeatsChance({.reflections = 5, .reflections_by_chance = 1}));
CHECK_FALSE(ValidationEvidenceBeatsChance({.reflections = 5, .reflections_by_chance = 1}, NO_CRYSTAL_SIGNIFICANCE));
// Against two by chance it takes 15 (tail 10^-9.7), and 14 is not enough (10^-8.96).
CHECK(ValidationEvidenceBeatsChance({.reflections = 15, .reflections_by_chance = 2}, NO_CRYSTAL_SIGNIFICANCE));
CHECK_FALSE(ValidationEvidenceBeatsChance({.reflections = 14, .reflections_by_chance = 2}, NO_CRYSTAL_SIGNIFICANCE));
// A weak real lattice on a few frames is still tens of decades past it.
CHECK(ValidationEvidenceBeatsChance({.reflections = 115, .reflections_by_chance = 4}, NO_CRYSTAL_SIGNIFICANCE));
}
TEST_CASE("ValidationEvidenceLog10Tail_Exact", "[RotationScale]") {
CHECK(ValidationEvidenceLog10Tail({.reflections = 4, .reflections_by_chance = 2}) == Catch::Approx(-2.0604).margin(1e-4));
CHECK(ValidationEvidenceLog10Tail({.reflections = 5, .reflections_by_chance = 2}) == Catch::Approx(-2.6981).margin(1e-4));
// Far below the smallest double, still finite.
CHECK(ValidationEvidenceLog10Tail({.reflections = 1354, .reflections_by_chance = 777}) == Catch::Approx(-509.691).margin(1e-3));
CHECK(ValidationEvidenceLog10Tail({}) == 0.0);
}