From a6c21bd84aee0db2ed54546481d6e70015dff8d2 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 18:58:09 +0200 Subject: [PATCH 01/11] Carry the twin-immune zone verdict as a value, not as report text The report decided whether to name a twin law by substring-matching ProcessResult::twin_zone_decision, a field every zone read overwrites. A run reads the zone up to four times, so the match saw the last comparison made rather than the refusal that stood, and TWIN_LAW and the _pdbx_reflns_twin mmCIF loop could appear on a crystal whose refusal was overturned - or be missing on one whose refusal stood. TwinZoneVerdict already returns the verdict; it now records it on the result, and the decision point reads that. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/Rugnux.cpp | 7 ++++++- rugnux/Rugnux.h | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 10a46f362..bfa6def4f 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1785,6 +1785,11 @@ namespace { z.tncs_normalised ? ", normalised per pseudo-translation class" : "", verdict < 0 ? "acentric, the added operators are a twin law or a pseudo-symmetry" : verdict > 0 ? "centric, the added operators are real" : "undecided"); + // The verdict as a VALUE, for the report to act on. Every caller acts on a decisive read - + // the refusal stands or it is overturned - so recording it here records the decision, where + // matching on the text records only the last comparison the run happened to make. + if (verdict != 0) + result.twin_zone_refusal_stood = verdict < 0; return verdict; } @@ -6924,7 +6929,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // refused group adds, at the fraction its own disagreement implies (a lower bound). if (twin_sg && result.space_group_search && result.space_group_search->refused_point_group_representative - && result.twin_zone_decision.find("acentric, the added") != std::string::npos) { + && result.twin_zone_refusal_stood) { const auto &sgs = *result.space_group_search; const gemmi::GroupOps hi = sgs.refused_point_group_representative->operations(); const gemmi::GroupOps lo = twin_sg->operations(); diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 1aadb6cb2..e80f35dd2 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -364,6 +364,12 @@ struct ProcessResult { // The twin-immune zone read for a promotion a twin gate refused (the H ratio or the added-operator // R), and what was decided on it. Empty when no twin gate fired. std::string twin_zone_decision; + // What that zone DECIDED, carried as a value beside the text: true where the refusal STOOD (the + // added operators read acentric and are recorded as a twin law), false where the zone overturned + // the refusal or decided nothing. A run reads the zone up to four times and each read rewrites + // the text, so the decision cannot be recovered from the text - matching on it named the twin law + // of whichever comparison happened to be made last rather than of the refusal that stood. + bool twin_zone_refusal_stood = false; // Per lattice twin law outside the adopted group: the operator's own H, the fraction it implies // and its correlation, on the final merge. Report-only. std::vector twin_law_evidence; From ccda9f043f84a17631ed1847c1b8669d31459522 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 18:58:46 +0200 Subject: [PATCH 02/11] Record the twin-immune zone only where it decided the refusal twin_zone_decision was documented as empty when no twin gate fired, but an undecided read - no zone reflections, or evidence inside the margin - wrote it too, and the report then printed "a twin gate refused this promotion, and the twin-immune zone of the operators it adds decided it" over a line reading "undecided". An undecided read now goes to the log and leaves the field alone, so the field holds what was decided and the prose is true of it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/Rugnux.cpp | 20 ++++++++++++++------ rugnux/Rugnux.h | 5 ++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index bfa6def4f..7d659b8f5 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -1767,16 +1767,17 @@ namespace { return 0; const auto z = AnalyzeTwinImmuneZone(p1_merged, cell, *higher.point_group_representative, *lower.point_group_representative, tncs_vector); + Logger logger("Rugnux"); if (z.zones.empty() || z.zones[0].n == 0) { - result.twin_zone_decision = fmt::format("{} over {}: no zone reflections in the shells with signal " - "({} merged reflections read) -> undecided", - higher.point_group_hm, lower.point_group_hm, p1_merged.size()); + logger.Info("Twin-immune zone {} over {}: no zone reflections in the shells with signal " + "({} merged reflections read) - undecided", higher.point_group_hm, + lower.point_group_hm, p1_merged.size()); return 0; } const auto &zone = z.zones[0]; const int verdict = zone.calibrated_evidence_nats <= -DECISIVE_NATS ? -1 : zone.calibrated_evidence_nats >= DECISIVE_NATS ? +1 : 0; - result.twin_zone_decision = fmt::format( + const std::string text = fmt::format( "{} over {}: operators {}; n {} <|E^2-1|> {:.3f} +- {:.3f} (control {:.3f}, {:+.3f} nats per " "reflection above acentric), evidence {:+.1f} nats, calibrated {:+.1f} ({:.2f}-{:.2f} A{}) -> {}", higher.point_group_hm, lower.point_group_hm, zone.operators, zone.n, zone.mean_abs_e2_minus_1, @@ -1785,11 +1786,18 @@ namespace { z.tncs_normalised ? ", normalised per pseudo-translation class" : "", verdict < 0 ? "acentric, the added operators are a twin law or a pseudo-symmetry" : verdict > 0 ? "centric, the added operators are real" : "undecided"); + // Recorded only where the zone DECIDED something. An undecided read decides nothing - the + // rule stays as it was - and recording it had the report announce "a twin gate refused this + // promotion, and the twin-immune zone decided it" over a line that says the opposite. + if (verdict == 0) { + logger.Info("Twin-immune zone {}", text); + return 0; + } + result.twin_zone_decision = text; // The verdict as a VALUE, for the report to act on. Every caller acts on a decisive read - // the refusal stands or it is overturned - so recording it here records the decision, where // matching on the text records only the last comparison the run happened to make. - if (verdict != 0) - result.twin_zone_refusal_stood = verdict < 0; + result.twin_zone_refusal_stood = verdict < 0; return verdict; } diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index e80f35dd2..8c43eb14e 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -362,7 +362,10 @@ struct ProcessResult { // or that it never confirmed the operators. Empty when the arms agreed. std::string space_group_arm_override; // The twin-immune zone read for a promotion a twin gate refused (the H ratio or the added-operator - // R), and what was decided on it. Empty when no twin gate fired. + // R), and what it decided. Empty where no zone DECIDED one: where no gate fired, and equally where + // a zone was read and came out undecided (too few reflections, or inside the margin), which leaves + // the rule as it was and is reported to the log rather than here. The report renders this field as + // the decision that was taken, so a read that took none must not appear in it. std::string twin_zone_decision; // What that zone DECIDED, carried as a value beside the text: true where the refusal STOOD (the // added operators read acentric and are recorded as a twin law), false where the zone overturned From d2edd7408978c6bf1b946ecef0c9ae486daf0e6c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 18:59:20 +0200 Subject: [PATCH 03/11] Do not report a lattice-arm comparison that was not made A lattice arm that indexed nothing (or post-refined nothing) measures no held-out residual, and PASS_DECISION then read "realised a held-out residual of nan against 1.234e-03 ..., so the metric symmetry stands" - a comparison the run could not make, stated as one it did. The fallback itself is right: the metric symmetry stands. Only the claim changes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/Rugnux.cpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 7d659b8f5..627a94efb 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -2190,14 +2190,33 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { IndexingSettings promoted_settings = experiment_.GetIndexingSettings(); promoted_settings.LatticeClassDemoted(false); experiment_.ImportIndexingSettings(promoted_settings); - logger.Info("Metric symmetry: re-integrated as {} the held-out residual is {:.3e}, " - "which {} does not beat ({:.3e}) - the higher symmetry stands", - class_text(promoted_arm), realised_promoted, class_text(demoted_arm), - realised_demoted); - lattice_arm = fmt::format( - "{} was run as well and realised a held-out residual of {:.3e} against {:.3e} " - "as {}, so the metric symmetry stands", class_text(demoted_arm), - realised_demoted, realised_promoted, class_text(promoted_arm)); + // An arm that indexed nothing, or post-refined nothing, measures no residual. The + // metric symmetry stands either way - that is the arm whose absence of evidence is + // not evidence against it - but the run must not report a comparison it could not + // make: written as a number, a missing residual came out as "realised a held-out + // residual of nan against 1.234e-03, so the metric symmetry stands". + const bool compared = std::isfinite(realised_promoted) && std::isfinite(realised_demoted); + if (compared) { + logger.Info("Metric symmetry: re-integrated as {} the held-out residual is {:.3e}, " + "which {} does not beat ({:.3e}) - the higher symmetry stands", + class_text(promoted_arm), realised_promoted, class_text(demoted_arm), + realised_demoted); + lattice_arm = fmt::format( + "{} was run as well and realised a held-out residual of {:.3e} against {:.3e} " + "as {}, so the metric symmetry stands", class_text(demoted_arm), + realised_demoted, realised_promoted, class_text(promoted_arm)); + } else { + const char *which = !std::isfinite(realised_demoted) && !std::isfinite(realised_promoted) + ? "neither arm measured a held-out residual" + : !std::isfinite(realised_demoted) + ? "the demoted arm measured no held-out residual" + : "the promoted arm measured no held-out residual"; + logger.Info("Metric symmetry: {}, so the two could not be compared - the higher " + "symmetry stands", which); + lattice_arm = fmt::format( + "{} was run as well, but {}, so the two could not be compared and the " + "metric symmetry stands untested", class_text(demoted_arm), which); + } } } else { logger.Info("Metric symmetry: {} asserts a = b and the free refinement leaves the two " From 5c6dedea71b0e7af29cabbf6a31780f25edd8ca7 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 18:59:59 +0200 Subject: [PATCH 04/11] Do not read the oscillation width as the rotation step SweepLayout rejected any inter-frame difference below half the header's oscillation width as read-back jitter. A series that steps by less than it exposes - overlapping wedges, or a writer that puts the whole sweep's range in that field - then had every difference rejected, no step was found, and the files fell into the "series that never turns" branch, which lays them out end to end and, unlike the gapped path, said nothing. The jitter bound is now absolute (1e-4 deg, below any step an instrument makes and above 32-bit float read-back noise), and the end-to-end fallback warns when the angles do span a rotation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- reader/SweepLayout.cpp | 24 ++++++++++++++++++++---- tests/SweepLayoutTest.cpp | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/reader/SweepLayout.cpp b/reader/SweepLayout.cpp index 981fa0137..67cda450f 100644 --- a/reader/SweepLayout.cpp +++ b/reader/SweepLayout.cpp @@ -125,14 +125,20 @@ Layout Place(const std::vector &frames, const std::string &logger_name) { // Angle_increment is not used for this: it is the oscillation WIDTH, which a series with // overlapping or spaced wedges does not step by. // - // A difference smaller than half the oscillation width is not a step but jitter in the recorded - // angle: no instrument slices finer than it exposes, so wedges overlapping twofold would be a - // read-back wobble, and taking one as the step would spread the sweep over millions of slots. + // A difference below this is jitter in the recorded angle rather than a step, and taking one as + // the step would spread the sweep over millions of slots. The bound is ABSOLUTE and far below + // any move an instrument makes: these headers write their angles as 32-bit floats, which is a + // few parts in 100000 - under 1e-4 deg even at 360 - while the finest rotation step collected is + // a thousandth of a degree. Deriving it from the oscillation width instead, as this used to, is + // the same mistake the paragraph above warns of: a series with overlapping wedges steps by far + // less than it exposes, and some writers put the whole sweep's range in that field, so every + // inter-frame difference was rejected, the guess stayed zero and a turning series fell into the + // "never turns" branch below and was laid out end to end. // // It is only a guess, and deliberately the crudest one: a single recorded difference, and the // one biased furthest low by the read-back noise. It is used below to count STEPS between // neighbours, never to place a frame outright. - const double too_fine = 0.5 * std::abs(f0.increment_deg); + constexpr double too_fine = 1e-4; // degrees double guess = 0; for (size_t i = 1; i < frames.size(); i++) { const double d = Fold(angle[i] - angle[i - 1]); @@ -144,6 +150,16 @@ Layout Place(const std::vector &frames, const std::string &logger_name) { // A series that never turns: a grid scan, a set of stills, or a single image. There is no sweep // to place anything on, so the files are the slots and the header's nominal increment stands. if (guess == 0) { + // ... unless the angles say it DOES turn, over the series if never between two neighbours. + // Then this layout is a guess - the files end to end, at the nominal increment - and it has + // to say so, as the gapped path below does. Nothing else in this branch reports anything. + const double turned = std::abs(angle.back() - angle[0]); + if (turned > std::max(std::abs(f0.increment_deg), 1e-3)) + logger.Warning("The angles of these {} images span {:.4f} deg, but no two neighbours are " + "more than {:g} deg apart, so no rotation step can be read from them. They " + "are laid out one after another at the header's {:.4f} deg - check that " + "this is the sweep the files describe.", + frames.size(), turned, too_fine, f0.increment_deg); out.files.reserve(frames.size()); for (const auto &f : frames) out.files.push_back(f.path); diff --git a/tests/SweepLayoutTest.cpp b/tests/SweepLayoutTest.cpp index ee472c4d2..a3ef530d3 100644 --- a/tests/SweepLayoutTest.cpp +++ b/tests/SweepLayoutTest.cpp @@ -153,3 +153,24 @@ TEST_CASE("SweepLayout_RepeatedAngleRefused") { // Two sweeps of the same crystal concatenated: the second covers angles the first already has. CHECK_THROWS_AS(sweep::Place(Series({0.0, 0.1, 0.2, 0.1, 0.2}, 0.1), "test"), JFJochException); } + +TEST_CASE("SweepLayout_StepFinerThanTheOscillationWidth") { + // The header field is the oscillation WIDTH, and a series can step by far less than it exposes - + // or, on some writers, hold the whole sweep's range in that field. Taking it for the step + // rejected every inter-frame difference as jitter: the sweep then read as one that never turns + // and was laid out end to end, every frame past the first at the wrong angle. + std::vector angles; + for (int i = 0; i < 100; i++) + angles.push_back(30.0 + 0.1 * i); + + const auto wedges = sweep::Place(Series(angles, 1.0), "test"); // 1 deg wedges, 0.1 deg apart + REQUIRE(wedges.files.size() == 100); + CHECK(wedges.present == 100); + CHECK(wedges.start_deg == Catch::Approx(30.0)); + CHECK(wedges.increment_deg == Catch::Approx(0.1)); + + const auto total = sweep::Place(Series(angles, 10.0), "test"); // the field holds the range + REQUIRE(total.files.size() == 100); + CHECK(total.present == 100); + CHECK(total.increment_deg == Catch::Approx(0.1)); +} From 404b233aa8ceb9934146c1f2bb201bae14bd5005 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:00:58 +0200 Subject: [PATCH 05/11] Let format autodetection fail as an answer, not as an exception MarCCD::CanRead and SMV::CanRead read the first plausible file of a directory outside their own try, and the CLI made all three CanRead calls outside the try that reports a bad input. "rugnux " whose alphabetically-first plausible file is unreadable therefore terminated with no message. CanRead now answers false for anything it cannot read, and the CLI asks the question where it can report the answer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- reader/MarCCD.cpp | 13 +++++++++---- reader/SMV.cpp | 13 +++++++++---- rugnux/rugnux_cli.cpp | 10 +++++++--- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/reader/MarCCD.cpp b/reader/MarCCD.cpp index 288d244c1..a5ee68d3a 100644 --- a/reader/MarCCD.cpp +++ b/reader/MarCCD.cpp @@ -251,13 +251,18 @@ std::vector CollectSweep(const std::string &path) { return out; } +// Asked of every input before anything is read, including inputs that are not marCCD at all, so it +// answers a question and never fails: a file it cannot open or make sense of is simply not one of +// ours. The directory arm has to be inside the try as much as the single-file one - it opens the +// first plausible file in the folder, which can be unreadable, and the throw came out of a call site +// that does not catch it, ending the run with no message at all. bool CanRead(const std::string &path) { std::error_code ec; - if (std::filesystem::is_directory(path, ec)) - return !CollectSweep(path).empty(); - if (!PlausibleExtension(std::filesystem::path(path))) - return false; try { + if (std::filesystem::is_directory(path, ec)) + return !CollectSweep(path).empty(); + if (!PlausibleExtension(std::filesystem::path(path))) + return false; return !InstrumentHeader(path).empty(); } catch (const JFJochException &) { return false; diff --git a/reader/SMV.cpp b/reader/SMV.cpp index 405582db1..d818cac6d 100644 --- a/reader/SMV.cpp +++ b/reader/SMV.cpp @@ -222,13 +222,18 @@ std::vector CollectSweep(const std::string &path) { return out; } +// Asked of every input before anything is read, including inputs that are not SMV at all, so it +// answers a question and never fails: a file it cannot open or make sense of is simply not one of +// ours. The directory arm has to be inside the try as much as the single-file one - it opens the +// first plausible file in the folder, which can be unreadable, and the throw came out of a call site +// that does not catch it, ending the run with no message at all. bool CanRead(const std::string &path) { std::error_code ec; - if (std::filesystem::is_directory(path, ec)) - return !CollectSweep(path).empty(); - if (!PlausibleExtension(std::filesystem::path(path))) - return false; try { + if (std::filesystem::is_directory(path, ec)) + return !CollectSweep(path).empty(); + if (!PlausibleExtension(std::filesystem::path(path))) + return false; const auto kv = HeaderBlock(path); return kv.has_value() && kv->count("SIZE1") && kv->count("SIZE2"); } catch (const JFJochException &) { diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 9c10383ec..69cbb1b70 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1420,10 +1420,14 @@ static int RunRugnux(int argc, char **argv) { JFJochMarCCDReader marccd_reader; JFJochSMVReader smv_reader; JFJochReader *reader_ptr = nullptr; - const bool input_is_cbf = JFJochCBFReader::CanRead(input_file); - const bool input_is_marccd = !input_is_cbf && JFJochMarCCDReader::CanRead(input_file); - const bool input_is_smv = !input_is_cbf && !input_is_marccd && JFJochSMVReader::CanRead(input_file); + // Which format this is. Asked INSIDE the try: these open and read the file - a directory whose + // first plausible member cannot be read throws here - and an input the program cannot read has to + // come out as the message below, not as an unhandled exception with nothing printed at all. + bool input_is_cbf = false, input_is_marccd = false, input_is_smv = false; try { + input_is_cbf = JFJochCBFReader::CanRead(input_file); + input_is_marccd = !input_is_cbf && JFJochMarCCDReader::CanRead(input_file); + input_is_smv = !input_is_cbf && !input_is_marccd && JFJochSMVReader::CanRead(input_file); if (input_is_cbf) { cbf_reader.ReadFiles(input_file); reader_ptr = &cbf_reader; From aecbcf3aaf506f46e45fafcf594cb1c4e8ea9eff Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:01:33 +0200 Subject: [PATCH 06/11] Clamp the marCCD header probe to the size of the file The header offset comes from TIFF tag 34710 unchecked and is handed to ReadPrefix, which allocates that many bytes before reading. This runs during format autodetection, on files nobody has said are marCCD, so a foreign or corrupt TIFF could have the program allocate ~4 GB for a probe. The probe now reads at most what the file holds; a short read was always the normal outcome and the caller already checks the size. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- reader/MarCCD.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/reader/MarCCD.cpp b/reader/MarCCD.cpp index a5ee68d3a..96af47893 100644 --- a/reader/MarCCD.cpp +++ b/reader/MarCCD.cpp @@ -60,6 +60,15 @@ std::vector ReadPrefix(const std::string &path, size_t bytes) { std::ifstream f(path, std::ios::binary); if (!f) throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path); + // Never allocate more than the file holds. One caller's count comes from the header-offset TIFF + // tag, which is a field of a file nobody has yet decided is ours - this runs on every input the + // program is given - and a foreign or corrupt TIFF can name four gigabytes there. Reading short + // is already the normal outcome (the caller checks the size it got back), so the clamp costs + // nothing. + f.seekg(0, std::ios::end); + const std::streamoff file_bytes = f.tellg(); + f.seekg(0, std::ios::beg); + bytes = std::min(bytes, file_bytes > 0 ? static_cast(file_bytes) : size_t(0)); std::vector out(bytes); f.read(reinterpret_cast(out.data()), static_cast(bytes)); out.resize(static_cast(f.gcount())); From ba5b7cda81e04d16987aaba2f1d319e3db3decab Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:02:12 +0200 Subject: [PATCH 07/11] Do not let a marCCD file name 2-theta as the scanned circle The scanned circle was read as the first start/end pair that differs, and two_theta is the first pair in the goniostat block. A file that leaves its end block at zero while the detector sits on a swung-out 2-theta arm therefore named the arm as the spindle and stopped before reaching phi; with no rotation range recorded either, the increment fell back to -start_angle, which differs per frame, and the sweep was refused outright. The header's own rotation_axis index (offset 732) is taken where it names a spindle circle, and the pair scan is the fallback for files that never filled it in - read past two_theta, which no sweep turns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- reader/MarCCD.cpp | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/reader/MarCCD.cpp b/reader/MarCCD.cpp index 96af47893..68ac7c171 100644 --- a/reader/MarCCD.cpp +++ b/reader/MarCCD.cpp @@ -38,6 +38,7 @@ constexpr size_t OFF_END_XTAL_TO_DETECTOR = 728; // ... and at its end constexpr size_t OFF_EXPOSURE_TIME = 656; // milliseconds constexpr size_t OFF_START_TWOTHETA = 668; // millidegrees, and the seven that follow it constexpr size_t OFF_END_TWOTHETA = 700; // the same seven, at the end of the exposure +constexpr size_t OFF_ROTATION_AXIS = 732; // which circle the sweep turned, as an index below constexpr size_t OFF_ROTATION_RANGE = 736; // millidegrees constexpr size_t OFF_PIXELSIZE_X = 772; // nanometres constexpr size_t OFF_PIXELSIZE_Y = 776; @@ -146,21 +147,33 @@ Header Parse(const std::vector &h, const std::string &path) { out.exposure_s = I32(h, OFF_EXPOSURE_TIME) * 1e-3; out.saturated_value = I32(h, OFF_SATURATED); - // Which circle moved. The header also carries a rotation_axis index, but the start/end pair - // that differs is the same answer read off the angles themselves, and it stays right on a file - // whose index field was never filled in. + // Which circle moved. The header names it as an index into the circle list, and that is taken + // where it names a spindle circle: reading the start/end pairs alone takes the FIRST pair that + // differs, and two_theta is the first of them, so a file that left its end block at zero while + // the detector sat on a swung-out 2-theta arm had the arm named as the spindle - and, with no + // rotation range recorded either, an increment of -start_angle that differs on every frame, + // which SweepLayout then refuses the whole sweep over. + // + // The index is not always filled in, and a file that leaves it at zero cannot be told from one + // that means two_theta by it, so the fallback is the pairs as before - read past two_theta, + // which is a detector circle and never the one a sweep turns. out.angle_increment_deg = I32(h, OFF_ROTATION_RANGE) * 1e-3; - for (size_t i = 0; i < N_CIRCLES; i++) { + const auto take_circle = [&](size_t i) { const double start = I32(h, OFF_START_TWOTHETA + 4 * i) * 1e-3; const double end = I32(h, OFF_END_TWOTHETA + 4 * i) * 1e-3; - if (start != end) { - out.axis_name = CIRCLE_NAMES[i]; - out.start_angle_deg = start; - if (out.angle_increment_deg == 0.0) - out.angle_increment_deg = end - start; - break; - } - } + if (start == end) + return false; + out.axis_name = CIRCLE_NAMES[i]; + out.start_angle_deg = start; + if (out.angle_increment_deg == 0.0) + out.angle_increment_deg = end - start; + return true; + }; + const int32_t named = I32(h, OFF_ROTATION_AXIS); + if (named <= 0 || static_cast(named) >= N_CIRCLES || !take_circle(static_cast(named))) + for (size_t i = 1; i < N_CIRCLES; i++) + if (take_circle(i)) + break; return out; } From fe3a28a1c0cce4f5fadd476833cd7149d5162607 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:02:47 +0200 Subject: [PATCH 08/11] Drop the centring-absent reflections from the reported twin statistics AnalyzeTwinning takes the lattice centring so that the reflections the centring extinguishes stay out of both statistics, but only the pre-promotion test passed it. The final, reported verdict and --mode scale did not, and a merged ASU in a centred group still carries keys in the absent class - noise, and an L-test partner step landing on one pairs a real reflection with an absent one. Both now pass the adopted group's centring. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/Rugnux.cpp | 10 +++++++++- rugnux/rugnux_cli.cpp | 6 +++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 627a94efb..883e1467e 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -6943,8 +6943,16 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b result.tncs.peak_percent, result.tncs.vector_frac[0], result.tncs.vector_frac[1], result.tncs.vector_frac[2], result.tncs.vector_length_A); + // Without the reflections the lattice centring extinguishes, as the pre-promotion test + // above is measured: the merged ASU of a centred group still holds keys in the absent + // class (the merge writes what it was given), they are noise rather than intensities, + // and an L-test partner step that lands on one pairs a real reflection with an absent. + // This is the number the user reads, so it is the one that must be treated. + const char twin_centring = twin_sg ? twin_sg->centring_type() + : end_msg.rotation_lattice_type.has_value() + ? end_msg.rotation_lattice_type->centering : 'P'; result.twinning = AnalyzeTwinning(sm.merged, twin_sg, 20, - TranslationalNCSVectorForLTest(result.tncs)); + TranslationalNCSVectorForLTest(result.tncs), twin_centring); // Set on the struct the text below is rendered from. AnalyzeTwinning returns a fresh result // with this flag clear, so setting it only afterwards left the statistics text asserting // "No twinning: the Laue class is holohedral" as a fact while the report, rendering the same diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 69cbb1b70..bc47bde04 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1868,8 +1868,12 @@ static int RunRugnux(int argc, char **argv) { tncs = AnalyzeTranslationalNCS(merged_reflections, gemmi::UnitCell(*uc), twin_sg); if (tncs.measurable) std::cout << std::endl << TranslationalNCSToText(tncs); + // Same as the full pipeline: without the reflections the lattice centring extinguishes. The + // merged ASU of a centred group still holds keys in the absent class, and they are noise, not + // intensities, in both statistics. const auto twinning = AnalyzeTwinning(merged_reflections, twin_sg, 20, - TranslationalNCSVectorForLTest(tncs)); + TranslationalNCSVectorForLTest(tncs), + twin_sg ? twin_sg->centring_type() : 'P'); std::cout << std::endl << TwinningAnalysisToText(twinning) << std::endl; // Diffraction anisotropy, exactly as the full pipeline computes it (Rugnux.cpp) - report-only, From 1b9b6d7405f71e90fadb6f5614e4978ea4aad43f Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:03:53 +0200 Subject: [PATCH 09/11] Correct three stale comments r_contrast_bound carried a calibration (0.70 against a bound of 0.65) that min_operator_r_contrast and the ledger text no longer agree with; TrimmedZoneSum's header still promised "its single largest member dropped", which the caller no longer supplies since the screw fix withholds a member standing at or above its row's mean; and mean_relative_scale still said "/ run median" though the gauge is now TypicalFrameScale. Comments only - no behaviour changes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- image_analysis/scale_merge/Merge.h | 4 +++- image_analysis/scale_merge/SearchSpaceGroup.h | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index cf7b6df80..32fb5a61f 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -87,7 +87,9 @@ struct SweepQualityRange { // 1 = nothing at all. The rest are supporting numbers, relative to the run unless stated. float severity = 0.0f; float rotation_deg = 0.0f; // width of the range - float mean_relative_scale = 1.0f; // / run median + // Gauged on the run's precision-weighted typical frame (TypicalFrameScale), not on its median + // frame, which on a sweep that spent most of its turn out of the beam is itself a dead one. + float mean_relative_scale = 1.0f; // / typical frame scale float mean_relative_cc = 1.0f; // / run median float indexed_fraction = 1.0f; // frames in the range that got a per-image scale at all float relative_b = NAN; // mean of the radiation-damage monitor's per-batch B over the range diff --git a/image_analysis/scale_merge/SearchSpaceGroup.h b/image_analysis/scale_merge/SearchSpaceGroup.h index c02478a8a..df23a484b 100644 --- a/image_analysis/scale_merge/SearchSpaceGroup.h +++ b/image_analysis/scale_merge/SearchSpaceGroup.h @@ -155,11 +155,16 @@ double AbsenceEvidence(double sum_u, int n_absent, int n_control); // candidate's absences could score lower. See SearchSpaceGroup.cpp. double ScrewZoneEvidence(double sum_u, int n_absent); -// A screw zone's sum with its single largest member dropped and rescaled to estimate the same -// quantity (dividing by n - H_n, the expected sum of the other n-1 under the null, and multiplying -// by n). ScrewZoneEvidence is a sum, so it is dominated by the largest one or two members of a class -// that holds only a handful of reflections, and a zone's verdict could hang on one reflection whose -// merged value moved with the scaling. See SearchSpaceGroup.cpp. +// A screw zone's sum with `max_u` dropped and rescaled to estimate the same quantity (dividing by +// n - H_n, the expected sum of the other n-1 under the null, and multiplying by n). ScrewZoneEvidence +// is a sum, so it is dominated by the largest one or two members of a class that holds only a handful +// of reflections, and a zone's verdict could hang on one reflection whose merged value moved with the +// scaling. max_u is the largest TRIMMABLE member, not simply the largest: the caller withholds a +// member that both violates the absence and stands at or above its row's own present mean, because +// that is a reflection that is there rather than a measurement that moved, and trimming it removed +// the single datum refuting the claim. The sum a withheld member leaves is therefore higher than a +// true trim would give, which UNDER-states the evidence: this estimator only ever refuses a screw, +// never asserts one. See SearchSpaceGroup.cpp. double TrimmedZoneSum(double sum_u, double max_u, int n_absent); struct SearchSpaceGroupOptions { @@ -763,8 +768,9 @@ struct SearchSpaceGroupResult { std::vector point_group_ledger; // The refusal bounds the ledger's own channels were judged against, copied from the options so the // report can print the MARGIN the adopted row had rather than only its value. The accept-side - // headroom is thin (measured: the lowest genuine adopted contrast on a 149-dataset corpus is 0.70 - // against a bound of 0.65), and a margin is the only form in which that is visible per run. + // headroom is thin (measured: the lowest genuine adopted contrast on the corpus is 0.77 against a + // bound of 0.72 - see min_operator_r_contrast, which is where these numbers are kept), and a + // margin is the only form in which that is visible per run. double r_contrast_bound = 0.0; double r_over_floor_bound = 0.0; From 65b9ae1259908bd51cc24b7f865fea57b667ce5c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:23:32 +0200 Subject: [PATCH 10/11] Reader tests for the marCCD header-offset clamp and the 2-theta circle Two cases on the existing byte-for-byte marCCD fixture: a file whose end-angle block is zeroed while the detector sits on a 2-theta arm must still name phi as the circle that turned, and a file whose tag 34710 names an offset far past its end must be refused rather than allocated for. Also drops a bound in SweepLayout that the absolute jitter threshold makes redundant. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- reader/SweepLayout.cpp | 2 +- tests/JFJochReaderTest.cpp | 43 +++++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/reader/SweepLayout.cpp b/reader/SweepLayout.cpp index 67cda450f..6feae1122 100644 --- a/reader/SweepLayout.cpp +++ b/reader/SweepLayout.cpp @@ -142,7 +142,7 @@ Layout Place(const std::vector &frames, const std::string &logger_name) { double guess = 0; for (size_t i = 1; i < frames.size(); i++) { const double d = Fold(angle[i] - angle[i - 1]); - if (std::abs(d) > std::max(too_fine, 1e-6) && (guess == 0 || std::abs(d) < std::abs(guess))) + if (std::abs(d) > too_fine && (guess == 0 || std::abs(d) < std::abs(guess))) guess = d; } diff --git a/tests/JFJochReaderTest.cpp b/tests/JFJochReaderTest.cpp index d6a61c86f..6b0c0cc19 100644 --- a/tests/JFJochReaderTest.cpp +++ b/tests/JFJochReaderTest.cpp @@ -4147,6 +4147,9 @@ namespace { int32_t wavelength_e5A = 97872; int32_t saturated = 65535; int32_t depth = 2; + int32_t rotation_axis = 4; // the circle that turned, as the header's own index (phi) + bool zero_end_block = false; // a writer that fills the start angles and nothing else + uint32_t header_offset_tag = MARCCD_HEADER_OFF; // what tag 34710 claims, true or not bool write_instrument_header = true; }; @@ -4169,7 +4172,7 @@ namespace { {277, 3, 1, 1}, // SamplesPerPixel {278, 4, 1, static_cast(ny)}, // RowsPerStrip: one strip {279, 4, 1, static_cast(npixel * 2)}, // StripByteCounts - {34710, 4, 1, MARCCD_HEADER_OFF}, // where the instrument header begins + {34710, 4, 1, f.header_offset_tag}, // where the instrument header begins }; PutU16(file, 8, static_cast(ifd.size())); size_t e = 10; @@ -4195,8 +4198,11 @@ namespace { PutI32(file, h + 656, 1000); // exposure, ms PutI32(file, h + 668, f.two_theta_mdeg); // start_twotheta PutI32(file, h + 684, f.start_phi_mdeg); // start_phi - PutI32(file, h + 700, f.two_theta_mdeg); // end_twotheta - PutI32(file, h + 716, f.end_phi_mdeg); // end_phi + if (!f.zero_end_block) { + PutI32(file, h + 700, f.two_theta_mdeg); // end_twotheta + PutI32(file, h + 716, f.end_phi_mdeg); // end_phi + } + PutI32(file, h + 732, f.rotation_axis); PutI32(file, h + 736, f.rotation_range_mdeg); PutI32(file, h + 772, f.pixel_nm); PutI32(file, h + 776, f.pixel_nm); @@ -4280,6 +4286,37 @@ TEST_CASE("JFJochMarCCDReader_Geometry", "[HDF5][Full]") { RemoveMarCCDSweep("marccdpix_", ".mccd"); } + SECTION("a swung-out 2-theta arm is not the circle the sweep turned") { + // Some writers fill the start angles and leave the whole end block at zero. Reading the + // scanned circle as the first start/end pair that differs then picks two_theta, which is the + // first pair in the block, and a detector on a 2-theta arm was named as the spindle - with a + // start angle that does not move from frame to frame. + MarCCDFields f; + f.two_theta_mdeg = 20000; // the detector swung out by 20 deg + f.zero_end_block = true; + f.rotation_axis = 0; // ... and the index never filled in either + WriteMarCCDSweep("marccd2t_", ".mccd", nx, ny, f); + JFJochMarCCDReader reader; + REQUIRE_NOTHROW(reader.ReadFiles("marccd2t_001.mccd")); + const auto x = reader.GetDataset()->experiment; + REQUIRE(x.GetGoniometer().has_value()); + CHECK(x.GetGoniometer()->GetName() == "phi"); + CHECK(x.GetGoniometer()->GetStart_deg() == Catch::Approx(0.0)); + CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.2)); + reader.Close(); + RemoveMarCCDSweep("marccd2t_", ".mccd"); + } + + SECTION("a header offset the file cannot hold is refused, not allocated") { + // The offset comes from TIFF tag 34710 and is read on every input the program is given, so a + // foreign or corrupt TIFF can name gigabytes on a file nobody has said is marCCD. + MarCCDFields f; + f.header_offset_tag = 4000000000u; + WriteMarCCDFrame("marccdfaroff_001.mccd", nx, ny, f); + CHECK_FALSE(JFJochMarCCDReader::CanRead("marccdfaroff_001.mccd")); + remove("marccdfaroff_001.mccd"); + } + SECTION("a TIFF that is not marCCD is refused, not misread") { // The name rule alone matches any numbered TIFF in the directory, so a file with no // instrument header must not be opened with a pixel size and a distance of zero. From 0f3dbe6e809e51dea7731135e83a197f0aad155a Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:33:26 +0200 Subject: [PATCH 11/11] Say why a directory could not be read, instead of opening it as HDF5 With the format probes no longer throwing, a directory no reader claims fell through to the HDF5 reader, which is never what a directory is: the answer - that no one-file-per-image sweep was found in it - came out buried under a page of HDF5 stack output. It is now stated. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/rugnux_cli.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index bc47bde04..55f38c0c0 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -1443,6 +1444,13 @@ static int RunRugnux(int argc, char **argv) { reader_ptr = &smv_reader; logger.Info("Read {} SMV images of the sweep containing {}", smv_reader.GetNumberOfImages(), input_file); + } else if (std::filesystem::is_directory(input_file)) { + // A directory is never an HDF5 master, so handing it to the HDF5 reader only buries the + // real answer - that no one-file-per-image sweep was found in it - under a page of HDF5 + // stack output. + logger.Error("{} holds no CBF, marCCD or SMV sweep this program can read (a directory is " + "read as the sweep of frames it holds)", input_file); + exit(EXIT_FAILURE); } else { hdf5_reader.ReadFile(input_file); reader_ptr = &hdf5_reader;