From b34f7b09a3d2bfa5532a12d831b1bdbd27cfa9a1 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 19:08:41 +0200 Subject: [PATCH] Rugnux: retire _image.dat, and read the diagnostic picture off the background _image.dat carried the per-image scaling table before _plot.txt existed, and the two were written from the same ScalingResult with six columns between them. Drop it: the writer, its call site and its documentation entries go, and _plot.txt gains the one column of the old file that was not already there and is not dead - cc_n, the number of observations cc_to_merge was taken over, without which that correlation cannot be read at the sparse end of a sweep. The old file's wedge_deg is NOT carried over: image_scale_wedge_deg is only ever reset, never assigned, so that column was nan on every run that ever wrote it. _plot.txt is now written on a stills run too - it is the only per-image scaling table there is, and the columns a sweep would fill come out nan - while the picture stays rotation-only, as only the rotation pre-scan builds the projection it draws. "merged" reads nan rather than 0 where no merge stands behind the run, so --no-merge no longer looks like a sweep every frame of which was thrown out. The picture's contrast now comes from the background instead of the peaks: the ramp saturates at eight times the median measured pixel, where it used to saturate at the brightest 0.1 % as the preview and the viewer do. That percentile belongs to the Bragg spots, so the background - which is what a shadow is read off, being a place where it is MISSING - sat in the first few per cent of the scale and rendered as a flat white. The median is pure background whatever the crystal, and eight times it falls between the 94th and the 99th percentile of the projection on every set measured. Two times the median, the first value tried, falls at the 73rd to 90th instead and floods the inner half of the detector to a flat indigo, hiding the deficit at the other end of the scale. The colours are untouched - they are the viewer's, from common/ColorScale.h - and only the intensity mapping moves. Processing is unchanged: p.hkl and p.mtz are byte-identical before and after on a rotation sweep, and the wall clock does not move. rugnux_stills_ab.py reads scale_G from _plot.txt instead of the retired file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- .gitignore | 3 +- docs/RUGNUX.md | 2 +- docs/RUGNUX_TUTORIAL.md | 26 +++--- image_analysis/scale_merge/Merge.h | 2 +- image_analysis/scale_merge/ScalingResult.cpp | 28 ------- image_analysis/scale_merge/ScalingResult.h | 2 - rugnux/DiagnosticOutput.cpp | 41 +++++++--- rugnux/DiagnosticOutput.h | 7 +- rugnux/Rugnux.cpp | 26 +++--- rugnux_stills_ab.py | 13 +-- tests/DiagnosticOutputTest.cpp | 85 +++++++++++++++----- 11 files changed, 139 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index 8ecf683a6..25ca7930a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,8 @@ rugnux_anomalous*/ *.cif *.pdb *.hkl -*_image.dat +*_plot.txt +*_detector.jpg # Test fixtures are checked in deliberately; add them with an explicit path. !tests/** diff --git a/docs/RUGNUX.md b/docs/RUGNUX.md index 05b8f9ea7..e8dc69f05 100644 --- a/docs/RUGNUX.md +++ b/docs/RUGNUX.md @@ -63,7 +63,7 @@ myrun_unmerged.mtz every observation before scaling, for pointless / aimless / myrun_P1.mtz the same observations merged in P1, so a wrong space-group call can be re-merged or re-refined without reprocessing (--no-p1-crosscheck skips it) myrun_report.txt what the run determined: cell, space group, statistics, warnings -myrun_image.dat one row per image, for plotting how the crystal behaved over the sweep +myrun_plot.txt one row per image, for plotting how the crystal behaved over the sweep ``` The two MTZ extras are most of the bytes a run writes — worth knowing when sizing a scratch diff --git a/docs/RUGNUX_TUTORIAL.md b/docs/RUGNUX_TUTORIAL.md index 5c3b61e20..90972a914 100644 --- a/docs/RUGNUX_TUTORIAL.md +++ b/docs/RUGNUX_TUTORIAL.md @@ -115,8 +115,8 @@ small move: under 1 % in distance, and the gauge-weak beam centre within 15 px o or of the run's own measured centre, whichever is nearer), and the second pass re-indexes de novo and re-integrates at the refined geometry. The refined pass is the canonical `_*` output; the header-geometry pass merges only to choose the space group and to judge the -refined pass against, and writes no merged files of its own — no `_01.mtz`, `.cif`, `.hkl` or -`_01_image.dat`, and no `_01_plot.txt` or `_01_detector.jpg`. (Where a process file is asked for at all, with `--no-merge` or +refined pass against, and writes no merged files of its own — no `_01.mtz`, `.cif` or +`.hkl`, and no `_01_plot.txt` or `_01_detector.jpg`. (Where a process file is asked for at all, with `--no-merge` or `--write-process-h5`, each pass still writes its own, so `_01_process.h5` appears beside `_process.h5`.) Disable it with `--rotation-no-postrefine`. @@ -246,7 +246,7 @@ goniometer axis but you want per-frame stills processing anyway, add `--force-st [The results report](RUGNUX_REPORT.md#the-results-report). - `_plot.txt` — **the per-image table**, one row per processed image, whitespace-separated, with a single `#` legend line over its own columns so gnuplot plots it as it stands - (`plot '_plot.txt' using 1:4`). Rotation runs only. The columns are: + (`plot '_plot.txt' using 1:4`). The columns are: | Column | Meaning | |--------|---------| @@ -258,10 +258,12 @@ goniometer axis but you want per-frame stills processing anyway, add `--force-st | `scale_G` | Per-image scale factor fitted by scaling. | | `sigma_M_deg` | Mosaicity as the rocking-curve **standard deviation** σM in degrees — not a FWHM, and not XDS's `REFLECTING_RANGE`. See the note below. | | `cc_to_merge` | Correlation of the image's own observations with the merged data. | - | `merged` | 1 where the image's observations are in the merged data, 0 where they are not — an image that never indexed, one an earlier guard dropped, and one ΔCC1/2 convicted all read 0. | + | `cc_n` | Observations that correlation was computed over; 0 where none was. | + | `merged` | 1 where the image's observations are in the merged data, 0 where they are not — an image that never indexed, one an earlier guard dropped, and one ΔCC1/2 convicted all read 0. `nan` on a run that merged nothing, `--no-merge` included, where the question has no answer. | A quantity nothing measured for an image is written `nan`, which gnuplot and numpy both skip; the - columns never shift and a row is never left out. + columns never shift and a row is never left out. A stills run writes the table too, with `nan` in + the columns a sweep would fill; `--no-merge` writes it with the scaling columns `nan`. **Reading `sigma_M_deg`.** It is fitted per image — a maximum-likelihood Gaussian rocking width over that image's own indexed spots — but it is then **smoothed in frame order** over the same @@ -279,14 +281,12 @@ goniometer axis but you want per-frame stills processing anyway, add `--force-st by default). The picture is the mean projection over the frames the beam-stop detection sampled — the very image it tested — coloured with `jfjoch_viewer`'s default map: white to indigo for the counts, **grey** for the module and chip gaps, **coral** for the shadow the run detected, and - **magenta** for everything else the pixel mask holds. A shadow drawn where the background is - unbroken, or a dark patch the run left uncoloured, is visible at a glance; read it against the - `Beam stop shadow: N pixels ...` line in the log. -- `_image.dat` — the per-image **scaling** table: `image_number scale_G mosaicity_deg - wedge_deg cc_to_merge cc_n`, whitespace-separated with a `#` header line. It predates `_plot.txt` - and is kept because the in-tree comparison scripts read it and because it carries the per-image - mosaicity and smoothing wedge, which `_plot.txt` does not. **For anything new, script against - `_plot.txt`.** + **magenta** for everything else the pixel mask holds. The scale saturates at eight times the + median measured pixel, not at the brightest pixels, so the background spans the ramp and the Bragg + spots clip: it is the background the picture is read for, since a shadow is a place the background + is missing. A shadow drawn where the + background is unbroken, or a dark patch the run left uncoloured, is visible at a glance; read it + against the `Beam stop shadow: N pixels ...` line in the log. Merged statistics (⟨I/σ⟩, CC1/2, completeness, …), the error model and timing are printed to the console. By default the written resolution is trimmed automatically where CC1/2 falls off diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index cf7b6df80..a1bff254f 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -78,7 +78,7 @@ enum class FrameDisposition { const char *FrameDispositionCode(FrameDisposition disposition); // "merged", "downgraded", "rejected" struct SweepQualityRange { - // Inclusive, in processed-image ordinals - the numbering of _image.dat and of every other + // Inclusive, in processed-image ordinals - the numbering of _plot.txt and of every other // per-image array rugnux writes. With -s/--stride the source image is start + ordinal * stride. int first_image = 0; int last_image = 0; diff --git a/image_analysis/scale_merge/ScalingResult.cpp b/image_analysis/scale_merge/ScalingResult.cpp index 1a02d09e0..d5537cc3c 100644 --- a/image_analysis/scale_merge/ScalingResult.cpp +++ b/image_analysis/scale_merge/ScalingResult.cpp @@ -3,9 +3,6 @@ #include "ScalingResult.h" #include -#include - -#include "../../common/JFJochException.h" ScalingResult::ScalingResult(size_t n) : image_scale_g(n, NAN), @@ -28,28 +25,3 @@ ScalingResult::ScalingResult(const std::vector &v) image_cc_n[i] = v[i].image_scale_cc_n.value_or(0); } } - -void ScalingResult::SaveToFile(const std::string &filename) { - const std::string img_path = filename + "_image.dat"; - std::ofstream img_file(img_path, std::ofstream::out | std::ofstream::trunc); - if (!img_file) { - throw JFJochException(JFJochExceptionCategory::FileWriteError, - "Cannot open " + img_path + " for writing"); - } - - // Header so the columns are self-describing (lines starting with '#' are comments for gnuplot/numpy). - img_file << "# image_number scale_G mosaicity_deg wedge_deg cc_to_merge cc_n\n"; - - for (size_t i = 0; i < image_scale_g.size(); ++i) { - img_file << i - << " " << image_scale_g[i] - << " " << mosaicity_deg[i] - << " " << rotation_wedge_deg[i] - << " " << image_cc[i] - << " " << image_cc_n[i] - << "\n"; - } - - img_file.close(); -} - diff --git a/image_analysis/scale_merge/ScalingResult.h b/image_analysis/scale_merge/ScalingResult.h index 717d1915a..466848600 100644 --- a/image_analysis/scale_merge/ScalingResult.h +++ b/image_analysis/scale_merge/ScalingResult.h @@ -4,7 +4,6 @@ #pragma once #include -#include #include "../IntegrationOutcome.h" struct ScalingResult { @@ -15,5 +14,4 @@ struct ScalingResult { std::vector image_cc_n; explicit ScalingResult(size_t n); explicit ScalingResult(const std::vector &v); - void SaveToFile(const std::string &filename); }; diff --git a/rugnux/DiagnosticOutput.cpp b/rugnux/DiagnosticOutput.cpp index 1c4d45e86..c7ba56fd3 100644 --- a/rugnux/DiagnosticOutput.cpp +++ b/rugnux/DiagnosticOutput.cpp @@ -16,9 +16,17 @@ #include "../preview/JFJochJPEG.h" namespace { - // The brightest 0.1 % of the measured pixels saturate, as the preview and the viewer set their - // contrast (preview/PreviewImage.cpp): a handful of hot pixels would otherwise take the whole - // range and leave the background - which is what the shadow is read off - a flat white. + // The top of the ramp is eight times the MEDIAN of the measured pixels, not the brightest 0.1 % + // the preview and the viewer saturate at (preview/PreviewImage.cpp). That percentile belongs to + // the Bragg peaks, and with it setting the range the background lands in the first few per cent + // of the scale and reads as a flat white - which is exactly what the picture has to be read for, + // since a shadow is a place the background is MISSING. The median is pure background whatever + // the crystal, and eight times it lands between the 94th and the 99th percentile of the + // projection on every test set measured, which is the top of the background: the background + // spans the ramp radially, and the spots clip, which costs nothing here as they are not what is + // being judged. Twice the median - the first value tried - lands at the 73rd to 90th percentile + // instead and floods the whole inner half of the detector to a flat indigo, hiding the deficit + // just as the old contrast did, at the other end of the scale. float SaturationValue(const std::vector &projection, const std::vector &mask) { std::vector valid; valid.reserve(projection.size()); @@ -27,14 +35,18 @@ namespace { valid.push_back(projection[i]); if (valid.empty()) return 1.0f; - const size_t rank = valid.size() - std::max(1, valid.size() / 1000); - std::nth_element(valid.begin(), valid.begin() + rank, valid.end()); - return std::max(valid[rank], 1.0f); + const size_t mid = valid.size() / 2; + std::nth_element(valid.begin(), valid.begin() + mid, valid.end()); + return std::max(8.0f * valid[mid], 1.0f); } float Element(const std::vector &v, size_t i) { return i < v.size() ? v[i] : NAN; } + + int Element(const std::vector &v, size_t i) { + return i < v.size() ? v[i] : 0; + } } std::vector DetectorDiagnosticRGB(const PixelMask &pixel_mask, @@ -101,9 +113,17 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo // One comment line, its names over their own columns, and nothing else a plotting program has // to be told to skip. The '#' takes the first character of the image-number field. - file << "#" << fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>7}\n", + file << "#" << fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}" + " {:>7}\n", "image", "angle_deg", "bkg", "resolution_A", "spots", "scale_G", - "sigma_M_deg", "cc_to_merge", "merged"); + "sigma_M_deg", "cc_to_merge", "cc_n", "merged"); + + // Whether "merged" has anything to answer at all. With no merge behind the run - --no-merge, or + // a sweep nothing indexed on - a column of zeros would read as a sweep every frame of which was + // thrown out, which is a different statement from "no merge was made". + const bool merge_ran = !plot.frame_disposition.empty() + || std::any_of(plot.scale_g.begin(), plot.scale_g.end(), + [](float g) { return std::isfinite(g); }); for (size_t i = 0; i < image_count; i++) { const float angle = goniometer @@ -120,9 +140,10 @@ void WritePerImagePlot(const std::string &output_prefix, const PerImagePlot &plo : std::isfinite(scale_g); file << fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}" - " {:>12.4f} {:>7}\n", + " {:>12.4f} {:>8} {:>7}\n", i, angle, Element(plot.bkg_estimate, i), Element(plot.resolution_A, i), Element(plot.spot_count, i), scale_g, Element(plot.mosaicity_deg, i), - Element(plot.cc_to_merge, i), merged); + Element(plot.cc_to_merge, i), Element(plot.cc_n, i), + merge_ran ? std::to_string(merged) : "nan"); } } diff --git a/rugnux/DiagnosticOutput.h b/rugnux/DiagnosticOutput.h index 21009f81d..0566b2da5 100644 --- a/rugnux/DiagnosticOutput.h +++ b/rugnux/DiagnosticOutput.h @@ -28,9 +28,9 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment, const std::vector &mean_projection); // The columns of _plot.txt. Every vector is indexed by the processed-image ordinal - the -// numbering of _image.dat and of every other per-image array rugnux writes - and a value -// nothing measured is NAN. frame_disposition holds FrameDisposition (Merge.h) and is empty when the -// sweep-quality diagnostic did not run. +// numbering of every per-image array rugnux writes - and a value nothing measured is NAN. +// frame_disposition holds FrameDisposition (Merge.h) and is empty when the sweep-quality diagnostic +// did not run. struct PerImagePlot { std::vector bkg_estimate; std::vector resolution_A; @@ -38,6 +38,7 @@ struct PerImagePlot { std::vector scale_g; std::vector mosaicity_deg; std::vector cc_to_merge; + std::vector cc_n; std::vector frame_disposition; }; diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 10a46f362..4c82463ff 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -7184,7 +7184,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // sweep (the per-image scaling the rotation merge already fits, binned by frame = dose). A // per-image CC that falls, and/or a mosaicity that rises, with frame number is the classic // radiation-damage signature - a data-quality-vs-dose read complementary to the fitted decay - // correction. The full per-image table is written to _scaling.txt for detail. Both this + // correction. The full per-image table is written to _plot.txt for detail. Both this // and the sweep-quality report below are text, so a superseded pass makes neither. if (experiment_.IsRotationIndexing() && !superseded) { const auto &outs = indexer->GetIntegrationOutcome(); @@ -7246,7 +7246,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // Sweep-quality report: the stretches of the sweep over which the crystal delivered much // less than the rest of the run, what each one looks like, what became of it and what // keeping it costs the merged intensities. Frame numbers are processed-image ordinals, - // inclusive, as in _image.dat. + // inclusive, as in _plot.txt. const auto &sq = sm.statistics.sweep_quality; if (sq.measured) { const int n_sweep = sq.frames_merged + sq.frames_downgraded + sq.frames_rejected; @@ -7383,11 +7383,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b WriteReflections(sm.merged, *result.consensus_cell, experiment_, sm.statistics, em_report, result.twinning, config_.output_prefix, static_cast(std::max(1, config_.nthreads))); - // Per-image scaling table (G, B-factor, mosaicity, wedge, CC) for inspection / XDS - // comparison. The offline self-scaling result is otherwise not exposed (process.h5's - // per-image arrays are only filled on the online per-image path). Sourced from the - // partials, which carry the first-pass per-image scale. - ScalingResult(indexer->GetIntegrationOutcome()).SaveToFile(config_.output_prefix); // P1 cross-check dataset. The group the files above are written in was chosen by the // search, and if that choice is wrong nothing in them says so - every statistic was @@ -7559,11 +7554,12 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b /*trusted_path=*/true); } - // The two files a person checks the run by eye with. Rotation only - a stills run has no spindle - // angle and no sweep to project - and not on the geometry pre-pass or a pass that is thrown - // away, which would otherwise leave a stale _01 copy beside the real one. - if (full && write_files && !cancelled_ && !geometry_prepass && !superseded - && experiment_.IsRotationIndexing()) { + // The two files a person checks the run by eye with. Not on the geometry pre-pass or a pass that + // is thrown away, which would otherwise leave a stale _01 copy beside the real one. The table is + // written on a stills run too - it is the only per-image scaling table there is, and the columns + // a stills run cannot fill come out as nan; the picture is rotation-only at its source, since + // only the rotation pre-scan builds the projection it draws. + if (full && write_files && !cancelled_ && !geometry_prepass && !superseded) { if (!detector_diagnostic_jpeg_.empty()) { const std::string path = config_.output_prefix + "_detector.jpg"; std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); @@ -7580,11 +7576,15 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b plots.GetPlotRaw(plot.resolution_A, PlotType::ResolutionEstimate, ""); plots.GetPlotRaw(plot.spot_count, PlotType::SpotCount, ""); if (indexer) { - // The same source as _image.dat, so the two files cannot disagree. + // The offline self-scaling result is otherwise not exposed - process.h5's per-image + // arrays are only filled on the online per-image path - so these columns are the only + // place a person or a comparison script can read it. Sourced from the partials, which + // carry the first-pass per-image scale. const ScalingResult scaling(indexer->GetIntegrationOutcome()); plot.scale_g = scaling.image_scale_g; plot.mosaicity_deg = scaling.mosaicity_deg; plot.cc_to_merge = scaling.image_cc; + plot.cc_n = scaling.image_cc_n; } if (result.has_merge_statistics) plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition; diff --git a/rugnux_stills_ab.py b/rugnux_stills_ab.py index eb8fa0366..c0190da1e 100755 --- a/rugnux_stills_ab.py +++ b/rugnux_stills_ab.py @@ -300,8 +300,8 @@ def parse_ring_loss(log): return {} -def parse_image_dat(path): - """Per-image scale health from _image.dat. +def parse_plot_txt(path): + """Per-image scale health from _plot.txt. This is here because it is what tells a real result from a spurious one. A stills merge is ScaleOnTheFly's per-image scale G, and images whose G collapses more than 50x below the run @@ -313,16 +313,17 @@ def parse_image_dat(path): being tested. Print both arms' numbers and the difference is obvious.""" if not path.exists(): return {} + # Column 6 (0-based 5) of _plot.txt is scale_G; see the file's own '#' legend line. g, n = [], 0 for line in path.read_text(errors="replace").splitlines(): if line.startswith("#"): continue f = line.split() - if len(f) < 2: + if len(f) < 6: continue n += 1 try: - v = float(f[1]) + v = float(f[5]) except ValueError: continue if v == v and v > 0: @@ -337,7 +338,7 @@ def parse_image_dat(path): def collect(cif, report, log=None): r = parse_cif(cif) - r.update(parse_image_dat(pathlib.Path(str(cif)[:-4] + "_image.dat"))) + r.update(parse_plot_txt(pathlib.Path(str(cif)[:-4] + "_plot.txt"))) rep = parse_report(report) r["index_rate"] = _f(rep.get("INDEXING_RATE"), 100.0) r["images"] = _i(rep.get("IMAGES_PROCESSED")) @@ -743,7 +744,7 @@ def main(): print(f" {'arm':<12} {'images scaled':>14} {'median G':>10} {'5th pct G':>11} " f"(a median G far from 1, or an arm-to-arm change in 'images scaled', means the") print(f" {'':<12} {'':>14} {'':>10} {'':>11} " - f"per-image scaling - not this change - is moving the merge; see parse_image_dat)") + f"per-image scaling - not this change - is moving the merge; see parse_plot_txt)") warn_scale = False for name, reps in per_arm.items(): for label, r, err, _ in reps: diff --git a/tests/DiagnosticOutputTest.cpp b/tests/DiagnosticOutputTest.cpp index bee2592fa..5cde8a597 100644 --- a/tests/DiagnosticOutputTest.cpp +++ b/tests/DiagnosticOutputTest.cpp @@ -36,8 +36,10 @@ namespace { TEST_CASE("DiagnosticImage_Colors") { ColorScale scale; - // One pixel of each kind, in the order the renderer tests them. + // One pixel of each kind, in the order the renderer tests them. Two measured pixels read zero, + // so the median is zero and the third one saturates. std::vector mask_bits{ + 0, // plain, no counts 0, // plain, no counts 0, // plain, the brightest pixel 1u << PixelMask::ModuleGapPixelBit, @@ -46,20 +48,44 @@ TEST_CASE("DiagnosticImage_Colors") { (1u << PixelMask::ModuleGapPixelBit) | (1u << PixelMask::BeamStopPixelBit), (1u << PixelMask::BeamStopPixelBit) | (1u << PixelMask::NoisyPixelBit)}; const PixelMask mask(mask_bits); - const std::vector projection{0.0f, 1000.0f, NAN, NAN, NAN, NAN, NAN}; + const std::vector projection{0.0f, 0.0f, 1000.0f, NAN, NAN, NAN, NAN, NAN}; const auto image = DetectorDiagnosticRGB(mask, projection); REQUIRE(image.size() == mask_bits.size()); CHECK(same(image[0], scale.Apply(0.0f))); // white: no counts - CHECK(same(image[1], scale.Apply(1.0f))); // indigo: the saturation value - CHECK(same(image[2], scale.Apply(ColorScaleSpecial::Gap))); // grey - CHECK(same(image[3], scale.Apply(ColorScaleSpecial::BeamStop))); // coral - CHECK(same(image[4], scale.Apply(ColorScaleSpecial::BadPixel))); // magenta + CHECK(same(image[2], scale.Apply(1.0f))); // indigo: the saturation value + CHECK(same(image[3], scale.Apply(ColorScaleSpecial::Gap))); // grey + CHECK(same(image[4], scale.Apply(ColorScaleSpecial::BeamStop))); // coral + CHECK(same(image[5], scale.Apply(ColorScaleSpecial::BadPixel))); // magenta // A gap cannot be shadowed - it carries no background for the detection to miss - so the gap // wins; every other mask bit loses to the shadow, which is what the picture is there to show. - CHECK(same(image[5], scale.Apply(ColorScaleSpecial::Gap))); - CHECK(same(image[6], scale.Apply(ColorScaleSpecial::BeamStop))); + CHECK(same(image[6], scale.Apply(ColorScaleSpecial::Gap))); + CHECK(same(image[7], scale.Apply(ColorScaleSpecial::BeamStop))); +} + +TEST_CASE("DiagnosticImage_Contrast") { + ColorScale scale; + + // The ramp saturates at eight times the MEDIAN of the measured pixels - 128 here, so 1024 - so + // the background spans the ramp and the peak clips. Set from the brightest pixels instead, as + // the preview and the viewer set their contrast, the peak would take the whole range and this + // background would sit at the very bottom of the scale, a flat white: a missing background is + // what a shadow is read off, so that picture cannot be judged. + std::vector mask_bits{0, 0, 0, 0, 0}; + const PixelMask mask(mask_bits); + const std::vector projection{0.0f, 128.0f, 128.0f, 1024.0f, 5000.0f}; + + const auto image = DetectorDiagnosticRGB(mask, projection); + const auto &lut = scale.LUTData(); + REQUIRE(image.size() == projection.size()); + + CHECK(same(image[0], lut.front())); + CHECK(same(image[1], lut[lut.size() / 8])); + CHECK(same(image[2], lut[lut.size() / 8])); + CHECK(same(image[3], lut.back())); + // Clipped, and that costs nothing here - the spots are not what the picture is judged on. + CHECK(same(image[4], lut.back())); } TEST_CASE("DiagnosticPlot_Format") { @@ -72,6 +98,7 @@ TEST_CASE("DiagnosticPlot_Format") { plot.scale_g = {1.0f, 1.1f, NAN, 0.9f}; plot.mosaicity_deg = {0.12f, 0.12f, NAN, 0.13f}; plot.cc_to_merge = {0.98f, 0.97f, NAN, 0.5f}; + plot.cc_n = {500, 480, 0, 120}; plot.frame_disposition = {static_cast(FrameDisposition::Merged), static_cast(FrameDisposition::Downgraded), static_cast(FrameDisposition::Rejected), @@ -89,11 +116,12 @@ TEST_CASE("DiagnosticPlot_Format") { CHECK(lines[i].find('#') == std::string::npos); const auto legend = Columns(lines[0]); - REQUIRE(legend.size() == 9); + REQUIRE(legend.size() == 10); CHECK(legend[0] == "#image"); CHECK(legend[1] == "angle_deg"); CHECK(legend[6] == "sigma_M_deg"); - CHECK(legend[8] == "merged"); + CHECK(legend[8] == "cc_n"); + CHECK(legend[9] == "merged"); for (size_t i = 1; i < lines.size(); i++) { const auto row = Columns(lines[i]); @@ -106,10 +134,13 @@ TEST_CASE("DiagnosticPlot_Format") { CHECK(std::stof(Columns(lines[3])[1]) == 11.25f); // Merged and downgraded frames are in the merged data; a rejected one is not. - CHECK(Columns(lines[1])[8] == "1"); - CHECK(Columns(lines[2])[8] == "1"); - CHECK(Columns(lines[3])[8] == "0"); - CHECK(Columns(lines[4])[8] == "0"); + CHECK(Columns(lines[1])[9] == "1"); + CHECK(Columns(lines[2])[9] == "1"); + CHECK(Columns(lines[3])[9] == "0"); + CHECK(Columns(lines[4])[9] == "0"); + + // The count the correlation beside it was taken over. + CHECK(Columns(lines[1])[8] == "500"); // A frame nothing measured leaves its own columns unplottable, not the row missing. CHECK(Columns(lines[3])[3] == "nan"); @@ -125,11 +156,29 @@ TEST_CASE("DiagnosticPlot_NoDisposition") { const auto lines = ReadLines(prefix + "_plot.txt"); REQUIRE(lines.size() == 4); - CHECK(Columns(lines[1])[8] == "1"); - CHECK(Columns(lines[2])[8] == "0"); - CHECK(Columns(lines[3])[8] == "1"); + CHECK(Columns(lines[1])[9] == "1"); + CHECK(Columns(lines[2])[9] == "0"); + CHECK(Columns(lines[3])[9] == "1"); // No goniometer: the angle column is present and unplottable, so the columns never shift. - CHECK(Columns(lines[1]).size() == 9); + CHECK(Columns(lines[1]).size() == 10); CHECK(Columns(lines[1])[1] == "nan"); } + +TEST_CASE("DiagnosticPlot_NoMerge") { + const std::string prefix = "diagnostic_plot_nomerge_test"; + + // Nothing was merged - --no-merge, or a sweep nothing indexed on - so "merged" has no answer to + // give and must not read as a sweep every frame of which was thrown out. + PerImagePlot plot; + plot.bkg_estimate = {12.0f, 13.0f}; + plot.scale_g = {NAN, NAN}; + WritePerImagePlot(prefix, plot, {}, 2); + + const auto lines = ReadLines(prefix + "_plot.txt"); + REQUIRE(lines.size() == 3); + CHECK(Columns(lines[1])[9] == "nan"); + CHECK(Columns(lines[2])[9] == "nan"); + // The columns a run without a merge still fills are there. + CHECK(std::stof(Columns(lines[1])[2]) == 12.0f); +}