Files
Jungfraujoch/tests/DiagnosticOutputTest.cpp
leonarski_fandClaude Opus 5.5 7bc98279a9 rugnux: image counts per stage and a per-image merged vector
The report now states how many images made it through each stage, as
counts beside the indexing rate: IMAGES_INDEXED and LATTICES_INDEXED
(section 2; a stills measure, --developer on rotation like
INDEXING_RATE), IMAGES_INTEGRATED (section 2) and IMAGES_MERGED
(section 3), plus an "Images" summary line and a console line. An image
is integrated on its main lattice only - extra lattices are indexed but
not integrated - so integrated images are also integrated lattices.

The merge records per image whether any of its observations reached
it (MergeStatistics::image_merged): on stills from MergeOnTheFly's
AddImages (past the cell form, scale and --min-image-cc), on rotation
from the disposition ledger (not rejected). It drives the merged column
of _plot.txt (which on stills used to be approximated by "has a scale"),
goes to the _process.h5 as /entry/MX/imageMerged (and through CBOR),
is read back by the reader, replaces MergedCrystals for --mode scale's
INPUT_nn rows, and --mode scale now writes a _plot.txt for its merge.
_plot.txt gains an indexed column at the end, so no column moves.

jfjoch_viewer's image list gets a Merged column and, from the Index and
Merged header context menus, Yes/No/N/A row filters that combine (e.g.
indexed but not merged); a file without imageMerged shows N/A.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3
2026-11-19 13:51:31 +01:00

293 lines
12 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_test_macros.hpp>
#include <algorithm>
#include <cmath>
#include <fstream>
#include <sstream>
#include "../rugnux/DiagnosticOutput.h"
#include "../rugnux/RugnuxOutputFiles.h"
#include "../image_analysis/scale_merge/Merge.h"
namespace {
bool same(const rgb &a, const rgb &b) {
return a.r == b.r && a.g == b.g && a.b == b.b;
}
// The columns of one row of <prefix>_plot.txt, whitespace-separated.
std::vector<std::string> Columns(const std::string &line) {
std::istringstream is(line);
std::vector<std::string> ret;
for (std::string token; is >> token;)
ret.push_back(token);
return ret;
}
// One input, its images in order: what a run of one file writes.
void OneDataset(PerImagePlot &plot, size_t n) {
plot.image_count = n;
plot.dataset.assign(n, 1);
plot.image.resize(n);
for (size_t i = 0; i < n; i++)
plot.image[i] = static_cast<int64_t>(i);
}
// The table's rows: the lines after its '#' header.
std::vector<std::string> Rows(const std::vector<std::string> &lines) {
std::vector<std::string> ret;
for (const auto &line : lines)
if (line[0] != '#')
ret.push_back(line);
return ret;
}
std::vector<std::string> ReadLines(const std::string &path) {
std::ifstream f(path);
std::vector<std::string> ret;
for (std::string line; std::getline(f, line);)
ret.push_back(line);
return ret;
}
}
TEST_CASE("DiagnosticImage_Colors") {
ColorScale scale;
// One pixel of each kind, in the order the renderer tests them. Two measured pixels read zero,
// so the median is zero and the third one saturates.
std::vector<uint32_t> mask_bits{
0, // plain, no counts
0, // plain, no counts
0, // plain, the brightest pixel
1u << PixelMask::ModuleGapPixelBit,
1u << PixelMask::BeamStopPixelBit,
1u << PixelMask::UserMaskedPixelBit,
(1u << PixelMask::ModuleGapPixelBit) | (1u << PixelMask::BeamStopPixelBit),
(1u << PixelMask::BeamStopPixelBit) | (1u << PixelMask::NoisyPixelBit)};
const PixelMask mask(mask_bits);
const std::vector<float> projection{0.0f, 0.0f, 1000.0f, NAN, NAN, NAN, NAN, NAN};
const auto image = DetectorDiagnosticRGB(mask, projection);
REQUIRE(image.size() == mask_bits.size());
CHECK(same(image[0], scale.Apply(0.0f))); // white: no counts
CHECK(same(image[2], scale.Apply(1.0f))); // indigo: the saturation value
CHECK(same(image[3], scale.Apply(ColorScaleSpecial::Gap))); // grey
CHECK(same(image[4], scale.Apply(ColorScaleSpecial::BeamStop))); // coral
CHECK(same(image[5], scale.Apply(ColorScaleSpecial::BadPixel))); // magenta
// A gap cannot be shadowed - it carries no background for the detection to miss - so the gap
// wins; every other mask bit loses to the shadow, which is what the picture is there to show.
CHECK(same(image[6], scale.Apply(ColorScaleSpecial::Gap)));
CHECK(same(image[7], scale.Apply(ColorScaleSpecial::BeamStop)));
}
TEST_CASE("DiagnosticImage_Contrast") {
ColorScale scale;
// The ramp saturates at eight times the MEDIAN of the measured pixels - 128 here, so 1024 - so
// the background spans the ramp and the peak clips. Set from the brightest pixels instead, as
// the preview and the viewer set their contrast, the peak would take the whole range and this
// background would sit at the very bottom of the scale, a flat white: a missing background is
// what a shadow is read off, so that picture cannot be judged.
std::vector<uint32_t> mask_bits{0, 0, 0, 0, 0};
const PixelMask mask(mask_bits);
const std::vector<float> projection{0.0f, 128.0f, 128.0f, 1024.0f, 5000.0f};
const auto image = DetectorDiagnosticRGB(mask, projection);
const auto &lut = scale.LUTData();
REQUIRE(image.size() == projection.size());
CHECK(same(image[0], lut.front()));
CHECK(same(image[1], lut[lut.size() / 8]));
CHECK(same(image[2], lut[lut.size() / 8]));
CHECK(same(image[3], lut.back()));
// Clipped, and that costs nothing here - the spots are not what the picture is judged on.
CHECK(same(image[4], lut.back()));
}
TEST_CASE("DiagnosticPlot_Format") {
const std::string prefix = "diagnostic_plot_format_test";
PerImagePlot plot;
OneDataset(plot, 4);
plot.image = {10, 12, 14, 16};
plot.angle_deg = {10.25f, 10.75f, 11.25f, 11.75f};
plot.bkg_estimate = {1.0f, 2.0f, 3.0f, 4.0f};
plot.resolution_A = {2.5f, 2.4f, NAN, 2.6f};
plot.spot_count = {100.0f, 120.0f, 0.0f, 90.0f};
plot.scale_g = {1.0f, 1.1f, NAN, 0.9f};
plot.mosaicity_deg = {0.12f, 0.12f, NAN, 0.13f};
plot.cc_to_merge = {0.98f, 0.97f, NAN, 0.5f};
plot.cc_n = {500, 480, 0, 120};
plot.frame_disposition = {static_cast<uint8_t>(FrameDisposition::Merged),
static_cast<uint8_t>(FrameDisposition::Downgraded),
static_cast<uint8_t>(FrameDisposition::Rejected),
static_cast<uint8_t>(FrameDisposition::Rejected)};
plot.merged = {1, 1, 0, 0};
plot.indexed = {1.0f, 1.0f, 0.0f, 1.0f};
WritePerImagePlot(prefix, plot);
const auto all = ReadLines(prefix + "_plot.txt");
// The header first, all of it comments, the last line of it the legend; no '#' after it.
size_t header = 0;
while (header < all.size() && all[header][0] == '#')
header++;
REQUIRE(header > 1);
const auto lines = std::vector<std::string>(all.begin() + static_cast<long>(header) - 1, all.end());
REQUIRE(lines.size() == 5);
for (size_t i = 1; i < lines.size(); i++)
CHECK(lines[i].find('#') == std::string::npos);
// Every column is described in the header.
const auto legend = Columns(lines[0]);
REQUIRE(legend.size() == 14);
for (size_t c = 1; c < legend.size(); c++) {
bool described = false;
for (size_t l = 0; l + 1 < header; l++)
described |= all[l].starts_with("# " + legend[c] + ":");
CHECK(described);
}
CHECK(legend[0] == "#ordinal");
CHECK(legend[1] == "dataset");
CHECK(legend[2] == "image");
CHECK(legend[3] == "angle_deg");
CHECK(legend[8] == "sigma_M_deg");
CHECK(legend[10] == "cc_n");
CHECK(legend[11] == "merged");
CHECK(legend[12] == "disposition");
CHECK(legend[13] == "indexed");
for (size_t i = 1; i < lines.size(); i++) {
const auto row = Columns(lines[i]);
REQUIRE(row.size() == legend.size());
CHECK(row[0] == std::to_string(i - 1));
CHECK(row[1] == "1");
CHECK(row[2] == std::to_string(10 + 2 * (i - 1)));
}
CHECK(std::stof(Columns(lines[1])[3]) == 10.25f);
CHECK(std::stof(Columns(lines[3])[3]) == 11.25f);
// Merged and downgraded frames are in the merged data; a rejected one is not.
CHECK(Columns(lines[1])[11] == "1");
CHECK(Columns(lines[2])[11] == "1");
CHECK(Columns(lines[3])[11] == "0");
CHECK(Columns(lines[4])[11] == "0");
// Graded: merged 1, downgraded 0.5, rejected 0 - and a rejected frame that never got a scale
// never reached the merge, so nothing is known of it.
CHECK(Columns(lines[1])[12] == "1");
CHECK(Columns(lines[2])[12] == "0.5");
CHECK(Columns(lines[3])[12] == "nan");
CHECK(Columns(lines[4])[12] == "0");
// Indexed by the per-image indexer, whatever became of the image in the merge.
CHECK(Columns(lines[3])[13] == "0");
CHECK(Columns(lines[4])[13] == "1");
// The count the correlation beside it was taken over.
CHECK(Columns(lines[1])[10] == "500");
// A frame nothing measured leaves its own columns unplottable, not the row missing.
CHECK(Columns(lines[3])[5] == "nan");
}
TEST_CASE("DiagnosticPlot_NoDisposition") {
const std::string prefix = "diagnostic_plot_nodisp_test";
// Stills, with no sweep-quality diagnostic: merged is the merge's own answer, and the disposition
// follows it - a crystal scaled but left out (here the third) is rejected, one never scaled is nan.
PerImagePlot plot;
OneDataset(plot, 3);
plot.scale_g = {1.0f, NAN, 0.8f};
plot.merged = {1, 0, 0};
WritePerImagePlot(prefix, plot);
const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
REQUIRE(lines.size() == 3);
CHECK(Columns(lines[0])[11] == "1");
CHECK(Columns(lines[1])[11] == "0");
CHECK(Columns(lines[2])[11] == "0");
CHECK(Columns(lines[0])[12] == "1");
CHECK(Columns(lines[1])[12] == "nan");
CHECK(Columns(lines[2])[12] == "0");
// No goniometer: the angle column is present and unplottable, so the columns never shift; no
// per-image indexing (--mode scale) leaves indexed nan.
CHECK(Columns(lines[0]).size() == 14);
CHECK(Columns(lines[0])[3] == "nan");
CHECK(Columns(lines[0])[13] == "nan");
}
TEST_CASE("DiagnosticPlot_NoMerge") {
const std::string prefix = "diagnostic_plot_nomerge_test";
// Nothing was merged - --no-merge, or a sweep nothing indexed on - so "merged" has no answer to
// give and must not read as a sweep every frame of which was thrown out.
PerImagePlot plot;
OneDataset(plot, 2);
plot.bkg_estimate = {12.0f, 13.0f};
plot.scale_g = {NAN, NAN};
WritePerImagePlot(prefix, plot);
const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
REQUIRE(lines.size() == 2);
CHECK(Columns(lines[0])[11] == "nan");
CHECK(Columns(lines[1])[11] == "nan");
CHECK(Columns(lines[0])[12] == "nan");
// The columns a run without a merge still fills are there.
CHECK(std::stof(Columns(lines[0])[4]) == 12.0f);
}
TEST_CASE("DiagnosticPlot_JointSweeps") {
const std::string prefix = "diagnostic_plot_joint_test";
// Two sweeps of 2 and 3 frames, as their own runs measured them.
PerImagePlot a, b;
OneDataset(a, 2);
a.angle_deg = {0.05f, 0.15f};
a.bkg_estimate = {5.0f, 6.0f};
OneDataset(b, 3);
b.angle_deg = {90.05f, 90.15f, 90.25f};
b.spot_count = {7.0f, 8.0f, 9.0f};
b.indexed = {1.0f, 0.0f, 1.0f};
// The joint merge's table: its rows are the sweeps' frames end to end, its scale the joint merge's.
PerImagePlot joint;
OneDataset(joint, 5);
joint.dataset_files = {"/data/a_master.h5", "/data/b_master.h5"};
joint.scale_g = {1.0f, 1.0f, 2.0f, 2.0f, 2.0f};
FillJointPlotRows(joint, 0, 1, a);
FillJointPlotRows(joint, 2, 2, b);
WritePerImagePlot(prefix, joint);
// The header names the file of each dataset, numbered as the dataset column.
const auto all = ReadLines(prefix + "_plot.txt");
CHECK(std::count(all.begin(), all.end(), "# dataset 1: /data/a_master.h5") == 1);
CHECK(std::count(all.begin(), all.end(), "# dataset 2: /data/b_master.h5") == 1);
const auto lines = Rows(all);
REQUIRE(lines.size() == 5);
const std::vector<std::string> dataset = {"1", "1", "2", "2", "2"};
const std::vector<std::string> image = {"0", "1", "0", "1", "2"};
for (size_t i = 0; i < lines.size(); i++) {
const auto row = Columns(lines[i]);
CHECK(row[0] == std::to_string(i));
CHECK(row[1] == dataset[i]);
CHECK(row[2] == image[i]);
}
CHECK(std::stof(Columns(lines[0])[3]) == 0.05f);
CHECK(std::stof(Columns(lines[4])[3]) == 90.25f);
CHECK(std::stof(Columns(lines[1])[4]) == 6.0f);
CHECK(Columns(lines[2])[4] == "nan");
CHECK(std::stof(Columns(lines[3])[6]) == 8.0f);
CHECK(std::stof(Columns(lines[2])[7]) == 2.0f);
// Each sweep's own per-image indexing.
CHECK(Columns(lines[0])[13] == "nan");
CHECK(Columns(lines[3])[13] == "0");
CHECK(Columns(lines[4])[13] == "1");
}