Files
Jungfraujoch/tests/JFJochReceiverLiteTest.cpp
leonarski_fandClaude Opus 5.5 15af14357d pixel_sum on the CPU and GPU path, under the device mask, and read back from HDF5
pixel_sum existed only on the FPGA path, so a DECTRIS master file carried an all-zero
/entry/image/pixel_sum. The CPU and GPU image preprocessors now compute it in the pass that takes the
minimum and maximum, as a signed 64-bit sum, and MXAnalysisWithoutFPGA reports it: Lite, live stills,
live rotation and rugnux. On the simulated card it equals the card's own sum, for 16-bit and for
summed 32-bit frames.

The sum excludes error and saturated pixels and the device mask (detector and calibration). The hot
pixels and the beam-stop shadow a run finds on its own frames do not change it, though they stay masked
in the image and in the other statistics, so the number does not depend on what a run decided. Those
two have bits of their own (PixelMask::RunMaskBits); the byte-per-pixel mask keeps the difference, 1 for
the device mask and 2 for the run's alone, and the CPU preprocessor gets a packed mask of the second.
Live stills reports this sum of the assembled image in place of the card's, and with the run's mask out
of it the two agree on the test frames.

Frames a live rotation run does not reach stay 0 in the master file, as before, and have no dataset in
the data files. The reader now reads pixel_sum into the dataset and the per-image message; a file
without it opens as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 15:19:34 +01:00

300 lines
12 KiB
C++

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <fstream>
#include "../writer/HDF5Objects.h"
#include "../receiver/JFJochReceiverService.h"
#include "../writer/StreamWriter.h"
#include "../image_pusher/HDF5FilePusher.h"
#include "../image_puller/TestImagePuller.h"
// PixelSigned(true) below is not incidental: the frames fed in come from compression_benchmark.h5,
// which stores signed int16, while a DECTRIS experiment declares unsigned by default. The receiver
// forwards images byte-for-byte, so the start message it re-emits has to describe them - otherwise
// the master file declares a type the data files do not have, which HDF5DataFile now refuses.
TEST_CASE("JFJochReceiverLite", "[JFJochReceiver]") {
Logger logger("JFJochReceiverLite");
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2)
.FilePrefix("crystal_test_lite").JungfrauConvPhotonCnt(false).PixelSigned(true).SetFileWriterFormat(
FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
PixelMask pixel_mask(experiment);
// Load example image
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
HDF5DataSet dataset(data, "/entry/data/data");
HDF5DataSpace file_space(dataset);
REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum());
REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum());
std::vector<int16_t> image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
std::vector<hsize_t> start = {4,0,0};
std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
dataset.ReadVector(image_conv, start, file_size);
HDF5FilePusher pusher;
auto puller = std::make_shared<TestImagePuller>();
StartMessage start_msg;
experiment.FillMessage(start_msg);
// As a DECTRIS detector sends it by default; the flags must reach the master file.
start_msg.countrate_correction_enabled = true;
start_msg.flatfield_enabled = true;
start_msg.countrate_correction_lookup_table = {0, 1, 3, 7};
start_msg.virtual_pixel_interpolation_enabled = true;
// The detector applied its own mask, not the one written with the data
start_msg.pixel_mask_enabled = true;
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)
});
DataMessage data_msg;
data_msg.image = CompressedImage(image_conv, file_space.GetDimensions()[2], file_space.GetDimensions()[1]);
for (int i = 0; i < 5; i++) {
data_msg.number = i;
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)
});
}
EndMessage end_msg{};
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(end_msg)
});
AcquisitionDeviceGroup group;
JFJochReceiverService service(group, logger, pusher);
service.NumThreads(nthreads);
// No progress value at the start of measurement
REQUIRE(!service.GetProgress().has_value());
SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings();
settings.signal_to_noise_threshold = 2.5;
settings.photon_count_threshold = 5;
settings.min_pix_per_spot = 1;
settings.max_pix_per_spot = 200;
settings.high_resolution_limit = 2.0;
settings.low_resolution_limit = 50.0;
service.SetSpotFindingSettings(settings);
service.Indexing(experiment.GetIndexingSettings());
service.Start(experiment, pixel_mask, nullptr, puller);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 1.0);
REQUIRE(receiver_out.status.indexing_rate);
CHECK(receiver_out.status.indexing_rate.value() == 1.0);
CHECK(receiver_out.status.images_sent == experiment.GetImageNum());
CHECK(!receiver_out.status.cancelled);
// No progress value at the end of the measurement
REQUIRE(!service.GetProgress().has_value());
HDF5ReadOnlyFile master("crystal_test_lite_master.h5");
CHECK(master.GetBool("/entry/instrument/detector/countrate_correction_applied"));
CHECK(master.GetBool("/entry/instrument/detector/flatfield_applied"));
CHECK(master.GetBool("/entry/instrument/detector/virtual_pixel_interpolation_applied"));
CHECK(!master.GetBool("/entry/instrument/detector/pixel_mask_applied"));
CHECK(master.ReadVector<uint32_t>("/entry/instrument/detector/countrate_correction_lookup_table")
== std::vector<uint32_t>{0, 1, 3, 7});
int64_t expected_sum = 0;
for (size_t i = 0; i < image_conv.size(); i++)
if (pixel_mask.GetMask()[i] == 0 && image_conv[i] != INT16_MIN && image_conv[i] < experiment.GetSaturationLimit())
expected_sum += image_conv[i];
CHECK(expected_sum != 0);
CHECK(master.ReadVector<int64_t>("/entry/image/pixel_sum") == std::vector<int64_t>(5, expected_sum));
}
TEST_CASE("JFJochReceiverLite_Cancel", "[JFJochReceiver]") {
Logger logger("JFJochReceiverLite_Cancel");
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2)
.FilePrefix("crystal_test_lite").JungfrauConvPhotonCnt(false).PixelSigned(true).SetFileWriterFormat(
FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
PixelMask pixel_mask(experiment);
HDF5FilePusher pusher;
auto puller = std::make_shared<TestImagePuller>();
AcquisitionDeviceGroup group;
JFJochReceiverService service(group, logger, pusher);
service.Indexing(experiment.GetIndexingSettings());
service.NumThreads(nthreads);
// No progress value at the start of measurement
REQUIRE(!service.GetProgress().has_value());
service.Start(experiment, pixel_mask, nullptr, puller);
std::this_thread::sleep_for(std::chrono::seconds(10));
service.Cancel(false);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 0.0);
CHECK(receiver_out.status.cancelled);
}
TEST_CASE("JFJochReceiverLite_DarkMask", "[JFJochReceiver]") {
Logger logger("JFJochReceiverLite_DarkMask");
RegisterHDF5Filter();
const uint16_t nthreads = 4;
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
DarkMaskSettings mask_settings;
mask_settings.NumberOfFrames(10).MaxCounts(1).MaxFramesWithCounts(5);
experiment.Mode(DetectorMode::DarkMask).ImportDarkMaskSettings(mask_settings);
PixelMask pixel_mask(experiment);
HDF5FilePusher pusher;
auto puller = std::make_shared<TestImagePuller>();
StartMessage start_msg;
experiment.FillMessage(start_msg);
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)
});
std::vector<uint16_t> image(experiment.GetPixelsNum(), 0);
image[1] = 1;
image[2] = 2;
image[3] = 3;
std::vector<uint16_t> image_last(experiment.GetPixelsNum(), 0);
image_last[1] = 0;
image_last[2] = 0;
image_last[3] = 0;
image_last[68] = 10000;
for (int i = 0; i < experiment.GetFrameNum(); i++) {
DataMessage data_msg;
if (i == experiment.GetFrameNum() - 1)
data_msg.image = CompressedImage(image_last, experiment.GetXPixelsNum(), experiment.GetYPixelsNum());
else
data_msg.image = CompressedImage(image, experiment.GetXPixelsNum(), experiment.GetYPixelsNum());
data_msg.number = i;
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)
});
}
EndMessage end_msg{};
puller->Put(ImagePullerOutput{
.cbor = std::make_shared<CBORStream2DeserializerOutput>(end_msg)
});
AcquisitionDeviceGroup group;
JFJochReceiverService service(group, logger, pusher);
service.NumThreads(nthreads);
// No progress value at the start of measurement
REQUIRE(!service.GetProgress().has_value());
service.Start(experiment, pixel_mask, nullptr, puller);
auto receiver_out = service.Stop();
CHECK(receiver_out.efficiency == 1.0);
CHECK(receiver_out.status.images_sent == 0);
CHECK(receiver_out.dark_mask_result.size() == experiment.GetPixelsNum());
CHECK(receiver_out.dark_mask_result[0] == 0);
CHECK(receiver_out.dark_mask_result[1] == 0);
CHECK(receiver_out.dark_mask_result[2] != 0);
CHECK(receiver_out.dark_mask_result[3] != 0);
CHECK(receiver_out.dark_mask_result[4] == 0);
CHECK(receiver_out.dark_mask_result[68] == 0);
CHECK(!receiver_out.status.cancelled);
// No progress value at the end of the measurement
REQUIRE(!service.GetProgress().has_value());
}
// Identical frames, so the run masks a large part of the detector as a shadow and as hot pixels. That mask
// is the run's own: the master file holds the sum under the device mask.
TEST_CASE("JFJochReceiverRugnux_PixelSumUnderDeviceMask", "[JFJochReceiver]") {
Logger logger(Catch::getResultCapture().getCurrentTestName());
RegisterHDF5Filter();
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
experiment.ImagesPerTrigger(20).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(10)
.FilePrefix("rugnux_live_pixel_sum").JungfrauConvPhotonCnt(false).PixelSigned(true)
.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, Coord(-1, 0, 0), {}));
PixelMask pixel_mask(experiment);
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
HDF5DataSet dataset(data, "/entry/data/data");
HDF5DataSpace file_space(dataset);
std::vector<int16_t> image_conv(file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
std::vector<hsize_t> start = {4, 0, 0};
std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
dataset.ReadVector(image_conv, start, file_size);
HDF5FilePusher pusher;
auto puller = std::make_shared<TestImagePuller>();
StartMessage start_msg;
experiment.FillMessage(start_msg);
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)});
DataMessage data_msg;
data_msg.image = CompressedImage(image_conv, file_space.GetDimensions()[2], file_space.GetDimensions()[1]);
for (int i = 0; i < experiment.GetImageNum(); i++) {
data_msg.number = i;
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)});
}
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(EndMessage{})});
AcquisitionDeviceGroup group;
JFJochReceiverService service(group, logger, pusher);
service.NumThreads(4);
service.RugnuxLive(true, nullptr, 4, 60);
service.SetSpotFindingSettings(DiffractionExperiment::DefaultDataProcessingSettings());
service.Indexing(experiment.GetIndexingSettings());
service.Start(experiment, pixel_mask, nullptr, puller);
service.Stop();
int64_t expected_sum = 0;
for (size_t i = 0; i < image_conv.size(); i++)
if (pixel_mask.GetMask()[i] == 0 && image_conv[i] != INT16_MIN && image_conv[i] < experiment.GetSaturationLimit())
expected_sum += image_conv[i];
CHECK(expected_sum != 0);
HDF5ReadOnlyFile master("rugnux_live_pixel_sum_master.h5");
CHECK(master.ReadVector<int64_t>("/entry/image/pixel_sum") == std::vector<int64_t>(20, expected_sum));
}