pixel_sum existed only on the FPGA path, so a DECTRIS master file carried an all-zero /entry/image/pixel_sum. The CPU and GPU image preprocessors now compute it in the pass that takes the minimum and maximum, as a signed 64-bit sum, and MXAnalysisWithoutFPGA reports it: Lite, live stills, live rotation and rugnux. On the simulated card it equals the card's own sum, for 16-bit and for summed 32-bit frames. The sum excludes error and saturated pixels and the device mask (detector and calibration). The hot pixels and the beam-stop shadow a run finds on its own frames do not change it, though they stay masked in the image and in the other statistics, so the number does not depend on what a run decided. Those two have bits of their own (PixelMask::RunMaskBits); the byte-per-pixel mask keeps the difference, 1 for the device mask and 2 for the run's alone, and the CPU preprocessor gets a packed mask of the second. Live stills reports this sum of the assembled image in place of the card's, and with the run's mask out of it the two agree on the test frames. Frames a live rotation run does not reach stay 0 in the master file, as before, and have no dataset in the data files. The reader now reads pixel_sum into the dataset and the per-image message; a file without it opens as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
300 lines
12 KiB
C++
300 lines
12 KiB
C++
// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
|
|
#include <fstream>
|
|
|
|
#include "../writer/HDF5Objects.h"
|
|
#include "../receiver/JFJochReceiverService.h"
|
|
#include "../writer/StreamWriter.h"
|
|
#include "../image_pusher/HDF5FilePusher.h"
|
|
#include "../image_puller/TestImagePuller.h"
|
|
|
|
// PixelSigned(true) below is not incidental: the frames fed in come from compression_benchmark.h5,
|
|
// which stores signed int16, while a DECTRIS experiment declares unsigned by default. The receiver
|
|
// forwards images byte-for-byte, so the start message it re-emits has to describe them - otherwise
|
|
// the master file declares a type the data files do not have, which HDF5DataFile now refuses.
|
|
TEST_CASE("JFJochReceiverLite", "[JFJochReceiver]") {
|
|
Logger logger("JFJochReceiverLite");
|
|
|
|
RegisterHDF5Filter();
|
|
|
|
const uint16_t nthreads = 4;
|
|
|
|
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
|
|
|
|
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2)
|
|
.FilePrefix("crystal_test_lite").JungfrauConvPhotonCnt(false).PixelSigned(true).SetFileWriterFormat(
|
|
FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
|
|
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
|
|
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
|
|
PixelMask pixel_mask(experiment);
|
|
|
|
// Load example image
|
|
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
|
|
HDF5DataSet dataset(data, "/entry/data/data");
|
|
HDF5DataSpace file_space(dataset);
|
|
|
|
REQUIRE(file_space.GetDimensions()[2] == experiment.GetXPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[1] == experiment.GetYPixelsNum());
|
|
std::vector<int16_t> image_conv (file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
|
|
|
|
std::vector<hsize_t> start = {4,0,0};
|
|
std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
|
|
dataset.ReadVector(image_conv, start, file_size);
|
|
|
|
HDF5FilePusher pusher;
|
|
|
|
auto puller = std::make_shared<TestImagePuller>();
|
|
|
|
StartMessage start_msg;
|
|
experiment.FillMessage(start_msg);
|
|
// As a DECTRIS detector sends it by default; the flags must reach the master file.
|
|
start_msg.countrate_correction_enabled = true;
|
|
start_msg.flatfield_enabled = true;
|
|
start_msg.countrate_correction_lookup_table = {0, 1, 3, 7};
|
|
start_msg.virtual_pixel_interpolation_enabled = true;
|
|
// The detector applied its own mask, not the one written with the data
|
|
start_msg.pixel_mask_enabled = true;
|
|
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)
|
|
});
|
|
|
|
DataMessage data_msg;
|
|
data_msg.image = CompressedImage(image_conv, file_space.GetDimensions()[2], file_space.GetDimensions()[1]);
|
|
|
|
for (int i = 0; i < 5; i++) {
|
|
data_msg.number = i;
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)
|
|
});
|
|
}
|
|
|
|
EndMessage end_msg{};
|
|
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(end_msg)
|
|
});
|
|
|
|
AcquisitionDeviceGroup group;
|
|
JFJochReceiverService service(group, logger, pusher);
|
|
|
|
service.NumThreads(nthreads);
|
|
|
|
// No progress value at the start of measurement
|
|
REQUIRE(!service.GetProgress().has_value());
|
|
|
|
SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings();
|
|
settings.signal_to_noise_threshold = 2.5;
|
|
settings.photon_count_threshold = 5;
|
|
settings.min_pix_per_spot = 1;
|
|
settings.max_pix_per_spot = 200;
|
|
settings.high_resolution_limit = 2.0;
|
|
settings.low_resolution_limit = 50.0;
|
|
service.SetSpotFindingSettings(settings);
|
|
service.Indexing(experiment.GetIndexingSettings());
|
|
|
|
service.Start(experiment, pixel_mask, nullptr, puller);
|
|
auto receiver_out = service.Stop();
|
|
|
|
CHECK(receiver_out.efficiency == 1.0);
|
|
REQUIRE(receiver_out.status.indexing_rate);
|
|
CHECK(receiver_out.status.indexing_rate.value() == 1.0);
|
|
CHECK(receiver_out.status.images_sent == experiment.GetImageNum());
|
|
CHECK(!receiver_out.status.cancelled);
|
|
|
|
// No progress value at the end of the measurement
|
|
REQUIRE(!service.GetProgress().has_value());
|
|
|
|
HDF5ReadOnlyFile master("crystal_test_lite_master.h5");
|
|
CHECK(master.GetBool("/entry/instrument/detector/countrate_correction_applied"));
|
|
CHECK(master.GetBool("/entry/instrument/detector/flatfield_applied"));
|
|
CHECK(master.GetBool("/entry/instrument/detector/virtual_pixel_interpolation_applied"));
|
|
CHECK(!master.GetBool("/entry/instrument/detector/pixel_mask_applied"));
|
|
CHECK(master.ReadVector<uint32_t>("/entry/instrument/detector/countrate_correction_lookup_table")
|
|
== std::vector<uint32_t>{0, 1, 3, 7});
|
|
|
|
int64_t expected_sum = 0;
|
|
for (size_t i = 0; i < image_conv.size(); i++)
|
|
if (pixel_mask.GetMask()[i] == 0 && image_conv[i] != INT16_MIN && image_conv[i] < experiment.GetSaturationLimit())
|
|
expected_sum += image_conv[i];
|
|
CHECK(expected_sum != 0);
|
|
CHECK(master.ReadVector<int64_t>("/entry/image/pixel_sum") == std::vector<int64_t>(5, expected_sum));
|
|
}
|
|
|
|
TEST_CASE("JFJochReceiverLite_Cancel", "[JFJochReceiver]") {
|
|
Logger logger("JFJochReceiverLite_Cancel");
|
|
|
|
RegisterHDF5Filter();
|
|
|
|
const uint16_t nthreads = 4;
|
|
|
|
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
|
|
|
|
experiment.ImagesPerTrigger(5).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(2)
|
|
.FilePrefix("crystal_test_lite").JungfrauConvPhotonCnt(false).PixelSigned(true).SetFileWriterFormat(
|
|
FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
|
|
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
|
|
.SetUnitCell(UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha =90, .beta = 90, .gamma = 90});
|
|
PixelMask pixel_mask(experiment);
|
|
|
|
HDF5FilePusher pusher;
|
|
|
|
auto puller = std::make_shared<TestImagePuller>();
|
|
|
|
AcquisitionDeviceGroup group;
|
|
JFJochReceiverService service(group, logger, pusher);
|
|
|
|
service.Indexing(experiment.GetIndexingSettings());
|
|
service.NumThreads(nthreads);
|
|
|
|
// No progress value at the start of measurement
|
|
REQUIRE(!service.GetProgress().has_value());
|
|
|
|
service.Start(experiment, pixel_mask, nullptr, puller);
|
|
std::this_thread::sleep_for(std::chrono::seconds(10));
|
|
service.Cancel(false);
|
|
|
|
auto receiver_out = service.Stop();
|
|
|
|
CHECK(receiver_out.efficiency == 0.0);
|
|
CHECK(receiver_out.status.cancelled);
|
|
}
|
|
|
|
TEST_CASE("JFJochReceiverLite_DarkMask", "[JFJochReceiver]") {
|
|
Logger logger("JFJochReceiverLite_DarkMask");
|
|
|
|
RegisterHDF5Filter();
|
|
|
|
const uint16_t nthreads = 4;
|
|
|
|
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
|
|
|
|
DarkMaskSettings mask_settings;
|
|
mask_settings.NumberOfFrames(10).MaxCounts(1).MaxFramesWithCounts(5);
|
|
experiment.Mode(DetectorMode::DarkMask).ImportDarkMaskSettings(mask_settings);
|
|
|
|
PixelMask pixel_mask(experiment);
|
|
|
|
HDF5FilePusher pusher;
|
|
|
|
auto puller = std::make_shared<TestImagePuller>();
|
|
|
|
StartMessage start_msg;
|
|
experiment.FillMessage(start_msg);
|
|
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)
|
|
});
|
|
|
|
|
|
std::vector<uint16_t> image(experiment.GetPixelsNum(), 0);
|
|
image[1] = 1;
|
|
image[2] = 2;
|
|
image[3] = 3;
|
|
|
|
std::vector<uint16_t> image_last(experiment.GetPixelsNum(), 0);
|
|
image_last[1] = 0;
|
|
image_last[2] = 0;
|
|
image_last[3] = 0;
|
|
image_last[68] = 10000;
|
|
|
|
for (int i = 0; i < experiment.GetFrameNum(); i++) {
|
|
DataMessage data_msg;
|
|
if (i == experiment.GetFrameNum() - 1)
|
|
data_msg.image = CompressedImage(image_last, experiment.GetXPixelsNum(), experiment.GetYPixelsNum());
|
|
else
|
|
data_msg.image = CompressedImage(image, experiment.GetXPixelsNum(), experiment.GetYPixelsNum());
|
|
data_msg.number = i;
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)
|
|
});
|
|
}
|
|
|
|
EndMessage end_msg{};
|
|
|
|
puller->Put(ImagePullerOutput{
|
|
.cbor = std::make_shared<CBORStream2DeserializerOutput>(end_msg)
|
|
});
|
|
|
|
AcquisitionDeviceGroup group;
|
|
JFJochReceiverService service(group, logger, pusher);
|
|
|
|
service.NumThreads(nthreads);
|
|
|
|
// No progress value at the start of measurement
|
|
REQUIRE(!service.GetProgress().has_value());
|
|
|
|
service.Start(experiment, pixel_mask, nullptr, puller);
|
|
auto receiver_out = service.Stop();
|
|
|
|
CHECK(receiver_out.efficiency == 1.0);
|
|
CHECK(receiver_out.status.images_sent == 0);
|
|
CHECK(receiver_out.dark_mask_result.size() == experiment.GetPixelsNum());
|
|
CHECK(receiver_out.dark_mask_result[0] == 0);
|
|
CHECK(receiver_out.dark_mask_result[1] == 0);
|
|
CHECK(receiver_out.dark_mask_result[2] != 0);
|
|
CHECK(receiver_out.dark_mask_result[3] != 0);
|
|
CHECK(receiver_out.dark_mask_result[4] == 0);
|
|
CHECK(receiver_out.dark_mask_result[68] == 0);
|
|
|
|
CHECK(!receiver_out.status.cancelled);
|
|
|
|
// No progress value at the end of the measurement
|
|
REQUIRE(!service.GetProgress().has_value());
|
|
}
|
|
// Identical frames, so the run masks a large part of the detector as a shadow and as hot pixels. That mask
|
|
// is the run's own: the master file holds the sum under the device mask.
|
|
TEST_CASE("JFJochReceiverRugnux_PixelSumUnderDeviceMask", "[JFJochReceiver]") {
|
|
Logger logger(Catch::getResultCapture().getCurrentTestName());
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment experiment(DetDECTRIS(2068, 2164, "Test", {}));
|
|
experiment.ImagesPerTrigger(20).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerFile(10)
|
|
.FilePrefix("rugnux_live_pixel_sum").JungfrauConvPhotonCnt(false).PixelSigned(true)
|
|
.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true)
|
|
.DetectorDistance_mm(75).BeamY_pxl(1136).BeamX_pxl(1090).IncidentEnergy_keV(12.4)
|
|
.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, Coord(-1, 0, 0), {}));
|
|
PixelMask pixel_mask(experiment);
|
|
|
|
HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
|
|
HDF5DataSet dataset(data, "/entry/data/data");
|
|
HDF5DataSpace file_space(dataset);
|
|
std::vector<int16_t> image_conv(file_space.GetDimensions()[1] * file_space.GetDimensions()[2]);
|
|
std::vector<hsize_t> start = {4, 0, 0};
|
|
std::vector<hsize_t> file_size = {1, file_space.GetDimensions()[1], file_space.GetDimensions()[2]};
|
|
dataset.ReadVector(image_conv, start, file_size);
|
|
|
|
HDF5FilePusher pusher;
|
|
auto puller = std::make_shared<TestImagePuller>();
|
|
StartMessage start_msg;
|
|
experiment.FillMessage(start_msg);
|
|
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(start_msg)});
|
|
DataMessage data_msg;
|
|
data_msg.image = CompressedImage(image_conv, file_space.GetDimensions()[2], file_space.GetDimensions()[1]);
|
|
for (int i = 0; i < experiment.GetImageNum(); i++) {
|
|
data_msg.number = i;
|
|
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(data_msg)});
|
|
}
|
|
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(EndMessage{})});
|
|
|
|
AcquisitionDeviceGroup group;
|
|
JFJochReceiverService service(group, logger, pusher);
|
|
service.NumThreads(4);
|
|
service.RugnuxLive(true, nullptr, 4, 60);
|
|
service.SetSpotFindingSettings(DiffractionExperiment::DefaultDataProcessingSettings());
|
|
service.Indexing(experiment.GetIndexingSettings());
|
|
service.Start(experiment, pixel_mask, nullptr, puller);
|
|
service.Stop();
|
|
|
|
int64_t expected_sum = 0;
|
|
for (size_t i = 0; i < image_conv.size(); i++)
|
|
if (pixel_mask.GetMask()[i] == 0 && image_conv[i] != INT16_MIN && image_conv[i] < experiment.GetSaturationLimit())
|
|
expected_sum += image_conv[i];
|
|
CHECK(expected_sum != 0);
|
|
|
|
HDF5ReadOnlyFile master("rugnux_live_pixel_sum_master.h5");
|
|
CHECK(master.ReadVector<int64_t>("/entry/image/pixel_sum") == std::vector<int64_t>(20, expected_sum));
|
|
}
|