Every test whose subject is not the card's pixel arithmetic now runs on the mock of the card, with what the card writes given by the test, precalculated on the CPU (tests/MockCardOutput.h): - JUNGFRAU conversion, threshold, Poisson, summation, storage cells, applied pixel mask, EIGER: the reference conversion of CheckImageOutput.h, rounded into the card's pixel format; - a signed 16-bit image passed through (the crystal tests): the image itself, INT16_MIN error and INT16_MAX saturated as the card reads them, the strong pixels from the CPU version of the card's spot finder (DetModuleSpotFinder_cpu.h); - pixel statistics and ROI sums from the same values, masked pixels left out; - pedestal runs and lost packets: the raw frames, the missing packet as UINT16_MAX. Integration bins and the ADU histogram are left at zero: no test asserts on their values. Moved: JFJochReceiverIntegrationTest.cpp (30 cases), JFJochReceiverProcessingTest.cpp (23), FPGAImagePullerTest.cpp (20), JFJochReceiverRugnuxTest.cpp (1), JFJochStateMachineTest.cpp (5), StreamWriterTest.cpp (4). JFJochReceiverTest_Raw_TwoDevices comes from MockAcquisitionDeviceTest.cpp, whose other ported receiver tests duplicated the originals now on the mock and are removed. What changes in meaning: the conversion tests check that the receiver carries the card's pixels into the image (geometry, multipixels, pixel format, compression), no longer that the card converts correctly; JFJochIntegrationTest_ZMQ_ROI checks the receiver's ROI and pixel counts against the hand sums with the card's sums calculated on the CPU. The card itself is compared with the mock in MockAcquisitionDeviceTest.cpp. The pedestal tests repeat 25-32 distinct frames instead of all distinct ones, the reference pedestal taken from the same frames. Waits sized for the HLS simulation are shortened: cancel after 1 s instead of 60 s / 5 s, the puller cancel after 2 s instead of 30 s, the dead-module puller gives up after 5 s without an image instead of 60 s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
794 lines
35 KiB
C++
794 lines
35 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
// The same frames through JFJochReceiverFPGA and through FPGAImagePuller + JFJochReceiverLite, on the mock of the
|
|
// card. The pixels must be the same; the analysis differs by design, and its numbers are printed.
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
|
|
#include <cmath>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <mutex>
|
|
#include <random>
|
|
#include <thread>
|
|
|
|
#include "../compression/JFJochDecompress.h"
|
|
#include "../frame_serialize/CBORStream2Deserializer.h"
|
|
#include "../image_pusher/HDF5FilePusher.h"
|
|
#include "../image_pusher/ImagePusher.h"
|
|
#include "../writer/HDF5Objects.h"
|
|
#include "../writer/StreamWriter.h"
|
|
#include "../receiver/FPGAImagePuller.h"
|
|
#include "../receiver/JFJochReceiverRugnuxRotation.h"
|
|
#include "../receiver/JFJochReceiverService.h"
|
|
#include "../rugnux/RugnuxDefaults.h"
|
|
#include "MockCardOutput.h"
|
|
|
|
using namespace std::chrono_literals;
|
|
|
|
namespace {
|
|
struct Received {
|
|
DataMessage message;
|
|
std::vector<uint8_t> compressed;
|
|
};
|
|
|
|
class CollectingPusher : public ImagePusher {
|
|
public:
|
|
std::mutex m;
|
|
std::map<int64_t, Received> images;
|
|
std::vector<int64_t> arrival;
|
|
std::vector<std::string> sequence;
|
|
EndMessage end;
|
|
std::chrono::milliseconds delay{0};
|
|
|
|
void StartDataCollection(StartMessage &) override {
|
|
std::unique_lock ul(m);
|
|
sequence.emplace_back("start");
|
|
}
|
|
bool EndDataCollection(const EndMessage &message) override {
|
|
std::unique_lock ul(m);
|
|
sequence.emplace_back("end");
|
|
end = message;
|
|
return true;
|
|
}
|
|
bool SendCalibration(const CompressedImage &c) override {
|
|
std::unique_lock ul(m);
|
|
sequence.emplace_back("calibration " + c.GetChannel());
|
|
return true;
|
|
}
|
|
bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override {
|
|
std::this_thread::sleep_for(delay);
|
|
auto d = CBORStream2Deserialize(image_data, image_size);
|
|
Received r;
|
|
r.message = d->data_message.value();
|
|
const uint8_t *p = r.message.image.GetCompressed();
|
|
r.compressed.assign(p, p + r.message.image.GetCompressedSize());
|
|
r.message.image = CompressedImage(r.compressed.data(), r.compressed.size(), r.message.image.GetWidth(),
|
|
r.message.image.GetHeight(), r.message.image.GetMode(),
|
|
r.message.image.GetCompressionAlgorithm());
|
|
std::unique_lock ul(m);
|
|
sequence.emplace_back("image");
|
|
arrival.push_back(image_number);
|
|
images.emplace(image_number, std::move(r));
|
|
return true;
|
|
}
|
|
std::string PrintSetup() const override { return "collecting"; }
|
|
ImagePusherType GetType() const override { return ImagePusherType::Test; }
|
|
};
|
|
|
|
float Pedestal(int module, int pixel) {
|
|
return 3000 + 10 * module + pixel % 50;
|
|
}
|
|
|
|
JFCalibration Calibration(const DiffractionExperiment &x) {
|
|
JFCalibration ret(x);
|
|
std::vector<float> g0(RAW_MODULE_SIZE), g1(RAW_MODULE_SIZE), g2(RAW_MODULE_SIZE);
|
|
for (int s = 0; s < x.GetStorageCellNumber(); s++)
|
|
for (int m = 0; m < x.GetModulesNum(); m++) {
|
|
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
|
|
g0[i] = Pedestal(m, i);
|
|
g1[i] = 15000 + 10 * m + i % 50;
|
|
g2[i] = 14000 + 10 * m + i % 50;
|
|
}
|
|
ret.Pedestal(m, 0, s).LoadPedestal(g0);
|
|
ret.Pedestal(m, 1, s).LoadPedestal(g1);
|
|
ret.Pedestal(m, 2, s).LoadPedestal(g2);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// Raw detector frame of a module: background of a few photons, some spots, one saturated and one bad
|
|
// pixel, different for each frame.
|
|
std::vector<uint16_t> Frame(int64_t frame, int module, bool jungfrau) {
|
|
std::vector<uint16_t> ret(RAW_MODULE_SIZE);
|
|
std::mt19937 g(static_cast<uint32_t>(frame * 100 + module));
|
|
std::uniform_int_distribution<int> background(0, jungfrau ? 300 : 3);
|
|
for (int i = 0; i < RAW_MODULE_SIZE; i++)
|
|
ret[i] = (jungfrau ? static_cast<uint16_t>(Pedestal(module, i)) : 0) + background(g);
|
|
|
|
std::uniform_int_distribution<int> where_x(20, RAW_MODULE_COLS - 20), where_y(20, RAW_MODULE_LINES - 20);
|
|
for (int s = 0; s < 40; s++) {
|
|
const int x0 = where_x(g), y0 = where_y(g);
|
|
for (int y = y0; y < y0 + 3; y++)
|
|
for (int x = x0; x < x0 + 3; x++)
|
|
ret[y * RAW_MODULE_COLS + x] = jungfrau ? 12000 : 2000;
|
|
}
|
|
ret[50 * RAW_MODULE_COLS + 500] = jungfrau ? 0xC000 : 0xFFFE;
|
|
ret[60 * RAW_MODULE_COLS + 600] = 0xFFFF;
|
|
return ret;
|
|
}
|
|
|
|
SpotFindingSettings Spots() {
|
|
SpotFindingSettings s = DiffractionExperiment::DefaultDataProcessingSettings();
|
|
s.signal_to_noise_threshold = 3;
|
|
s.photon_count_threshold = 2;
|
|
s.min_pix_per_spot = 1;
|
|
s.indexing = false;
|
|
return s;
|
|
}
|
|
|
|
// What the card writes for the first frames: their conversion
|
|
std::vector<CardImage> CardImages(const DiffractionExperiment &x, int64_t distinct_frames) {
|
|
const bool jungfrau = x.GetDetectorType() == DetectorType::JUNGFRAU;
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
std::vector<CardImage> ret;
|
|
for (int64_t f = 0; f < distinct_frames; f++) {
|
|
std::vector<uint16_t> raw;
|
|
for (int m = 0; m < x.GetModulesNum(); m++) {
|
|
const auto frame = Frame(f, m, jungfrau);
|
|
raw.insert(raw.end(), frame.begin(), frame.end());
|
|
}
|
|
ret.push_back(ConvertedImage(x, calib, mask, raw));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
// The card writing zeros
|
|
AcquisitionDeviceGroup Devices(const DiffractionExperiment &x) {
|
|
return MockDevices(x, {});
|
|
}
|
|
|
|
struct PathResult {
|
|
std::map<int64_t, Received> images;
|
|
std::vector<int64_t> arrival;
|
|
std::vector<std::string> sequence;
|
|
EndMessage end;
|
|
JFJochReceiverOutput output;
|
|
};
|
|
|
|
PathResult Run(const DiffractionExperiment &x, bool puller, int64_t distinct_frames, bool rugnux = false) {
|
|
auto devices = MockDevices(x, CardImages(x, distinct_frames));
|
|
Logger logger("FPGAImagePullerTest");
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(4);
|
|
service.SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(puller);
|
|
service.RugnuxLive(rugnux, nullptr, 2, 30);
|
|
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
PathResult ret;
|
|
ret.output = service.Stop();
|
|
ret.images = std::move(pusher.images);
|
|
ret.arrival = std::move(pusher.arrival);
|
|
ret.sequence = std::move(pusher.sequence);
|
|
ret.end = pusher.end;
|
|
return ret;
|
|
}
|
|
|
|
std::vector<uint8_t> Pixels(const CompressedImage &image) {
|
|
std::vector<uint8_t> ret;
|
|
image.GetUncompressed(ret);
|
|
return ret;
|
|
}
|
|
|
|
template<class T>
|
|
void Row(const std::string &name, const std::optional<T> &a, const std::optional<T> &b) {
|
|
if (a == b)
|
|
return;
|
|
std::cout << " differs " << name << ": FPGA ";
|
|
if (a) std::cout << *a; else std::cout << "-";
|
|
std::cout << " puller ";
|
|
if (b) std::cout << *b; else std::cout << "-";
|
|
std::cout << std::endl;
|
|
}
|
|
|
|
// Must be the same: asserted. Analysis: printed for the first image and summed over all.
|
|
void Compare(const std::string &name, const PathResult &a, const PathResult &b) {
|
|
std::cout << "=== " << name << std::endl;
|
|
REQUIRE(a.images.size() == b.images.size());
|
|
REQUIRE(!a.images.empty());
|
|
|
|
int64_t spots_a = 0, spots_b = 0, sat_a = 0, sat_b = 0, err_a = 0, err_b = 0, strong_a = 0, strong_b = 0;
|
|
double dot = 0, norm_a = 0, norm_b = 0, ratio_sum = 0;
|
|
size_t ratio_n = 0, bins_a = 0, bins_b = 0;
|
|
|
|
for (const auto &[n, ia]: a.images) {
|
|
INFO("image " << n);
|
|
REQUIRE(b.images.contains(n));
|
|
const auto &ib = b.images.at(n);
|
|
const auto &ma = ia.message;
|
|
const auto &mb = ib.message;
|
|
|
|
CHECK(ma.image.GetMode() == mb.image.GetMode());
|
|
CHECK(ma.image.GetCompressionAlgorithm() == mb.image.GetCompressionAlgorithm());
|
|
CHECK(ma.image.GetWidth() == mb.image.GetWidth());
|
|
CHECK(ma.image.GetHeight() == mb.image.GetHeight());
|
|
CHECK(Pixels(ma.image) == Pixels(mb.image));
|
|
CHECK(ia.compressed.size() == ib.compressed.size());
|
|
|
|
// From the source
|
|
CHECK(ma.number == mb.number);
|
|
CHECK(ma.original_number == mb.original_number);
|
|
CHECK(ma.packets_expected == mb.packets_expected);
|
|
CHECK(ma.packets_received == mb.packets_received);
|
|
CHECK(ma.image_collection_efficiency == mb.image_collection_efficiency);
|
|
CHECK(ma.timestamp == mb.timestamp);
|
|
CHECK(ma.timestamp_base == mb.timestamp_base);
|
|
CHECK(ma.exptime == mb.exptime);
|
|
CHECK(ma.exptime_base == mb.exptime_base);
|
|
CHECK(ma.jf_info == mb.jf_info);
|
|
CHECK(ma.storage_cell == mb.storage_cell);
|
|
CHECK(ma.xfel_pulse_id == mb.xfel_pulse_id);
|
|
CHECK(ma.xfel_event_code == mb.xfel_event_code);
|
|
CHECK(ma.pixel_sum == mb.pixel_sum);
|
|
CHECK(ma.user_data == mb.user_data);
|
|
CHECK(ma.run_number == mb.run_number);
|
|
CHECK(ma.run_name == mb.run_name);
|
|
|
|
if (n == a.images.begin()->first) {
|
|
Row("saturated_pixel_count", ma.saturated_pixel_count, mb.saturated_pixel_count);
|
|
Row("error_pixel_count", ma.error_pixel_count, mb.error_pixel_count);
|
|
Row("min_viable_pixel_value", ma.min_viable_pixel_value, mb.min_viable_pixel_value);
|
|
Row("max_viable_pixel_value", ma.max_viable_pixel_value, mb.max_viable_pixel_value);
|
|
Row("strong_pixel_count", ma.strong_pixel_count, mb.strong_pixel_count);
|
|
Row("spot_count", ma.spot_count, mb.spot_count);
|
|
Row("bkg_estimate", ma.bkg_estimate, mb.bkg_estimate);
|
|
Row("ice_ring_score", ma.ice_ring_score, mb.ice_ring_score);
|
|
Row("receiver_aq_dev_delay", ma.receiver_aq_dev_delay, mb.receiver_aq_dev_delay);
|
|
Row("receiver_buf_in_preparation", ma.receiver_buf_in_preparation, mb.receiver_buf_in_preparation);
|
|
Row("receiver_buf_in_sending", ma.receiver_buf_in_sending, mb.receiver_buf_in_sending);
|
|
std::cout << " compression_ratio FPGA " << ma.compression_ratio.value_or(0)
|
|
<< " puller " << mb.compression_ratio.value_or(0) << std::endl;
|
|
}
|
|
spots_a += ma.spots.size();
|
|
spots_b += mb.spots.size();
|
|
sat_a += ma.saturated_pixel_count.value_or(0);
|
|
sat_b += mb.saturated_pixel_count.value_or(0);
|
|
err_a += ma.error_pixel_count.value_or(0);
|
|
err_b += mb.error_pixel_count.value_or(0);
|
|
strong_a += ma.strong_pixel_count.value_or(0);
|
|
strong_b += mb.strong_pixel_count.value_or(0);
|
|
|
|
bins_a = ma.az_int_profile.size();
|
|
bins_b = mb.az_int_profile.size();
|
|
for (size_t i = 0; i < std::min(bins_a, bins_b); i++) {
|
|
if (std::isfinite(ma.az_int_profile[i]) && std::isfinite(mb.az_int_profile[i])) {
|
|
dot += ma.az_int_profile[i] * mb.az_int_profile[i];
|
|
norm_a += ma.az_int_profile[i] * ma.az_int_profile[i];
|
|
norm_b += mb.az_int_profile[i] * mb.az_int_profile[i];
|
|
if (ma.az_int_profile[i] != 0) {
|
|
ratio_sum += mb.az_int_profile[i] / ma.az_int_profile[i];
|
|
ratio_n++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
std::cout << " over " << a.images.size() << " images:"
|
|
<< " spots FPGA " << spots_a << " puller " << spots_b
|
|
<< " | strong " << strong_a << " / " << strong_b
|
|
<< " | saturated " << sat_a << " / " << sat_b
|
|
<< " | error " << err_a << " / " << err_b
|
|
<< " | az bins " << bins_a << " / " << bins_b
|
|
<< " cosine " << (norm_a * norm_b > 0 ? dot / std::sqrt(norm_a * norm_b) : 0)
|
|
<< " mean ratio puller/FPGA " << (ratio_n > 0 ? ratio_sum / ratio_n : 0) << std::endl;
|
|
|
|
const auto &sa = a.output.status;
|
|
const auto &sb = b.output.status;
|
|
std::cout << " status: efficiency " << a.output.efficiency << " / " << b.output.efficiency
|
|
<< " | collected " << sa.images_collected << " / " << sb.images_collected
|
|
<< " | sent " << sa.images_sent << " / " << sb.images_sent
|
|
<< " | dropped " << sa.images_dropped << " / " << sb.images_dropped
|
|
<< " | max_image_sent " << sa.max_image_number_sent << " / " << sb.max_image_number_sent
|
|
<< " | compressed_size " << sa.compressed_size << " / " << sb.compressed_size
|
|
<< " | ratio " << sa.compressed_ratio.value_or(0) << " / " << sb.compressed_ratio.value_or(0)
|
|
<< " | max_delay " << sa.max_receive_delay.value_or(0) << " / " << sb.max_receive_delay.value_or(0)
|
|
<< std::endl;
|
|
|
|
CHECK(a.output.efficiency == b.output.efficiency);
|
|
CHECK(sa.images_collected == sb.images_collected);
|
|
CHECK(sa.images_sent == sb.images_sent);
|
|
CHECK(sb.images_dropped == 0);
|
|
CHECK(sa.max_image_number_sent == sb.max_image_number_sent);
|
|
CHECK(sa.compressed_size == sb.compressed_size);
|
|
CHECK(a.end.images_collected_count == b.end.images_collected_count);
|
|
CHECK(a.end.images_sent_to_write_count == b.end.images_sent_to_write_count);
|
|
CHECK(a.end.max_image_number == b.end.max_image_number);
|
|
CHECK(a.end.adu_histogram == b.end.adu_histogram);
|
|
|
|
// Calibration goes to the writer only from the FPGA receiver
|
|
CHECK(a.sequence.front() == "start");
|
|
CHECK(b.sequence.front() == "start");
|
|
CHECK(a.sequence.back() == "end");
|
|
CHECK(b.sequence.back() == "end");
|
|
auto sequence_a = a.sequence;
|
|
std::erase_if(sequence_a, [](const std::string &s) { return s.starts_with("calibration"); });
|
|
CHECK(sequence_a == b.sequence);
|
|
|
|
int64_t displacement = 0;
|
|
for (size_t i = 0; i < b.arrival.size(); i++)
|
|
displacement = std::max(displacement, std::abs(b.arrival[i] - static_cast<int64_t>(i)));
|
|
std::cout << " max displacement at the pusher (Lite's threads): " << displacement << " images" << std::endl;
|
|
}
|
|
|
|
void CompareBoth(const std::string &name, const DiffractionExperiment &x, int64_t distinct_frames) {
|
|
Compare(name, Run(x, false, distinct_frames), Run(x, true, distinct_frames));
|
|
}
|
|
|
|
DiffractionExperiment Base(const DetectorSetup &detector, int64_t images, int64_t distinct_frames) {
|
|
DiffractionExperiment x(detector);
|
|
x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).InternalPacketGeneratorImages(distinct_frames)
|
|
.ImagesPerTrigger(images).ImagesPerFile(10).IncidentEnergy_keV(12.4)
|
|
.DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600);
|
|
return x;
|
|
}
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 6, 3);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
CompareBoth("JF conversion, int16, BSHUF_LZ4", x, 3);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_ApplyMask_ROI", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 4, 2);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_ZSTD).ApplyPixelMask(true);
|
|
x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}});
|
|
const auto a = Run(x, false, 2);
|
|
const auto b = Run(x, true, 2);
|
|
Compare("JF conversion, applied mask, ROI, BSHUF_ZSTD", a, b);
|
|
const auto &ra = a.images.begin()->second.message.roi.at("box");
|
|
const auto &rb = b.images.begin()->second.message.roi.at("box");
|
|
std::cout << " ROI sum " << ra.sum << " / " << rb.sum << " pixels " << ra.pixels << " / " << rb.pixels
|
|
<< " max " << ra.max_count << " / " << rb.max_count << std::endl;
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Unsigned", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 4, 2);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false);
|
|
CompareBoth("JF conversion, uint16, BSHUF_LZ4", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Summation", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 3, 2);
|
|
x.Conversion();
|
|
x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
CompareBoth("JF conversion, card sums 2 frames, int32", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_NoGeometry", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 4, 2);
|
|
x.Conversion();
|
|
x.GeometryTransformation(false).Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
CompareBoth("JF conversion, no geometry transform", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 4, 2);
|
|
x.Raw();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
CompareBoth("JF raw, BSHUF_LZ4", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw_NoCompression", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 3, 1);
|
|
x.Raw();
|
|
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
CompareBoth("JF raw, no compression", x, 1);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Conversion", "[FPGAImagePuller]") {
|
|
auto x = Base(DetEIGER(2), 4, 2);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
CompareBoth("EIGER assembled, 16 bit, BSHUF_LZ4", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Raw", "[FPGAImagePuller]") {
|
|
auto x = Base(DetEIGER(2), 4, 2);
|
|
x.Raw();
|
|
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
|
|
CompareBoth("EIGER raw layout, 16 bit, BSHUF_ZSTD", x, 2);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") {
|
|
auto x = Base(DetEIGER(2), 3, 1);
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
x.EigerBitDepth(32);
|
|
// The 16-bit raw frames do not map onto a 32-bit stream: the card writes zeros
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_EIGER32");
|
|
for (const bool puller: {false, true}) {
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(4).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(puller);
|
|
PixelMask mask(x);
|
|
service.Start(x, mask, nullptr);
|
|
const auto out = service.Stop();
|
|
CHECK(out.efficiency == 1.0f);
|
|
CHECK(pusher.images.size() == 3);
|
|
CHECK(pusher.images.at(0).message.image.GetMode() == x.GetImageMode());
|
|
}
|
|
}
|
|
|
|
// Nobody takes the images: the slots run out and later images are dropped whole, without any waiting.
|
|
TEST_CASE("FPGAImagePuller_SlowConsumer_Puller", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 20, 1);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_SlowConsumer");
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2, 4);
|
|
|
|
// Not read until the card is through, however long the simulation takes
|
|
while (!devices[0].Counters().IsAcquisitionFinished())
|
|
std::this_thread::sleep_for(10ms);
|
|
|
|
std::vector<int64_t> numbers;
|
|
bool end = false;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
while (!end && std::chrono::steady_clock::now() - t0 < 60s) {
|
|
const auto m = puller.PollImage(10ms);
|
|
if (!m)
|
|
continue;
|
|
if (m->cbor->data_message)
|
|
numbers.push_back(m->cbor->data_message->number);
|
|
end = m->cbor->end_message.has_value();
|
|
}
|
|
CHECK(end);
|
|
puller.Stop();
|
|
std::cout << "slow consumer, puller alone: " << numbers.size() << " images, "
|
|
<< puller.GetDroppedImages() << " dropped" << std::endl;
|
|
CHECK(numbers.size() + puller.GetDroppedImages() == 20);
|
|
CHECK(puller.GetDroppedImages() > 0);
|
|
for (size_t i = 1; i < numbers.size(); i++)
|
|
CHECK(numbers[i] > numbers[i - 1]);
|
|
}
|
|
|
|
// The same behind Lite: a writer that takes its time keeps Lite's only thread, and the drops reach the statistics.
|
|
TEST_CASE("FPGAImagePuller_SlowConsumer_Receiver", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 24, 1);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_SlowReceiver");
|
|
CollectingPusher pusher;
|
|
pusher.delay = 3s;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(1).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true, 2);
|
|
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
const auto out = service.Stop();
|
|
|
|
std::cout << "slow consumer, Lite: collected " << out.status.images_collected << " dropped "
|
|
<< out.status.images_dropped << " efficiency " << out.efficiency << std::endl;
|
|
CHECK(out.status.images_dropped > 0);
|
|
CHECK(out.status.images_collected + out.status.images_dropped == 24);
|
|
CHECK(out.efficiency == Catch::Approx(static_cast<float>(out.status.images_collected) / 24.0f));
|
|
CHECK(pusher.sequence.back() == "end");
|
|
}
|
|
|
|
// A module frame missing in the middle, and a module that dies: the images keep coming with no help
|
|
// from the end of the run, and the missing module is filled, as in JFJochReceiverFPGA.
|
|
TEST_CASE("FPGAImagePuller_MissingModule", "[FPGAImagePuller]") {
|
|
constexpr int64_t images = 62;
|
|
constexpr int64_t dies_at = 3;
|
|
auto x = Base(DetJF(2), images, 1);
|
|
x.UseInternalPacketGenerator(false).Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
|
|
auto device = std::make_unique<MockAcquisitionDevice>(0, 64);
|
|
SetOutputs(*device, x, 0, CardImages(x, 1));
|
|
for (int64_t f = 0; f < images; f++)
|
|
for (int m = 0; m < 2; m++) {
|
|
if ((m == 1) && (f == 1 || f >= dies_at))
|
|
continue;
|
|
device->CreateJFPackets(x, f + 1, 1, m, nullptr);
|
|
}
|
|
AcquisitionDeviceGroup devices;
|
|
devices.Add(std::move(device));
|
|
|
|
Logger logger("FPGAImagePuller_MissingModule");
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
|
|
|
|
// No final packet and no cancel yet: images must come through anyway, up to the lag behind the card
|
|
std::vector<int64_t> numbers;
|
|
std::vector<float> efficiency;
|
|
bool stop_requested = false, end = false;
|
|
size_t images_before_cancel = 0;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
auto last = t0;
|
|
while (!end && std::chrono::steady_clock::now() - t0 < 900s) {
|
|
const auto m = puller.PollImage(10ms);
|
|
if (!m) {
|
|
if (!stop_requested && std::chrono::steady_clock::now() - last > 5s) {
|
|
stop_requested = true;
|
|
std::cout << "dead module, before cancel: fastest " << devices[0].Counters().GetFastestFrameNumber()
|
|
<< " module 0 at " << devices[0].Counters().GetCurrFrameNumber(0)
|
|
<< " module 1 at " << devices[0].Counters().GetCurrFrameNumber(1)
|
|
<< " images so far " << numbers.size() << std::endl;
|
|
puller.Cancel();
|
|
}
|
|
continue;
|
|
}
|
|
last = std::chrono::steady_clock::now();
|
|
if (m->cbor->data_message) {
|
|
const auto &d = *m->cbor->data_message;
|
|
numbers.push_back(d.number);
|
|
efficiency.push_back(d.image_collection_efficiency.value_or(-1));
|
|
if (!stop_requested)
|
|
images_before_cancel = numbers.size();
|
|
|
|
if (d.number == 1) {
|
|
// Module 1 absent from this frame: its pixels are the fill value
|
|
std::vector<uint8_t> tmp;
|
|
const auto *p = reinterpret_cast<const int16_t *>(d.image.GetUncompressedPtr(tmp));
|
|
CHECK(p[x.GetPixel0OfModuleConv(1)] == x.GetImageFillValue());
|
|
CHECK(p[x.GetPixel0OfModuleConv(0)] != x.GetImageFillValue());
|
|
}
|
|
}
|
|
end = m->cbor->end_message.has_value();
|
|
}
|
|
CHECK(end);
|
|
REQUIRE_NOTHROW(puller.Stop());
|
|
std::cout << "dead module: " << numbers.size() << " images, before any cancel: "
|
|
<< images_before_cancel << std::endl;
|
|
// Image i is let go once the card reports frame i + 2 + MissingAfterFrames of the other module
|
|
CHECK(images_before_cancel >= 4);
|
|
REQUIRE(numbers.size() >= 4);
|
|
for (size_t i = 0; i < numbers.size(); i++)
|
|
CHECK(numbers[i] == static_cast<int64_t>(i));
|
|
CHECK(efficiency[0] == 1.0f);
|
|
CHECK(efficiency[1] == 0.5f);
|
|
// Frame 2 of module 1 is sent, but the card holds a module's last frames until newer ones push them
|
|
// out: given up on, it comes with the cancel. Its buffer went back.
|
|
CHECK(efficiency[3] == 0.5f);
|
|
CHECK(efficiency.back() == 0.5f);
|
|
}
|
|
|
|
TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") {
|
|
for (const bool silent: {false, true}) {
|
|
auto x = Base(DetJF(2), 6, 1);
|
|
x.UseInternalPacketGenerator(false).Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
|
|
auto device = std::make_unique<MockAcquisitionDevice>(0, 64);
|
|
SetOutputs(*device, x, 0, CardImages(x, 3));
|
|
for (int64_t f = 0; f < 3; f++)
|
|
for (int m = 0; m < 2; m++)
|
|
device->CreateJFPackets(x, f + 1, 1, m, nullptr);
|
|
AcquisitionDeviceGroup devices;
|
|
devices.Add(std::move(device));
|
|
|
|
Logger logger("FPGAImagePuller_Cancel");
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true);
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
|
|
std::this_thread::sleep_for(2s);
|
|
service.Cancel(silent);
|
|
const auto out = service.Stop();
|
|
|
|
std::cout << "cancel silent=" << silent << ": cancelled " << out.status.cancelled << " collected "
|
|
<< out.status.images_collected << " sent " << out.status.images_sent << " efficiency "
|
|
<< out.efficiency << std::endl;
|
|
CHECK(out.status.cancelled == !silent);
|
|
CHECK(pusher.sequence.front() == "start");
|
|
CHECK(pusher.sequence.back() == "end");
|
|
CHECK(out.status.images_collected <= 3);
|
|
if (silent) {
|
|
CHECK(out.status.images_collected == 3);
|
|
CHECK(out.efficiency == Catch::Approx(0.5f));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Default stays the FPGA receiver; pedestal and summation above the card's limit never take the puller
|
|
TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 2, 1);
|
|
x.Conversion();
|
|
x.Summation(2).CPUSummation(true).Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
REQUIRE(x.IsCPUSummation());
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_HostSummation");
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(4).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true);
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
const auto out = service.Stop();
|
|
CHECK(out.efficiency == 1.0f);
|
|
CHECK(!out.received_packets.empty()); // only the FPGA receiver fills these
|
|
CHECK(out.status.images_collected == 2);
|
|
}
|
|
|
|
// JFJochReceiverRugnuxRotation takes the same stream, with its own forwarding thread.
|
|
TEST_CASE("FPGAImagePuller_RugnuxReceiver", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 6, 3);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
|
|
x.FilePrefix("fpga_puller_rugnux");
|
|
REQUIRE(RugnuxRotationData(x));
|
|
|
|
auto devices = MockDevices(x, CardImages(x, 3));
|
|
Logger logger("FPGAImagePuller_Rugnux");
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true);
|
|
service.RugnuxLive(true, nullptr, 2, 30);
|
|
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
const auto out = service.Stop();
|
|
std::cout << "rugnux receiver: efficiency " << out.efficiency << " sent " << out.status.images_sent
|
|
<< " warning '" << out.analysis_warning << "'" << std::endl;
|
|
CHECK(out.efficiency == 1.0f);
|
|
CHECK(out.status.images_sent == 6);
|
|
CHECK(pusher.images.size() == 6);
|
|
CHECK(pusher.sequence.front() == "start");
|
|
CHECK(pusher.sequence.back() == "end");
|
|
CHECK(pusher.end.adu_histogram.contains("module0"));
|
|
CHECK_FALSE(pusher.sequence[1].starts_with("calibration")); // the writer gets calibration only from the FPGA receiver
|
|
const auto a = Pixels(pusher.images.at(0).message.image);
|
|
const auto b = Pixels(pusher.images.at(3).message.image);
|
|
CHECK(a == b); // the card output repeats after the third frame
|
|
}
|
|
|
|
// In an unsigned image the card gives saturated and error pixels one value: the counts are the card's, on every receiver.
|
|
TEST_CASE("FPGAImagePuller_SaturatedAndErrorCountsFromTheCard", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(2), 4, 2);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false).FilePrefix("fpga_puller_counts");
|
|
const auto card = Run(x, false, 2);
|
|
|
|
PathResult puller;
|
|
SECTION("JFJochReceiverLite") {
|
|
puller = Run(x, true, 2);
|
|
}
|
|
SECTION("JFJochReceiverRugnuxStills") {
|
|
puller = Run(x, true, 2, true);
|
|
}
|
|
SECTION("JFJochReceiverRugnuxRotation") {
|
|
auto rotation = x;
|
|
rotation.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
|
|
puller = Run(rotation, true, 2, true);
|
|
}
|
|
|
|
REQUIRE(card.images.size() == 4);
|
|
REQUIRE(puller.images.size() == 4);
|
|
int64_t saturated = 0, error = 0;
|
|
for (const auto &[n, image]: puller.images) {
|
|
INFO("image " << n);
|
|
CHECK(image.message.saturated_pixel_count == card.images.at(n).message.saturated_pixel_count);
|
|
CHECK(image.message.error_pixel_count == card.images.at(n).message.error_pixel_count);
|
|
saturated += image.message.saturated_pixel_count.value_or(0);
|
|
error += image.message.error_pixel_count.value_or(0);
|
|
}
|
|
CHECK(saturated == 8);
|
|
CHECK(error == 8);
|
|
}
|
|
|
|
// Rotation forwards the images unanalysed, so nothing would replace what the card's ROI map, not loaded for this run, gave.
|
|
TEST_CASE("FPGAImagePuller_NoStaleROI", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 4, 1);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
|
|
x.FilePrefix("fpga_puller_roi");
|
|
x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}});
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_NoStaleROI");
|
|
CollectingPusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true);
|
|
service.RugnuxLive(true, nullptr, 2, 30);
|
|
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
service.Stop();
|
|
REQUIRE(pusher.images.size() == 4);
|
|
for (const auto &[n, image]: pusher.images)
|
|
CHECK(image.message.roi.empty());
|
|
}
|
|
|
|
// An image is many frames: efficiency and progress count images.
|
|
TEST_CASE("FPGAImagePuller_RugnuxReceiver_Summation", "[FPGAImagePuller]") {
|
|
auto x = Base(DetJF(1), 3, 2);
|
|
x.Conversion();
|
|
x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4);
|
|
x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt));
|
|
x.FilePrefix("fpga_puller_rugnux_sum");
|
|
REQUIRE(x.GetImageNum() == 3);
|
|
REQUIRE(x.GetFrameNum() == 6);
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_RugnuxSum");
|
|
CollectingPusher pusher;
|
|
ImageBuffer buffer(256 * 1024 * 1024);
|
|
JFJochReceiverCurrentStatus status;
|
|
JFJochReceiverPlots plots;
|
|
RugnuxLiveStatus live;
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
|
|
FPGAImagePuller puller(x, mask, &calib, devices, logger, 2);
|
|
JFJochReceiverRugnuxRotation receiver(x, mask, puller, pusher, logger, 2, 2, nullptr, 30, live, Spots(), status,
|
|
plots, buffer);
|
|
receiver.StopReceiver();
|
|
puller.Stop();
|
|
CHECK(receiver.GetEfficiency() == 1.0f);
|
|
CHECK(receiver.GetProgress() == 1.0f);
|
|
CHECK(pusher.images.size() == 3);
|
|
}
|
|
|
|
// The real writer: the master file states what the card did, not what a DECTRIS stream implies.
|
|
TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") {
|
|
RegisterHDF5Filter();
|
|
auto x = Base(DetJF(2), 5, 2);
|
|
x.Conversion();
|
|
x.Compression(CompressionAlgorithm::BSHUF_LZ4).ApplyPixelMask(true).ImagesPerFile(2)
|
|
.FilePrefix("fpga_image_puller_writer").SetFileWriterFormat(FileWriterFormat::NXmxVDS)
|
|
.OverwriteExistingFiles(true).SaveCalibration(true);
|
|
|
|
auto devices = Devices(x);
|
|
Logger logger("FPGAImagePuller_Writer");
|
|
HDF5FilePusher pusher;
|
|
JFJochReceiverService service(devices, logger, pusher, 256);
|
|
service.NumThreads(2).SetSpotFindingSettings(Spots());
|
|
service.UseFPGAImagePuller(true);
|
|
PixelMask mask(x);
|
|
const JFCalibration calib = Calibration(x);
|
|
service.Start(x, mask, &calib);
|
|
const auto out = service.Stop();
|
|
CHECK(out.efficiency == 1.0f);
|
|
CHECK(out.writer_err.empty());
|
|
CHECK(out.status.images_sent == 5);
|
|
|
|
HDF5ReadOnlyFile master("fpga_image_puller_writer_master.h5");
|
|
CHECK(master.GetBool("/entry/instrument/detector/pixel_mask_applied"));
|
|
HDF5DataSet dataset(master, "/entry/data/data");
|
|
HDF5DataSpace space(dataset);
|
|
CHECK(space.GetDimensions()[0] == 5);
|
|
CHECK(space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
}
|