Files
leonarski_fandClaude Opus 5.5 ce89180627 Tests: receiver, puller, state machine and writer tests on MockAcquisitionDevice
Every test whose subject is not the card's pixel arithmetic now runs on the mock of the card,
with what the card writes given by the test, precalculated on the CPU (tests/MockCardOutput.h):
- JUNGFRAU conversion, threshold, Poisson, summation, storage cells, applied pixel mask, EIGER:
  the reference conversion of CheckImageOutput.h, rounded into the card's pixel format;
- a signed 16-bit image passed through (the crystal tests): the image itself, INT16_MIN error and
  INT16_MAX saturated as the card reads them, the strong pixels from the CPU version of the card's
  spot finder (DetModuleSpotFinder_cpu.h);
- pixel statistics and ROI sums from the same values, masked pixels left out;
- pedestal runs and lost packets: the raw frames, the missing packet as UINT16_MAX.
Integration bins and the ADU histogram are left at zero: no test asserts on their values.

Moved: JFJochReceiverIntegrationTest.cpp (30 cases), JFJochReceiverProcessingTest.cpp (23),
FPGAImagePullerTest.cpp (20), JFJochReceiverRugnuxTest.cpp (1), JFJochStateMachineTest.cpp (5),
StreamWriterTest.cpp (4). JFJochReceiverTest_Raw_TwoDevices comes from MockAcquisitionDeviceTest.cpp,
whose other ported receiver tests duplicated the originals now on the mock and are removed.

What changes in meaning: the conversion tests check that the receiver carries the card's pixels
into the image (geometry, multipixels, pixel format, compression), no longer that the card converts
correctly; JFJochIntegrationTest_ZMQ_ROI checks the receiver's ROI and pixel counts against the
hand sums with the card's sums calculated on the CPU. The card itself is compared with the mock in
MockAcquisitionDeviceTest.cpp. The pedestal tests repeat 25-32 distinct frames instead of all
distinct ones, the reference pedestal taken from the same frames. Waits sized for the HLS
simulation are shortened: cancel after 1 s instead of 60 s / 5 s, the puller cancel after 2 s
instead of 30 s, the dead-module puller gives up after 5 s without an image instead of 60 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
2026-11-08 08:12:56 +01:00

221 lines
10 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
// What the FPGA card writes, precalculated on the CPU for the tests on MockAcquisitionDevice. It does not matter how
// it is calculated here: that the mock gives the host what the card gives is checked against the HLS simulation in
// MockAcquisitionDeviceTest.cpp.
#include "CheckImageOutput.h"
#include "../acquisition_device/AcquisitionDeviceGroup.h"
#include "../acquisition_device/MockAcquisitionDevice.h"
#include "../common/PixelMask.h"
#include "../image_analysis/spot_finding/DetModuleSpotFinder_cpu.h"
#include "../receiver/JFJochReceiverTest.h"
// The card output of one image, one entry per module (absolute module number)
using CardImage = std::vector<std::unique_ptr<DeviceOutput>>;
inline std::unique_ptr<DeviceOutput> OutputWithPixels(const uint16_t *pixels) {
auto ret = std::make_unique<DeviceOutput>();
memcpy(ret->pixels, pixels, RAW_MODULE_SIZE * sizeof(uint16_t));
return ret;
}
template<class T>
void StorePixels(DeviceOutput &output, const std::vector<int64_t> &values) {
auto pixels = reinterpret_cast<T *>(output.pixels);
for (int i = 0; i < RAW_MODULE_SIZE; i++)
pixels[i] = static_cast<T>(values[i]);
}
// The values in the pixel format the card writes for the experiment
inline void StorePixels(DeviceOutput &output, const DiffractionExperiment &x, const std::vector<int64_t> &values) {
switch (x.GetByteDepthFPGA()) {
case 1:
x.IsPixelSigned() ? StorePixels<int8_t>(output, values) : StorePixels<uint8_t>(output, values);
break;
case 2:
x.IsPixelSigned() ? StorePixels<int16_t>(output, values) : StorePixels<uint16_t>(output, values);
break;
default:
x.IsPixelSigned() ? StorePixels<int32_t>(output, values) : StorePixels<uint32_t>(output, values);
break;
}
}
// Error and saturated pixels counted, the others summed, masked pixels left out; and the same for each ROI of
// the experiment's ROI map (one bit per ROI)
inline void PixelStatistics(DeviceOutput &output, const DiffractionExperiment &x, const PixelMask &mask,
const std::vector<uint16_t> &experiment_roi_map, size_t module,
const std::vector<int64_t> &values, int64_t err, int64_t sat) {
std::vector<uint16_t> roi_map(RAW_MODULE_SIZE);
ConvertedToRawGeometry(x, module, roi_map.data(), experiment_roi_map.data());
const uint32_t *module_mask = mask.GetMaskRaw().data() + module * RAW_MODULE_SIZE;
auto &stats = output.module_statistics;
stats.max_value = INT24_MIN;
stats.min_value = INT24_MAX;
for (int r = 0; r < FPGA_ROI_COUNT; r++)
output.roi_counts[r].max_value = INT24_MIN;
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
const int64_t v = values[i];
if (module_mask[i] != 0) {
stats.masked_pixels++;
continue;
}
if (v == err)
stats.err_pixels++;
else if (v == sat)
stats.saturated_pixels++;
else {
stats.pixel_sum += v;
stats.max_value = std::max<int64_t>(stats.max_value, v);
stats.min_value = std::min<int64_t>(stats.min_value, v);
}
// roi_calc.cpp: the coordinates include the multipixels
const int64_t line = i / RAW_MODULE_COLS + (i / RAW_MODULE_COLS / 256) * 2;
const int64_t col = i % RAW_MODULE_COLS + (i % RAW_MODULE_COLS / 256) * 2;
for (int r = 0; r < FPGA_ROI_COUNT; r++) {
if (((roi_map[i] >> r) & 1) == 0 || v == err)
continue;
ROICount &roi = output.roi_counts[r];
if (v != sat) {
roi.sum += v;
roi.sum2 += v * v;
roi.good_pixels++;
roi.sum_x_weighted += v * col;
roi.sum_y_weighted += v * line;
}
roi.max_value = std::max<int64_t>(roi.max_value, v);
}
}
}
// The strong pixels of the card's spot finder, by its CPU version, on the pixels in the output
inline void FindStrongPixels(DeviceOutput &output, const DiffractionExperiment &x, const SpotFindingSettings &settings,
size_t module) {
std::vector<float> d_map(RAW_MODULE_SIZE), mean(RAW_MODULE_SIZE), stddev(RAW_MODULE_SIZE);
std::vector<uint32_t> valid_count(RAW_MODULE_SIZE), strong_pixel(RAW_MODULE_SIZE);
x.CalcSpotFinderResolutionMap(d_map.data(), module);
switch (x.GetByteDepthFPGA()) {
case 1:
FindSpots<int8_t>(output, settings, d_map.data(), mean.data(), stddev.data(), valid_count.data(),
strong_pixel.data());
break;
case 2:
FindSpots<int16_t>(output, settings, d_map.data(), mean.data(), stddev.data(), valid_count.data(),
strong_pixel.data());
break;
default:
FindSpots<int32_t>(output, settings, d_map.data(), mean.data(), stddev.data(), valid_count.data(),
strong_pixel.data());
break;
}
}
// JUNGFRAU conversion of a raw image (all modules) as CheckImageOutput.h computes it; with host summation the card
// converts single frames to 16 bit. With conversion, error and saturated pixels are counted from the conversion into
// signed 32 bit, as the card counts them before the pixel is written in the image's format, where unsigned they have
// one value; without, the card reads UINT16_MAX from the detector as saturated.
inline CardImage ConvertedImage(const DiffractionExperiment &x, const JFCalibration &calib, const PixelMask &mask,
const std::vector<uint16_t> &raw, size_t storage_cell = 0) {
DiffractionExperiment card_x(x);
if (x.IsCPUSummation())
card_x.Summation(1).BitDepthImage(16);
DiffractionExperiment signed_x(card_x);
signed_x.PixelSigned(true).BitDepthImage(32);
const auto ref = ConversionRef(card_x, calib, raw.data(), x.GetModulesNum(), storage_cell);
const auto ref_signed = ConversionRef(signed_x, calib, raw.data(), x.GetModulesNum(), storage_cell);
const auto roi_map = x.ExportROIMap();
CardImage ret;
std::vector<int64_t> values(RAW_MODULE_SIZE), stats_values(RAW_MODULE_SIZE);
for (int m = 0; m < x.GetModulesNum(); m++) {
for (size_t i = 0; i < RAW_MODULE_SIZE; i++) {
const size_t pxl = m * RAW_MODULE_SIZE + i;
values[i] = std::llround(ref[pxl]);
if (x.IsJungfrauConvPhotonCnt())
stats_values[i] = std::llround(ref_signed[pxl]);
else
stats_values[i] = (raw[pxl] == UINT16_MAX) ? INT32_MAX : values[i];
}
auto output = std::make_unique<DeviceOutput>();
StorePixels(*output, card_x, values);
PixelStatistics(*output, x, mask, roi_map, m, stats_values, INT32_MIN, INT32_MAX);
ret.push_back(std::move(output));
}
return ret;
}
// The card passing a signed 16-bit image (raw geometry, all modules) through, INT16_MIN marking an error and
// INT16_MAX a saturated pixel, with the strong pixels of its spot finder
inline CardImage Int16Image(const DiffractionExperiment &x, const PixelMask &mask, const std::vector<int16_t> &image,
const SpotFindingSettings &settings) {
const int64_t err = (x.GetByteDepthFPGA() == 4) ? INT32_MIN : INT16_MIN;
const int64_t sat = (x.GetByteDepthFPGA() == 4) ? INT32_MAX : INT16_MAX;
const auto roi_map = x.ExportROIMap();
CardImage ret;
std::vector<int64_t> values(RAW_MODULE_SIZE);
for (int m = 0; m < x.GetModulesNum(); m++) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
const int16_t v = image[m * RAW_MODULE_SIZE + i];
values[i] = (v == INT16_MIN) ? err : ((v == INT16_MAX) ? sat : v);
}
auto output = std::make_unique<DeviceOutput>();
StorePixels(*output, x, values);
PixelStatistics(*output, x, mask, roi_map, m, values, err, sat);
FindStrongPixels(*output, x, settings, m);
ret.push_back(std::move(output));
}
return ret;
}
// Frame F of the mock of data stream d writes images[F % images.size()]
inline void SetOutputs(MockAcquisitionDevice &mock, const DiffractionExperiment &x, uint16_t data_stream,
const std::vector<CardImage> &images) {
for (size_t i = 0; i < images.size(); i++)
for (int m = 0; m < x.GetModulesNum(data_stream); m++)
mock.SetOutput(i, m, *images[i][x.GetFirstModuleOfDataStream(data_stream) + m]);
}
// One mock per data stream
inline AcquisitionDeviceGroup MockDevices(const DiffractionExperiment &x, const std::vector<CardImage> &images) {
AcquisitionDeviceGroup ret;
for (int d = 0; d < x.GetDataStreamsNum(); d++) {
auto mock = std::make_unique<MockAcquisitionDevice>(d, 64);
SetOutputs(*mock, x, d, images);
ret.Add(std::move(mock));
}
return ret;
}
// The mocks for JFJochReceiverTest(): the conversion of its raw image with its calibration, the pixel mask applied
// if the experiment asks for it, one image per storage cell
inline AcquisitionDeviceGroup ConvertingMocks(const DiffractionExperiment &x, const PixelMask &mask,
std::vector<uint16_t> raw) {
if (x.IsApplyPixelMask()) {
for (int i = 0; i < raw.size(); i++)
if (mask.GetMaskRaw().at(i) != 0)
raw[i] = UINT16_MAX;
}
const JFCalibration calib = GeneratePedestalCalibration(x);
std::vector<CardImage> images;
for (int s = 0; s < x.GetStorageCellNumber(); s++)
images.push_back(ConvertedImage(x, calib, mask, raw, s));
return MockDevices(x, images);
}
// The default raw image of JFJochReceiverTest()
inline AcquisitionDeviceGroup ConvertingMocks(const DiffractionExperiment &x, const PixelMask &mask) {
std::vector<uint16_t> raw(RAW_MODULE_SIZE * x.GetModulesNum());
for (int i = 0; i < raw.size(); i++)
raw[i] = i % 65536;
return ConvertingMocks(x, mask, raw);
}