diff --git a/tests/MockAcquisitionDeviceTest.cpp b/tests/MockAcquisitionDeviceTest.cpp index 1f5289f76..0c4148ebe 100644 --- a/tests/MockAcquisitionDeviceTest.cpp +++ b/tests/MockAcquisitionDeviceTest.cpp @@ -3,33 +3,48 @@ // MockAcquisitionDevice against HLSSimulatedDevice: the same scenario runs on the HLS simulation, what the card // wrote is recorded and given to the mock, and the host must see the same on both - the buffers, which module -// went to which buffer (i.e. the order of completions), counters and, for the receiver, its results. +// went to which buffer (i.e. the order of completions), counters and, for the receiver, its results. One case +// for each kind of output and mode the tests on the mock give it (tests/MockCardOutput.h); [HLSvsMock] selects them. #include +#include #include #include "../acquisition_device/HLSSimulatedDevice.h" #include "../acquisition_device/MockAcquisitionDevice.h" +#include "../common/AzimuthalIntegrationMapping.h" #include "../image_pusher/TestImagePusher.h" #include "../receiver/JFJochReceiverService.h" +#include "../receiver/JFJochReceiverTest.h" namespace { - // HLS simulation that can give away everything it wrote to the host - as long as no buffer was used twice - class RecordingHLSDevice : public HLSSimulatedDevice { + // A device that can give away everything it wrote to the host, by frame and module - as long as no buffer was + // used twice + template + class Recording : public Device { public: - using HLSSimulatedDevice::HLSSimulatedDevice; + using Device::Device; + + std::map, const DeviceOutput *> Outputs() const { + std::map, const DeviceOutput *> ret; + for (size_t h = 0; h < this->buffer_device.size(); h++) { + const DeviceOutput *output = this->GetDeviceOutput(h); + if (output->module_statistics.packet_count > 0) + ret[{output->module_statistics.frame_number, output->module_statistics.module_number}] = output; + } + return ret; + } void CopyOutputs(MockAcquisitionDevice &mock) const { - for (size_t h = 0; h < buffer_device.size(); h++) { - const DeviceOutput *output = GetDeviceOutput(h); - if (output->module_statistics.packet_count > 0) - mock.SetOutput(output->module_statistics.frame_number, output->module_statistics.module_number, - *output); - } + for (const auto &[key, output]: Outputs()) + mock.SetOutput(key.first, key.second, *output); } }; - void CompareDevices(const AcquisitionDevice &hls, const AcquisitionDevice &mock, const DiffractionExperiment &x) { + using RecordingHLSDevice = Recording; + + void CompareDevices(const AcquisitionDevice &hls, const AcquisitionDevice &mock, const DiffractionExperiment &x, + uint16_t data_stream) { CHECK(hls.GetBytesReceived() == mock.GetBytesReceived()); CHECK(hls.Counters().GetSlowestFrameNumber() == mock.Counters().GetSlowestFrameNumber()); CHECK(hls.Counters().GetFastestFrameNumber() == mock.Counters().GetFastestFrameNumber()); @@ -42,7 +57,7 @@ namespace { int64_t frames = x.GetFrameNum() / x.GetFPGASummation(); for (int64_t f = 0; f < frames; f++) { - for (int m = 0; m < x.GetModulesNum(); m++) { + for (int m = 0; m < x.GetModulesNum(data_stream); m++) { INFO("frame " << f << " module " << m); auto handle = hls.Counters().GetBufferHandle(f, m); // The same buffer means the same order of completions @@ -57,24 +72,79 @@ namespace { // Runs the same packets on both devices, the mock with what the HLS simulation wrote template - void CompareOnPackets(const DiffractionExperiment &x, Packets packets) { - RecordingHLSDevice hls(0, 64); + void CompareOnPackets(const DiffractionExperiment &x, Packets packets, uint16_t data_stream = 0) { + RecordingHLSDevice hls(data_stream, 64); packets(hls); REQUIRE_NOTHROW(hls.StartAction(x)); REQUIRE_NOTHROW(hls.WaitForActionComplete()); REQUIRE(hls.GetBytesReceived() > 0); - MockAcquisitionDevice mock(0, 64); + MockAcquisitionDevice mock(data_stream, 64); hls.CopyOutputs(mock); packets(mock); REQUIRE_NOTHROW(mock.StartAction(x)); REQUIRE_NOTHROW(mock.WaitForActionComplete()); - CompareDevices(hls, mock, x); + CompareDevices(hls, mock, x, data_stream); + } + + // Raw JUNGFRAU frame of a module: the G0 pedestal of GeneratePedestalCalibration() with a few photons, spots, + // pixels in G1 and G2, one saturated and one bad pixel + std::vector RawFrame(int module) { + std::vector ret(RAW_MODULE_SIZE); + std::mt19937 g(1900 + module); + std::uniform_int_distribution background(0, 300); + for (int i = 0; i < RAW_MODULE_SIZE; i++) + ret[i] = 3000 + 10 * module + i % 50 + background(g); + std::uniform_int_distribution where(20 * RAW_MODULE_COLS, (RAW_MODULE_LINES - 20) * RAW_MODULE_COLS); + for (int s = 0; s < 40; s++) { + const int i0 = where(g); + for (int y = 0; y < 3; y++) + for (int i = 0; i < 3; i++) + ret[i0 + y * RAW_MODULE_COLS + i] = 12000; + } + for (int i = 0; i < RAW_MODULE_SIZE; i += 1001) + ret[i] = 0x4000 | (14000 + i % 50); + for (int i = 0; i < RAW_MODULE_SIZE; i += 1777) + ret[i] = 0xC000 | 13000; + ret[50 * RAW_MODULE_COLS + 500] = 0xC000; + ret[60 * RAW_MODULE_COLS + 600] = 0xFFFF; + return ret; + } + + // JUNGFRAU, internal packet generator: the card loaded as the receiver loads it - calibration, pixel mask, ROI + // map, integration and resolution maps, spot finder - so all its outputs are there + void CompareOnGenerator(const DiffractionExperiment &x, uint16_t data_stream = 0) { + const JFCalibration calib = GeneratePedestalCalibration(x); + const PixelMask mask(x); + const AzimuthalIntegrationMapping azint(x, mask); + SpotFindingSettings settings = DiffractionExperiment::DefaultDataProcessingSettings(); + settings.signal_to_noise_threshold = 3; + settings.photon_count_threshold = 2; + settings.min_pix_per_spot = 1; + + CompareOnPackets(x, [&](AcquisitionDevice &d) { + d.InitializeCalibration(x, calib); + d.InitializePixelMask(x, mask); + d.InitializeROIMap(x, x.ExportROIMap()); + d.InitializeDataProcessing(x, azint); + d.SetSpotFinderParameters(settings); + for (int m = 0; m < x.GetModulesNum(data_stream); m++) + d.SetInternalGeneratorFrame(RawFrame(x.GetFirstModuleOfDataStream(data_stream) + m).data(), m); + }, data_stream); + } + + DiffractionExperiment GeneratorExperiment(const DetectorSetup &detector) { + DiffractionExperiment x(detector); + x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(2).UseInternalPacketGenerator(true) + .IncidentEnergy_keV(12.4).DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(250); + x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 200, 240)}, + .circles = {ROICircle("circle", 600, 300, 20)}, .azimuthal = {}}); + return x; } } -TEST_CASE("MockAcquisitionDevice_SameAsHLS_Packets", "[MockAcquisitionDevice][HLS]") { +TEST_CASE("MockAcquisitionDevice_SameAsHLS_Packets", "[MockAcquisitionDevice][HLSvsMock]") { const uint16_t nmodules = 4; DiffractionExperiment x(DetJF(nmodules)); x.Raw(); @@ -85,6 +155,7 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Packets", "[MockAcquisitionDevice][HL module[i] = (7 * i + 534 + 5 * i * i) % UINT16_MAX; CompareOnPackets(x, [&](auto &d) { + d.CreateXfelBunchIDPacket(1234.0, 0); // Frame 1: module 0 incomplete, both halves; module 1 missing; modules 2 and 3 complete d.CreateJFPacket(x, 1, 0, 0, module.data()); d.CreateJFPacket(x, 1, 64, 0, module.data()); @@ -103,7 +174,7 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Packets", "[MockAcquisitionDevice][HL }); } -TEST_CASE("MockAcquisitionDevice_SameAsHLS_Summation", "[MockAcquisitionDevice][HLS]") { +TEST_CASE("MockAcquisitionDevice_SameAsHLS_Summation", "[MockAcquisitionDevice][HLSvsMock]") { const uint16_t nmodules = 2; DiffractionExperiment x(DetJF(nmodules)); x.PedestalG0Frames(0).ImagesPerTrigger(3).NumTriggers(1).Summation(2) @@ -127,7 +198,7 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Summation", "[MockAcquisitionDevice][ }); } -TEST_CASE("MockAcquisitionDevice_SameAsHLS_EIGER", "[MockAcquisitionDevice][HLS]") { +TEST_CASE("MockAcquisitionDevice_SameAsHLS_EIGER", "[MockAcquisitionDevice][HLSvsMock]") { DiffractionExperiment x(DetEIGER(1)); x.Raw(); x.PedestalG0Frames(0).ImagesPerTrigger(1).NumTriggers(1); @@ -149,7 +220,7 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_EIGER", "[MockAcquisitionDevice][HLS] }); } -TEST_CASE("MockAcquisitionDevice_SameAsHLS_InternalGenerator", "[MockAcquisitionDevice][HLS]") { +TEST_CASE("MockAcquisitionDevice_SameAsHLS_InternalGenerator", "[MockAcquisitionDevice][HLSvsMock]") { const uint16_t nmodules = 2; DiffractionExperiment x(DetJF(nmodules)); x.Raw(); @@ -158,7 +229,98 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_InternalGenerator", "[MockAcquisition CompareOnPackets(x, [&](auto &d) {}); } -TEST_CASE("MockAcquisitionDevice_SameAsHLS_Cancel", "[MockAcquisitionDevice][HLS]") { +// JUNGFRAU modes, with every output of the card: pixels, spot finder, integration, ADU histogram, ROI, pixel counts +TEST_CASE("MockAcquisitionDevice_SameAsHLS_Modes", "[MockAcquisitionDevice][HLSvsMock]") { + DiffractionExperiment x = GeneratorExperiment(DetJF(1)); + x.Conversion(); + + SECTION("conversion, int16") {} + SECTION("conversion, fixed G1") { + x.FixedGainG1(true); + } + SECTION("conversion, uint16") { + x.BitDepthImage(16).PixelSigned(false); + } + SECTION("conversion, uint8") { + x.BitDepthImage(8).PixelSigned(false); + } + SECTION("conversion, int32") { + x.BitDepthImage(32).PixelSigned(true); + } + SECTION("conversion, Poisson") { + x.LossyCompressionPoisson(1); + } + SECTION("conversion, threshold") { + x.PixelValueLowThreshold(50); + } + SECTION("conversion, applied pixel mask") { + x.ApplyPixelMask(true).MaskChipEdges(true); + } + SECTION("conversion, storage cells") { + x.StorageCells(4); + REQUIRE(x.GetStorageCellNumber() == 4); + } + SECTION("conversion, card summation") { + x.Summation(2); + REQUIRE(x.GetFPGASummation() == 2); + } + SECTION("conversion, host summation") { + x.Summation(3).CPUSummation(true); + REQUIRE(x.GetFPGASummation() == 1); + } + SECTION("raw") { + x.Raw(); + } + SECTION("pedestal G0") { + x.Mode(DetectorMode::PedestalG0).PedestalG0Frames(4); + } + SECTION("pedestal G1, fixed G1, storage cells") { + x.Mode(DetectorMode::PedestalG1).FixedGainG1(true).StorageCells(4).PedestalG1Frames(1); + REQUIRE(x.GetStorageCellNumber() == 4); + } + SECTION("pedestal G2, storage cells") { + x.Mode(DetectorMode::PedestalG2).StorageCells(16).PedestalG2Frames(2); + REQUIRE(x.GetStorageCellNumber() == 2); + } + + CompareOnGenerator(x); +} + +TEST_CASE("MockAcquisitionDevice_SameAsHLS_EIGER_BitDepth", "[MockAcquisitionDevice][HLSvsMock]") { + DiffractionExperiment x = GeneratorExperiment(DetEIGER(1)); + x.Raw(); + + SECTION("8 bit") { + x.EigerBitDepth(8); + } + SECTION("16 bit") {} + SECTION("32 bit") { + x.EigerBitDepth(32); + } + + std::vector frame(RAW_MODULE_SIZE * 2); + std::mt19937 g(1387); + std::uniform_int_distribution dist(0, UINT16_MAX); + for (auto &i: frame) + i = dist(g); + + CompareOnPackets(x, [&](AcquisitionDevice &d) { + d.SetInternalGeneratorFrame(frame.data(), 0); + }); +} + +// Each card ends its own data collection: the second of two data streams, then the first +TEST_CASE("MockAcquisitionDevice_SameAsHLS_TwoDataStreams", "[MockAcquisitionDevice][HLSvsMock]") { + DiffractionExperiment x = GeneratorExperiment(DetJF(2)); + x.Conversion(); + x.DataStreams(2); + REQUIRE(x.GetDataStreamsNum() == 2); + + CompareOnGenerator(x, 1); + CompareOnGenerator(x, 0); +} + +TEST_CASE("MockAcquisitionDevice_SameAsHLS_Cancel", "[MockAcquisitionDevice][HLSvsMock]") { DiffractionExperiment x(DetJF(4)); x.Raw(); x.PedestalG0Frames(0).ImagesPerTrigger(5).NumTriggers(1); @@ -175,13 +337,14 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Cancel", "[MockAcquisitionDevice][HLS } // The receiver on the HLS simulation, and on the mock with what the HLS simulation wrote during that run -TEST_CASE("MockAcquisitionDevice_SameAsHLS_Receiver", "[MockAcquisitionDevice][HLS]") { +TEST_CASE("MockAcquisitionDevice_SameAsHLS_Receiver", "[MockAcquisitionDevice][HLSvsMock]") { const uint16_t nmodules = 2; DiffractionExperiment x(DetJF(nmodules)); x.Conversion(); x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).ImagesPerTrigger(4) .IncidentEnergy_keV(12.4).DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600) .Compression(CompressionAlgorithm::BSHUF_LZ4); + x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}}); JFCalibration calib(x); std::vector pedestal(RAW_MODULE_SIZE); @@ -234,6 +397,9 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Receiver", "[MockAcquisitionDevice][H std::vector azint; service.GetPlotRaw(azint, PlotType::AzInt, ""); plots.push_back(azint); + std::vector roi; + service.GetPlotRaw(roi, PlotType::ROISum, "box"); + plots.push_back(roi); return output; }; @@ -243,8 +409,9 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Receiver", "[MockAcquisitionDevice][H std::vector> hls_plots; auto hls_output = run(hls_devices, hls_image, hls_plots); - auto mock = std::make_unique(0, 64); - dynamic_cast(hls_devices[0]).CopyOutputs(*mock); + const auto &hls = dynamic_cast(hls_devices[0]); + auto mock = std::make_unique>(0, 64); + hls.CopyOutputs(*mock); AcquisitionDeviceGroup mock_devices; mock_devices.Add(std::move(mock)); std::vector mock_image; @@ -259,6 +426,20 @@ TEST_CASE("MockAcquisitionDevice_SameAsHLS_Receiver", "[MockAcquisitionDevice][H CHECK(mock_output.received_packets == hls_output.received_packets); CHECK(mock_output.expected_packets == hls_output.expected_packets); CHECK(mock_image == hls_image); + + // The frames of the run; how many of those the generator sends after them to end the data collection are + // written before the receiver cancels the card depends on timing + const auto hls_outputs = hls.Outputs(); + const auto mock_outputs = dynamic_cast &>(mock_devices[0]).Outputs(); + for (uint64_t f = 0; f < x.GetFrameNum(); f++) { + for (uint32_t m = 0; m < nmodules; m++) { + INFO("frame " << f << " module " << m); + REQUIRE(hls_outputs.contains({f, m})); + REQUIRE(mock_outputs.contains({f, m})); + CHECK(memcmp(mock_outputs.at({f, m}), hls_outputs.at({f, m}), sizeof(DeviceOutput)) == 0); + } + } + REQUIRE(mock_plots.size() == hls_plots.size()); for (int i = 0; i < hls_plots.size(); i++) { INFO("plot " << i);