diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index b2670647..acf4ecea 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -73,7 +73,3 @@ INSTALL(TARGETS jfjoch_receiver RUNTIME) ADD_EXECUTABLE(jfjoch_action_test jfjoch_action_test.cpp) TARGET_LINK_LIBRARIES(jfjoch_action_test JungfraujochHost JFJochReceiver) INSTALL(TARGETS jfjoch_action_test RUNTIME) - -ADD_EXECUTABLE(jfjoch_action_test_mock jfjoch_action_test_mock.cpp) -TARGET_LINK_LIBRARIES(jfjoch_action_test_mock JungfraujochHost JFJochReceiver) -INSTALL(TARGETS jfjoch_action_test_mock RUNTIME) \ No newline at end of file diff --git a/receiver/jfjoch_action_test.cpp b/receiver/jfjoch_action_test.cpp index 0b759bc9..9e76402d 100644 --- a/receiver/jfjoch_action_test.cpp +++ b/receiver/jfjoch_action_test.cpp @@ -4,49 +4,98 @@ #include #include "PCIExpressDevice.h" +#include "MockAcquisitionDevice.h" #include "JFJochReceiverTest.h" #include "../tests/FPGAUnitTest.h" +void print_usage(Logger &logger) { + logger.Info("Usage ./jfjoch_action_test {} "); + logger.Info("Options:"); + logger.Info(" -C conversion on CPU"); + logger.Info(" -M use mock device"); + logger.Info(" -B blocking mode (FPGA)"); + logger.Info(" -v verbose"); + logger.Info(" -s number of data streams (acquisition devices)"); + logger.Info(" -m number of modules per data stream"); + logger.Info(" -i number of images"); + logger.Info(" -p data processing period"); + logger.Info(" -N number of image processing threads"); +} + int main(int argc, char **argv) { + Logger logger("ActionTest"); + logger.Verbose(true); + + constexpr uint64_t clock_MHz = 200; uint16_t nstreams = 1; uint16_t nmodules = 1; size_t nimages = 2; uint64_t processing_period = 20; + uint16_t nthreads = 64; + bool conversion_on_cpu = false; + bool use_mock_device = false; + bool nonblocking_mode = true; + bool verbose = false; - Logger logger("ActionTest"); - logger.Verbose(true); + if (argc == 1) + print_usage(logger); - bool abort_test = false; - - if ((argc == 1) || (argc > 6)) { - logger.Error("Usage ./jfjoch_action_test {<# of images> {<# of modules> {<# of streams> {}}}}"); - exit(EXIT_FAILURE); + int opt; + while ((opt = getopt(argc, argv, "s:i:m:p:N:CMBv")) != -1) { + switch (opt) { + case 'C': + conversion_on_cpu = true; + break; + case 'M': + use_mock_device = true; + break; + case 'B': + nonblocking_mode = false; + break; + case 'i': + nimages = atol(optarg); + break; + case 'm': + nmodules = atol(optarg); + break; + case 's': + nstreams = atol(optarg); + break; + case 'p': + processing_period = atol(optarg); + break; + case 'N': + nthreads = atol(optarg); + break; + case 'v': + verbose = true; + break; + default: /* '?' */ + print_usage(logger); + exit(EXIT_FAILURE); + } } - if (argc >= 3) nimages = atol(argv[2]); - if (argc >= 4) nmodules = atol(argv[3]); - if (argc >= 5) nstreams = atoi(argv[4]); - if (argc >= 6) processing_period = atoi(argv[5]); + if (optind != argc - 1) { + print_usage(logger); + exit(EXIT_FAILURE); + } DiffractionExperiment x(DetectorGeometry(nmodules*nstreams, 2, 8, 36, true)); x.Mode(DetectorMode::Conversion); x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1); - x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false); + x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false).ConversionOnCPU(conversion_on_cpu); x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams); + logger.Info("Data streams {} Total modules {} Total images {} Threads {}", nstreams, nstreams * nmodules, nimages, nthreads); + std::vector dev_name = { "/dev/jfjoch0", "/dev/jfjoch2", "/dev/jfjoch1", "/dev/jfjoch3" }; - std::vector numa_node = {0,1,0,1}; - bool verbose = false; - bool print_status_updates = true; - uint16_t nthreads = 64; - uint64_t clock_MHz = 200; - bool nonblocking_mode = true; logger.Verbose(verbose); @@ -56,29 +105,44 @@ int main(int argc, char **argv) { } std::vector> pcie_devices; + std::vector> mock_devices; std::vector aq_devices; - std::string image_path = std::string(argv[1]) + "/tests/test_data/mod5_raw0.bin"; + std::string image_path = std::string(argv[optind]) + "/tests/test_data/mod5_raw0.bin"; std::vector input(RAW_MODULE_SIZE, 0); LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE); - for (int i = 0; i < nstreams; i++) { - pcie_devices.push_back(std::make_unique(dev_name[i], i)); - pcie_devices[i]->SetCustomInternalGeneratorFrame(input); - pcie_devices[i]->EnableLogging(&logger); - pcie_devices[i]->SetFPGANonBlockingMode(nonblocking_mode); - aq_devices.push_back(pcie_devices[i].get()); - } + if (use_mock_device) { + if (nmodules > 1) { + logger.Warning("Conversion results will be wrong with more than 1 module per stream"); + } + for (int i = 0; i < nstreams; i++) { + mock_devices.push_back(std::make_unique(i, 1024)); + mock_devices[i]->SetCustomInternalGeneratorFrame(input); + mock_devices[i]->EnableLogging(&logger); + aq_devices.push_back(mock_devices[i].get()); + } - if (!nonblocking_mode) - logger.Warning("FPGA uses blocking mode - in case data acquisition is aborted, it is necessary to cold reboot the machine"); + } else { + for (int i = 0; i < nstreams; i++) { + pcie_devices.push_back(std::make_unique(dev_name[i], i)); + pcie_devices[i]->SetCustomInternalGeneratorFrame(input); + pcie_devices[i]->EnableLogging(&logger); + pcie_devices[i]->SetFPGANonBlockingMode(nonblocking_mode); + aq_devices.push_back(pcie_devices[i].get()); + } + + if (!nonblocking_mode) + logger.Warning( + "FPGA uses blocking mode - in case data acquisition is aborted, it is necessary to cold reboot the machine"); + } volatile bool done = false; JFJochProtoBuf::ReceiverOutput output; bool ret; std::thread run_thread([&] { try { - ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads,abort_test, verbose); + ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads, false, verbose); } catch (std::exception &e) { logger.Error(e.what()); ret = false; @@ -86,7 +150,7 @@ int main(int argc, char **argv) { done = true; }); - if (print_status_updates) { + if (!use_mock_device) { while (!done) { for (int i = 0; i < nstreams; i++) { auto status = pcie_devices[i]->GetStatus(); @@ -111,27 +175,34 @@ int main(int argc, char **argv) { logger.Info("Frame rate: {} Hz", static_cast(nimages)/receiving_time); logger.Info("Total throughput: {:.2f} GB/s", static_cast(nimages*nstreams*nmodules*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9)); - logger.Info(""); - for (int i = 0; i < nstreams; i++) { - auto stalls_hbm = output.device_statistics(i).fpga_status().stalls_hbm(); - auto stalls_host = output.device_statistics(i).fpga_status().stalls_host(); - uint64_t throughput_MBs = nimages * nmodules * RAW_MODULE_SIZE*sizeof(uint16_t) * clock_MHz / (nimages * nmodules * 128 * 128 + stalls_hbm); - double performance = static_cast(throughput_MBs) / 1000; - // Assuming 250 MHz clock - logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm, stalls_host, performance); + if (!use_mock_device) { + logger.Info(""); + for (int i = 0; i < nstreams; i++) { + auto stalls_hbm = output.device_statistics(i).fpga_status().stalls_hbm(); + auto stalls_host = output.device_statistics(i).fpga_status().stalls_host(); - for (const auto& iter: output.device_statistics(i).fpga_status().host_writer_err()) - logger.Error("Device {}: FPGA host writer error {}", i, iter); + uint64_t throughput_MBs = nimages * nmodules * RAW_MODULE_SIZE * sizeof(uint16_t) * clock_MHz / + (nimages * nmodules * 128 * 128 + stalls_hbm); + double performance = static_cast(throughput_MBs) / 1000; - if (output.device_statistics(i).fpga_status().mailbox_err_reg() != 0) - logger.Error("Device {}: Mailbox error {:x}", i, - output.device_statistics(i).fpga_status().mailbox_err_reg()); + logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm, + stalls_host, performance); + + for (const auto &iter: output.device_statistics(i).fpga_status().host_writer_err()) + logger.Error("Device {}: FPGA host writer error {}", i, iter); + + if (output.device_statistics(i).fpga_status().mailbox_err_reg() != 0) + logger.Error("Device {}: Mailbox error {:x}", i, + output.device_statistics(i).fpga_status().mailbox_err_reg()); + } } - if (ret) { + logger.Info(""); logger.Info("Test properly executed! (check stall values manually)"); exit(EXIT_SUCCESS); - } else + } else { + logger.Info("Test finished with errors! (check stall values manually)"); exit(EXIT_FAILURE); + } } diff --git a/receiver/jfjoch_action_test_mock.cpp b/receiver/jfjoch_action_test_mock.cpp deleted file mode 100644 index bb39b251..00000000 --- a/receiver/jfjoch_action_test_mock.cpp +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (2019-2022) Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-or-later - -#include - -#include "MockAcquisitionDevice.h" -#include "JFJochReceiverTest.h" -#include "../tests/FPGAUnitTest.h" - -int main(int argc, char **argv) { - uint16_t nstreams = 1; - const uint16_t nmodules = 1; - size_t nimages = 2; - uint64_t processing_period = 20; - - Logger logger("ActionTest"); - logger.Verbose(true); - - bool abort_test = false; - - if ((argc == 1) || (argc > 6)) { - logger.Error("Usage ./jfjoch_action_test {<# of images> {<# of modules> {<# of streams> {}}}}"); - exit(EXIT_FAILURE); - } - - if (argc >= 3) nimages = atol(argv[2]); - if (argc >= 4) nstreams = atoi(argv[3]); - if (argc >= 5) processing_period = atoi(argv[4]); - - DiffractionExperiment x(DetectorGeometry(nmodules*nstreams, 2, 8, 36, true)); - - x.Mode(DetectorMode::Conversion); - x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1); - x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false); - x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams); - - bool verbose = false; - bool print_status_updates = true; - uint16_t nthreads = 16; - - logger.Verbose(verbose); - - std::vector> pcie_devices; - std::vector aq_devices; - - std::string image_path = std::string(argv[1]) + "/tests/test_data/mod5_raw0.bin"; - std::vector input(RAW_MODULE_SIZE, 0); - LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE); - - for (int i = 0; i < nstreams; i++) { - pcie_devices.push_back(std::make_unique(i, 1024)); - pcie_devices[i]->SetCustomInternalGeneratorFrame(input); - pcie_devices[i]->EnableLogging(&logger); - aq_devices.push_back(pcie_devices[i].get()); - } - - volatile bool done = false; - JFJochProtoBuf::ReceiverOutput output; - bool ret; - std::thread run_thread([&] { - try { - ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads,abort_test, verbose); - } catch (std::exception &e) { - logger.Error(e.what()); - ret = false; - } - done = true; - }); - - - run_thread.join(); - - double receiving_time = static_cast(output.end_time_ms() - output.start_time_ms())/1000.0; - - logger.Info("Efficiency: {:.2f}%", output.efficiency() * 100.f); - logger.Info("Max delay: {}",output.max_receive_delay()); - logger.Info("Compression factor: {}x", output.compressed_ratio()); - logger.Info("Receiving time: {} s", receiving_time); - logger.Info("Frame rate: {} Hz", static_cast(nimages)/receiving_time); - logger.Info("Total throughput: {:.2f} GB/s", - static_cast(nimages*nstreams*nmodules*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9)); - logger.Info(""); - - if (ret) { - logger.Info("Test properly executed! (check stall values manually)"); - exit(EXIT_SUCCESS); - } else - exit(EXIT_FAILURE); -}