mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 42 Wiki Activity

jfjoch_action_test: Command line options for more flexibility in testing

Browse Source
This commit is contained in:
leonarski_f
2023-07-27 14:09:03 +02:00
parent 8def7d624c
commit 1717887d4a
3 changed files with 116 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
receiver/CMakeLists.txt
View File

@@ -73,7 +73,3 @@ INSTALL(TARGETS jfjoch_receiver RUNTIME)
ADD_EXECUTABLE(jfjoch_action_test jfjoch_action_test.cpp)
TARGET_LINK_LIBRARIES(jfjoch_action_test JungfraujochHost JFJochReceiver)
INSTALL(TARGETS jfjoch_action_test RUNTIME)
ADD_EXECUTABLE(jfjoch_action_test_mock jfjoch_action_test_mock.cpp)
TARGET_LINK_LIBRARIES(jfjoch_action_test_mock JungfraujochHost JFJochReceiver)
INSTALL(TARGETS jfjoch_action_test_mock RUNTIME)

161
receiver/jfjoch_action_test.cpp
View File

@@ -4,49 +4,98 @@
#include <iostream>
#include "PCIExpressDevice.h"
#include "MockAcquisitionDevice.h"
#include "JFJochReceiverTest.h"
#include "../tests/FPGAUnitTest.h"
void print_usage(Logger &logger) {
logger.Info("Usage ./jfjoch_action_test {<options>} <path to repository>");
logger.Info("Options:");
logger.Info(" -C conversion on CPU");
logger.Info(" -M use mock device");
logger.Info(" -B blocking mode (FPGA)");
logger.Info(" -v verbose");
logger.Info(" -s<num> number of data streams (acquisition devices)");
logger.Info(" -m<num> number of modules per data stream");
logger.Info(" -i<num> number of images");
logger.Info(" -p<num> data processing period");
logger.Info(" -N<num> number of image processing threads");
}
int main(int argc, char **argv) {
Logger logger("ActionTest");
logger.Verbose(true);
constexpr uint64_t clock_MHz = 200;
uint16_t nstreams = 1;
uint16_t nmodules = 1;
size_t nimages = 2;
uint64_t processing_period = 20;
uint16_t nthreads = 64;
bool conversion_on_cpu = false;
bool use_mock_device = false;
bool nonblocking_mode = true;
bool verbose = false;
Logger logger("ActionTest");
logger.Verbose(true);
if (argc == 1)
print_usage(logger);
bool abort_test = false;
if ((argc == 1) || (argc > 6)) {
logger.Error("Usage ./jfjoch_action_test <path to JFjoch source> {<# of images> {<# of modules> {<# of streams> {<processing period>}}}}");
exit(EXIT_FAILURE);
int opt;
while ((opt = getopt(argc, argv, "s:i:m:p:N:CMBv")) != -1) {
switch (opt) {
case 'C':
conversion_on_cpu = true;
break;
case 'M':
use_mock_device = true;
break;
case 'B':
nonblocking_mode = false;
break;
case 'i':
nimages = atol(optarg);
break;
case 'm':
nmodules = atol(optarg);
break;
case 's':
nstreams = atol(optarg);
break;
case 'p':
processing_period = atol(optarg);
break;
case 'N':
nthreads = atol(optarg);
break;
case 'v':
verbose = true;
break;
default: /* '?' */
print_usage(logger);
exit(EXIT_FAILURE);
}
}
if (argc >= 3) nimages = atol(argv[2]);
if (argc >= 4) nmodules = atol(argv[3]);
if (argc >= 5) nstreams = atoi(argv[4]);
if (argc >= 6) processing_period = atoi(argv[5]);
if (optind != argc - 1) {
print_usage(logger);
exit(EXIT_FAILURE);
}
DiffractionExperiment x(DetectorGeometry(nmodules*nstreams, 2, 8, 36, true));
x.Mode(DetectorMode::Conversion);
x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1);
x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false);
x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false).ConversionOnCPU(conversion_on_cpu);
x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams);
logger.Info("Data streams {} Total modules {} Total images {} Threads {}", nstreams, nstreams * nmodules, nimages, nthreads);
std::vector<std::string> dev_name = {
"/dev/jfjoch0",
"/dev/jfjoch2",
"/dev/jfjoch1",
"/dev/jfjoch3"
};
std::vector<int16_t> numa_node = {0,1,0,1};
bool verbose = false;
bool print_status_updates = true;
uint16_t nthreads = 64;
uint64_t clock_MHz = 200;
bool nonblocking_mode = true;
logger.Verbose(verbose);
@@ -56,29 +105,44 @@ int main(int argc, char **argv) {
}
std::vector<std::unique_ptr<PCIExpressDevice>> pcie_devices;
std::vector<std::unique_ptr<MockAcquisitionDevice>> mock_devices;
std::vector<AcquisitionDevice *> aq_devices;
std::string image_path = std::string(argv[1]) + "/tests/test_data/mod5_raw0.bin";
std::string image_path = std::string(argv[optind]) + "/tests/test_data/mod5_raw0.bin";
std::vector<uint16_t> input(RAW_MODULE_SIZE, 0);
LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE);
for (int i = 0; i < nstreams; i++) {
pcie_devices.push_back(std::make_unique<PCIExpressDevice>(dev_name[i], i));
pcie_devices[i]->SetCustomInternalGeneratorFrame(input);
pcie_devices[i]->EnableLogging(&logger);
pcie_devices[i]->SetFPGANonBlockingMode(nonblocking_mode);
aq_devices.push_back(pcie_devices[i].get());
}
if (use_mock_device) {
if (nmodules > 1) {
logger.Warning("Conversion results will be wrong with more than 1 module per stream");
}
for (int i = 0; i < nstreams; i++) {
mock_devices.push_back(std::make_unique<MockAcquisitionDevice>(i, 1024));
mock_devices[i]->SetCustomInternalGeneratorFrame(input);
mock_devices[i]->EnableLogging(&logger);
aq_devices.push_back(mock_devices[i].get());
}
if (!nonblocking_mode)
logger.Warning("FPGA uses blocking mode - in case data acquisition is aborted, it is necessary to cold reboot the machine");
} else {
for (int i = 0; i < nstreams; i++) {
pcie_devices.push_back(std::make_unique<PCIExpressDevice>(dev_name[i], i));
pcie_devices[i]->SetCustomInternalGeneratorFrame(input);
pcie_devices[i]->EnableLogging(&logger);
pcie_devices[i]->SetFPGANonBlockingMode(nonblocking_mode);
aq_devices.push_back(pcie_devices[i].get());
}
if (!nonblocking_mode)
logger.Warning(
"FPGA uses blocking mode - in case data acquisition is aborted, it is necessary to cold reboot the machine");
}
volatile bool done = false;
JFJochProtoBuf::ReceiverOutput output;
bool ret;
std::thread run_thread([&] {
try {
ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads,abort_test, verbose);
ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads, false, verbose);
} catch (std::exception &e) {
logger.Error(e.what());
ret = false;
@@ -86,7 +150,7 @@ int main(int argc, char **argv) {
done = true;
});
if (print_status_updates) {
if (!use_mock_device) {
while (!done) {
for (int i = 0; i < nstreams; i++) {
auto status = pcie_devices[i]->GetStatus();
@@ -111,27 +175,34 @@ int main(int argc, char **argv) {
logger.Info("Frame rate: {} Hz", static_cast<double>(nimages)/receiving_time);
logger.Info("Total throughput: {:.2f} GB/s",
static_cast<double>(nimages*nstreams*nmodules*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9));
logger.Info("");
for (int i = 0; i < nstreams; i++) {
auto stalls_hbm = output.device_statistics(i).fpga_status().stalls_hbm();
auto stalls_host = output.device_statistics(i).fpga_status().stalls_host();
uint64_t throughput_MBs = nimages * nmodules * RAW_MODULE_SIZE*sizeof(uint16_t) * clock_MHz / (nimages * nmodules * 128 * 128 + stalls_hbm);
double performance = static_cast<double>(throughput_MBs) / 1000;
// Assuming 250 MHz clock
logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm, stalls_host, performance);
if (!use_mock_device) {
logger.Info("");
for (int i = 0; i < nstreams; i++) {
auto stalls_hbm = output.device_statistics(i).fpga_status().stalls_hbm();
auto stalls_host = output.device_statistics(i).fpga_status().stalls_host();
for (const auto& iter: output.device_statistics(i).fpga_status().host_writer_err())
logger.Error("Device {}: FPGA host writer error {}", i, iter);
uint64_t throughput_MBs = nimages * nmodules * RAW_MODULE_SIZE * sizeof(uint16_t) * clock_MHz /
(nimages * nmodules * 128 * 128 + stalls_hbm);
double performance = static_cast<double>(throughput_MBs) / 1000;
if (output.device_statistics(i).fpga_status().mailbox_err_reg() != 0)
logger.Error("Device {}: Mailbox error {:x}", i,
output.device_statistics(i).fpga_status().mailbox_err_reg());
logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm,
stalls_host, performance);
for (const auto &iter: output.device_statistics(i).fpga_status().host_writer_err())
logger.Error("Device {}: FPGA host writer error {}", i, iter);
if (output.device_statistics(i).fpga_status().mailbox_err_reg() != 0)
logger.Error("Device {}: Mailbox error {:x}", i,
output.device_statistics(i).fpga_status().mailbox_err_reg());
}
}
if (ret) {
logger.Info("");
logger.Info("Test properly executed! (check stall values manually)");
exit(EXIT_SUCCESS);
} else
} else {
logger.Info("Test finished with errors! (check stall values manually)");
exit(EXIT_FAILURE);
}
}

89
receiver/jfjoch_action_test_mock.cpp
View File

@@ -1,89 +0,0 @@
// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later
#include <iostream>
#include "MockAcquisitionDevice.h"
#include "JFJochReceiverTest.h"
#include "../tests/FPGAUnitTest.h"
int main(int argc, char **argv) {
uint16_t nstreams = 1;
const uint16_t nmodules = 1;
size_t nimages = 2;
uint64_t processing_period = 20;
Logger logger("ActionTest");
logger.Verbose(true);
bool abort_test = false;
if ((argc == 1) || (argc > 6)) {
logger.Error("Usage ./jfjoch_action_test <path to JFjoch source> {<# of images> {<# of modules> {<# of streams> {<processing period>}}}}");
exit(EXIT_FAILURE);
}
if (argc >= 3) nimages = atol(argv[2]);
if (argc >= 4) nstreams = atoi(argv[3]);
if (argc >= 5) processing_period = atoi(argv[4]);
DiffractionExperiment x(DetectorGeometry(nmodules*nstreams, 2, 8, 36, true));
x.Mode(DetectorMode::Conversion);
x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1);
x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false);
x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams);
bool verbose = false;
bool print_status_updates = true;
uint16_t nthreads = 16;
logger.Verbose(verbose);
std::vector<std::unique_ptr<MockAcquisitionDevice>> pcie_devices;
std::vector<AcquisitionDevice *> aq_devices;
std::string image_path = std::string(argv[1]) + "/tests/test_data/mod5_raw0.bin";
std::vector<uint16_t> input(RAW_MODULE_SIZE, 0);
LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE);
for (int i = 0; i < nstreams; i++) {
pcie_devices.push_back(std::make_unique<MockAcquisitionDevice>(i, 1024));
pcie_devices[i]->SetCustomInternalGeneratorFrame(input);
pcie_devices[i]->EnableLogging(&logger);
aq_devices.push_back(pcie_devices[i].get());
}
volatile bool done = false;
JFJochProtoBuf::ReceiverOutput output;
bool ret;
std::thread run_thread([&] {
try {
ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads,abort_test, verbose);
} catch (std::exception &e) {
logger.Error(e.what());
ret = false;
}
done = true;
});
run_thread.join();
double receiving_time = static_cast<double>(output.end_time_ms() - output.start_time_ms())/1000.0;
logger.Info("Efficiency: {:.2f}%", output.efficiency() * 100.f);
logger.Info("Max delay: {}",output.max_receive_delay());
logger.Info("Compression factor: {}x", output.compressed_ratio());
logger.Info("Receiving time: {} s", receiving_time);
logger.Info("Frame rate: {} Hz", static_cast<double>(nimages)/receiving_time);
logger.Info("Total throughput: {:.2f} GB/s",
static_cast<double>(nimages*nstreams*nmodules*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9));
logger.Info("");
if (ret) {
logger.Info("Test properly executed! (check stall values manually)");
exit(EXIT_SUCCESS);
} else
exit(EXIT_FAILURE);
}