Jungfraujoch/receiver/jfjoch_action_test.cpp

// Copyright (2019-2022) Paul Scherrer Institute
// SPDX-License-Identifier: GPL-3.0-or-later

#include <iostream>

#include "PCIExpressDevice.h"
#include "JFJochReceiverTest.h"
#include "../tests/FPGAUnitTest.h"

int main(int argc, char **argv) {
    uint16_t nstreams = 1;
    uint16_t nmodules = 1;
    size_t nimages = 2;
    uint64_t processing_period = 20;

    Logger logger("ActionTest");
    logger.Verbose(true);

    bool abort_test = false;

    if ((argc == 1) || (argc > 6)) {
        logger.Error("Usage ./jfjoch_action_test <path to JFjoch source> {<# of images> {<# of modules> {<# of streams> {<processing period>}}}}");
        exit(EXIT_FAILURE);
    }

    if (argc >= 3) nimages = atol(argv[2]);
    if (argc >= 4) nmodules = atol(argv[3]);
    if (argc >= 5) nstreams = atoi(argv[4]);
    if (argc >= 6) processing_period = atoi(argv[5]);

    DiffractionExperiment x(DetectorGeometry(nmodules*nstreams, 2, 8, 36, true));

    x.Mode(DetectorMode::Conversion);
    x.ImagesPerTrigger(nimages).PedestalG0Frames(0).UseInternalPacketGenerator(true).PhotonEnergy_keV(12.4).NumTriggers(1);
    x.SpotFindingPeriod(std::chrono::milliseconds(processing_period)).MaskModuleEdges(false).MaskChipEdges(false);
    x.Compression(JFJochProtoBuf::BSHUF_LZ4).DataStreams(nstreams);

    std::vector<std::string> dev_name = {
            "/dev/jfjoch0",
            "/dev/jfjoch2",
            "/dev/jfjoch1",
            "/dev/jfjoch3"
    };
    std::vector<int16_t> numa_node = {0,1,0,1};
    bool verbose = false;
    bool print_status_updates = true;
    uint16_t nthreads = 64;
    uint64_t clock_MHz = 200;
    bool nonblocking_mode = true;

    logger.Verbose(verbose);

    if (nstreams > dev_name.size()) {
        logger.Error("Only {} data streams allowed on this platform", dev_name.size());
        exit(EXIT_FAILURE);
    }

    std::vector<std::unique_ptr<PCIExpressDevice>> pcie_devices;
    std::vector<AcquisitionDevice *> aq_devices;

    std::string image_path = std::string(argv[1]) + "/tests/test_data/mod5_raw0.bin";
    std::vector<uint16_t> input(RAW_MODULE_SIZE, 0);
    LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE);

    for (int i = 0; i < nstreams; i++) {
        pcie_devices.push_back(std::make_unique<PCIExpressDevice>(dev_name[i], i));
        pcie_devices[i]->SetCustomInternalGeneratorFrame(input);
        pcie_devices[i]->EnableLogging(&logger);
        pcie_devices[i]->SetFPGANonBlockingMode(nonblocking_mode);
        aq_devices.push_back(pcie_devices[i].get());
    }

    if (!nonblocking_mode)
        logger.Warning("FPGA uses blocking mode - in case data acquisition is aborted, it is necessary to cold reboot the machine");

    volatile bool done = false;
    JFJochProtoBuf::ReceiverOutput output;
    bool ret;
    std::thread run_thread([&] {
        try {
            ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads,abort_test, verbose);
        } catch (std::exception &e) {
            logger.Error(e.what());
            ret = false;
        }
        done = true;
    });

    if (print_status_updates) {
        while (!done) {
            for (int i = 0; i < nstreams; i++) {
                auto status = pcie_devices[i]->GetStatus();
                logger.Info("Device {}:  Head packet: {:8d}  Power: {:5.1f} W  FPGA Temp: {:d} degC  HBM Temp: {:d}/{:d} degC  Stalls: {:15d}",
                            i, status.slowest_head(), status.current_edge_12v_a() * status.voltage_edge_12v_v() +
                                                      status.current_edge_3p3v_a() * status.voltage_edge_3p3v_v(),
                            std::lround(status.fpga_temp_degc()), status.hbm_temp_0_degc(), status.hbm_temp_1_degc(),
                            status.stalls_hbm());
            }
            std::this_thread::sleep_for(std::chrono::seconds(1));
        }
    }

    run_thread.join();

    double receiving_time = static_cast<double>(output.end_time_ms() - output.start_time_ms())/1000.0;

    logger.Info("Efficiency: {:.2f}%", output.efficiency() * 100.f);
    logger.Info("Max delay: {}",output.max_receive_delay());
    logger.Info("Compression factor: {}x", output.compressed_ratio());
    logger.Info("Receiving time: {} s", receiving_time);
    logger.Info("Frame rate: {} Hz", static_cast<double>(nimages)/receiving_time);
    logger.Info("Total throughput: {:.2f} GB/s",
            static_cast<double>(nimages*nstreams*nmodules*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9));
    logger.Info("");
    for (int i = 0; i < nstreams; i++) {
        auto stalls_hbm = output.device_statistics(i).fpga_status().stalls_hbm();
        auto stalls_host = output.device_statistics(i).fpga_status().stalls_host();

        uint64_t throughput_MBs = nimages * nmodules * RAW_MODULE_SIZE*sizeof(uint16_t) * clock_MHz / (nimages * nmodules * 128 * 128 + stalls_hbm);
        double performance = static_cast<double>(throughput_MBs) / 1000;
        // Assuming 250 MHz clock
        logger.Info("Device {}:  stalls HBM: {}  stalls host: {}   est. performance: {:.2f} GB/s", i, stalls_hbm, stalls_host, performance);

        if (output.device_statistics(i).fpga_status().frame_statistics_alignment_err())
            logger.Error("Device {}: memory alignment error", i);
        if (output.device_statistics(i).fpga_status().frame_statistics_tlast_err())
            logger.Error("Device {}: error in AXI-Stream sequence", i);
        if (output.device_statistics(i).fpga_status().frame_statistics_work_req_err())
            logger.Error("Device {}: parity error in work request", i);
         if (output.device_statistics(i).fpga_status().mailbox_err_reg() != 0)
            logger.Error("Device {}: Mailbox error {:x}", i,
                         output.device_statistics(i).fpga_status().mailbox_err_reg());
    }

    if (ret) {
        logger.Info("Test properly executed! (check stall values manually)");
        exit(EXIT_SUCCESS);
    } else
        exit(EXIT_FAILURE);
}