263 lines
11 KiB
C++
263 lines
11 KiB
C++
// Copyright (2019-2023) Paul Scherrer Institute
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
#include "FPGAAcquisitionDevice.h"
|
|
#include <random>
|
|
#include <bitset>
|
|
|
|
void FPGAAcquisitionDevice::StartSendingWorkRequests() {
|
|
stop_work_requests = false;
|
|
send_work_request_future = std::async(std::launch::async, &FPGAAcquisitionDevice::SendWorkRequestThread, this);
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::Finalize() {
|
|
read_work_completion_future.get();
|
|
|
|
stop_work_requests = true;
|
|
send_work_request_future.get();
|
|
|
|
FPGA_EndAction();
|
|
|
|
while (!HW_IsIdle())
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::ReadWorkCompletionThread() {
|
|
uint32_t values[16];
|
|
|
|
Completion c{};
|
|
bool quit_loop = false;
|
|
do {
|
|
while (!HW_ReadMailbox(values))
|
|
std::this_thread::sleep_for(std::chrono::microseconds(10));
|
|
|
|
c = parse_hw_completion(values);
|
|
if (c.data_collection_id == data_collection_id) {
|
|
work_completion_queue.PutBlocking(c);
|
|
if (c.type == Completion::Type::End)
|
|
quit_loop = true;
|
|
}
|
|
} while (!quit_loop);
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::SendWorkRequestThread() {
|
|
while (!stop_work_requests) {
|
|
WorkRequest wr{};
|
|
if (work_request_queue.Get(wr)) {
|
|
if ( !HW_SendWorkRequest(wr.handle)) {
|
|
work_request_queue.Put(wr);
|
|
std::this_thread::sleep_for(std::chrono::microseconds(10));
|
|
}
|
|
} else {
|
|
std::this_thread::sleep_for(std::chrono::microseconds(10));
|
|
}
|
|
}
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) {
|
|
auto offset = experiment.GetFirstModuleOfDataStream(data_stream);
|
|
|
|
if (calib.GetModulesNum() != experiment.GetModulesNum())
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Mismatch regarding module count in calibration and experiment description");
|
|
|
|
if (calib.GetStorageCellNum() != experiment.GetStorageCellNumber())
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Mismatch regarding storage cell count in calibration and experiment description");
|
|
|
|
size_t modules = experiment.GetModulesNum(data_stream);
|
|
|
|
if (1 + modules * (3 + 3 * experiment.GetStorageCellNumber()) > buffer_device.size())
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Not enough host/FPGA buffers to load all calibration constants");
|
|
|
|
for (int m = 0; m < modules; m++) {
|
|
calib.GainCalibration(m).ExportG0(buffer_device[1 + m]);
|
|
calib.GainCalibration(m).ExportG1(buffer_device[1 + m + modules]);
|
|
calib.GainCalibration(m).ExportG2(buffer_device[1 + m + modules * 2]);
|
|
}
|
|
|
|
for (int s = 0; s < experiment.GetStorageCellNumber(); s++) {
|
|
auto mask = calib.CalculateMask(experiment, s);
|
|
for (int m = 0; m < modules; m++) {
|
|
auto pedestal_g0 = calib.Pedestal(offset + m, 0, s).GetPedestal();
|
|
auto pedestal_g1 = calib.Pedestal(offset + m, 1, s).GetPedestal();
|
|
auto pedestal_g2 = calib.Pedestal(offset + m, 2, s).GetPedestal();
|
|
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
|
|
if (experiment.GetApplyPixelMaskInFPGA() && (mask[(offset + m) * RAW_MODULE_SIZE + i] != 0)) {
|
|
buffer_device[1 + m + (3 + 0 * 16 + s) * modules][i] = 16384;
|
|
buffer_device[1 + m + (3 + 1 * 16 + s) * modules][i] = 16384;
|
|
buffer_device[1 + m + (3 + 2 * 16 + s) * modules][i] = 16384;
|
|
} else {
|
|
buffer_device[1 + m + (3 + 0 * 16 + s) * modules][i] = pedestal_g0[i];
|
|
buffer_device[1 + m + (3 + 1 * 16 + s) * modules][i] = pedestal_g1[i];
|
|
buffer_device[1 + m + (3 + 2 * 16 + s) * modules][i] = pedestal_g2[i];
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, ActionConfig &job) {
|
|
std::random_device rd;
|
|
std::uniform_int_distribution<uint16_t> dist;
|
|
data_collection_id = dist(rd);
|
|
|
|
job.nmodules = x.GetModulesNum(data_stream);
|
|
job.nframes = x.GetFrameNum();
|
|
job.one_over_energy = std::lround((1<<20)/ x.GetPhotonEnergy_keV());
|
|
job.nstorage_cells = x.GetStorageCellNumber() - 1;
|
|
job.mode = data_collection_id << 16;
|
|
|
|
if (fpga_non_blocking_mode)
|
|
job.mode |= MODE_NONBLOCKING_ON_WR;
|
|
|
|
if ((x.GetDetectorMode() == DetectorMode::Conversion) && x.GetConversionOnFPGA())
|
|
job.mode |= MODE_CONV;
|
|
|
|
if (x.IsUsingInternalPacketGen())
|
|
job.mode |= MODE_INTERNAL_PACKET_GEN;
|
|
}
|
|
|
|
|
|
void FPGAAcquisitionDevice::Start(const DiffractionExperiment &experiment) {
|
|
if (!HW_IsIdle())
|
|
throw(JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
|
|
"Hardware action running prior to start of data acquisition"));
|
|
ActionConfig cfg_in{}, cfg_out{};
|
|
|
|
FillActionRegister(experiment, cfg_in);
|
|
HW_WriteActionRegister(&cfg_in);
|
|
HW_ReadActionRegister(&cfg_out);
|
|
|
|
if (experiment.IsUsingInternalPacketGen())
|
|
memcpy(buffer_device[0], internal_pkt_gen_frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t));
|
|
|
|
if (cfg_out.mode != cfg_in.mode)
|
|
throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
|
|
"Mismatch between expected and actual values of configuration registers (mode)");
|
|
if (cfg_out.nframes != cfg_in.nframes)
|
|
throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
|
|
"Mismatch between expected and actual values of configuration registers (Frames per trigger)");
|
|
if (cfg_out.nmodules != cfg_in.nmodules)
|
|
throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
|
|
"Mismatch between expected and actual values of configuration registers (#modules)");
|
|
|
|
FPGA_StartAction();
|
|
|
|
read_work_completion_future = std::async(std::launch::async, &FPGAAcquisitionDevice::ReadWorkCompletionThread, this);
|
|
}
|
|
|
|
ActionConfig FPGAAcquisitionDevice::ReadActionRegister() {
|
|
ActionConfig cfg{};
|
|
HW_ReadActionRegister(&cfg);
|
|
return cfg;
|
|
}
|
|
|
|
inline void FIFO_check(JFJochProtoBuf::FPGAStatus &fpga_status,
|
|
const std::string &name,
|
|
uint32_t fifo_register,
|
|
uint16_t pos_empty,
|
|
uint16_t pos_full) {
|
|
auto fifo_status = fpga_status.add_fifo_status();
|
|
fifo_status->set_name(name);
|
|
|
|
if (std::bitset<32>(fifo_register).test(pos_empty))
|
|
fifo_status->set_value(JFJochProtoBuf::FPGAFIFOStatusEnum::EMPTY);
|
|
if (std::bitset<32>(fifo_register).test(pos_full))
|
|
fifo_status->set_value(JFJochProtoBuf::FPGAFIFOStatusEnum::FULL);
|
|
fifo_status->set_value(JFJochProtoBuf::FPGAFIFOStatusEnum::PARTIAL);
|
|
}
|
|
|
|
JFJochProtoBuf::FPGAStatus FPGAAcquisitionDevice::GetStatus() const {
|
|
|
|
ActionStatus status{};
|
|
ActionEnvParams env{};
|
|
|
|
HW_GetStatus(&status);
|
|
HW_GetEnvParams(&env);
|
|
|
|
JFJochProtoBuf::FPGAStatus ret;
|
|
auto full_status_register = status.ctrl_reg;
|
|
ret.set_full_status_register(full_status_register);
|
|
|
|
ret.set_stalls_hbm(status.pipeline_stalls_hbm);
|
|
ret.set_stalls_host(status.pipeline_stalls_host);
|
|
|
|
ret.set_max_modules(status.max_modules);
|
|
ret.set_git_sha1(status.git_sha1);
|
|
|
|
FIFO_check(ret, "Conversion input (data)", status.fifo_status, 0, 1);
|
|
FIFO_check(ret, "Conversion input (cmd)", status.fifo_status, 2, 3);
|
|
FIFO_check(ret, "UDP", status.fifo_status, 6, 7);
|
|
FIFO_check(ret, "Work Request", status.fifo_status, 12, 13);
|
|
FIFO_check(ret, "Work Completion", status.fifo_status, 14, 15);
|
|
FIFO_check(ret, "Host mem (data)", status.fifo_status, 8, 9);
|
|
FIFO_check(ret, "Host mem (cmd)", status.fifo_status, 10, 11);
|
|
FIFO_check(ret, "Data FIFO #8", status.fifo_status, 16, 17);
|
|
FIFO_check(ret, "Addr FIFO #3", status.fifo_status, 18, 19);
|
|
|
|
ret.set_fpga_idle(HW_IsIdle());
|
|
|
|
ret.set_packets_ether(status.packets_eth);
|
|
ret.set_packets_udp(status.packets_udp);
|
|
ret.set_packets_icmp(status.packets_icmp);
|
|
ret.set_packets_jfjoch(status.packets_processed);
|
|
ret.set_packets_sls(status.packets_sls);
|
|
ret.set_error_eth(status.udp_err_eth);
|
|
ret.set_error_packet_len(status.udp_err_len);
|
|
ret.set_cancel_bit(full_status_register & (1<<2));
|
|
ret.set_host_writer_idle(full_status_register & (1<<4));
|
|
ret.set_frame_statistics_alignment_err(full_status_register & (1 << 24));
|
|
ret.set_frame_statistics_tlast_err(full_status_register & (1 << 25));
|
|
ret.set_frame_statistics_work_req_err(full_status_register & (1 << 26));
|
|
|
|
ret.set_mailbox_status_reg(env.mailbox_status_reg);
|
|
ret.set_mailbox_err_reg(env.mailbox_err_reg);
|
|
|
|
ret.set_fpga_temp_degc(env.fpga_temp_C);
|
|
|
|
ret.set_current_edge_12v_a(static_cast<double>(env.fpga_pcie_12V_I_mA) / 1000.0);
|
|
ret.set_voltage_edge_12v_v(static_cast<double>(env.fpga_pcie_12V_V_mV) / 1000.0);
|
|
|
|
ret.set_current_edge_3p3v_a(static_cast<double>(env.fpga_pcie_3p3V_I_mA) / 1000.0);
|
|
ret.set_voltage_edge_3p3v_v(static_cast<double>(env.fpga_pcie_3p3V_V_mV) / 1000.0);
|
|
|
|
ret.set_pcie_c2h_beats(env.pcie_c2h_beats);
|
|
ret.set_pcie_h2c_beats(env.pcie_h2c_beats);
|
|
ret.set_pcie_c2h_descriptors(env.pcie_c2h_descriptors);
|
|
ret.set_pcie_h2c_descriptors(env.pcie_h2c_descriptors);
|
|
ret.set_pcie_c2h_status(env.pcie_c2h_status);
|
|
ret.set_pcie_h2c_status(env.pcie_h2c_status);
|
|
|
|
ret.set_ethernet_rx_aligned(env.ethernet_aligned);
|
|
ret.set_hbm_temp_0_degc(env.hbm_0_temp_C);
|
|
ret.set_hbm_temp_1_degc(env.hbm_1_temp_C);
|
|
ret.set_slowest_head(counters.GetSlowestHead());
|
|
return ret;
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::SetFPGANonBlockingMode(bool input) {
|
|
fpga_non_blocking_mode = input;
|
|
}
|
|
|
|
void FPGAAcquisitionDevice::SetCustomInternalGeneratorFrame(const std::vector<uint16_t> &v) {
|
|
if (v.size() != RAW_MODULE_SIZE)
|
|
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
|
"Error in size of custom internal generator frame");
|
|
for (int i = 0; i < RAW_MODULE_SIZE; i++)
|
|
internal_pkt_gen_frame[i] = v[i];
|
|
}
|
|
|
|
std::vector<uint16_t> FPGAAcquisitionDevice::GetInternalGeneratorFrame() const {
|
|
return internal_pkt_gen_frame;
|
|
}
|
|
|
|
FPGAAcquisitionDevice::FPGAAcquisitionDevice(uint16_t data_stream)
|
|
: AcquisitionDevice(data_stream),
|
|
internal_pkt_gen_frame(RAW_MODULE_SIZE) {
|
|
for (int i = 0; i < RAW_MODULE_SIZE; i++)
|
|
internal_pkt_gen_frame[i] = i % 65536;
|
|
} |