Files
Jungfraujoch/fpga/pcie_driver/jfjoch_function.c
2025-03-02 13:15:28 +01:00

427 lines
16 KiB
C

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "jfjoch_drv.h"
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/delay.h>
uint32_t jfjoch_read_register(struct jfjoch_drvdata *drvdata, uint32_t addr) {
return ioread32(drvdata->bar0 + addr);
}
void jfjoch_start(struct jfjoch_drvdata *drvdata) {
u32 tmp1, tmp2;
// Drain work completion queue
mutex_lock(&drvdata->work_compl_read_mutex);
while(!kfifo_is_empty(&drvdata->work_compl))
tmp2 = kfifo_get(&drvdata->work_compl, &tmp1);
mutex_unlock(&drvdata->work_compl_read_mutex);
// Set PCIe beats counters
iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
// Start DMA
// Run C2H
iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x04);
// Write Start value to action config register
iowrite32(0x1, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
void jfjoch_end(struct jfjoch_drvdata *drvdata) {
// Write cancel register
iowrite32(0x4, drvdata->bar0 + ACTION_CONFIG_OFFSET);
// RUN ==> C2H channel 0 control register
iowrite32(0, drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x04);
}
void jfjoch_cancel(struct jfjoch_drvdata *drvdata) {
iowrite32(0x4, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
int jfjoch_send_wr(struct jfjoch_drvdata *drvdata, u32 handle) {
u32 full;
if (handle >= nbuffer)
return -EFAULT;
spin_lock(&drvdata->work_request_submit_spinlock);
full = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_FULL;
if (full) {
spin_unlock(&drvdata->work_request_submit_spinlock);
return -EAGAIN;
}
iowrite32(handle, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA);
atomic_inc(&drvdata->active_handles);
spin_unlock(&drvdata->work_request_submit_spinlock);
return 0;
}
int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
int ret, tmp;
mutex_lock(&drvdata->work_compl_read_mutex);
ret = wait_event_interruptible_timeout(drvdata->work_compl_wait_queue, !kfifo_is_empty(&drvdata->work_compl), HZ);
if (ret > 0)
tmp = kfifo_get(&drvdata->work_compl, output);
mutex_unlock(&drvdata->work_compl_read_mutex);
if (ret < 0)
return ret;
else if ((ret == 0) || (tmp == 0))
return -EAGAIN;
return 0;
}
int jfjoch_set_config(struct jfjoch_drvdata *drvdata, const struct DataCollectionConfig *config) {
if (config->nframes > MAX_FRAMES)
return -EINVAL;
memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_NMODULES, config, sizeof(struct DataCollectionConfig));
return 0;
}
void jfjoch_get_config(struct jfjoch_drvdata *drvdata, struct DataCollectionConfig *config) {
memcpy_fromio(config, (drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_NMODULES, sizeof(struct DataCollectionConfig));
}
void jfjoch_get_status(struct jfjoch_drvdata *drvdata, struct DataCollectionStatus *status) {
memcpy_fromio(status, drvdata->bar0 + ACTION_CONFIG_OFFSET, sizeof(struct DataCollectionStatus));
}
int jfjoch_get_net_cfg(struct jfjoch_drvdata *drvdata, u32 net_if, struct NetworkStatus *status) {
if ((net_if >= drvdata->eth_links) && (net_if != NET_IF_FRAME_GENERATOR))
return -EINVAL;
memcpy_fromio(status, (drvdata->bar0) + NET_CFG_OFFSET + net_if * 0x10000, sizeof(struct NetworkStatus));
return 0;
}
int jfjoch_set_mac_addr(struct jfjoch_drvdata *drvdata, u32 net_if, u64 *mac_addr) {
if ((net_if >= drvdata->eth_links) && (net_if != NET_IF_FRAME_GENERATOR))
return -EINVAL;
memcpy_toio((drvdata->bar0) + NET_CFG_OFFSET + net_if * 0x10000 + ADDR_NET_MAC_ADDR_LO, mac_addr, sizeof(uint64_t));
return 0;
}
int jfjoch_get_mac_addr(struct jfjoch_drvdata *drvdata, u32 net_if, u64 *mac_addr) {
struct NetworkStatus status;
int ret = jfjoch_get_net_cfg(drvdata, net_if, &status);
if (ret == 0)
*mac_addr = status.mac_addr;
return ret;
}
int jfjoch_set_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 net_if, const u32 *addr) {
if ((net_if >= drvdata->eth_links) && (net_if != NET_IF_FRAME_GENERATOR))
return -EINVAL;
iowrite32(*addr, (drvdata->bar0) + NET_CFG_OFFSET + net_if * 0x10000 + ADDR_NET_IPV4_ADDR);
return 0;
}
int jfjoch_set_net_mode(struct jfjoch_drvdata *drvdata, u32 net_if, const u32 *net_mode) {
if ((net_if >= drvdata->eth_links) && (net_if != NET_IF_FRAME_GENERATOR))
return -EINVAL;
iowrite32(*net_mode, (drvdata->bar0) + NET_CFG_OFFSET + net_if * 0x10000 + ADDR_NET_MODE);
return 0;
}
int jfjoch_get_net_mode(struct jfjoch_drvdata *drvdata, u32 net_if, u32 *net_mode) {
struct NetworkStatus status;
int ret = jfjoch_get_net_cfg(drvdata, net_if, &status);
if (ret == 0)
*net_mode = status.jfjoch_net_mode;
return ret;
}
int jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 net_if, u32 *addr) {
struct NetworkStatus status;
int ret = jfjoch_get_net_cfg(drvdata, net_if, &status);
if (ret == 0)
*addr = status.ipv4_addr;
return ret;
}
u64 jfjoch_read_cms_default_config(struct jfjoch_drvdata *drvdata) {
struct device *const dev = &drvdata->pdev->dev;
u32 tmp, host_msg_offset, msg_len, opcode;
u32 field_type, field_len;
u64 i, j;
u8 output[256];
u64 mac = 0;
if (ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5)) {
dev_err(dev, "Mailbox not available");
return 0;
}
host_msg_offset = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28300);
iowrite32(0x04 << 24, drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset);
iowrite32(1<<5, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG);
i = 0;
while (i < 50) {
if (!(ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5)))
break;
msleep(100);
i++;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5);
if (tmp) {
dev_err(dev, "Mailbox not ready %x", tmp);
return 0;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28304);
if (tmp) {
dev_err(dev, "Error in mailbox %x", tmp);
return 0;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset);
opcode = (tmp >> 24) & 0xFF;
msg_len = tmp & 0xFFF;
if (opcode != 0x04) {
dev_err(dev, "Opcode in return message 0x%x doesn't match 0x%x", opcode, 0x04);
return 0;
}
i = 0;
while (i < msg_len) {
field_type = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i);
field_len = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i + 1);
for (j = 0; j < field_len; j++)
output[j] = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i + 2 + j);
output[field_len] = 0x0;
if (field_type == 0x21) {
memcpy(drvdata->fpga_serial_number, output, 63);
drvdata->fpga_serial_number[63] = 0;
}
if (field_type == 0x4B) {
mac = ((u64)(output[2]))
| ((u64)(output[3]) << 8)
| ((u64)(output[4]) << 16)
| ((u64)(output[5]) << 24)
| ((u64)(output[6]) << 32)
| ((u64)(output[7]) << 40);
drvdata->default_mac = mac;
drvdata->number_of_seq_mac_addrs = output[0];
for (j = 0; j < drvdata->eth_links; j++) {
u64 local_mac = drvdata->default_mac + (j << 40);
jfjoch_set_mac_addr(drvdata, j, &local_mac);
}
jfjoch_set_mac_addr(drvdata, NET_IF_FRAME_GENERATOR, &drvdata->default_mac);
}
i += field_len + 2;
}
return mac;
}
void jfjoch_is_idle(struct jfjoch_drvdata *drvdata, uint32_t *output) {
if (ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_CTRL_REGISTER) & CTRL_REGISTER_IDLE)
*output = 1;
else
*output = 0;
}
void jfjoch_setup_cms(struct jfjoch_drvdata *drvdata) {
struct device *const dev = &drvdata->pdev->dev;
u32 reg_map_ready, i;
iowrite32(0x1, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_MB_RESETN_REG);
iowrite32(1 << 27, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG); // enable HBM monitoring
reg_map_ready = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HOST_STATUS2_REG) & 0x1;
i = 0;
while (i < 10000) {
if (reg_map_ready)
break;
msleep(1);
reg_map_ready = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HOST_STATUS2_REG) & 0x1;
i++;
}
if (i == 10000)
dev_err(dev, "CMS not ready after 10s");
}
void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *env_params) {
u16 pcie_link_status;
struct DataCollectionStatus status;
jfjoch_get_status(drvdata, &status);
env_params->packets_sls = status.packets_sls;
env_params->packets_udp = status.packets_udp;
env_params->mailbox_status_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS);
env_params->mailbox_err_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_ERR);
env_params->mailbox_interrupt_status = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_IS);
env_params->fpga_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_FPGA_TEMP_INS_REG);
env_params->fpga_pcie_3p3V_I_mA = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_3V3PEX_I_IN_INS_REG);
env_params->fpga_pcie_12V_I_mA = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_12VPEX_I_IN_INS_REG);
env_params->fpga_pcie_3p3V_V_mV = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_3V3_PEX_INS_REG);
env_params->fpga_pcie_12V_V_mV = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_12V_PEX_INS_REG);
env_params->pcie_h2c_descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x48);
env_params->pcie_h2c_beats = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xCC);
env_params->pcie_h2c_status = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x40);
env_params->pcie_c2h_descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x48);
env_params->pcie_c2h_beats = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xCC);
env_params->pcie_c2h_status = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x40);
env_params->pcie_user_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IE_MASK);
env_params->pcie_dma_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IE_MASK);
env_params->pcie_user_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IP);
env_params->pcie_dma_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IP);
env_params->pcie_user_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IR);
env_params->pcie_dma_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IR);
env_params->hbm_0_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP1_INS_REG);
env_params->hbm_1_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP2_INS_REG);
env_params->qsfp_cage_0_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CAGE_TEMP0_INS_REG);
env_params->qsfp_cage_1_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CAGE_TEMP1_INS_REG);
env_params->eth_link_count = drvdata->eth_links;
env_params->eth_link_status = jfjoch_network_check_link(drvdata);
env_params->work_compl_fifo_avail = kfifo_avail(&drvdata->work_compl);
env_params->active_handles = atomic_read(&drvdata->active_handles);
env_params->fpga_default_mac_addr = drvdata->default_mac;
env_params->fpga_default_mac_addrs_count = drvdata->number_of_seq_mac_addrs;
memcpy(env_params->serial_number, drvdata->fpga_serial_number, 63);
env_params->serial_number[63] = 0;
scnprintf(env_params->device_number, 64, "%02x:%02x:%01x",
drvdata->pdev->bus->number, PCI_SLOT( drvdata->pdev->devfn), PCI_FUNC( drvdata->pdev->devfn));
scnprintf(env_params->fpga_firmware_version, 64, "%s", drvdata->fpga_version);
if (ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_CTRL_REGISTER) & CTRL_REGISTER_IDLE)
env_params->idle = true;
else
env_params->idle = false;
pcie_link_status = jfjoch_get_pcie_link_status(drvdata);
env_params->pcie_link_speed = pcie_link_status & PCI_EXP_LNKSTA_CLS;
env_params->pcie_link_width = (pcie_link_status & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT;
}
void jfjoch_clr_net_counters(struct jfjoch_drvdata *drvdata) {
iowrite32(1 << 3, drvdata->bar0 + ACTION_CONFIG_OFFSET);
iowrite32(0, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct LoadCalibrationConfig *config) {
struct device *const dev = &drvdata->pdev->dev;
u32 i, tmp, hls_ctrl_reg;
hls_ctrl_reg = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
if ((hls_ctrl_reg & (1<<2)) == 0) {
dev_err(dev, "Load calibration not ready for operation");
return -EBUSY;
}
if (config->handle >= drvdata->nbuf) {
dev_err(dev, "Not enough buffers to support this card\n");
return -EINVAL;
}
// Start DMA
// Clear counters and RUN H2C
iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0);
iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0);
iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04);
// Setup HLS core and start operation
memcpy_toio(drvdata->bar0 + ADDR_LOAD_CALIBRATION_HANDLE, config, sizeof(struct LoadCalibrationConfig));
iowrite32(0x1, drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
i = 0;
while (i < 1000) {
tmp = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
if (tmp & (1 << 1))
break;
udelay(5);
i++;
}
// STOP H2C channel
iowrite32(0, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04);
if (i == 5000) {
dev_err(dev, "Loading didn't finish in 5 milliseconds\n");
return -ETIMEDOUT;
} else {
u32 ret = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_RETURN);
if (ret != 0) {
dev_err(dev, "Loading error\n");
return -EINVAL;
}
}
return 0;
}
int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config) {
struct device *const dev = &drvdata->pdev->dev;
if (ioread32(drvdata->bar0 + ADDR_FRAME_GEN_CTRL) & 0x1) {
dev_err(dev, "Frame generator busy\n");
return -EBUSY;
}
memcpy_toio(drvdata->bar0 + ADDR_FRAME_GEN_CONFIG, config, sizeof(struct FrameGeneratorConfig));
iowrite32(0x1, drvdata->bar0 + ADDR_FRAME_GEN_CTRL);
return 0;
}
void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
params,
sizeof(struct SpotFinderParameters));
}
void jfjoch_get_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
memcpy_fromio(params,
(drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
sizeof(struct SpotFinderParameters));
}
u16 jfjoch_get_pcie_link_status(struct jfjoch_drvdata *drvdata) {
int pos;
u16 link_status;
/* Find PCI Express Capability offset */
pos = pci_find_capability(drvdata->pdev, PCI_CAP_ID_EXP);
if (!pos) {
pr_err("Device does not support PCI Express\n");
return 0;
}
/* Read the Link Status register (part of PCIe capability structure) */
pci_read_config_word(drvdata->pdev, pos + PCI_EXP_LNKSTA, &link_status);
return link_status;
}