Files
Jungfraujoch/fpga/pcie_driver/jfjoch_function.c

440 lines
17 KiB
C

// Copyright (2019-2023) Paul Scherrer Institute
#include "jfjoch_drv.h"
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/delay.h>
uint32_t jfjoch_read_register(struct jfjoch_drvdata *drvdata, uint32_t addr) {
return ioread32(drvdata->bar0 + addr);
}
void jfjoch_start(struct jfjoch_drvdata *drvdata) {
u32 tmp1, tmp2;
// Drain work completion queue
mutex_lock(&drvdata->work_compl_read_mutex);
while(!kfifo_is_empty(&drvdata->work_compl))
tmp2 = kfifo_get(&drvdata->work_compl, &tmp1);
// Reset work completion queue
drvdata->work_compl_count = 0;
mutex_unlock(&drvdata->work_compl_read_mutex);
// Set PCIe beats counters
iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xC0);
// Start DMA
// Run C2H
iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x04);
// Write Start value to action config register
iowrite32(0x1, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
void jfjoch_end(struct jfjoch_drvdata *drvdata) {
// Write cancel register
iowrite32(0x4, drvdata->bar0 + ACTION_CONFIG_OFFSET);
// RUN ==> C2H channel 0 control register
iowrite32(0, drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x04);
}
void jfjoch_cancel(struct jfjoch_drvdata *drvdata) {
iowrite32(0x4, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
int jfjoch_send_wr(struct jfjoch_drvdata *drvdata, u32 handle) {
u32 full;
if (handle >= nbuffer)
return -EFAULT;
spin_lock(&drvdata->work_request_submit_spinlock);
full = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_FULL;
if (full) {
spin_unlock(&drvdata->work_request_submit_spinlock);
return -EAGAIN;
}
iowrite32(handle, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA);
atomic_inc(&drvdata->active_handles);
spin_unlock(&drvdata->work_request_submit_spinlock);
return 0;
}
int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) {
struct device *const dev = &drvdata->pdev->dev;
int ret, tmp, i;
u32 curr_compl_count, handle;
u32 data_collection_id;
mutex_lock(&drvdata->work_compl_read_mutex);
ret = wait_event_interruptible_timeout(drvdata->work_compl_wait_queue, !kfifo_is_empty(&drvdata->work_compl), HZ);
if (ret > 0) {
tmp = kfifo_get(&drvdata->work_compl, output);
handle = (*output) & 0xFFFF;
data_collection_id = (*output >> 16) & 0xFFFF;
if (handle == HANDLE_START) {
drvdata->work_compl_count = 0;
curr_compl_count = 0;
} else if ((handle != HANDLE_END) && (data_collection_id != DATA_COLLECTION_ID_PURGE)) {
curr_compl_count = drvdata->work_compl_count;
drvdata->work_compl_count++;
}
}
mutex_unlock(&drvdata->work_compl_read_mutex);
if (ret < 0)
return ret;
else if ((ret == 0) || (tmp == 0))
return -EAGAIN;
// Guarantee that data are in host memory
if ((handle != HANDLE_START) && (handle != HANDLE_END)) {
u32 descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
i = 0;
while ((descriptors < curr_compl_count * DMA_DESCRIPTORS_PER_MODULE) && (i < 100)) {
udelay(10);
descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1 << 12) + 0x48);
i++;
}
if (descriptors < DMA_DESCRIPTORS_PER_MODULE * curr_compl_count) {
dev_err(dev, "Late completion count SW %u, HW %u HANDLE %x", curr_compl_count, descriptors, handle);
return -EIO;
} else if (i > 0) {
dev_warn(dev, "Late completion count SW %u, HW %u HANDLE %x delay %d", curr_compl_count, descriptors, handle, i);
}
}
return 0;
}
int jfjoch_set_config(struct jfjoch_drvdata *drvdata, const struct DataCollectionConfig *config) {
if (config->nframes > MAX_FRAMES)
return -EINVAL;
memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_NMODULES, config, sizeof(struct DataCollectionConfig));
return 0;
}
void jfjoch_get_config(struct jfjoch_drvdata *drvdata, struct DataCollectionConfig *config) {
memcpy_fromio(config, (drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_NMODULES, sizeof(struct DataCollectionConfig));
}
void jfjoch_get_status(struct jfjoch_drvdata *drvdata, struct DataCollectionStatus *status) {
memcpy_fromio(status, drvdata->bar0 + ACTION_CONFIG_OFFSET, sizeof(struct DataCollectionStatus));
}
void jfjoch_set_mac_addr(struct jfjoch_drvdata *drvdata, u64 *mac_addr) {
memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_MAC_ADDR_LO, mac_addr, sizeof(uint64_t));
}
void jfjoch_get_mac_addr(struct jfjoch_drvdata *drvdata, u64 *mac_addr) {
memcpy_fromio(mac_addr, (drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_MAC_ADDR_LO, sizeof(uint64_t));
}
void jfjoch_set_ipv4_addr(struct jfjoch_drvdata *drvdata, const u32 *addr) {
iowrite32(*addr, (drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_IPV4_ADDR);
}
void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr) {
*addr = ioread32((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_IPV4_ADDR);
}
u64 jfjoch_read_mac_addr(struct jfjoch_drvdata *drvdata) {
struct device *const dev = &drvdata->pdev->dev;
u32 tmp, host_msg_offset, msg_len, opcode;
u32 field_type, field_len;
u32 i, j;
u8 output[256];
u64 mac = 0;
u64 design_number;
// Check which card is this
if (ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET) & (1<<7))
design_number = 1;
else
design_number = 0;
if (ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5)) {
dev_err(dev, "Mailbox not available");
return 0;
}
host_msg_offset = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28300);
dev_info(dev, "Host msg offset %x", host_msg_offset);
iowrite32(0x04 << 24, drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset);
iowrite32(1<<5, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG);
i = 0;
while (i < 50) {
if (!(ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5)))
break;
msleep(100);
i++;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG) & (1 << 5);
if (tmp) {
dev_err(dev, "Mailbox not ready %x", tmp);
return 0;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28304);
if (tmp) {
dev_err(dev, "Error in mailbox %x", tmp);
return 0;
}
tmp = ioread32(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset);
opcode = (tmp >> 24) & 0xFF;
msg_len = tmp & 0xFFF;
dev_info(dev, "Mailbox response %x opcode %x len %x", tmp, opcode, msg_len);
if (opcode != 0x04) {
dev_err(dev, "Opcode in return message 0x%x doesn't match 0x%x", opcode, 0x04);
return 0;
}
i = 0;
while (i < msg_len) {
field_type = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i);
field_len = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i + 1);
for (j = 0; j < field_len; j++)
output[j] = ioread8(drvdata->bar0 + CMS_OFFSET + 0x28000 + host_msg_offset + 0x4 + i + 2 + j);
output[field_len] = 0x0;
if (field_type == 0x21)
dev_info(dev, "Card serial number %s", output);
if (field_type == 0x4B) {
dev_info(dev, "MAC Addr %d %02x:%02x:%02x:%02x:%02x:%02x",
output[0], output[2], output[3], output[4], output[5], output[6], output[7]);
mac = ((u64)(output[2]))
| ((u64)(output[3]) << 8)
| ((u64)(output[4]) << 16)
| ((u64)(output[5]) << 24)
| ((u64)(output[6]) << 32)
| ((u64)(output[7]) << 40);
mac += (design_number << 40);
jfjoch_set_mac_addr(drvdata, &mac);
}
i += field_len + 2;
}
return mac;
}
void jfjoch_is_idle(struct jfjoch_drvdata *drvdata, uint32_t *output) {
if (ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_CTRL_REGISTER) & CTRL_REGISTER_IDLE)
*output = 1;
else
*output = 0;
}
void jfjoch_setup_cms(struct jfjoch_drvdata *drvdata) {
struct device *const dev = &drvdata->pdev->dev;
u32 reg_map_ready, i;
iowrite32(0x1, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_MB_RESETN_REG);
iowrite32(1 << 27, drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CONTROL_REG); // enable HBM monitoring
reg_map_ready = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HOST_STATUS2_REG) & 0x1;
i = 0;
while (i < 10000) {
if (reg_map_ready)
break;
msleep(1);
reg_map_ready = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HOST_STATUS2_REG) & 0x1;
i++;
}
if (i == 10000)
dev_err(dev, "CMS not ready after 10s");
else
dev_info(dev, "CMS ready after restart in %d ms", i);
}
void jfjoch_setup_network(struct jfjoch_drvdata *drvdata) {
// Enable RS_FEC
iowrite32(0x03, drvdata->bar0 + CMAC_OFFSET + 0x107C);
// Restart TX and RX cores + deassert resets
iowrite32((1 << 31) + (1 << 30), drvdata->bar0 + CMAC_OFFSET + 0x0004);
iowrite32(0x0, drvdata->bar0 + CMAC_OFFSET + 0x0004);
// TX Send RFI
iowrite32(0x10, drvdata->bar0 + CMAC_OFFSET + 0x000C);
// Enable RX
iowrite32(0x1, drvdata->bar0 + CMAC_OFFSET + 0x0014);
// start communication
iowrite32(0x1, drvdata->bar0 + CMAC_OFFSET + 0x000C);
// enable counters
iowrite32(0, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *env_params) {
int i;
env_params->mailbox_status_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS);
env_params->mailbox_err_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_ERR);
env_params->mailbox_interrupt_status = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_IS);
env_params->fpga_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_FPGA_TEMP_INS_REG);
env_params->fpga_pcie_3p3V_I_mA = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_3V3PEX_I_IN_INS_REG);
env_params->fpga_pcie_12V_I_mA = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_12VPEX_I_IN_INS_REG);
env_params->fpga_pcie_3p3V_V_mV = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_3V3_PEX_INS_REG);
env_params->fpga_pcie_12V_V_mV = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_12V_PEX_INS_REG);
env_params->pcie_h2c_descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x48);
env_params->pcie_h2c_beats = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xCC);
env_params->pcie_h2c_status = ioread32(drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x40);
env_params->pcie_c2h_descriptors = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x48);
env_params->pcie_c2h_beats = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xCC);
env_params->pcie_c2h_status = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x40);
env_params->pcie_user_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IE_MASK);
env_params->pcie_dma_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IE_MASK);
env_params->pcie_user_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IP);
env_params->pcie_dma_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IP);
env_params->pcie_user_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IR);
env_params->pcie_dma_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IR);
env_params->hbm_0_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP1_INS_REG);
env_params->hbm_1_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP2_INS_REG);
env_params->qsfp_cage_0_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CAGE_TEMP0_INS_REG);
env_params->qsfp_cage_1_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_CAGE_TEMP1_INS_REG);
// These are latched low register, so need to ask twice, to get current value
for (i = 0; i < 2; i++) {
env_params->eth_100G_link = ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204) & 0x1; // stat_rx_status
env_params->eth_10G_link[0] = ioread32(drvdata->bar0 + MAC_10G_0_OFFSET + 0x0404) & 0x1; // stat_rx_status
env_params->eth_10G_link[1] = ioread32(drvdata->bar0 + MAC_10G_1_OFFSET + 0x0404) & 0x1; // stat_rx_status
env_params->eth_10G_link[2] = ioread32(drvdata->bar0 + MAC_10G_2_OFFSET + 0x0404) & 0x1; // stat_rx_status
env_params->eth_10G_link[3] = ioread32(drvdata->bar0 + MAC_10G_3_OFFSET + 0x0404) & 0x1; // stat_rx_status
}
env_params->work_compl_fifo_avail = kfifo_avail(&drvdata->work_compl);
env_params->active_handles = atomic_read(&drvdata->active_handles);
}
void jfjoch_clr_net_counters(struct jfjoch_drvdata *drvdata) {
iowrite32(1 << 3, drvdata->bar0 + ACTION_CONFIG_OFFSET);
iowrite32(0, drvdata->bar0 + ACTION_CONFIG_OFFSET);
}
int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct LoadCalibrationConfig *config) {
struct device *const dev = &drvdata->pdev->dev;
u32 i, tmp, hls_ctrl_reg;
hls_ctrl_reg = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
if ((hls_ctrl_reg & (1<<2)) == 0) {
dev_err(dev, "Load calibration not ready for operation");
return -EBUSY;
}
if (config->handle >= drvdata->nbuf) {
dev_err(dev, "Not enough buffers to support this card\n");
return -EINVAL;
}
// Start DMA
// Clear counters and RUN H2C
iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0);
iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0);
iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04);
// Setup HLS core and start operation
memcpy_toio(drvdata->bar0 + ADDR_LOAD_CALIBRATION_HANDLE, config, sizeof(struct LoadCalibrationConfig));
iowrite32(0x1, drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
i = 0;
while (i < 1000) {
tmp = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
if (tmp & (1 << 1))
break;
udelay(5);
i++;
}
// STOP H2C channel
iowrite32(0, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04);
if (i == 5000) {
dev_err(dev, "Loading didn't finish in 5 milliseconds\n");
return -ETIMEDOUT;
} else {
u32 ret = ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_RETURN);
if (ret != 0) {
dev_err(dev, "Loading error\n");
return -EINVAL;
}
}
return 0;
}
int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config) {
struct device *const dev = &drvdata->pdev->dev;
if (ioread32(drvdata->bar0 + ADDR_FRAME_GEN_CTRL) & 0x1) {
dev_err(dev, "Frame generator busy\n");
return -EBUSY;
}
iowrite32(config->frames, drvdata->bar0 + ADDR_FRAME_GEN_FRAMES);
iowrite32(config->modules, drvdata->bar0 + ADDR_FRAME_GEN_MODULES);
iowrite32(config->dest_ipv4_addr, drvdata->bar0 + ADDR_FRAME_GEN_DEST_IPV4_ADDR);
iowrite32(config->dest_mac_addr & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_LO);
iowrite32(config->dest_mac_addr >> 32, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_HI);
iowrite32(config->bunchid & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_LO);
iowrite32(config->bunchid >> 32, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_HI);
iowrite32(config->exptime, drvdata->bar0 + ADDR_FRAME_GEN_EXPTIME);
iowrite32(config->debug, drvdata->bar0 + ADDR_FRAME_GEN_DEBUG);
iowrite32(0x1, drvdata->bar0 + ADDR_FRAME_GEN_CTRL);
return 0;
}
u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata) {
return ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x48);
}
void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
params,
sizeof(struct SpotFinderParameters));
}
void jfjoch_get_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) {
memcpy_fromio(params,
(drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_SPOT_FINDER_THRESHOLD,
sizeof(struct SpotFinderParameters));
}
u32 jfjoch_get_data_source(struct jfjoch_drvdata *drvdata) {
return ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_DATA_SOURCE);
}
void jfjoch_set_data_source(struct jfjoch_drvdata *drvdata, const u32 *val) {
iowrite32(*val, drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_DATA_SOURCE);
}