mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

Adapt PCIe driver and tests for the new frame generator

Browse Source
This commit is contained in:
leonarski_f
2023-09-13 21:44:20 +02:00
parent f3e85deb31
commit 0b95456d3d
13 changed files with 99 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/Definitions.h
View File

@@ -50,11 +50,10 @@
// For FPGA
#define ACTION_TYPE 0x52324158
#define RELEASE_LEVEL 0x003B
#define RELEASE_LEVEL 0x003C
#define MODE_CONV 0x0001L
#define MODE_INTERNAL_PACKET_GEN 0x0002L
#define MODE_NONBLOCKING_ON_WR 0x0004L // Don't block acquisition if there is no WR available
#define MODE_NONBLOCKING_ON_WR 0x0002L // Don't block acquisition if there is no WR available
#define TASK_NO_DATA_STREAM UINT16_MAX

11
fpga/pcie_driver/ActionConfig.h
View File

@@ -69,6 +69,17 @@ struct ActionEnvParams {
bool ethernet_aligned;
};
struct FrameGeneratorConfig {
uint32_t frames;
uint32_t modules;
uint64_t dest_mac_addr;
uint32_t dest_ipv4_addr;
uint64_t bunchid;
uint32_t exptime;
uint32_t debug;
};
#pragma pack(pop)
#endif //JUNGFRAUJOCH_ACTIONCONFIG_H

21
fpga/pcie_driver/jfjoch_drv.h
View File

@@ -42,11 +42,23 @@
#define CMAC_OFFSET (0x020000)
#define PCIE_OFFSET (0x090000)
#define INT_PKT_GEN_OFFSET (0x100000)
#define FRAME_GEN_OFFSET (0x080000)
#define ADDR_LOAD_CALIBRATION_CTRL (0x000000)
#define ADDR_LOAD_CALIBRATION_MOD (0x000010)
#define ADDR_LOAD_CALIBRATION_SC (0x000018)
#define ADDR_LOAD_CALIBRATION_MEM (0x002000)
#define ADDR_LOAD_CALIBRATION_CTRL (LOAD_CALIBRATION_OFFSET | 0x000000)
#define ADDR_LOAD_CALIBRATION_MOD (LOAD_CALIBRATION_OFFSET | 0x000010)
#define ADDR_LOAD_CALIBRATION_SC (LOAD_CALIBRATION_OFFSET | 0x000018)
#define ADDR_LOAD_CALIBRATION_MEM (LOAD_CALIBRATION_OFFSET | 0x002000)
#define ADDR_FRAME_GEN_CTRL (FRAME_GEN_OFFSET | 0x000000)
#define ADDR_FRAME_GEN_FRAMES (FRAME_GEN_OFFSET | 0x000010)
#define ADDR_FRAME_GEN_MODULES (FRAME_GEN_OFFSET | 0x000018)
#define ADDR_FRAME_GEN_DEST_MAC_LO (FRAME_GEN_OFFSET | 0x000020)
#define ADDR_FRAME_GEN_DEST_MAC_HI (FRAME_GEN_OFFSET | 0x000024)
#define ADDR_FRAME_GEN_DEST_IPV4_ADDR (FRAME_GEN_OFFSET | 0x00002C)
#define ADDR_FRAME_GEN_BUNCHID_LO (FRAME_GEN_OFFSET | 0x000034)
#define ADDR_FRAME_GEN_BUNCHID_HI (FRAME_GEN_OFFSET | 0x000038)
#define ADDR_FRAME_GEN_EXPTIME (FRAME_GEN_OFFSET | 0x000040)
#define ADDR_FRAME_GEN_DEBUG (FRAME_GEN_OFFSET | 0x000048)
#define JFJOCH_DMA_SETTINGS (XDMA_CTRL_RUN_STOP | XDMA_CTRL_IE_DESC_ALIGN_MISMATCH | XDMA_CTRL_IE_DESC_ERROR | XDMA_CTRL_IE_READ_ERROR \
| XDMA_CTRL_IE_WRITE_ERROR | XDMA_CTRL_IE_DESC_COMPLETED | XDMA_CTRL_STM_MODE_WB)
@@ -113,6 +125,7 @@ void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr);
void jfjoch_load_int_pkt_gen(struct jfjoch_drvdata *drvdata, char* output);
void jfjoch_save_int_pkt_gen(struct jfjoch_drvdata *drvdata, const char* input);
int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig *config);
int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config);
u64 jfjoch_read_mac_addr(struct jfjoch_drvdata *drvdata);

32
fpga/pcie_driver/jfjoch_function.c
View File

@@ -297,8 +297,8 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig
for (i = 0; i < cell_count; i++) {
u64 addr = drvdata->bufs[i].dma_address;
iowrite32(PCI_DMA_L(addr), drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MEM + i * 2 * 4);
iowrite32(PCI_DMA_H(addr), drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MEM + (i * 2 + 1) * 4);
iowrite32(PCI_DMA_L(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + i * 2 * 4);
iowrite32(PCI_DMA_H(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + (i * 2 + 1) * 4);
}
// Start DMA
@@ -307,13 +307,13 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig
iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0);
iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04);
iowrite32(config->nmodules, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MOD);
iowrite32(config->nstorage_cells, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_SC);
iowrite32(0x1, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_CTRL);
iowrite32(config->nmodules, drvdata->bar0 + ADDR_LOAD_CALIBRATION_MOD);
iowrite32(config->nstorage_cells, drvdata->bar0 + ADDR_LOAD_CALIBRATION_SC);
iowrite32(0x1, drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL);
i = 0;
while (i < 1000) {
if (ioread32(drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_CTRL) & (1 << 1))
if (ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL) & (1 << 1))
break;
msleep(10);
i++;
@@ -328,4 +328,24 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig
}
return 0;
}
int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config) {
struct device *const dev = &drvdata->pdev->dev;
if (ioread32(drvdata->bar0 + ADDR_FRAME_GEN_CTRL) & 0x1) {
dev_err(dev, "Frame generator busy\n");
return -EBUSY;
}
iowrite32(config->frames, drvdata->bar0 + ADDR_FRAME_GEN_FRAMES);
iowrite32(config->modules, drvdata->bar0 + ADDR_FRAME_GEN_MODULES);
iowrite32(config->dest_ipv4_addr, drvdata->bar0 + ADDR_FRAME_GEN_DEST_IPV4_ADDR);
iowrite32(config->dest_mac_addr & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_LO);
iowrite32(config->dest_mac_addr >> 32, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_HI);
iowrite32(config->bunchid & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_LO);
iowrite32(config->bunchid >> 32, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_HI);
iowrite32(config->exptime, drvdata->bar0 + ADDR_FRAME_GEN_EXPTIME);
iowrite32(config->debug, drvdata->bar0 + ADDR_FRAME_GEN_DEBUG);
iowrite32(0x1, drvdata->bar0 + ADDR_FRAME_GEN_CTRL);
}

5
fpga/pcie_driver/jfjoch_ioctl.c
View File

@@ -9,6 +9,7 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) {
struct ActionStatus status;
struct ActionConfig config;
struct ActionEnvParams env_params;
struct FrameGeneratorConfig frame_generator_config;
u32 exchange[16];
int err;
void *tmp = NULL;
@@ -101,6 +102,10 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) {
case IOCTL_JFJOCH_DEFAULT_MAC:
jfjoch_read_mac_addr(drvdata);
return 0;
case IOCTL_JFJOCH_RUN_FRAME_GEN:
if (copy_from_user(&frame_generator_config, (char *) arg, sizeof(struct FrameGeneratorConfig)) != 0)
return -EFAULT;
return jfjoch_run_frame_gen(drvdata, &frame_generator_config);
case IOCTL_JFJOCH_SET_INT_PKT:
tmp = vmalloc(INT_PKT_GEN_FRAME_SIZE_BYTES);
if (tmp == NULL)

1
fpga/pcie_driver/jfjoch_ioctl.h
View File

@@ -36,5 +36,6 @@
#define IOCTL_JFJOCH_SET_INT_PKT _IOW(IOCTL_JFJOCH_MAGIC, 19, char *)
#define IOCTL_JFJOCH_GET_INT_PKT _IOR(IOCTL_JFJOCH_MAGIC, 20, char *)
#define IOCTL_JFJOCH_LOAD_CALIB _IOW(IOCTL_JFJOCH_MAGIC, 21, struct ActionConfig)
#define IOCTL_JFJOCH_RUN_FRAME_GEN _IOW(IOCTL_JFJOCH_MAGIC, 22, struct FrameGeneratorConfig)
#endif //JUNGFRAUJOCH_JFJOCH_IOCTL_H

5
receiver/FPGAAcquisitionDevice.cpp
View File

@@ -118,9 +118,6 @@ void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, A
if ((x.GetDetectorMode() == DetectorMode::Conversion) && x.GetConversionOnFPGA())
job.mode |= MODE_CONV;
if (x.IsUsingInternalPacketGen())
job.mode |= MODE_INTERNAL_PACKET_GEN;
}
@@ -144,7 +141,7 @@ void FPGAAcquisitionDevice::Start(const DiffractionExperiment &experiment) {
throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError,
"Mismatch between expected and actual values of configuration registers (#modules)");
FPGA_StartAction();
FPGA_StartAction(experiment);
read_work_completion_future = std::async(std::launch::async, &FPGAAcquisitionDevice::ReadWorkCompletionThread, this);
}

2
receiver/FPGAAcquisitionDevice.h
View File

@@ -11,7 +11,7 @@ class FPGAAcquisitionDevice : public AcquisitionDevice {
uint16_t data_collection_id = 0;
bool fpga_non_blocking_mode = true;
virtual void FPGA_StartAction() = 0;
virtual void FPGA_StartAction(const DiffractionExperiment &experiment) = 0;
virtual void FPGA_EndAction() = 0;
virtual void HW_WriteActionRegister(const ActionConfig *job) = 0;

31
receiver/HLSSimulatedDevice.cpp
View File

@@ -127,13 +127,26 @@ void HLSSimulatedDevice::HW_WriteActionRegister(const ActionConfig *job) {
memcpy(&cfg, job, sizeof(ActionConfig));
}
void HLSSimulatedDevice::FPGA_StartAction() {
void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experiment) {
if (action_thread.joinable())
action_thread.join();
run_data_collection = 1;
cancel_data_collection = 0;
idle = false;
if (experiment.IsUsingInternalPacketGen()) {
frame_generator(din_eth,
reinterpret_cast<ap_uint<512> *>(internal_pkt_gen_frame.data()),
experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1,
experiment.GetModulesNum(data_stream),
mac_addr,
mac_addr,
ipv4_addr,
ipv4_addr,
INT_PKT_GEN_BUNCHID,
INT_PKT_GEN_EXPTTIME,
INT_PKT_GEN_DEBUG);
}
action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this );
}
@@ -213,22 +226,6 @@ void HLSSimulatedDevice::HLSMainThread() {
ap_uint<8> err_reg;
if (cfg.mode & MODE_INTERNAL_PACKET_GEN) {
frame_generator(din_eth,
reinterpret_cast<ap_uint<512> *>(internal_pkt_gen_frame.data()),
cfg.nframes + DELAY_FRAMES_STOP_AND_QUIT + 1,
cfg.nmodules,
mac_addr,
mac_addr,
ipv4_addr,
ipv4_addr,
INT_PKT_GEN_BUNCHID,
INT_PKT_GEN_EXPTTIME,
INT_PKT_GEN_DEBUG);
if (logger)
logger->Info("Packets ready {}", din_eth.size());
}
while(!din_eth.empty())
ethernet(din_eth, ip1, arp1, mac_addr, eth_packets, clear_counters);

5
receiver/HLSSimulatedDevice.h
View File

@@ -41,14 +41,15 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice {
void HW_ReadActionRegister(ActionConfig *job) override;
void HW_WriteActionRegister(const ActionConfig *job) override;
void FPGA_StartAction() override;
void FPGA_StartAction(const DiffractionExperiment &experiment) override;
void FPGA_EndAction() override;
bool HW_IsIdle() const override;
bool HW_ReadMailbox(uint32_t values[16]);
bool HW_ReadMailbox(uint32_t values[16]) override;
bool HW_SendWorkRequest(uint32_t handle) override;
void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) override;
void HW_GetStatus(ActionStatus *status) const override;
void HLSMainThread() ;
void RunFrameGenerator(const FrameGeneratorConfig& config);
public:
HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1);
~HLSSimulatedDevice();

23
receiver/PCIExpressDevice.cpp
View File

@@ -92,14 +92,25 @@ bool PCIExpressDevice::HW_SendWorkRequest(uint32_t handle) {
return true;
}
void PCIExpressDevice::FPGA_StartAction() {
if (ioctl(fd, IOCTL_JFJOCH_SET_INT_PKT, internal_pkt_gen_frame.data()) != 0)
throw JFJochException(JFJochExceptionCategory::PCIeError,
"Failed loading internal packet generator frame", errno);
void PCIExpressDevice::FPGA_StartAction(const DiffractionExperiment &experiment) {
if (experiment.IsUsingInternalPacketGen()) {
if (ioctl(fd, IOCTL_JFJOCH_SET_INT_PKT, internal_pkt_gen_frame.data()) != 0)
throw JFJochException(JFJochExceptionCategory::PCIeError,
"Failed loading internal packet generator frame", errno);
}
if (ioctl(fd, IOCTL_JFJOCH_START) != 0)
throw JFJochException(JFJochExceptionCategory::PCIeError,
"Failed starting action", errno);
throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed starting action", errno);
if (experiment.IsUsingInternalPacketGen()) {
FrameGeneratorConfig config{};
config.frames = experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1;
config.modules = experiment.GetModulesNum(data_stream);
config.dest_ipv4_addr = ipv4_addr;
config.dest_mac_addr = mac_addr;
if (ioctl(fd, IOCTL_JFJOCH_RUN_FRAME_GEN) != 0)
throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed starting frame generator", errno);
}
}
void PCIExpressDevice::FPGA_EndAction() {

2
receiver/PCIExpressDevice.h
View File

@@ -11,7 +11,7 @@ class PCIExpressDevice : public FPGAAcquisitionDevice {
bool HW_ReadMailbox(uint32_t values[16]) override;
bool HW_SendWorkRequest(uint32_t handle) override;
void FPGA_StartAction() override;
void FPGA_StartAction(const DiffractionExperiment &experiment) override;
bool HW_IsIdle() const final;
void HW_WriteActionRegister(const ActionConfig *job) override;
void HW_ReadActionRegister(ActionConfig *job) override;

4
tests/FPGAIntegrationTest.cpp
View File

@@ -403,7 +403,6 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") {
TEST_CASE("HLS_C_Simulation_internal_packet_generator_convert_full_range", "[FPGA][Full]") {
double energy = 6.0;
Logger logger("logger");
const uint16_t nmodules = 4;
DiffractionExperiment x((DetectorGeometry(nmodules, 2, 8, 36, true)));
@@ -435,7 +434,6 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_convert_full_range", "[FPG
}
HLSSimulatedDevice test(0, 64);
test.EnableLogging(&logger);
REQUIRE_NOTHROW(test.InitializeCalibration(x, c));
REQUIRE_NOTHROW(test.StartAction(x));
@@ -545,8 +543,6 @@ TEST_CASE("HLS_C_Simulation_check_2_trigger_convert", "[FPGA][Full]") {
x.PedestalG0Frames(0).NumTriggers(2).ImagesPerTrigger(5);
HLSSimulatedDevice test(0, 64);
Logger logger("bla");
test.EnableLogging(&logger);
JFCalibration c(x);
REQUIRE_NOTHROW(c.Pedestal(0, 0).LoadPedestal(pedestal_g0));