diff --git a/common/Definitions.h b/common/Definitions.h index 0ec88938..cb311547 100644 --- a/common/Definitions.h +++ b/common/Definitions.h @@ -50,11 +50,10 @@ // For FPGA #define ACTION_TYPE 0x52324158 -#define RELEASE_LEVEL 0x003B +#define RELEASE_LEVEL 0x003C #define MODE_CONV 0x0001L -#define MODE_INTERNAL_PACKET_GEN 0x0002L -#define MODE_NONBLOCKING_ON_WR 0x0004L // Don't block acquisition if there is no WR available +#define MODE_NONBLOCKING_ON_WR 0x0002L // Don't block acquisition if there is no WR available #define TASK_NO_DATA_STREAM UINT16_MAX diff --git a/fpga/pcie_driver/ActionConfig.h b/fpga/pcie_driver/ActionConfig.h index 8a31793b..fa691569 100644 --- a/fpga/pcie_driver/ActionConfig.h +++ b/fpga/pcie_driver/ActionConfig.h @@ -69,6 +69,17 @@ struct ActionEnvParams { bool ethernet_aligned; }; + +struct FrameGeneratorConfig { + uint32_t frames; + uint32_t modules; + uint64_t dest_mac_addr; + uint32_t dest_ipv4_addr; + uint64_t bunchid; + uint32_t exptime; + uint32_t debug; +}; + #pragma pack(pop) #endif //JUNGFRAUJOCH_ACTIONCONFIG_H diff --git a/fpga/pcie_driver/jfjoch_drv.h b/fpga/pcie_driver/jfjoch_drv.h index e5d29cd0..abf097c5 100644 --- a/fpga/pcie_driver/jfjoch_drv.h +++ b/fpga/pcie_driver/jfjoch_drv.h @@ -42,11 +42,23 @@ #define CMAC_OFFSET (0x020000) #define PCIE_OFFSET (0x090000) #define INT_PKT_GEN_OFFSET (0x100000) +#define FRAME_GEN_OFFSET (0x080000) -#define ADDR_LOAD_CALIBRATION_CTRL (0x000000) -#define ADDR_LOAD_CALIBRATION_MOD (0x000010) -#define ADDR_LOAD_CALIBRATION_SC (0x000018) -#define ADDR_LOAD_CALIBRATION_MEM (0x002000) +#define ADDR_LOAD_CALIBRATION_CTRL (LOAD_CALIBRATION_OFFSET | 0x000000) +#define ADDR_LOAD_CALIBRATION_MOD (LOAD_CALIBRATION_OFFSET | 0x000010) +#define ADDR_LOAD_CALIBRATION_SC (LOAD_CALIBRATION_OFFSET | 0x000018) +#define ADDR_LOAD_CALIBRATION_MEM (LOAD_CALIBRATION_OFFSET | 0x002000) + +#define ADDR_FRAME_GEN_CTRL (FRAME_GEN_OFFSET | 0x000000) +#define ADDR_FRAME_GEN_FRAMES (FRAME_GEN_OFFSET | 0x000010) +#define ADDR_FRAME_GEN_MODULES (FRAME_GEN_OFFSET | 0x000018) +#define ADDR_FRAME_GEN_DEST_MAC_LO (FRAME_GEN_OFFSET | 0x000020) +#define ADDR_FRAME_GEN_DEST_MAC_HI (FRAME_GEN_OFFSET | 0x000024) +#define ADDR_FRAME_GEN_DEST_IPV4_ADDR (FRAME_GEN_OFFSET | 0x00002C) +#define ADDR_FRAME_GEN_BUNCHID_LO (FRAME_GEN_OFFSET | 0x000034) +#define ADDR_FRAME_GEN_BUNCHID_HI (FRAME_GEN_OFFSET | 0x000038) +#define ADDR_FRAME_GEN_EXPTIME (FRAME_GEN_OFFSET | 0x000040) +#define ADDR_FRAME_GEN_DEBUG (FRAME_GEN_OFFSET | 0x000048) #define JFJOCH_DMA_SETTINGS (XDMA_CTRL_RUN_STOP | XDMA_CTRL_IE_DESC_ALIGN_MISMATCH | XDMA_CTRL_IE_DESC_ERROR | XDMA_CTRL_IE_READ_ERROR \ | XDMA_CTRL_IE_WRITE_ERROR | XDMA_CTRL_IE_DESC_COMPLETED | XDMA_CTRL_STM_MODE_WB) @@ -113,6 +125,7 @@ void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr); void jfjoch_load_int_pkt_gen(struct jfjoch_drvdata *drvdata, char* output); void jfjoch_save_int_pkt_gen(struct jfjoch_drvdata *drvdata, const char* input); int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig *config); +int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config); u64 jfjoch_read_mac_addr(struct jfjoch_drvdata *drvdata); diff --git a/fpga/pcie_driver/jfjoch_function.c b/fpga/pcie_driver/jfjoch_function.c index f5f8e451..6907bd80 100644 --- a/fpga/pcie_driver/jfjoch_function.c +++ b/fpga/pcie_driver/jfjoch_function.c @@ -297,8 +297,8 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig for (i = 0; i < cell_count; i++) { u64 addr = drvdata->bufs[i].dma_address; - iowrite32(PCI_DMA_L(addr), drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MEM + i * 2 * 4); - iowrite32(PCI_DMA_H(addr), drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MEM + (i * 2 + 1) * 4); + iowrite32(PCI_DMA_L(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + i * 2 * 4); + iowrite32(PCI_DMA_H(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + (i * 2 + 1) * 4); } // Start DMA @@ -307,13 +307,13 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig iowrite32((1 << 2), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0); iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0x04); - iowrite32(config->nmodules, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_MOD); - iowrite32(config->nstorage_cells, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_SC); - iowrite32(0x1, drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_CTRL); + iowrite32(config->nmodules, drvdata->bar0 + ADDR_LOAD_CALIBRATION_MOD); + iowrite32(config->nstorage_cells, drvdata->bar0 + ADDR_LOAD_CALIBRATION_SC); + iowrite32(0x1, drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL); i = 0; while (i < 1000) { - if (ioread32(drvdata->bar0 + LOAD_CALIBRATION_OFFSET + ADDR_LOAD_CALIBRATION_CTRL) & (1 << 1)) + if (ioread32(drvdata->bar0 + ADDR_LOAD_CALIBRATION_CTRL) & (1 << 1)) break; msleep(10); i++; @@ -328,4 +328,24 @@ int jfjoch_load_calibration(struct jfjoch_drvdata *drvdata, struct ActionConfig } return 0; +} + +int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config) { + struct device *const dev = &drvdata->pdev->dev; + + if (ioread32(drvdata->bar0 + ADDR_FRAME_GEN_CTRL) & 0x1) { + dev_err(dev, "Frame generator busy\n"); + return -EBUSY; + } + + iowrite32(config->frames, drvdata->bar0 + ADDR_FRAME_GEN_FRAMES); + iowrite32(config->modules, drvdata->bar0 + ADDR_FRAME_GEN_MODULES); + iowrite32(config->dest_ipv4_addr, drvdata->bar0 + ADDR_FRAME_GEN_DEST_IPV4_ADDR); + iowrite32(config->dest_mac_addr & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_LO); + iowrite32(config->dest_mac_addr >> 32, drvdata->bar0 + ADDR_FRAME_GEN_DEST_MAC_HI); + iowrite32(config->bunchid & 0xFFFFFFFF, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_LO); + iowrite32(config->bunchid >> 32, drvdata->bar0 + ADDR_FRAME_GEN_BUNCHID_HI); + iowrite32(config->exptime, drvdata->bar0 + ADDR_FRAME_GEN_EXPTIME); + iowrite32(config->debug, drvdata->bar0 + ADDR_FRAME_GEN_DEBUG); + iowrite32(0x1, drvdata->bar0 + ADDR_FRAME_GEN_CTRL); } \ No newline at end of file diff --git a/fpga/pcie_driver/jfjoch_ioctl.c b/fpga/pcie_driver/jfjoch_ioctl.c index d66db572..592f36bd 100644 --- a/fpga/pcie_driver/jfjoch_ioctl.c +++ b/fpga/pcie_driver/jfjoch_ioctl.c @@ -9,6 +9,7 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct ActionStatus status; struct ActionConfig config; struct ActionEnvParams env_params; + struct FrameGeneratorConfig frame_generator_config; u32 exchange[16]; int err; void *tmp = NULL; @@ -101,6 +102,10 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { case IOCTL_JFJOCH_DEFAULT_MAC: jfjoch_read_mac_addr(drvdata); return 0; + case IOCTL_JFJOCH_RUN_FRAME_GEN: + if (copy_from_user(&frame_generator_config, (char *) arg, sizeof(struct FrameGeneratorConfig)) != 0) + return -EFAULT; + return jfjoch_run_frame_gen(drvdata, &frame_generator_config); case IOCTL_JFJOCH_SET_INT_PKT: tmp = vmalloc(INT_PKT_GEN_FRAME_SIZE_BYTES); if (tmp == NULL) diff --git a/fpga/pcie_driver/jfjoch_ioctl.h b/fpga/pcie_driver/jfjoch_ioctl.h index 5ccdfed2..6a239b28 100644 --- a/fpga/pcie_driver/jfjoch_ioctl.h +++ b/fpga/pcie_driver/jfjoch_ioctl.h @@ -36,5 +36,6 @@ #define IOCTL_JFJOCH_SET_INT_PKT _IOW(IOCTL_JFJOCH_MAGIC, 19, char *) #define IOCTL_JFJOCH_GET_INT_PKT _IOR(IOCTL_JFJOCH_MAGIC, 20, char *) #define IOCTL_JFJOCH_LOAD_CALIB _IOW(IOCTL_JFJOCH_MAGIC, 21, struct ActionConfig) +#define IOCTL_JFJOCH_RUN_FRAME_GEN _IOW(IOCTL_JFJOCH_MAGIC, 22, struct FrameGeneratorConfig) #endif //JUNGFRAUJOCH_JFJOCH_IOCTL_H diff --git a/receiver/FPGAAcquisitionDevice.cpp b/receiver/FPGAAcquisitionDevice.cpp index ee0d770c..3bf17c71 100644 --- a/receiver/FPGAAcquisitionDevice.cpp +++ b/receiver/FPGAAcquisitionDevice.cpp @@ -118,9 +118,6 @@ void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, A if ((x.GetDetectorMode() == DetectorMode::Conversion) && x.GetConversionOnFPGA()) job.mode |= MODE_CONV; - - if (x.IsUsingInternalPacketGen()) - job.mode |= MODE_INTERNAL_PACKET_GEN; } @@ -144,7 +141,7 @@ void FPGAAcquisitionDevice::Start(const DiffractionExperiment &experiment) { throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, "Mismatch between expected and actual values of configuration registers (#modules)"); - FPGA_StartAction(); + FPGA_StartAction(experiment); read_work_completion_future = std::async(std::launch::async, &FPGAAcquisitionDevice::ReadWorkCompletionThread, this); } diff --git a/receiver/FPGAAcquisitionDevice.h b/receiver/FPGAAcquisitionDevice.h index 23354e54..6708d733 100644 --- a/receiver/FPGAAcquisitionDevice.h +++ b/receiver/FPGAAcquisitionDevice.h @@ -11,7 +11,7 @@ class FPGAAcquisitionDevice : public AcquisitionDevice { uint16_t data_collection_id = 0; bool fpga_non_blocking_mode = true; - virtual void FPGA_StartAction() = 0; + virtual void FPGA_StartAction(const DiffractionExperiment &experiment) = 0; virtual void FPGA_EndAction() = 0; virtual void HW_WriteActionRegister(const ActionConfig *job) = 0; diff --git a/receiver/HLSSimulatedDevice.cpp b/receiver/HLSSimulatedDevice.cpp index bb53a7e4..cf535550 100644 --- a/receiver/HLSSimulatedDevice.cpp +++ b/receiver/HLSSimulatedDevice.cpp @@ -127,13 +127,26 @@ void HLSSimulatedDevice::HW_WriteActionRegister(const ActionConfig *job) { memcpy(&cfg, job, sizeof(ActionConfig)); } -void HLSSimulatedDevice::FPGA_StartAction() { +void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experiment) { if (action_thread.joinable()) action_thread.join(); run_data_collection = 1; cancel_data_collection = 0; idle = false; + if (experiment.IsUsingInternalPacketGen()) { + frame_generator(din_eth, + reinterpret_cast *>(internal_pkt_gen_frame.data()), + experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1, + experiment.GetModulesNum(data_stream), + mac_addr, + mac_addr, + ipv4_addr, + ipv4_addr, + INT_PKT_GEN_BUNCHID, + INT_PKT_GEN_EXPTTIME, + INT_PKT_GEN_DEBUG); + } action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this ); } @@ -213,22 +226,6 @@ void HLSSimulatedDevice::HLSMainThread() { ap_uint<8> err_reg; - if (cfg.mode & MODE_INTERNAL_PACKET_GEN) { - frame_generator(din_eth, - reinterpret_cast *>(internal_pkt_gen_frame.data()), - cfg.nframes + DELAY_FRAMES_STOP_AND_QUIT + 1, - cfg.nmodules, - mac_addr, - mac_addr, - ipv4_addr, - ipv4_addr, - INT_PKT_GEN_BUNCHID, - INT_PKT_GEN_EXPTTIME, - INT_PKT_GEN_DEBUG); - if (logger) - logger->Info("Packets ready {}", din_eth.size()); - } - while(!din_eth.empty()) ethernet(din_eth, ip1, arp1, mac_addr, eth_packets, clear_counters); diff --git a/receiver/HLSSimulatedDevice.h b/receiver/HLSSimulatedDevice.h index 1cb70032..5e7e93b4 100644 --- a/receiver/HLSSimulatedDevice.h +++ b/receiver/HLSSimulatedDevice.h @@ -41,14 +41,15 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { void HW_ReadActionRegister(ActionConfig *job) override; void HW_WriteActionRegister(const ActionConfig *job) override; - void FPGA_StartAction() override; + void FPGA_StartAction(const DiffractionExperiment &experiment) override; void FPGA_EndAction() override; bool HW_IsIdle() const override; - bool HW_ReadMailbox(uint32_t values[16]); + bool HW_ReadMailbox(uint32_t values[16]) override; bool HW_SendWorkRequest(uint32_t handle) override; void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) override; void HW_GetStatus(ActionStatus *status) const override; void HLSMainThread() ; + void RunFrameGenerator(const FrameGeneratorConfig& config); public: HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1); ~HLSSimulatedDevice(); diff --git a/receiver/PCIExpressDevice.cpp b/receiver/PCIExpressDevice.cpp index a074f180..654f74e2 100644 --- a/receiver/PCIExpressDevice.cpp +++ b/receiver/PCIExpressDevice.cpp @@ -92,14 +92,25 @@ bool PCIExpressDevice::HW_SendWorkRequest(uint32_t handle) { return true; } -void PCIExpressDevice::FPGA_StartAction() { - if (ioctl(fd, IOCTL_JFJOCH_SET_INT_PKT, internal_pkt_gen_frame.data()) != 0) - throw JFJochException(JFJochExceptionCategory::PCIeError, - "Failed loading internal packet generator frame", errno); +void PCIExpressDevice::FPGA_StartAction(const DiffractionExperiment &experiment) { + if (experiment.IsUsingInternalPacketGen()) { + if (ioctl(fd, IOCTL_JFJOCH_SET_INT_PKT, internal_pkt_gen_frame.data()) != 0) + throw JFJochException(JFJochExceptionCategory::PCIeError, + "Failed loading internal packet generator frame", errno); + } if (ioctl(fd, IOCTL_JFJOCH_START) != 0) - throw JFJochException(JFJochExceptionCategory::PCIeError, - "Failed starting action", errno); + throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed starting action", errno); + + if (experiment.IsUsingInternalPacketGen()) { + FrameGeneratorConfig config{}; + config.frames = experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1; + config.modules = experiment.GetModulesNum(data_stream); + config.dest_ipv4_addr = ipv4_addr; + config.dest_mac_addr = mac_addr; + if (ioctl(fd, IOCTL_JFJOCH_RUN_FRAME_GEN) != 0) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed starting frame generator", errno); + } } void PCIExpressDevice::FPGA_EndAction() { diff --git a/receiver/PCIExpressDevice.h b/receiver/PCIExpressDevice.h index 8af755b5..c6ec7dc8 100644 --- a/receiver/PCIExpressDevice.h +++ b/receiver/PCIExpressDevice.h @@ -11,7 +11,7 @@ class PCIExpressDevice : public FPGAAcquisitionDevice { bool HW_ReadMailbox(uint32_t values[16]) override; bool HW_SendWorkRequest(uint32_t handle) override; - void FPGA_StartAction() override; + void FPGA_StartAction(const DiffractionExperiment &experiment) override; bool HW_IsIdle() const final; void HW_WriteActionRegister(const ActionConfig *job) override; void HW_ReadActionRegister(ActionConfig *job) override; diff --git a/tests/FPGAIntegrationTest.cpp b/tests/FPGAIntegrationTest.cpp index a390de67..0a862276 100644 --- a/tests/FPGAIntegrationTest.cpp +++ b/tests/FPGAIntegrationTest.cpp @@ -403,7 +403,6 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") { TEST_CASE("HLS_C_Simulation_internal_packet_generator_convert_full_range", "[FPGA][Full]") { double energy = 6.0; - Logger logger("logger"); const uint16_t nmodules = 4; DiffractionExperiment x((DetectorGeometry(nmodules, 2, 8, 36, true))); @@ -435,7 +434,6 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_convert_full_range", "[FPG } HLSSimulatedDevice test(0, 64); - test.EnableLogging(&logger); REQUIRE_NOTHROW(test.InitializeCalibration(x, c)); REQUIRE_NOTHROW(test.StartAction(x)); @@ -545,8 +543,6 @@ TEST_CASE("HLS_C_Simulation_check_2_trigger_convert", "[FPGA][Full]") { x.PedestalG0Frames(0).NumTriggers(2).ImagesPerTrigger(5); HLSSimulatedDevice test(0, 64); - Logger logger("bla"); - test.EnableLogging(&logger); JFCalibration c(x); REQUIRE_NOTHROW(c.Pedestal(0, 0).LoadPedestal(pedestal_g0));