From b3eceef7cdcd55200e1f643c031f8b9e71701c95 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 1 Nov 2023 15:55:06 +0100 Subject: [PATCH] FPGA: Max module number is 32 --- common/Definitions.h | 4 +- fpga/hdl/action_config.v | 4 +- fpga/hls/data_collection_fsm.cpp | 2 +- fpga/hls/frame_generator.cpp | 2 +- fpga/hls/hls_jfjoch.h | 6 +- fpga/hls/load_from_hbm.cpp | 2 +- fpga/hls/save_to_hbm.cpp | 2 +- receiver/HLSSimulatedDevice.cpp | 102 ++++++++++++++++++------------- tests/FPGAIntegrationTest.cpp | 2 +- 9 files changed, 71 insertions(+), 55 deletions(-) diff --git a/common/Definitions.h b/common/Definitions.h index 44f13eac..ea917c76 100644 --- a/common/Definitions.h +++ b/common/Definitions.h @@ -49,7 +49,7 @@ // For FPGA #define ACTION_TYPE 0x52324158 -#define RELEASE_LEVEL 0x0044 +#define RELEASE_LEVEL 0x0045 #define MODE_CONV 0x0001L #define MODE_BITSHUFFLE_FPGA 0x0002L @@ -133,7 +133,7 @@ #define FPGA_INTEGRATION_BIN_COUNT 1024 -#define MAX_MODULES_FPGA 16 +#define MAX_MODULES_FPGA 32 #define ADU_HISTO_BIN_WIDTH 32 #define ADU_HISTO_BIN_COUNT (65536/ ADU_HISTO_BIN_WIDTH) diff --git a/fpga/hdl/action_config.v b/fpga/hdl/action_config.v index 7305aaa0..6cef147c 100644 --- a/fpga/hdl/action_config.v +++ b/fpga/hdl/action_config.v @@ -89,7 +89,7 @@ module action_config output reg [31:0] fpga_ipv4_addr , output reg [31:0] one_over_energy , output reg [31:0] nframes , - output reg [7:0] nmodules , + output reg [4:0] nmodules , output reg [3:0] nstorage_cells , output reg [6:0] nsummation , output wire [31:0] hbm_size_bytes , @@ -511,7 +511,7 @@ always @(posedge clk) begin nmodules <= 0; else if (reg_data_collection_idle) begin if (w_hs && waddr == `ADDR_NMODULES ) - nmodules <= (s_axi_WDATA[7:0] & wmask[7:0]) | (nmodules & !wmask[7:0]); + nmodules <= (s_axi_WDATA[4:0] & wmask[4:0]) | (nmodules & !wmask[4:0]); end end diff --git a/fpga/hls/data_collection_fsm.cpp b/fpga/hls/data_collection_fsm.cpp index 8a6becb0..05feb231 100644 --- a/fpga/hls/data_collection_fsm.cpp +++ b/fpga/hls/data_collection_fsm.cpp @@ -13,7 +13,7 @@ void data_collection_fsm(AXI_STREAM ð_in, ap_uint<32> mode, ap_uint<32> one_over_energy, ap_uint<32> nframes, - ap_uint<8> nmodules, + ap_uint<5> nmodules, ap_uint<4> nstorage_cells, ap_uint<7> nsummation) { #pragma HLS INTERFACE ap_ctrl_none port=return diff --git a/fpga/hls/frame_generator.cpp b/fpga/hls/frame_generator.cpp index ceb68a30..3e08acc5 100644 --- a/fpga/hls/frame_generator.cpp +++ b/fpga/hls/frame_generator.cpp @@ -78,7 +78,7 @@ int frame_generator(STREAM_512 &data_out, ap_uint<256> *d_hbm_p1, ap_uint<32> hbm_size_bytes, ap_uint<32> frames, - ap_uint<5> modules, + ap_uint<6> modules, ap_uint<48> src_mac_addr, ap_uint<48> dest_mac_addr, ap_uint<32> src_ipv4_addr, diff --git a/fpga/hls/hls_jfjoch.h b/fpga/hls/hls_jfjoch.h index e88ff38e..53d587f3 100644 --- a/fpga/hls/hls_jfjoch.h +++ b/fpga/hls/hls_jfjoch.h @@ -79,7 +79,7 @@ struct axis_completion { ap_uint<32> debug; ap_uint<16> handle; ap_uint<16> packet_count; - ap_uint<7> module; + ap_uint<5> module; ap_uint<1> last; ap_uint<1> ignore; }; @@ -272,7 +272,7 @@ void data_collection_fsm(AXI_STREAM ð_in, ap_uint<32> mode, ap_uint<32> one_over_energy, ap_uint<32> nframes, - ap_uint<8> nmodules, + ap_uint<5> nmodules, ap_uint<4> nstorage_cells, ap_uint<7> nsummation); @@ -322,7 +322,7 @@ int frame_generator(STREAM_512 &data_out, ap_uint<256> *d_hbm_p1, ap_uint<32> hbm_size_bytes, ap_uint<32> frames, - ap_uint<5> modules, + ap_uint<6> modules, ap_uint<48> src_mac_addr, ap_uint<48> dest_mac_addr, ap_uint<32> src_ipv4_addr, diff --git a/fpga/hls/load_from_hbm.cpp b/fpga/hls/load_from_hbm.cpp index 5086ef78..ff9fe2a5 100644 --- a/fpga/hls/load_from_hbm.cpp +++ b/fpga/hls/load_from_hbm.cpp @@ -29,7 +29,7 @@ void load_from_hbm(STREAM_512 &data_in, data_in >> packet; data_out << packet; - for (ap_uint<16> i = 0; i < hbm_size_bytes * 4 / (RAW_MODULE_SIZE * sizeof(uint32_t)); i++) + for (ap_uint<16> i = 0; i < hbm_size_bytes * 4 / (RAW_MODULE_SIZE * sizeof(uint16_t)); i++) m_axis_free_handles << i; ap_uint<32> offset_hbm_0 = 12 * hbm_size_bytes / 32; diff --git a/fpga/hls/save_to_hbm.cpp b/fpga/hls/save_to_hbm.cpp index 8cf6b5a4..83244cac 100644 --- a/fpga/hls/save_to_hbm.cpp +++ b/fpga/hls/save_to_hbm.cpp @@ -52,7 +52,7 @@ void save_to_hbm(STREAM_512 &data_in, ap_uint<64> frame_number = addr.frame_number; ap_uint<5> module_number = addr.module; ap_uint<7> eth_packet = addr.eth_packet; - ap_uint<5> id = module_number * 2 + (frame_number % 2); + ap_uint<6> id = module_number * 2 + (frame_number % 2); ap_uint<16> curr_handle = 0; if (cmpl[id].frame_number != frame_number) { diff --git a/receiver/HLSSimulatedDevice.cpp b/receiver/HLSSimulatedDevice.cpp index 15e4d358..f6657c1d 100644 --- a/receiver/HLSSimulatedDevice.cpp +++ b/receiver/HLSSimulatedDevice.cpp @@ -24,7 +24,7 @@ uint16_t checksum(const uint16_t *addr, size_t count) { while (sum>>16) sum = (sum & 0xffff) + (sum >> 16); - return ~sum; + return ~sum; } HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node) @@ -138,19 +138,22 @@ void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experimen cancel_data_collection = 0; idle = false; if (experiment.IsUsingInternalPacketGen()) { - frame_generator(din_frame_generator, - hbm.data(), - hbm.data(), - hbm_if_size, - experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1, - experiment.GetModulesNum(data_stream), - mac_addr, - mac_addr, - ipv4_addr, - ipv4_addr, - INT_PKT_GEN_BUNCHID, - INT_PKT_GEN_EXPTTIME, - INT_PKT_GEN_DEBUG); + auto ret = frame_generator(din_frame_generator, + hbm.data(), + hbm.data(), + hbm_if_size, + experiment.GetFrameNum() + DELAY_FRAMES_STOP_AND_QUIT + 1, + experiment.GetModulesNum(data_stream), + mac_addr, + mac_addr, + ipv4_addr, + ipv4_addr, + INT_PKT_GEN_BUNCHID, + INT_PKT_GEN_EXPTTIME, + INT_PKT_GEN_DEBUG); + if (ret) + throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, + "Error running internal packet generator"); } action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this ); } @@ -258,7 +261,7 @@ void HLSSimulatedDevice::HLSMainThread() { while ((!din_eth.empty()) || (!din_frame_generator.empty())) stream_merge(din_eth, din_frame_generator, network0); - while(!network0.empty()) + while(!network0.empty()) ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters); while(!ip1.empty()) @@ -473,7 +476,7 @@ void HLSSimulatedDevice::HLSMainThread() { if (!datamover_in.GetDataStream().empty()) throw std::runtime_error("Datamover queue is not empty"); - if (err_reg != 0) + if (err_reg != 0) throw std::runtime_error("Error reg for frame_statistics not zero, val=" + std::to_string(err_reg)); while (!datamover_out.IsIdle()) @@ -506,15 +509,18 @@ void HLSSimulatedDevice::HW_LoadCalibration(uint32_t modules, uint32_t storage_c in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; } - load_calibration(hbm.data(), - hbm.data(), - modules, - storage_cells, - hbm_if_size, - LOAD_CALIBRATION_DEST_CALIB, - datamover_in.GetCtrlStream(), - datamover_in.GetDataStream(), - calibration_addr_bram); + int ret = load_calibration(hbm.data(), + hbm.data(), + modules, + storage_cells, + hbm_if_size, + LOAD_CALIBRATION_DEST_CALIB, + datamover_in.GetCtrlStream(), + datamover_in.GetDataStream(), + calibration_addr_bram); + if (ret) + throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, + "Error in loading calibration"); if (logger) logger->Info("Load calibration done"); @@ -533,15 +539,20 @@ void HLSSimulatedDevice::HW_LoadIntegrationMap(uint32_t modules) { in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; } - load_calibration(hbm.data(), - hbm.data(), - modules, - 0, - hbm_if_size, - LOAD_CALIBRATION_DEST_INTEGRATION, - datamover_in.GetCtrlStream(), - datamover_in.GetDataStream(), - calibration_addr_bram); + int ret = load_calibration(hbm.data(), + hbm.data(), + modules, + 0, + hbm_if_size, + LOAD_CALIBRATION_DEST_INTEGRATION, + datamover_in.GetCtrlStream(), + datamover_in.GetDataStream(), + calibration_addr_bram); + + if (ret) + throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, + "Error in loading calibration"); + if (logger) logger->Info("Load integration_map"); @@ -560,15 +571,20 @@ void HLSSimulatedDevice::HW_LoadInternalGeneratorFrame(uint32_t modules) { in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; } - load_calibration(hbm.data(), - hbm.data(), - modules, - 0, - hbm_if_size, - LOAD_CALIBRATION_DEST_FRAME_GEN, - datamover_in.GetCtrlStream(), - datamover_in.GetDataStream(), - calibration_addr_bram); + int ret = load_calibration(hbm.data(), + hbm.data(), + modules, + 0, + hbm_if_size, + LOAD_CALIBRATION_DEST_FRAME_GEN, + datamover_in.GetCtrlStream(), + datamover_in.GetDataStream(), + calibration_addr_bram); + + if (ret) + throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, + "Error in loading calibration"); + if (logger) logger->Info("Load to frame generator done"); diff --git a/tests/FPGAIntegrationTest.cpp b/tests/FPGAIntegrationTest.cpp index 41ebcd8e..df531f50 100644 --- a/tests/FPGAIntegrationTest.cpp +++ b/tests/FPGAIntegrationTest.cpp @@ -40,7 +40,7 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator", "[FPGA][Full]") { } TEST_CASE("HLS_C_Simulation_internal_packet_generator_custom_frame", "[FPGA][Full]") { - const uint16_t nmodules = 16; + const uint16_t nmodules = 32; const size_t nframes = 2; DiffractionExperiment x((DetectorGeometry(nmodules)));