FPGA: Save to HBM operates in parallel to host writer

fpga/hls/CMakeLists.txt

@@ -13,7 +13,8 @@ ADD_LIBRARY( HLSSimulation STATIC
         ip_header_checksum.h
         udp.cpp
         sls_detector.cpp
-        save_to_hbm.cpp)
+        save_to_hbm.cpp
+        writer_split.cpp)
 
 TARGET_INCLUDE_DIRECTORIES(HLSSimulation PUBLIC ../include)
 TARGET_LINK_LIBRARIES(HLSSimulation CommonFunctions)
@@ -47,6 +48,7 @@ MAKE_HLS_MODULE(arp.cpp arp)
 MAKE_HLS_MODULE(udp.cpp udp)
 MAKE_HLS_MODULE(sls_detector.cpp sls_detector)
 MAKE_HLS_MODULE(save_to_hbm.cpp save_to_hbm)
+MAKE_HLS_MODULE(writer_split.cpp writer_split)
 
 SET (HLS_IPS psi_ch_hls_data_collection_fsm_1_0.zip
              psi_ch_hls_timer_host_1_0.zip
@@ -61,7 +63,9 @@ SET (HLS_IPS psi_ch_hls_data_collection_fsm_1_0.zip
              psi_ch_hls_sls_detector_1_0.zip
              psi_ch_hls_icmp_1_0.zip
              psi_ch_hls_host_writer_1_0.zip
-             psi_ch_hls_save_to_hbm_1_0.zip)
+             psi_ch_hls_save_to_hbm_1_0.zip
+             psi_ch_hls_writer_split_1_0.zip
+             )
 
 SET (HLS_IPS ${HLS_IPS} PARENT_SCOPE)
 ADD_CUSTOM_TARGET(hls DEPENDS ${HLS_IPS})

fpga/hls/hls_jfjoch.h

@@ -112,6 +112,13 @@ void host_writer(STREAM_512 &data_in,
 void timer_hbm(STREAM_512 &in, STREAM_512 &data_out, uint64_t &counter);
 void timer_host(STREAM_512 &data_in, STREAM_512 &data_out, uint64_t &counter);
 
+void writer_split(STREAM_512 &data_in,
+                  STREAM_512 &data_out_1,
+                  STREAM_512 &data_out_2,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_in,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out_1,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out_2);
+
 void internal_packet_generator(STREAM_512 &data_in, STREAM_512 &data_out,
                                hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_in,
                                hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out,
@@ -120,11 +127,9 @@ void internal_packet_generator(STREAM_512 &data_in, STREAM_512 &data_out,
 
 void save_to_hbm(STREAM_512 &data_in,
                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_in,
-                 STREAM_512 &data_out,
-                 hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out,
+                 hls::stream<ap_uint<32> > &completion_out,
                  hls::burst_maxi<hbm256_t> d_hbm_p0, hls::burst_maxi<hbm256_t> d_hbm_p1,
                  hls::burst_maxi<hbm256_t> d_hbm_p2, hls::burst_maxi<hbm256_t> d_hbm_p3,
-                 STREAM_512  &completion_out,
                  volatile uint64_t &packets_processed,
                  volatile ap_uint<1> &idle,
                  ap_uint<8> &err_reg,

fpga/hls/save_to_hbm.cpp

@@ -8,9 +8,8 @@
 #endif
 
 #define PACKET_SIZE 8192
-#define HBM_BURST_SIZE 64
 
-inline void write_completion(STREAM_512 &m_axis_completion,
+inline void write_completion(hls::stream<ap_uint<32> > &m_axis_completion,
                              const ap_uint<32> &handle,
                              const ap_uint<8> &module_number,
                              const ap_uint<64> &frame_num,
@@ -32,7 +31,7 @@ inline void write_completion(STREAM_512 &m_axis_completion,
     status[2] = flushing;
     ap_uint<128> tmp = (handle, packet_count, status, module_number, frame_num);
     status[7] = tmp.xor_reduce(); // ensure completion has even parity
-/*
+
     if (handle != HANDLE_SKIP_FRAME) {
         m_axis_completion << handle;
         m_axis_completion << (packet_count, status, module_number);
@@ -53,17 +52,14 @@ inline void write_completion(STREAM_512 &m_axis_completion,
         m_axis_completion << packet_mask( 95,64);
         m_axis_completion << packet_mask( 63,32);
         m_axis_completion << packet_mask( 31, 0);
-    } */
-
+    }
 }
 
 void save_to_hbm(STREAM_512 &data_in,
                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_in,
-                 STREAM_512 &data_out,
-                 hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out,
+                 hls::stream<ap_uint<32> > &completion_out,
                  hls::burst_maxi<hbm256_t> d_hbm_p0, hls::burst_maxi<hbm256_t> d_hbm_p1,
                  hls::burst_maxi<hbm256_t> d_hbm_p2, hls::burst_maxi<hbm256_t> d_hbm_p3,
-                 STREAM_512  &completion_out,
                  volatile uint64_t &packets_processed,
                  volatile ap_uint<1> &idle,
                  ap_uint<8> &err_reg,
@@ -71,8 +67,6 @@ void save_to_hbm(STREAM_512 &data_in,
 #pragma HLS INTERFACE ap_ctrl_none port=return
 #pragma HLS INTERFACE register both axis port=data_in
 #pragma HLS INTERFACE register both axis port=addr_in
-#pragma HLS INTERFACE register both axis port=data_out
-#pragma HLS INTERFACE register both axis port=addr_out
 #pragma HLS INTERFACE register both axis port=completion_out
 #pragma HLS INTERFACE register ap_vld port=packets_processed
 #pragma HLS INTERFACE register ap_vld port=err_reg
@@ -123,11 +117,9 @@ void save_to_hbm(STREAM_512 &data_in,
 
     ap_uint<ADDR_STREAM_WIDTH> addr;
     addr_in >> addr;
-    addr_out << addr;
 
     packet_512_t packet_in;
     data_in >> packet_in;
-    data_out << packet_in;
 
     ap_uint<5> nmodules              = ACT_REG_NMODULES(packet_in.data);
     ap_uint<32> data_collection_mode = ACT_REG_MODE(packet_in.data);
@@ -142,7 +134,6 @@ void save_to_hbm(STREAM_512 &data_in,
     uint64_t total_counter = 0;
     packets_processed = 0;
     addr_in >> addr;
-    addr_out << addr;
 
     Loop_good_packet:
     while (!addr_last_flag(addr)) {
@@ -180,8 +171,16 @@ void save_to_hbm(STREAM_512 &data_in,
 
             packet_mask[id]  = ap_uint<128>(1) << eth_packet;
             packet_count[id] = 1;
+            
+            if (hbm_size == 64)
+                handle_val = (handle_val + 1) % 64;
+            else if (hbm_size == 512)
+                handle_val = (handle_val + 1) % 512;
+            else if (hbm_size == 1024)
+                handle_val = (handle_val + 1) % 1024;
+            else
+                handle_val = (handle_val + 1) % 32;
 
-            handle_val = (handle_val + 1) % hbm_size;
         } else {
             packet_count[id]++;
             packet_mask[id] |= ap_uint<128>(1) << eth_packet;
@@ -198,12 +197,10 @@ void save_to_hbm(STREAM_512 &data_in,
             }
 
             data_in >> packet_in;
-            data_out << packet_in;
             d_hbm_p0.write(packet_in.data(255, 0));
             d_hbm_p1.write(packet_in.data(511, 256));
 
             data_in >> packet_in;
-            data_out << packet_in;
             d_hbm_p2.write(packet_in.data(255, 0));
             d_hbm_p3.write(packet_in.data(511, 256));
 
@@ -221,7 +218,6 @@ void save_to_hbm(STREAM_512 &data_in,
         total_counter++;
         packets_processed = total_counter;
         addr_in >> addr;
-        addr_out << addr;
         err_reg = internal_err_reg;
     }
 
@@ -235,7 +231,6 @@ void save_to_hbm(STREAM_512 &data_in,
     }
 
     data_in >> packet_in;
-    data_out << packet_in;
 
     idle = 1;
 }

fpga/hls/writer_split.cpp

@@ -0,0 +1,53 @@
+// Copyright (2019-2023) Paul Scherrer Institute
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "hls_jfjoch.h"
+
+void writer_split(STREAM_512 &data_in,
+                  STREAM_512 &data_out_1,
+                  STREAM_512 &data_out_2,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_in,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out_1,
+                  hls::stream<ap_uint<ADDR_STREAM_WIDTH> > &addr_out_2) {
+#pragma HLS INTERFACE register both axis port=data_in
+#pragma HLS INTERFACE register both axis port=data_out_1
+#pragma HLS INTERFACE register both axis port=data_out_2
+
+#pragma HLS INTERFACE register both axis port=addr_in
+#pragma HLS INTERFACE register both axis port=addr_out_1
+#pragma HLS INTERFACE register both axis port=addr_out_2
+
+#pragma HLS INTERFACE ap_ctrl_none port=return
+    packet_512_t packet;
+    ap_uint<ADDR_STREAM_WIDTH> addr;
+
+    addr_in >> addr;
+    addr_out_1 << addr;
+    addr_out_2 << addr;
+
+    data_in >> packet;
+    data_out_1 << packet;
+    data_out_2 << packet;
+
+    addr_in >> addr;
+    addr_out_1 << addr;
+    addr_out_2 << addr;
+
+    Loop_good_packet:
+    while (!addr_last_flag(addr)) {
+#pragma HLS PIPELINE II=128
+        for (int i = 0; i < 128; i++) {
+            data_in >> packet;
+            data_out_1 << packet;
+            data_out_2 << packet;
+        }
+        addr_in >> addr;
+        addr_out_1 << addr;
+        addr_out_2 << addr;
+    }
+
+    data_in >> packet;
+    data_out_1 << packet;
+    data_out_2 << packet;
+}
+

fpga/scripts/bd_pcie.tcl

@@ -143,7 +143,6 @@ xilinx.com:ip:xlconcat:2.1\
 xilinx.com:ip:axi_protocol_converter:2.1\
 xilinx.com:ip:axi_register_slice:2.1\
 xilinx.com:ip:hbm:1.0\
-xilinx.com:ip:util_vector_logic:2.0\
 xilinx.com:ip:axi_bram_ctrl:4.1\
 xilinx.com:ip:axis_data_fifo:2.0\
 xilinx.com:ip:axis_register_slice:1.1\
@@ -154,9 +153,12 @@ psi.ch:hls:internal_packet_generator:1.0\
 psi.ch:hls:jf_conversion:1.0\
 psi.ch:hls:load_calibration:1.0\
 xilinx.com:ip:mailbox:2.1\
+psi.ch:hls:save_to_hbm:1.0\
 psi.ch:hls:timer_hbm:1.0\
 psi.ch:hls:timer_host:1.0\
+psi.ch:hls:writer_split:1.0\
 xilinx.com:ip:cmac_usplus:3.1\
+xilinx.com:ip:util_vector_logic:2.0\
 xilinx.com:ip:axi_firewall:1.2\
 xilinx.com:ip:axis_clock_converter:1.1\
 xilinx.com:ip:util_ds_buf:2.2\

fpga/scripts/jfjoch.tcl

@@ -88,7 +88,7 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   create_bd_pin -dir I -type clk axi_clk
   create_bd_pin -dir I -type rst axi_rst_n
 
-  # Create instance: action_config_0, and set properties
+ # Create instance: action_config_0, and set properties
   set block_name action_config
   set block_cell_name action_config_0
   if { [catch {set action_config_0 [create_bd_cell -type module -reference $block_name $block_cell_name] } errmsg] } {
@@ -154,6 +154,24 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
    CONFIG.HAS_AFULL {1} \
  ] $axis_addr_fifo_2
 
+  # Create instance: axis_addr_fifo_3, and set properties
+  set axis_addr_fifo_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_addr_fifo_3 ]
+  set_property -dict [ list \
+   CONFIG.FIFO_DEPTH {16} \
+   CONFIG.FIFO_MEMORY_TYPE {block} \
+   CONFIG.HAS_AEMPTY {0} \
+   CONFIG.HAS_AFULL {0} \
+ ] $axis_addr_fifo_3
+
+  # Create instance: axis_addr_fifo_4, and set properties
+  set axis_addr_fifo_4 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_addr_fifo_4 ]
+  set_property -dict [ list \
+   CONFIG.FIFO_DEPTH {16} \
+   CONFIG.FIFO_MEMORY_TYPE {block} \
+   CONFIG.HAS_AEMPTY {0} \
+   CONFIG.HAS_AFULL {0} \
+ ] $axis_addr_fifo_4
+
   # Create instance: axis_data_fifo_0, and set properties
   set axis_data_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_0 ]
   set_property -dict [ list \
@@ -200,6 +218,18 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
    CONFIG.HAS_AFULL {1} \
  ] $axis_data_fifo_5
 
+  # Create instance: axis_data_fifo_6, and set properties
+  set axis_data_fifo_6 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_6 ]
+  set_property -dict [ list \
+   CONFIG.FIFO_DEPTH {4096} \
+ ] $axis_data_fifo_6
+
+  # Create instance: axis_data_fifo_7, and set properties
+  set axis_data_fifo_7 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_7 ]
+  set_property -dict [ list \
+   CONFIG.FIFO_DEPTH {4096} \
+ ] $axis_data_fifo_7
+
   # Create instance: axis_data_fifo_c2h_cmd, and set properties
   set axis_data_fifo_c2h_cmd [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_c2h_cmd ]
   set_property -dict [ list \
@@ -370,6 +400,12 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   # Create instance: network_stack
   create_hier_cell_network_stack $hier_obj network_stack
 
+  # Create instance: one, and set properties
+  set one [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 one ]
+
+  # Create instance: save_to_hbm_0, and set properties
+  set save_to_hbm_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:save_to_hbm:1.0 save_to_hbm_0 ]
+
   # Create instance: smartconnect_0, and set properties
   set smartconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_0 ]
   set_property -dict [ list \
@@ -384,6 +420,16 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   # Create instance: timer_host_0, and set properties
   set timer_host_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:timer_host:1.0 timer_host_0 ]
 
+  # Create instance: writer_split_0, and set properties
+  set writer_split_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:writer_split:1.0 writer_split_0 ]
+
+  # Create instance: xlconstant_hbm_size, and set properties
+  set xlconstant_hbm_size [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 xlconstant_hbm_size ]
+  set_property -dict [ list \
+   CONFIG.CONST_VAL {64} \
+   CONFIG.CONST_WIDTH {32} \
+ ] $xlconstant_hbm_size
+
   # Create interface connections
   connect_bd_intf_net -intf_net Conn2 [get_bd_intf_pins eth_out] [get_bd_intf_pins network_stack/M00_AXIS]
   connect_bd_intf_net -intf_net Conn3 [get_bd_intf_pins eth_in] [get_bd_intf_pins network_stack/eth_in]
@@ -393,13 +439,17 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   connect_bd_intf_net -intf_net axi_bram_ctrl_internal_packet_generator_BRAM_PORTA [get_bd_intf_pins axi_bram_ctrl_internal_packet_generator_0/BRAM_PORTA] [get_bd_intf_pins internal_packet_generator_uram/BRAM_PORTA]
   connect_bd_intf_net -intf_net axis_addr_fifo_0_M_AXIS [get_bd_intf_pins axis_addr_fifo_0/M_AXIS] [get_bd_intf_pins internal_packet_generator_0/addr_in]
   connect_bd_intf_net -intf_net axis_addr_fifo_2_M_AXIS [get_bd_intf_pins axis_addr_fifo_1/M_AXIS] [get_bd_intf_pins jf_conversion_0/addr_in]
-  connect_bd_intf_net -intf_net axis_addr_fifo_3_M_AXIS [get_bd_intf_pins axis_addr_fifo_2/M_AXIS] [get_bd_intf_pins host_writer_0/addr_in]
+  connect_bd_intf_net -intf_net axis_addr_fifo_2_M_AXIS1 [get_bd_intf_pins axis_addr_fifo_2/M_AXIS] [get_bd_intf_pins writer_split_0/addr_in]
+  connect_bd_intf_net -intf_net axis_addr_fifo_3_M_AXIS1 [get_bd_intf_pins axis_addr_fifo_3/M_AXIS] [get_bd_intf_pins save_to_hbm_0/addr_in]
+  connect_bd_intf_net -intf_net axis_addr_fifo_4_M_AXIS [get_bd_intf_pins axis_addr_fifo_4/M_AXIS] [get_bd_intf_pins host_writer_0/addr_in]
   connect_bd_intf_net -intf_net axis_data_fifo_0_M_AXIS [get_bd_intf_pins axis_data_fifo_0/M_AXIS] [get_bd_intf_pins load_calibration_0/data_in]
   connect_bd_intf_net -intf_net axis_data_fifo_1_M_AXIS [get_bd_intf_pins axis_data_fifo_1/M_AXIS] [get_bd_intf_pins internal_packet_generator_0/data_in]
   connect_bd_intf_net -intf_net axis_data_fifo_2_M_AXIS [get_bd_intf_pins axis_data_fifo_2/M_AXIS] [get_bd_intf_pins timer_hbm_0/data_in]
   connect_bd_intf_net -intf_net axis_data_fifo_4_M_AXIS [get_bd_intf_pins axis_data_fifo_3/M_AXIS] [get_bd_intf_pins jf_conversion_0/data_in]
   connect_bd_intf_net -intf_net axis_data_fifo_4_M_AXIS1 [get_bd_intf_pins axis_data_fifo_4/M_AXIS] [get_bd_intf_pins timer_host_0/data_in]
-  connect_bd_intf_net -intf_net axis_data_fifo_8_M_AXIS [get_bd_intf_pins axis_data_fifo_5/M_AXIS] [get_bd_intf_pins host_writer_0/data_in]
+  connect_bd_intf_net -intf_net axis_data_fifo_5_M_AXIS [get_bd_intf_pins axis_data_fifo_5/M_AXIS] [get_bd_intf_pins writer_split_0/data_in]
+  connect_bd_intf_net -intf_net axis_data_fifo_6_M_AXIS [get_bd_intf_pins axis_data_fifo_6/M_AXIS] [get_bd_intf_pins save_to_hbm_0/data_in]
+  connect_bd_intf_net -intf_net axis_data_fifo_7_M_AXIS [get_bd_intf_pins axis_data_fifo_7/M_AXIS] [get_bd_intf_pins host_writer_0/data_in]
   connect_bd_intf_net -intf_net axis_data_fifo_c2h_cmd_M_AXIS [get_bd_intf_pins m_axis_c2h_datamover_cmd] [get_bd_intf_pins axis_data_fifo_c2h_cmd/M_AXIS]
   connect_bd_intf_net -intf_net axis_data_fifo_c2h_data_M_AXIS [get_bd_intf_pins m_axis_c2h_data] [get_bd_intf_pins axis_data_fifo_c2h_data/M_AXIS]
   connect_bd_intf_net -intf_net axis_data_fifo_h2c_cmd_M_AXIS [get_bd_intf_pins m_axis_h2c_datamover_cmd] [get_bd_intf_pins axis_data_fifo_h2c_cmd/M_AXIS]
@@ -444,12 +494,20 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   connect_bd_intf_net -intf_net network_stack_udp_addr_out [get_bd_intf_pins axis_udp_addr_fifo_0/S_AXIS] [get_bd_intf_pins network_stack/udp_addr_out]
   connect_bd_intf_net -intf_net network_stack_udp_out [get_bd_intf_pins axis_udp_fifo_0/S_AXIS] [get_bd_intf_pins network_stack/udp_out]
   connect_bd_intf_net -intf_net s_axi_1 [get_bd_intf_pins s_axi] [get_bd_intf_pins smartconnect_0/S00_AXI]
+  connect_bd_intf_net -intf_net save_to_hbm_0_m_axi_d_hbm_p0 [get_bd_intf_pins m_axi_d_hbm_p12] [get_bd_intf_pins save_to_hbm_0/m_axi_d_hbm_p0]
+  connect_bd_intf_net -intf_net save_to_hbm_0_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p13] [get_bd_intf_pins save_to_hbm_0/m_axi_d_hbm_p1]
+  connect_bd_intf_net -intf_net save_to_hbm_0_m_axi_d_hbm_p2 [get_bd_intf_pins m_axi_d_hbm_p14] [get_bd_intf_pins save_to_hbm_0/m_axi_d_hbm_p2]
+  connect_bd_intf_net -intf_net save_to_hbm_0_m_axi_d_hbm_p3 [get_bd_intf_pins m_axi_d_hbm_p15] [get_bd_intf_pins save_to_hbm_0/m_axi_d_hbm_p3]
   connect_bd_intf_net -intf_net smartconnect_0_M00_AXI [get_bd_intf_pins action_config_0/s_axi] [get_bd_intf_pins smartconnect_0/M00_AXI]
   connect_bd_intf_net -intf_net smartconnect_0_M01_AXI [get_bd_intf_pins mailbox_0/S0_AXI] [get_bd_intf_pins smartconnect_0/M01_AXI]
   connect_bd_intf_net -intf_net smartconnect_0_M02_AXI [get_bd_intf_pins axi_bram_ctrl_calibration_addr/S_AXI] [get_bd_intf_pins smartconnect_0/M02_AXI]
   connect_bd_intf_net -intf_net smartconnect_0_M03_AXI [get_bd_intf_pins axi_bram_ctrl_internal_packet_generator_0/S_AXI] [get_bd_intf_pins smartconnect_0/M03_AXI]
   connect_bd_intf_net -intf_net timer_hbm_0_data_out [get_bd_intf_pins axis_data_fifo_3/S_AXIS] [get_bd_intf_pins timer_hbm_0/data_out]
   connect_bd_intf_net -intf_net timer_host_0_data_out [get_bd_intf_pins axis_register_slice_data_1/S_AXIS] [get_bd_intf_pins timer_host_0/data_out]
+  connect_bd_intf_net -intf_net writer_split_0_addr_out_1 [get_bd_intf_pins axis_addr_fifo_3/S_AXIS] [get_bd_intf_pins writer_split_0/addr_out_1]
+  connect_bd_intf_net -intf_net writer_split_0_addr_out_2 [get_bd_intf_pins axis_addr_fifo_4/S_AXIS] [get_bd_intf_pins writer_split_0/addr_out_2]
+  connect_bd_intf_net -intf_net writer_split_0_data_out_1 [get_bd_intf_pins axis_data_fifo_6/S_AXIS] [get_bd_intf_pins writer_split_0/data_out_1]
+  connect_bd_intf_net -intf_net writer_split_0_data_out_2 [get_bd_intf_pins axis_data_fifo_7/S_AXIS] [get_bd_intf_pins writer_split_0/data_out_2]
 
   # Create port connections
   connect_bd_net -net action_config_0_clear_counters [get_bd_pins action_config_0/clear_counters] [get_bd_pins network_stack/clear_counters]
@@ -462,7 +520,7 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   connect_bd_net -net action_config_0_nmodules [get_bd_pins action_config_0/nmodules] [get_bd_pins data_collection_fsm_0/nmodules]
   connect_bd_net -net action_config_0_nstorage_cells [get_bd_pins action_config_0/nstorage_cells] [get_bd_pins data_collection_fsm_0/nstorage_cells]
   connect_bd_net -net action_config_0_one_over_energy [get_bd_pins action_config_0/one_over_energy] [get_bd_pins data_collection_fsm_0/one_over_energy]
-  connect_bd_net -net ap_clk_1 [get_bd_pins axi_clk] [get_bd_pins action_config_0/clk] [get_bd_pins axi_bram_ctrl_calibration_addr/s_axi_aclk] [get_bd_pins axi_bram_ctrl_internal_packet_generator_0/s_axi_aclk] [get_bd_pins axi_bram_ctrl_internal_packet_generator_1/s_axi_aclk] [get_bd_pins axis_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_addr_fifo_1/s_axis_aclk] [get_bd_pins axis_addr_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aclk] [get_bd_pins axis_register_slice_addr_0/aclk] [get_bd_pins axis_register_slice_addr_1/aclk] [get_bd_pins axis_register_slice_data_0/aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_in_0/aclk] [get_bd_pins axis_register_slice_host_mem/aclk] [get_bd_pins axis_register_slice_udp/aclk] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_udp_fifo_0/s_axis_aclk] [get_bd_pins axis_work_completion_fifo_0/s_axis_aclk] [get_bd_pins axis_work_request_fifo_0/s_axis_aclk] [get_bd_pins data_collection_fsm_0/ap_clk] [get_bd_pins host_writer_0/ap_clk] [get_bd_pins internal_packet_generator_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins load_calibration_0/ap_clk] [get_bd_pins mailbox_0/M1_AXIS_ACLK] [get_bd_pins mailbox_0/S0_AXI_ACLK] [get_bd_pins mailbox_0/S1_AXIS_ACLK] [get_bd_pins network_stack/axiclk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins timer_hbm_0/ap_clk] [get_bd_pins timer_host_0/ap_clk]
+  connect_bd_net -net ap_clk_1 [get_bd_pins axi_clk] [get_bd_pins action_config_0/clk] [get_bd_pins axi_bram_ctrl_calibration_addr/s_axi_aclk] [get_bd_pins axi_bram_ctrl_internal_packet_generator_0/s_axi_aclk] [get_bd_pins axi_bram_ctrl_internal_packet_generator_1/s_axi_aclk] [get_bd_pins axis_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_addr_fifo_1/s_axis_aclk] [get_bd_pins axis_addr_fifo_2/s_axis_aclk] [get_bd_pins axis_addr_fifo_3/s_axis_aclk] [get_bd_pins axis_addr_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_6/s_axis_aclk] [get_bd_pins axis_data_fifo_7/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aclk] [get_bd_pins axis_register_slice_addr_0/aclk] [get_bd_pins axis_register_slice_addr_1/aclk] [get_bd_pins axis_register_slice_data_0/aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_in_0/aclk] [get_bd_pins axis_register_slice_host_mem/aclk] [get_bd_pins axis_register_slice_udp/aclk] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_udp_fifo_0/s_axis_aclk] [get_bd_pins axis_work_completion_fifo_0/s_axis_aclk] [get_bd_pins axis_work_request_fifo_0/s_axis_aclk] [get_bd_pins data_collection_fsm_0/ap_clk] [get_bd_pins host_writer_0/ap_clk] [get_bd_pins internal_packet_generator_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins load_calibration_0/ap_clk] [get_bd_pins mailbox_0/M1_AXIS_ACLK] [get_bd_pins mailbox_0/S0_AXI_ACLK] [get_bd_pins mailbox_0/S1_AXIS_ACLK] [get_bd_pins network_stack/axiclk] [get_bd_pins save_to_hbm_0/ap_clk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins timer_hbm_0/ap_clk] [get_bd_pins timer_host_0/ap_clk] [get_bd_pins writer_split_0/ap_clk]
   connect_bd_net -net axis_addr_fifo_2_almost_empty [get_bd_pins action_config_0/calib_addr_fifo_empty] [get_bd_pins axis_addr_fifo_1/almost_empty]
   connect_bd_net -net axis_addr_fifo_2_almost_full [get_bd_pins action_config_0/calib_addr_fifo_full] [get_bd_pins axis_addr_fifo_1/almost_full]
   connect_bd_net -net axis_addr_fifo_4_almost_empty [get_bd_pins action_config_0/last_addr_fifo_empty] [get_bd_pins axis_addr_fifo_2/almost_empty]
@@ -501,12 +559,14 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
   connect_bd_net -net network_stack_packets_sls_ap_vld [get_bd_pins action_config_0/packets_sls_valid] [get_bd_pins network_stack/packets_sls_ap_vld]
   connect_bd_net -net network_stack_packets_udp [get_bd_pins action_config_0/packets_udp] [get_bd_pins network_stack/packets_udp]
   connect_bd_net -net network_stack_packets_udp_ap_vld [get_bd_pins action_config_0/packets_udp_valid] [get_bd_pins network_stack/packets_udp_ap_vld]
-  connect_bd_net -net reset_axi [get_bd_pins axi_rst_n] [get_bd_pins action_config_0/resetn] [get_bd_pins axis_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_addr_fifo_1/s_axis_aresetn] [get_bd_pins axis_addr_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aresetn] [get_bd_pins axis_register_slice_addr_0/aresetn] [get_bd_pins axis_register_slice_addr_1/aresetn] [get_bd_pins axis_register_slice_data_0/aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_in_0/aresetn] [get_bd_pins axis_register_slice_host_mem/aresetn] [get_bd_pins axis_register_slice_udp/aresetn] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_udp_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_completion_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_request_fifo_0/s_axis_aresetn] [get_bd_pins network_stack/resetn] [get_bd_pins smartconnect_0/aresetn]
-  connect_bd_net -net reset_hls [get_bd_pins ap_rst_n] [get_bd_pins axi_bram_ctrl_calibration_addr/s_axi_aresetn] [get_bd_pins axi_bram_ctrl_internal_packet_generator_0/s_axi_aresetn] [get_bd_pins axi_bram_ctrl_internal_packet_generator_1/s_axi_aresetn] [get_bd_pins data_collection_fsm_0/ap_rst_n] [get_bd_pins host_writer_0/ap_rst_n] [get_bd_pins internal_packet_generator_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins load_calibration_0/ap_rst_n] [get_bd_pins mailbox_0/S0_AXI_ARESETN] [get_bd_pins network_stack/ap_rst_n] [get_bd_pins timer_hbm_0/ap_rst_n] [get_bd_pins timer_host_0/ap_rst_n]
+  connect_bd_net -net one_dout [get_bd_pins one/dout] [get_bd_pins save_to_hbm_0/completion_out_TREADY]
+  connect_bd_net -net reset_axi [get_bd_pins axi_rst_n] [get_bd_pins action_config_0/resetn] [get_bd_pins axis_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_addr_fifo_1/s_axis_aresetn] [get_bd_pins axis_addr_fifo_2/s_axis_aresetn] [get_bd_pins axis_addr_fifo_3/s_axis_aresetn] [get_bd_pins axis_addr_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_6/s_axis_aresetn] [get_bd_pins axis_data_fifo_7/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aresetn] [get_bd_pins axis_register_slice_addr_0/aresetn] [get_bd_pins axis_register_slice_addr_1/aresetn] [get_bd_pins axis_register_slice_data_0/aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_in_0/aresetn] [get_bd_pins axis_register_slice_host_mem/aresetn] [get_bd_pins axis_register_slice_udp/aresetn] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_udp_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_completion_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_request_fifo_0/s_axis_aresetn] [get_bd_pins network_stack/resetn] [get_bd_pins smartconnect_0/aresetn]
+  connect_bd_net -net reset_hls [get_bd_pins ap_rst_n] [get_bd_pins axi_bram_ctrl_calibration_addr/s_axi_aresetn] [get_bd_pins axi_bram_ctrl_internal_packet_generator_0/s_axi_aresetn] [get_bd_pins axi_bram_ctrl_internal_packet_generator_1/s_axi_aresetn] [get_bd_pins data_collection_fsm_0/ap_rst_n] [get_bd_pins host_writer_0/ap_rst_n] [get_bd_pins internal_packet_generator_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins load_calibration_0/ap_rst_n] [get_bd_pins mailbox_0/S0_AXI_ARESETN] [get_bd_pins network_stack/ap_rst_n] [get_bd_pins save_to_hbm_0/ap_rst_n] [get_bd_pins timer_hbm_0/ap_rst_n] [get_bd_pins timer_host_0/ap_rst_n] [get_bd_pins writer_split_0/ap_rst_n]
   connect_bd_net -net timer_hbm_0_counter [get_bd_pins action_config_0/stalls_hbm] [get_bd_pins timer_hbm_0/counter]
   connect_bd_net -net timer_hbm_0_counter_ap_vld [get_bd_pins action_config_0/stalls_hbm_valid] [get_bd_pins timer_hbm_0/counter_ap_vld]
   connect_bd_net -net timer_host_0_counter [get_bd_pins action_config_0/stalls_host] [get_bd_pins timer_host_0/counter]
   connect_bd_net -net timer_host_0_counter_ap_vld [get_bd_pins action_config_0/stalls_host_valid] [get_bd_pins timer_host_0/counter_ap_vld]
+  connect_bd_net -net xlconstant_0_dout [get_bd_pins save_to_hbm_0/hbm_size] [get_bd_pins xlconstant_hbm_size/dout]
 
   # Restore current instance
   current_bd_instance $oldCurInst

receiver/HLSSimulatedDevice.cpp

@@ -217,12 +217,16 @@ void HLSSimulatedDevice::HLSMainThread() {
     STREAM_512 converted_1;
     STREAM_512 converted_2;
     STREAM_512 converted_3;
+    STREAM_512 converted_4;
+    STREAM_512 converted_5;
 
     hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr0;
     hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr1;
     hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr2;
     hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr3;
     hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr4;
+    hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr5;
+    hls::stream<ap_uint<ADDR_STREAM_WIDTH> > addr6;
 
     hls::stream<ap_uint<UDP_METADATA_STREAM_WIDTH> > udp_metadata;
     ap_uint<1> idle_data_collection;
@@ -308,25 +312,29 @@ void HLSSimulatedDevice::HLSMainThread() {
     // Timer procedure - count how many times write_data is not accepting input (to help track down latency issues)
     hls_cores.emplace_back([&] { timer_host(converted_1, converted_2, counter_host); });
 
-    STREAM_512 save_to_hbm_completion;
+    // Temporary - split streams so data can be written to both HBM and internal memory
+    hls_cores.emplace_back([&] { writer_split(converted_2, converted_3, converted_4,
+                                              addr3, addr4, addr5); });
+
+    hls::stream<ap_uint<32> > save_to_hbm_completion;
     ap_uint<8> save_to_hbm_err_reg;
     uint64_t save_to_hbm_packets_processed;
     ap_uint<1> save_to_hbm_idle;
 
-    hls_cores.emplace_back([&] { save_to_hbm(converted_2, addr3, converted_3, addr4,
+    // 3. Write images to HBM
+    hls_cores.emplace_back([&] { save_to_hbm(converted_3, addr4, save_to_hbm_completion,
                                              (hbm256_t *) (hbm_memory[12].data()),
                                              (hbm256_t *) (hbm_memory[13].data()),
                                              (hbm256_t *) (hbm_memory[14].data()),
                                              (hbm256_t *) (hbm_memory[15].data()),
-                                                  save_to_hbm_completion,
-                                                  save_to_hbm_packets_processed,
-                                                  save_to_hbm_idle,
-                                                  save_to_hbm_err_reg,
-                                                  16); });
+                                             save_to_hbm_packets_processed,
+                                             save_to_hbm_idle,
+                                             save_to_hbm_err_reg,
+                                             16); });
 
-    // 3. Prepare data to write to host memory
+    // 4. Prepare data to write to host memory
     hls_cores.emplace_back([&] {
-        host_writer(converted_3, addr4, datamover_out.GetDataStream(),
+        host_writer(converted_4, addr5, datamover_out.GetDataStream(),
                     datamover_out.GetCtrlStream(), work_request_stream, completion_stream,
                     packets_processed, host_writer_idle, err_reg); });
 
@@ -348,6 +356,9 @@ void HLSSimulatedDevice::HLSMainThread() {
     if (!addr4.empty())
         throw std::runtime_error("Addr4 queue not empty");
 
+    if (!addr5.empty())
+        throw std::runtime_error("Addr5 queue not empty");
+
     if (!raw1.empty())
         throw std::runtime_error("Raw1 queue not empty");
 
@@ -372,6 +383,9 @@ void HLSSimulatedDevice::HLSMainThread() {
     if (!converted_3.empty())
         throw std::runtime_error("Converted_3 queue not empty");
 
+    if (!converted_4.empty())
+        throw std::runtime_error("Converted_4 queue not empty");
+
     if (!datamover_in.GetDataStream().empty())
         throw std::runtime_error("Datamover queue is not empty");