mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

FPGA: Integration on FPGA allows for per pixel weights (in range 1.99 - 3e-5)

Browse Source
This commit is contained in:
leonarski_f
2023-10-21 15:37:46 +02:00
parent d91eb6bdd5
commit 3b65e6bf88
15 changed files with 104 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fpga/hls/axis_256_to_512.cpp
+1 -1
View File
@@ -13,7 +13,7 @@ void axis_128_to_512(hls::stream<ap_axiu<128,1,1,1>> &data_in,
data_in >> packet_128[0];
while (!packet_128[0].user) {
#pragma HLS PIPELINE II=2
#pragma HLS PIPELINE II=4
data_in >> packet_128[1];
data_in >> packet_128[2];
data_in >> packet_128[3];
fpga/hls/frame_generator.cpp
+2 -2
View File
@@ -52,8 +52,8 @@ void generate_packet(STREAM_512 &data_out,
data_out << packet;
ap_uint<32> offset_hbm_0 = 18 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_1 = 19 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_0 = 20 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_1 = 21 * hbm_size_bytes / 32;
ap_uint<208> remainder = header(719, 512);
for (int i = 0; i < 128; i++) {
fpga/hls/hls_jfjoch.h
+2
View File
@@ -243,6 +243,8 @@ void integration(STREAM_512 &data_in,
hls::stream<axis_completion > &m_axis_completion,
ap_uint<256> *d_hbm_p0,
ap_uint<256> *d_hbm_p1,
ap_uint<256> *d_hbm_p2,
ap_uint<256> *d_hbm_p3,
ap_uint<32> hbm_size_bytes);
// Packet stream handling
fpga/hls/integration.cpp
+23 -6
View File
@@ -9,6 +9,8 @@ void integration(STREAM_512 &data_in,
hls::stream<axis_completion > &m_axis_completion,
ap_uint<256> *d_hbm_p0,
ap_uint<256> *d_hbm_p1,
ap_uint<256> *d_hbm_p2,
ap_uint<256> *d_hbm_p3,
ap_uint<32> hbm_size_bytes) {
#pragma HLS INTERFACE ap_ctrl_none port=return
#pragma HLS INTERFACE register both axis port=data_in
@@ -21,8 +23,12 @@ void integration(STREAM_512 &data_in,
max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8
#pragma HLS INTERFACE m_axi port=d_hbm_p1 bundle=d_hbm_p1 depth=512 offset=off \
max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8
#pragma HLS INTERFACE m_axi port=d_hbm_p2 bundle=d_hbm_p2 depth=512 offset=off \
max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8
#pragma HLS INTERFACE m_axi port=d_hbm_p3 bundle=d_hbm_p3 depth=512 offset=off \
max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8
ap_int<30> sum[64][FPGA_INTEGRATION_BIN_COUNT];
ap_fixed<46,30, AP_RND_CONV> sum[64][FPGA_INTEGRATION_BIN_COUNT];
// log2(32768*512*1024/64) = 28 + sign 1 bit
#pragma HLS BIND_STORAGE variable=sum type=ram_t2p impl=bram
#pragma HLS ARRAY_PARTITION variable=sum type=complete dim=1
@@ -41,7 +47,9 @@ void integration(STREAM_512 &data_in,
ap_int<16> in_val[32];
ap_uint<16> in_bin[32];
ap_uint<256> bins_0, bins_1;
ap_ufixed<16,1, AP_RND_CONV> in_coeff[32];
ap_uint<256> bins_0, bins_1, coeff_0, coeff_1;
packet_512_t packet_in;
data_in >> packet_in;
@@ -49,6 +57,8 @@ void integration(STREAM_512 &data_in,
ap_uint<32> offset_hbm_0 = 16 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_1 = 17 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_2 = 18 * hbm_size_bytes / 32;
ap_uint<32> offset_hbm_3 = 19 * hbm_size_bytes / 32;
axis_completion cmpl;
s_axis_completion >> cmpl;
@@ -61,13 +71,18 @@ void integration(STREAM_512 &data_in,
data_out << packet_in;
bins_0 = d_hbm_p0[offset_hbm_0 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k];
bins_1 = d_hbm_p1[offset_hbm_1 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k];
coeff_0 = d_hbm_p2[offset_hbm_2 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k];
coeff_1 = d_hbm_p3[offset_hbm_2 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k];
unpack_2xhbm_to_32x16bit(bins_0, bins_1, in_bin);
unpack_2xhbm_to_32x16bit(coeff_0, coeff_1, in_coeff);
unpack32(packet_in.data, in_val);
for (int j = 0; j < 32; j++) {
ap_uint<44> tmp = in_val[j] * in_val[j];
ap_fixed<32,16, AP_RND_CONV> tmp = in_val[j] * in_coeff[j];
if ((in_val[j] != INT16_MAX) && (in_val[j] != INT16_MIN) && (in_bin[j] < FPGA_INTEGRATION_BIN_COUNT)) {
sum[k * 32 + j][in_bin[j]] += in_val[j];
sum[k * 32 + j][in_bin[j]] += tmp;
count[k * 32 + j][in_bin[j]] += 1;
}
}
@@ -77,7 +92,7 @@ void integration(STREAM_512 &data_in,
for (int i = 0; i < FPGA_INTEGRATION_BIN_COUNT; i++) {
#pragma HLS PIPELINE II=1
ap_axiu<128,1,1,1> res;
ap_int<38> main_sum = 0;
ap_fixed<64, 40, AP_RND_CONV> main_sum = 0;
ap_int<22> main_count = 0;
for (int j = 0; j < 64; j++) {
main_sum += sum[j][i];
@@ -86,7 +101,9 @@ void integration(STREAM_512 &data_in,
count[j][i] = 0;
}
res.data(63, 0) = main_count;
res.data(127, 64) = main_sum;
for (int j = 0; j < 64; j++) {
res.data[64 + j] = main_sum[j];
}
res.dest = 0;
res.id = 0;
res.keep = UINT32_MAX;
fpga/hls/load_calibration.cpp
+11 -2
View File
@@ -78,6 +78,7 @@ void load_calibration(ap_uint<256> *d_hbm_p0,
}
}
} else if (destination == LOAD_CALIBRATION_DEST_INTEGRATION) {
// load maps
for (int m = 0; m < modules; m++) {
#pragma HLS PIPELINE OFF
setup_datamover(datamover_in_cmd, in_mem_location[m], RAW_MODULE_SIZE * sizeof(int16_t));
@@ -85,13 +86,21 @@ void load_calibration(ap_uint<256> *d_hbm_p0,
size_t offset_hbm_1 = 17 * hbm_size_bytes / 32 + m * RAW_MODULE_SIZE * sizeof(int16_t) / 64;
read_module(d_hbm_p0, d_hbm_p1, host_memory_in, offset_hbm_0, offset_hbm_1);
}
} else if (destination == LOAD_CALIBRATION_DEST_FRAME_GEN) {
// load weights
for (int m = 0; m < modules; m++) {
#pragma HLS PIPELINE OFF
setup_datamover(datamover_in_cmd, in_mem_location[m], RAW_MODULE_SIZE * sizeof(int16_t));
setup_datamover(datamover_in_cmd, in_mem_location[modules + m], RAW_MODULE_SIZE * sizeof(int16_t));
size_t offset_hbm_0 = 18 * hbm_size_bytes / 32 + m * RAW_MODULE_SIZE * sizeof(int16_t) / 64;
size_t offset_hbm_1 = 19 * hbm_size_bytes / 32 + m * RAW_MODULE_SIZE * sizeof(int16_t) / 64;
read_module(d_hbm_p0, d_hbm_p1, host_memory_in, offset_hbm_0, offset_hbm_1);
}
} else if (destination == LOAD_CALIBRATION_DEST_FRAME_GEN) {
for (int m = 0; m < modules; m++) {
#pragma HLS PIPELINE OFF
setup_datamover(datamover_in_cmd, in_mem_location[m], RAW_MODULE_SIZE * sizeof(int16_t));
size_t offset_hbm_0 = 20 * hbm_size_bytes / 32 + m * RAW_MODULE_SIZE * sizeof(int16_t) / 64;
size_t offset_hbm_1 = 21 * hbm_size_bytes / 32 + m * RAW_MODULE_SIZE * sizeof(int16_t) / 64;
read_module(d_hbm_p0, d_hbm_p1, host_memory_in, offset_hbm_0, offset_hbm_1);
}
}
}
fpga/scripts/bd_pcie.tcl
+2
View File
@@ -404,6 +404,8 @@ proc create_root_design { parentCell } {
connect_bd_intf_net -intf_net jungfraujoch_0_m_axi_d_hbm_p17 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_17] [get_bd_intf_pins jungfraujoch_0/m_axi_d_hbm_p17]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axi_d_hbm_p18 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_18] [get_bd_intf_pins jungfraujoch_0/m_axi_d_hbm_p18]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axi_d_hbm_p19 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_19] [get_bd_intf_pins jungfraujoch_0/m_axi_d_hbm_p19]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axi_d_hbm_p20 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_20] [get_bd_intf_pins jungfraujoch_0/m_axi_d_hbm_p20]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axi_d_hbm_p21 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_21] [get_bd_intf_pins jungfraujoch_0/m_axi_d_hbm_p21]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axis_c2h_data [get_bd_intf_pins jungfraujoch_0/m_axis_c2h_data] [get_bd_intf_pins pcie_dma_0/s_axis_c2h_data]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axis_c2h_datamover_cmd [get_bd_intf_pins jungfraujoch_0/m_axis_c2h_datamover_cmd] [get_bd_intf_pins pcie_dma_0/s_axis_c2h_cmd]
connect_bd_intf_net -intf_net jungfraujoch_0_m_axis_h2c_datamover_cmd [get_bd_intf_pins jungfraujoch_0/m_axis_h2c_datamover_cmd] [get_bd_intf_pins pcie_dma_0/s_axis_h2c_cmd]
fpga/scripts/hbm_u55c.tcl
+1 -4
View File
@@ -217,7 +217,6 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } {
CONFIG.USER_MC9_LOOKAHEAD_SBRF {true} \
CONFIG.USER_MC9_REF_TEMP_COMP {false} \
CONFIG.USER_MC9_TRAFFIC_OPTION {Linear} \
CONFIG.USER_MC_ENABLE_10 {FALSE} \
CONFIG.USER_MC_ENABLE_11 {FALSE} \
CONFIG.USER_MC_ENABLE_12 {FALSE} \
CONFIG.USER_MC_ENABLE_13 {FALSE} \
@@ -233,8 +232,6 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } {
CONFIG.USER_PHY_ENABLE_13 {TRUE} \
CONFIG.USER_PHY_ENABLE_14 {TRUE} \
CONFIG.USER_PHY_ENABLE_15 {TRUE} \
CONFIG.USER_SAXI_20 {false} \
CONFIG.USER_SAXI_21 {false} \
CONFIG.USER_SAXI_22 {false} \
CONFIG.USER_SAXI_23 {false} \
CONFIG.USER_SAXI_24 {false} \
@@ -281,7 +278,7 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } {
connect_bd_net [get_bd_pins axi_resetn] [get_bd_pins hbm/APB_0_PRESET_N] [get_bd_pins hbm/APB_1_PRESET_N]
connect_bd_net [get_bd_pins refclk100] [get_bd_pins hbm/APB_0_PCLK] [get_bd_pins hbm/APB_1_PCLK] [get_bd_pins hbm/HBM_REF_CLK_0] [get_bd_pins hbm/HBM_REF_CLK_1]
for {set i 0} {$i < 20} {incr i} {
for {set i 0} {$i < 22} {incr i} {
create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:aximm_rtl:1.0 s_axi_hbm_$i
set cell [create_bd_cell -type ip -vlnv {xilinx.com:ip:axi_register_slice:*} axi_register_slice_$i ]
fpga/scripts/jfjoch.tcl
+8 -2
View File
@@ -79,6 +79,10 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p19
create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p20
create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p21
create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis_c2h_data
create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis_c2h_datamover_cmd
@@ -619,14 +623,16 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } {
connect_bd_intf_net -intf_net data_collection_fsm_0_data_out [get_bd_intf_pins axis_data_fifo_0/S_AXIS] [get_bd_intf_pins data_collection_fsm_0/data_out]
connect_bd_intf_net -intf_net eth_in_1 [get_bd_intf_pins eth_in] [get_bd_intf_pins stream_merge_0/input_1]
connect_bd_intf_net -intf_net frame_generator_0_data_out [get_bd_intf_pins axis_frame_generator_fifo_0/S_AXIS] [get_bd_intf_pins frame_generator_0/data_out]
connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p0 [get_bd_intf_pins m_axi_d_hbm_p18] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p0]
connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p19] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p1]
connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p0 [get_bd_intf_pins m_axi_d_hbm_p20] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p0]
connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p21] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p1]
connect_bd_intf_net -intf_net host_writer_0_datamover_out_cmd [get_bd_intf_pins axis_data_fifo_c2h_cmd/S_AXIS] [get_bd_intf_pins host_writer_0/datamover_out_cmd]
connect_bd_intf_net -intf_net host_writer_0_host_memory_out [get_bd_intf_pins axis_register_slice_host_mem/S_AXIS] [get_bd_intf_pins host_writer_0/host_memory_out]
connect_bd_intf_net -intf_net host_writer_0_m_axis_completion [get_bd_intf_pins axis_work_completion_fifo_0/S_AXIS] [get_bd_intf_pins host_writer_0/m_axis_completion]
connect_bd_intf_net -intf_net integration_0_data_out [get_bd_intf_pins axis_data_fifo_8/S_AXIS] [get_bd_intf_pins integration_0/data_out]
connect_bd_intf_net -intf_net integration_0_m_axi_d_hbm_p0 [get_bd_intf_pins m_axi_d_hbm_p16] [get_bd_intf_pins integration_0/m_axi_d_hbm_p0]
connect_bd_intf_net -intf_net integration_0_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p17] [get_bd_intf_pins integration_0/m_axi_d_hbm_p1]
connect_bd_intf_net -intf_net integration_0_m_axi_d_hbm_p2 [get_bd_intf_pins m_axi_d_hbm_p18] [get_bd_intf_pins integration_0/m_axi_d_hbm_p2]
connect_bd_intf_net -intf_net integration_0_m_axi_d_hbm_p3 [get_bd_intf_pins m_axi_d_hbm_p19] [get_bd_intf_pins integration_0/m_axi_d_hbm_p3]
connect_bd_intf_net -intf_net integration_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_5/S_AXIS] [get_bd_intf_pins integration_0/m_axis_completion]
connect_bd_intf_net -intf_net integration_0_result_out [get_bd_intf_pins axis_integration_result_fifo_0/S_AXIS] [get_bd_intf_pins integration_0/result_out]
connect_bd_intf_net -intf_net jf_conversion_0_data_out [get_bd_intf_pins axis_data_fifo_6/S_AXIS] [get_bd_intf_pins jf_conversion_0/data_out]
receiver/AcquisitionDevice.cpp
+5 -1
View File
@@ -181,7 +181,11 @@ int16_t *AcquisitionDevice::GetDeviceBuffer(size_t handle) {
void AcquisitionDevice::InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) {}
void AcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment &experiment,
const std::vector<uint16_t> &v) {}
const std::vector<uint16_t> &v) {}
void AcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment &experiment,
const std::vector<uint16_t> &v,
const std::vector<double> &weights) {}
void AcquisitionDevice::MapBuffersStandard(size_t c2h_buffer_count, size_t h2c_buffer_count, int16_t numa_node) {
try {
receiver/AcquisitionDevice.h
+2 -1
View File
@@ -105,7 +105,8 @@ public:
// Calibration
virtual void InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib);
virtual void InitializeIntegrationMap(const DiffractionExperiment &experiment, const std::vector<uint16_t> &v);
void InitializeIntegrationMap(const DiffractionExperiment& experiment);
virtual void InitializeIntegrationMap(const DiffractionExperiment &experiment, const std::vector<uint16_t> &v,
const std::vector<double> &weights);
const AcquisitionCounters& Counters() const;
virtual void SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) {};
receiver/FPGAAcquisitionDevice.cpp
+37 -1
View File
@@ -71,13 +71,49 @@ void FPGAAcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment
size_t modules = experiment.GetModulesNum(data_stream);
if (modules > buffer_device.size())
if (modules > 2 * buffer_device.size())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Not enough host/FPGA buffers to load all integration map values");
for (int m = 0; m < modules; m++)
memcpy(buffer_device[m], v.data() + (offset + m) * RAW_MODULE_SIZE, RAW_MODULE_SIZE * sizeof(uint16_t));
for (int m = 0; m < modules; m++) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
buffer_device[modules + m][i] = to_fixed(1.0, 15);
}
}
HW_LoadIntegrationMap(modules);
}
void FPGAAcquisitionDevice::InitializeIntegrationMap(const DiffractionExperiment &experiment,
const std::vector<uint16_t> &v,
const std::vector<double> &weights) {
auto offset = experiment.GetFirstModuleOfDataStream(data_stream);
if (v.size() != experiment.GetModulesNum() * RAW_MODULE_SIZE)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Mismatch regarding integration map array");
if (weights.size() != experiment.GetModulesNum() * RAW_MODULE_SIZE)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Mismatch regarding weights array");
size_t modules = experiment.GetModulesNum(data_stream);
if (modules > 2 * buffer_device.size())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Not enough host/FPGA buffers to load all integration map values");
for (int m = 0; m < modules; m++)
memcpy(buffer_device[m], v.data() + (offset + m) * RAW_MODULE_SIZE, RAW_MODULE_SIZE * sizeof(uint16_t));
for (int m = 0; m < modules; m++) {
for (int i = 0; i < RAW_MODULE_SIZE; i++) {
buffer_device[modules + m][i] = to_fixed(weights[m * RAW_MODULE_SIZE + i], 15);
}
}
HW_LoadIntegrationMap(modules);
}
receiver/FPGAAcquisitionDevice.h
+3
View File
@@ -46,6 +46,9 @@ public:
JFJochProtoBuf::FPGAStatus GetStatus() const override;
void InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) override;
void InitializeIntegrationMap(const DiffractionExperiment &experiment, const std::vector<uint16_t> &v) override;
void InitializeIntegrationMap(const DiffractionExperiment &experiment, const std::vector<uint16_t> &v,
const std::vector<double> &weights) override;
void SetInternalGeneratorFrame(const std::vector<uint16_t> &v);
void SetInternalGeneratorFrame();
std::vector<uint16_t> GetInternalGeneratorFrame() const override;
receiver/HLSSimulatedDevice.cpp
+1 -1
View File
@@ -354,7 +354,7 @@ void HLSSimulatedDevice::HLSMainThread() {
// 7. Integration of pixels
hls_cores.emplace_back([&] { integration(converted_6, converted_7, integration_result_0, compl4, compl5,
hbm.data(), hbm.data(), hbm_if_size);});
hbm.data(), hbm.data(), hbm.data(), hbm.data(), hbm_if_size);});
hls_cores.emplace_back([&] { axis_128_to_512(integration_result_0, integration_result_1);});
receiver/HLSSimulatedDevice.h
+1 -1
View File
@@ -25,7 +25,7 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice {
volatile bool idle;
constexpr static const size_t hbm_if_count = 20;
constexpr static const size_t hbm_if_count = 22;
constexpr static const size_t hbm_if_size = 32*1024*1024LU;
std::vector<ap_uint<256>> hbm;
tests/FPGAIntegrationTest.cpp
+5 -5
View File
@@ -1117,23 +1117,23 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_integration", "[FPGA][Full
REQUIRE(memcmp(imageBuf, frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0);
auto integration_result = test.GetDeviceOutput(0, 0)->integration_result;
CHECK(integration_result[0].sum == 32754LU * (RAW_MODULE_SIZE / 2 - 1));
CHECK(integration_result[0].sum == 32754LU * (RAW_MODULE_SIZE / 2 - 1) * (1LU<<24));
CHECK(integration_result[0].count == RAW_MODULE_SIZE / 2 - 1);
CHECK(integration_result[1].sum == 0);
CHECK(integration_result[1].count == 0);
CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].sum == 32754LU * (RAW_MODULE_SIZE / 2 - 2));
CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].sum == 32754LU * (RAW_MODULE_SIZE / 2 - 2)* (1LU<<24));
CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].count == RAW_MODULE_SIZE / 2 - 2);
integration_result = test.GetDeviceOutput(0, 1)->integration_result;
CHECK(integration_result[54].sum == 32754LU * (RAW_MODULE_SIZE - 2));
CHECK(integration_result[54].sum == 32754LU * (RAW_MODULE_SIZE - 2) * (1LU<<24));
CHECK(integration_result[54].count == RAW_MODULE_SIZE - 2);
integration_result = test.GetDeviceOutput(0, 2)->integration_result;
CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE - 2));
CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE - 2) * (1LU<<24));
CHECK(integration_result[54].count == RAW_MODULE_SIZE - 2);
integration_result = test.GetDeviceOutput(0, 3)->integration_result;
CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE - 2));
CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE - 2) * (1LU<<24));
CHECK(integration_result[54].count == RAW_MODULE_SIZE - 2);
}