From 1798de247b1abb5fcadde690c78fbe9364400131 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Sat, 9 Dec 2023 12:08:39 +0100 Subject: [PATCH] Extend FPGA functionality --- .gitlab-ci.yml | 17 +- README.md | 5 +- acquisition_device/AcquisitionCounters.cpp | 39 +- acquisition_device/AcquisitionCounters.h | 4 +- acquisition_device/AcquisitionDevice.cpp | 54 +- acquisition_device/AcquisitionDevice.h | 41 +- acquisition_device/AcquisitionDeviceGroup.cpp | 5 - acquisition_device/CMakeLists.txt | 1 - acquisition_device/Completion.cpp | 41 +- acquisition_device/Completion.h | 9 +- acquisition_device/FPGAAcquisitionDevice.cpp | 93 +- acquisition_device/FPGAAcquisitionDevice.h | 11 +- acquisition_device/HLSSimulatedDevice.cpp | 113 +- acquisition_device/HLSSimulatedDevice.h | 10 +- acquisition_device/MockAcquisitionDevice.cpp | 187 --- acquisition_device/MockAcquisitionDevice.h | 44 - acquisition_device/PCIExpressDevice.cpp | 13 +- acquisition_device/PCIExpressDevice.h | 4 +- broker/JFJochBrokerParser.cpp | 3 - common/CMakeLists.txt | 3 +- common/Definitions.h | 42 +- common/DeviceOutput.h | 49 + common/NUMAHWPolicy.cpp | 3 + fpga/CMakeLists.txt | 2 +- fpga/README.md | 67 +- fpga/hdl/action_config.v | 42 +- fpga/hdl/check_eth_busy.v | 47 - fpga/hls/CMakeLists.txt | 31 +- fpga/hls/add_multipixel.cpp | 294 ----- fpga/hls/add_multipixel_tb.cpp | 66 - fpga/hls/adu_histo.cpp | 7 +- fpga/hls/adu_histo_tb.cpp | 57 + fpga/hls/axis_helpers.cpp | 24 +- fpga/hls/datamover_model.h | 4 + fpga/hls/ethernet.cpp | 3 +- fpga/hls/frame_summation.cpp | 24 +- fpga/hls/frame_summation_tb.cpp | 22 +- fpga/hls/frame_summation_tb_2.cpp | 98 ++ fpga/hls/hls_bitshuffle.cpp | 346 ----- fpga/hls/hls_jfjoch.h | 52 +- fpga/hls/host_writer.cpp | 186 ++- fpga/hls/integration.cpp | 34 +- fpga/hls/integration_tb.cpp | 64 + fpga/hls/ipv4.cpp | 2 +- fpga/hls/jf_conversion.cpp | 8 +- fpga/hls/load_calibration.cpp | 17 +- fpga/hls/module_upside_down.cpp | 74 -- fpga/hls/module_upside_down_tb.cpp | 71 - fpga/hls/pedestal.cpp | 254 ++++ fpga/hls/save_to_hbm.cpp | 2 +- fpga/hls/spot_finder.cpp | 1 - fpga/hls/stream_merge.cpp | 69 +- fpga/host_library/JungfraujochDevice.cpp | 26 +- fpga/host_library/JungfraujochDevice.h | 4 + fpga/host_library/jfjoch_pcie_status.cpp | 30 +- fpga/pcie_driver/ActionConfig.h | 10 + fpga/pcie_driver/CMakeLists.txt | 3 +- fpga/pcie_driver/Makefile | 2 +- fpga/pcie_driver/README.md | 57 +- fpga/pcie_driver/dkms.conf | 10 + fpga/pcie_driver/install_dkms.sh | 12 + fpga/pcie_driver/jfjoch_drv.c | 37 +- fpga/pcie_driver/jfjoch_drv.h | 35 +- fpga/pcie_driver/jfjoch_function.c | 100 +- fpga/pcie_driver/jfjoch_int.c | 53 + fpga/pcie_driver/jfjoch_ioctl.c | 14 +- fpga/pcie_driver/jfjoch_ioctl.h | 4 +- fpga/pcie_driver/jfjoch_memory.c | 28 +- fpga/pcie_driver/jfjoch_miscdev.c | 23 +- fpga/pcie_driver/jfjoch_pcie_setup.c | 6 +- fpga/pcie_driver/jfjoch_sysfs.c | 7 +- fpga/scripts/bd_pcie.tcl | 104 +- fpga/scripts/build_pcie_design.tcl | 2 +- fpga/scripts/hbm_u55c.tcl | 11 +- fpga/scripts/image_processing.tcl | 180 ++- fpga/scripts/jfjoch.tcl | 193 +-- fpga/scripts/mac_100g_pcie.tcl | 36 +- fpga/scripts/mac_4x10g.tcl | 214 +++ fpga/scripts/pcie_dma.tcl | 1 + fpga/xdc/pcie_timing.xdc | 17 +- image_analysis/RadialIntegrationProfile.cpp | 8 +- image_analysis/RadialIntegrationProfile.h | 4 +- jungfrau/JFModulePedestal.cpp | 57 +- jungfrau/JFModulePedestal.h | 23 +- receiver/JFJochReceiver.cpp | 138 +- receiver/JFJochReceiver.h | 4 +- receiver/jfjoch_action_test.cpp | 117 +- tests/AcquisitionCountersTest.cpp | 20 +- tests/CMakeLists.txt | 6 +- tests/FPGAAddMultipixelTest.cpp | 51 - tests/FPGAHLSBitshuffleTest.cpp | 51 - tests/FPGAIntegrationTest.cpp | 137 +- tests/JFJochFullIntegrationTest.cpp | 1155 ----------------- tests/MockAcquisitionDeviceTest.cpp | 102 -- tests/RadialIntegrationTest.cpp | 24 +- tools/CMakeLists.txt | 6 +- tools/JFCalibrationPerfTest.cpp | 93 -- 97 files changed, 2302 insertions(+), 3672 deletions(-) delete mode 100644 acquisition_device/MockAcquisitionDevice.cpp delete mode 100644 acquisition_device/MockAcquisitionDevice.h create mode 100644 common/DeviceOutput.h delete mode 100644 fpga/hdl/check_eth_busy.v delete mode 100644 fpga/hls/add_multipixel.cpp delete mode 100644 fpga/hls/add_multipixel_tb.cpp create mode 100644 fpga/hls/adu_histo_tb.cpp create mode 100644 fpga/hls/frame_summation_tb_2.cpp delete mode 100644 fpga/hls/hls_bitshuffle.cpp create mode 100644 fpga/hls/integration_tb.cpp delete mode 100644 fpga/hls/module_upside_down.cpp delete mode 100644 fpga/hls/module_upside_down_tb.cpp create mode 100644 fpga/hls/pedestal.cpp create mode 100644 fpga/pcie_driver/dkms.conf create mode 100755 fpga/pcie_driver/install_dkms.sh create mode 100644 fpga/pcie_driver/jfjoch_int.c create mode 100644 fpga/scripts/mac_4x10g.tcl delete mode 100644 tests/FPGAAddMultipixelTest.cpp delete mode 100644 tests/FPGAHLSBitshuffleTest.cpp delete mode 100644 tests/JFJochFullIntegrationTest.cpp delete mode 100644 tests/MockAcquisitionDeviceTest.cpp delete mode 100644 tools/JFCalibrationPerfTest.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f45fce27..e0a2c738 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,20 @@ build:x86:aocc: - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON .. - make -j48 jfjoch +build:x86:driver: + stage: build + variables: + GIT_SUBMODULE_STRATEGY: recursive + CC: gcc + CXX: g++ + tags: + - gcc + - x86 + needs: [] + script: + - cd fpga/pcie_driver + - make + build:x86:vitis_hls: stage: build variables: @@ -96,13 +110,12 @@ test:x86:gcc: - mkdir -p build - cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - - make -j48 CatchTest CompressionBenchmark HDF5DatasetWriteTest JFCalibrationPerfTest + - make -j48 CatchTest CompressionBenchmark HDF5DatasetWriteTest - cd tests - ./CatchTest -r junit -o report.xml - cd ../tools - ./HDF5DatasetWriteTest ../../tests/test_data/compression_benchmark.h5 - numactl -m 0 -N 0 ./CompressionBenchmark ../../tests/test_data/compression_benchmark.h5 - - numactl -m 0 -N 0 ./JFCalibrationPerfTest artifacts: expire_in: 1 week reports: diff --git a/README.md b/README.md index eb314f4a..b066a539 100644 --- a/README.md +++ b/README.md @@ -109,14 +109,11 @@ make jfjoch ``` ## Tests - -To enable compiling test routines use parameter `-DJFJOCH_COMPILE_TESTS=ON` for `cmake`. + Automated test routine is then accessible as `tests/CatchTest`. There are also benchmark routines: * `CompressionBenchmark` to measure compression bandwidth (single threaded) * `HDF5DatasetWriteTest` to measure HDF5 dataset writing speed (single threaded) -* `DataAnalysisPerfTest` to measure data analysis performance (single threaded) -* `JFCalibrationPerfTest` to measure pedestal calculation and online conversion performance In addition, tests are executed to verify that datasets written by Jungfraujoch are readable with XDS Durin plugin and CrystFEL. Input files for these programs are placed in `xds_durin` and `crystfel` folders. See `.gitlab-ci.yml` for details. diff --git a/acquisition_device/AcquisitionCounters.cpp b/acquisition_device/AcquisitionCounters.cpp index 9191022d..47f7cbaa 100644 --- a/acquisition_device/AcquisitionCounters.cpp +++ b/acquisition_device/AcquisitionCounters.cpp @@ -29,7 +29,8 @@ void AcquisitionCounters::Reset(const DiffractionExperiment &experiment, uint16_ for (int i = 0; i < max_modules; i++) curr_frame_number[i] = 0; - handle_for_frame = std::vector((expected_frames+1) * nmodules, HandleNotFound); + handle_for_frame = std::vector(expected_frames * nmodules, HandleNotFound); + handle_for_pedestal = std::vector(nmodules * 16, HandleNotFound); // 16 = storage cells packets_collected = std::vector(expected_frames * nmodules); saved_completions = std::vector(expected_frames * nmodules); packets_per_module = std::vector(nmodules); @@ -40,6 +41,17 @@ void AcquisitionCounters::Reset(const DiffractionExperiment &experiment, uint16_ void AcquisitionCounters::UpdateCounters(const Completion *c) { std::unique_lock ul(m); + if (c->pedestal) { + if (c->module_number >= nmodules) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "UpdateCounters wrong module number: " + std::to_string(c->module_number) + " for pedestal SC" + std::to_string(c->frame_number)); + if (c->frame_number >= 16) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "UpdateCounters pedestal frame number is out of bounds for storage cells"); + handle_for_pedestal.at(c->frame_number * nmodules + c->module_number) = c->handle; + return; + } + if (c->module_number >= nmodules) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "UpdateCounters wrong module number: " + std::to_string(c->module_number) + " for frame " + std::to_string(c->frame_number)); @@ -103,6 +115,18 @@ uint64_t AcquisitionCounters::GetBufferHandleAndClear(size_t frame, uint16_t mod return ret_val; } +uint64_t AcquisitionCounters::GetPedestalBufferHandle(size_t storage_cell, uint16_t module_number) const { + std::unique_lock ul(m); + if (storage_cell >= 16) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, + "GetBufferHandleAndClear Wrong frame number: " + std::to_string(storage_cell)); + if (module_number >= nmodules) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, + "GetBufferHandleAndClear Wrong module number: " + std::to_string(module_number) + + " for SC " + std::to_string(storage_cell)); + return handle_for_pedestal.at(storage_cell * nmodules + module_number); +} + uint64_t AcquisitionCounters::GetCurrFrameNumber(uint16_t module_number) const { if (module_number >= max_modules) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, @@ -167,19 +191,6 @@ bool AcquisitionCounters::IsAnyPacketCollected(size_t frame, uint16_t module_num return packets_collected[frame * nmodules + module_number] > 0; } -Completion AcquisitionCounters::GetCompletion(size_t frame, uint16_t module_number) const { - if (frame >= expected_frames) - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, - "GetBunchID Wrong frame number: " + std::to_string(frame)); - - if (module_number >= nmodules) - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, - "GetBunchID Wrong module number: " + std::to_string(module_number) - + " for frame " + std::to_string(frame)); - - return saved_completions[frame * nmodules + module_number]; -} - uint64_t AcquisitionCounters::GetTotalPackets() const { return total_packets; } diff --git a/acquisition_device/AcquisitionCounters.h b/acquisition_device/AcquisitionCounters.h index 5f0c7637..67b4c84d 100644 --- a/acquisition_device/AcquisitionCounters.h +++ b/acquisition_device/AcquisitionCounters.h @@ -22,6 +22,7 @@ class AcquisitionCounters { mutable std::condition_variable_any data_updated; std::vector handle_for_frame; + std::vector handle_for_pedestal; std::vector packets_collected; std::vector saved_completions; @@ -53,12 +54,11 @@ public: void WaitForFrame(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; int64_t CalculateDelay(size_t curr_frame, uint16_t module_number = UINT16_MAX) const; // mutex acquired indirectly uint64_t GetBufferHandle(size_t frame, uint16_t module_number) const; + uint64_t GetPedestalBufferHandle(size_t storage_cell, uint16_t module_number) const; bool IsFullModuleCollected(size_t frame, uint16_t module_number) const; bool IsAnyPacketCollected(size_t frame, uint16_t module_number) const; bool IsAcquisitionFinished() const; - Completion GetCompletion(size_t curr_frame, uint16_t module_number) const; - uint64_t GetTotalPackets() const; uint64_t GetTotalPackets(uint16_t module_number) const; diff --git a/acquisition_device/AcquisitionDevice.cpp b/acquisition_device/AcquisitionDevice.cpp index 186ef82d..1130b92e 100644 --- a/acquisition_device/AcquisitionDevice.cpp +++ b/acquisition_device/AcquisitionDevice.cpp @@ -82,30 +82,39 @@ void AcquisitionDevice::StartAction(const DiffractionExperiment &experiment, uin start_time = std::chrono::system_clock::now(); } -void AcquisitionDevice::WaitForActionComplete() { +void AcquisitionDevice::WaitForActionComplete(bool pedestal_mode) { auto c = work_completion_queue.GetBlocking(); while (c.type != Completion::Type::End) { work_completion_count++; - while (2 * work_completion_count > GetCompletedDescriptors()) // Two descriptors per module + while (work_completion_count > GetCompletedDescriptors() / GetExpectedDescriptorsPerModule() ) std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (c.frame_number >= expected_frames) { + auto output = GetDeviceOutput(c.handle); + c.module_number = output->module_statistics.module_number; + c.packet_count = output->module_statistics.packet_count; + c.frame_number = output->module_statistics.frame_number; + c.pedestal = output->module_statistics.pedestal; + + if ((c.frame_number >= expected_frames) && !c.pedestal) { Cancel(); // this frame is not of any interest, therefore its location can be immediately released SendWorkRequest(c.handle); } else if (c.module_number >= max_modules) { // Module number out of bounds, don't process if (logger != nullptr) - logger->Error("Completion with wrong module number data stream {} completion frame number {} module {} handle {} timestamp {} status {}", - data_stream, c.frame_number, c.module_number, c.handle, c.timestamp, c.status); + logger->Error("Completion with wrong module number data stream {} completion frame number {} module {} handle {}", + data_stream, c.frame_number, c.module_number, c.handle); + SendWorkRequest(c.handle); + } else if (pedestal_mode && !c.pedestal) { + counters.UpdateCounters(&c); SendWorkRequest(c.handle); } else counters.UpdateCounters(&c); if (logger != nullptr) - logger->Debug("Data stream {} completion frame number {} module {} handle {} timestamp {} status {}", - data_stream, c.frame_number, c.module_number, c.handle, c.timestamp, c.status); + logger->Debug("Data stream {} completion frame number {} module {} handle {}", + data_stream, c.frame_number, c.module_number, c.handle); c = work_completion_queue.GetBlocking(); } @@ -130,7 +139,15 @@ uint64_t AcquisitionDevice::GetBytesReceived() const { const DeviceOutput *AcquisitionDevice::GetDeviceOutput(size_t frame_number, uint16_t module_number) const { auto handle = counters.GetBufferHandle(frame_number, module_number); if (handle != HandleNotValid) - return (DeviceOutput *) buffer_device.at(handle); + return GetDeviceOutput(handle); + else + return (DeviceOutput *) GetErrorFrameBuffer(); +} + +const DeviceOutput *AcquisitionDevice::GetDeviceOutputPedestal(size_t storage_cell, uint16_t module_number) const { + auto handle = counters.GetPedestalBufferHandle(storage_cell, module_number); + if (handle != HandleNotValid) + return GetDeviceOutput(handle); else return (DeviceOutput *) GetErrorFrameBuffer(); } @@ -139,11 +156,18 @@ const int16_t *AcquisitionDevice::GetErrorFrameBuffer() const { return buffer_err.data(); } -int16_t *AcquisitionDevice::GetDeviceBuffer(size_t handle) { +const DeviceOutput *AcquisitionDevice::GetDeviceOutput(size_t handle) const { if (handle >= buffer_device.size()) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Handle outside of range"); else - return (int16_t *) buffer_device.at(handle); + return (DeviceOutput *) buffer_device.at(handle); +} + +DeviceOutput *AcquisitionDevice::GetDeviceOutput(size_t handle) { + if (handle >= buffer_device.size()) + throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Handle outside of range"); + else + return (DeviceOutput *) buffer_device.at(handle); } void AcquisitionDevice::InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) {} @@ -241,4 +265,12 @@ AcquisitionDeviceNetConfig AcquisitionDevice::GetNetConfig() const { .ipv4_addr = GetIPv4Address(), .udp_port = GetUDPPort() }; -} \ No newline at end of file +} + +void AcquisitionDevice::SetDefaultDataSource(AcquisitionDeviceSource id) { + +} + +AcquisitionDeviceSource AcquisitionDevice::GetDataSource() { + return AcquisitionDeviceSource::NONE; +} diff --git a/acquisition_device/AcquisitionDevice.h b/acquisition_device/AcquisitionDevice.h index 98305944..5f533003 100644 --- a/acquisition_device/AcquisitionDevice.h +++ b/acquisition_device/AcquisitionDevice.h @@ -18,31 +18,7 @@ #include "AcquisitionCounters.h" #include "Completion.h" - -#pragma pack(push) -#pragma pack(1) - -struct IntegrationResult { - int64_t count; - int64_t sum; -}; - -struct SpotFindingResult { - char strong_pixel[RAW_MODULE_SIZE / 8]; - int32_t count_threshold; - int32_t snr_threshold; - int32_t strong_pixel_count; - int32_t reserved[13]; -}; - -struct DeviceOutput { - int16_t pixels[512 * 64 * 32]; - SpotFindingResult spot_finding_result; - IntegrationResult integration_result[FPGA_INTEGRATION_BIN_COUNT]; - uint32_t adu_histogram[ADU_HISTO_BIN_COUNT]; -}; - -#pragma pack(pop) +#include "../common/DeviceOutput.h" struct AcquisitionDeviceStatistics { uint64_t good_packets; @@ -55,6 +31,8 @@ struct AcquisitionDeviceStatistics { std::vector packets_received_per_module; }; +enum class AcquisitionDeviceSource {MAC_100G, MAC_4x10G, FRAME_GENERATOR, NONE}; + class AcquisitionDevice { std::vector buffer_err; @@ -81,11 +59,12 @@ protected: uint64_t mac_addr; uint32_t ipv4_addr; std::atomic work_completion_count; - explicit AcquisitionDevice(uint16_t data_stream); void UnmapBuffers(); void MapBuffersStandard(size_t c2h_buffer_count, size_t h2c_buffer_count, int16_t numa_node); + const DeviceOutput *GetDeviceOutput(size_t handle) const; + DeviceOutput *GetDeviceOutput(size_t handle); public: AcquisitionDevice(const AcquisitionDevice &other) = delete; AcquisitionDevice &operator=(const AcquisitionDevice &other) = delete; @@ -97,7 +76,7 @@ public: void StartAction(const DiffractionExperiment &experiment, uint32_t optional_flags = 0); void PrepareAction(const DiffractionExperiment &experiment); - void WaitForActionComplete(); + void WaitForActionComplete(bool pedestal_mode = false); virtual void Cancel() = 0; void EnableLogging(Logger *logger); @@ -109,12 +88,11 @@ public: virtual DeviceStatus GetDeviceStatus() const; AcquisitionDeviceStatistics GetStatistics() const; const DeviceOutput *GetDeviceOutput(size_t frame_number, uint16_t module_number) const; + const DeviceOutput *GetDeviceOutputPedestal(size_t frame_number, uint16_t module_number) const; void FrameBufferRelease(size_t frame_number, uint16_t module_number); const int16_t *GetErrorFrameBuffer() const; - int16_t *GetDeviceBuffer(size_t handle); - // Calibration virtual void InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib); virtual void InitializeIntegrationMap(const DiffractionExperiment &experiment, const std::vector &v); @@ -133,7 +111,12 @@ public: virtual std::vector GetInternalGeneratorFrame() const { return {}; } + virtual void SetDefaultDataSource(AcquisitionDeviceSource id); + virtual AcquisitionDeviceSource GetDataSource(); + AcquisitionDeviceNetConfig GetNetConfig() const; + + virtual uint32_t GetExpectedDescriptorsPerModule() const = 0; }; diff --git a/acquisition_device/AcquisitionDeviceGroup.cpp b/acquisition_device/AcquisitionDeviceGroup.cpp index c9b1f056..5a0178b4 100644 --- a/acquisition_device/AcquisitionDeviceGroup.cpp +++ b/acquisition_device/AcquisitionDeviceGroup.cpp @@ -2,7 +2,6 @@ #include "AcquisitionDeviceGroup.h" #include "PCIExpressDevice.h" -#include "MockAcquisitionDevice.h" #include "HLSSimulatedDevice.h" AcquisitionDevice &AcquisitionDeviceGroup::operator[](int idx) { @@ -29,10 +28,6 @@ void AcquisitionDeviceGroup::AddHLSDevice(int64_t buffer_size_modules) { aq_devices.emplace_back(std::make_unique(aq_devices.size(), buffer_size_modules)); } -void AcquisitionDeviceGroup::AddMockDevice(int64_t buffer_size_modules) { - aq_devices.emplace_back(std::make_unique(aq_devices.size(), buffer_size_modules)); -} - std::vector AcquisitionDeviceGroup::GetNetworkConfig() { std::vector ret; for (const auto &i: aq_devices) diff --git a/acquisition_device/CMakeLists.txt b/acquisition_device/CMakeLists.txt index 43b66255..f0d51fba 100644 --- a/acquisition_device/CMakeLists.txt +++ b/acquisition_device/CMakeLists.txt @@ -1,7 +1,6 @@ ADD_LIBRARY(JungfraujochAcqusitionDevice STATIC AcquisitionDevice.cpp AcquisitionDevice.h AcquisitionCounters.cpp AcquisitionCounters.h - MockAcquisitionDevice.cpp MockAcquisitionDevice.h HLSSimulatedDevice.cpp HLSSimulatedDevice.h Completion.cpp Completion.h ../fpga/pcie_driver/ActionConfig.h PCIExpressDevice.cpp PCIExpressDevice.h diff --git a/acquisition_device/Completion.cpp b/acquisition_device/Completion.cpp index 97d9c9b4..416ce36f 100644 --- a/acquisition_device/Completion.cpp +++ b/acquisition_device/Completion.cpp @@ -1,49 +1,20 @@ // Copyright (2019-2023) Paul Scherrer Institute -#include - #include "Completion.h" -#include "../common/JFJochException.h" #include "../common/Definitions.h" -inline uint64_t bit_concat(uint32_t high, uint32_t low) { - return (uint64_t(high) << 32) | low; -} - -Completion parse_hw_completion(uint32_t tmp[16]) { +Completion parse_hw_completion(uint32_t tmp) { Completion c{}; - c.handle = tmp[0]; - c.module_number = tmp[1] & 0xFF; - c.packet_count = (tmp[1] & (0xFFFF0000)) >> 16; - c.status = (tmp[1] >> 8) & 0xFF; - c.data_collection_id = tmp[11] & UINT16_MAX; + c.handle = tmp & UINT16_MAX; + c.data_collection_id = (tmp>>16) & UINT16_MAX; - uint64_t detector_frame_number = bit_concat(tmp[2], tmp[3]); - - uint32_t parity = (std::bitset<32>(tmp[0]).count() + std::bitset<32>(tmp[1]).count() - + std::bitset<32>(tmp[2]).count() + std::bitset<32>(tmp[3]).count()) % 2; - - if (parity == 1) - throw JFJochException(JFJochExceptionCategory::HardwareParityError, "Wrong parity in work completion"); - - if (c.handle == HANDLE_START) { + if (c.handle == HANDLE_START) c.type = Completion::Type::Start; - } else if (c.handle == HANDLE_END) { + else if (c.handle == HANDLE_END) c.type = Completion::Type::End; - c.frame_number = detector_frame_number; - } else { + else c.type = Completion::Type::Image; - c.frame_number = detector_frame_number; - c.timestamp = bit_concat(tmp[4], tmp[5]); - c.bunchid = bit_concat(tmp[6], tmp[7]); - - c.exptime = tmp[8]; - c.debug = tmp[9]; - - c.packet_mask[0] = bit_concat(tmp[14], tmp[15]); - c.packet_mask[1] = bit_concat(tmp[12], tmp[13]); - } return c; } \ No newline at end of file diff --git a/acquisition_device/Completion.h b/acquisition_device/Completion.h index 66ce8170..ddcc894e 100644 --- a/acquisition_device/Completion.h +++ b/acquisition_device/Completion.h @@ -14,18 +14,13 @@ struct Completion { enum class Type {Start, End, Image} type; uint64_t frame_number; - uint64_t packet_mask[2]; - uint64_t bunchid; - uint64_t timestamp; - uint32_t exptime; - uint32_t debug; uint32_t handle; uint16_t packet_count; uint16_t data_collection_id; - uint16_t status; uint16_t module_number; + bool pedestal; }; -Completion parse_hw_completion(uint32_t hw_input[16]); +Completion parse_hw_completion(uint32_t hw_input); #endif //JUNGFRAUJOCH_COMPLETION_H diff --git a/acquisition_device/FPGAAcquisitionDevice.cpp b/acquisition_device/FPGAAcquisitionDevice.cpp index 096883c5..2317214b 100644 --- a/acquisition_device/FPGAAcquisitionDevice.cpp +++ b/acquisition_device/FPGAAcquisitionDevice.cpp @@ -20,15 +20,17 @@ void FPGAAcquisitionDevice::Finalize() { while (!HW_IsIdle()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + SetDataSource(default_data_source); } void FPGAAcquisitionDevice::ReadWorkCompletionThread() { - uint32_t values[16]; + uint32_t values; Completion c{}; bool quit_loop = false; do { - while (!HW_ReadMailbox(values)) + while (!HW_ReadMailbox(&values)) std::this_thread::sleep_for(std::chrono::microseconds(10)); c = parse_hw_completion(values); @@ -162,11 +164,6 @@ void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &e throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Not enough host/FPGA buffers to load all calibration constants"); - if (modules * (3 + 3 * storage_cells) > LOAD_CALIBRATION_BRAM_SIZE) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "FPGA is not compatible with that many calibration constants"); - - for (int m = 0; m < modules; m++) { calib.GainCalibration(m).ExportG0(buffer_device[m]); calib.GainCalibration(m).ExportG1(buffer_device[m + modules]); @@ -196,6 +193,16 @@ void FPGAAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &e HW_LoadCalibration(modules, storage_cells); } +union float_uint32 { + float f; + uint32_t u; +}; + +inline uint32_t float2uint(float f) { + float_uint32 fu; + fu.f = f; + return fu.u; +} void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, DataCollectionConfig &job) { std::random_device rd; @@ -204,13 +211,30 @@ void FPGAAcquisitionDevice::FillActionRegister(const DiffractionExperiment& x, D job.nmodules = x.GetModulesNum(data_stream) - 1; job.nframes = x.GetFrameNum(); - job.one_over_energy = std::lround((1<<20)/ x.GetPhotonEnergy_keV()); + job.one_over_energy = float2uint(1 / x.GetPhotonEnergy_keV()); job.nstorage_cells = x.GetStorageCellNumber() - 1; job.mode = data_collection_id << 16; job.nsummation = x.GetSummation() - 1; - if (x.GetDetectorMode() == DetectorMode::Conversion) - job.mode |= MODE_CONV; + expected_descriptors_per_module = 5; + + switch (x.GetDetectorMode()) { + case DetectorMode::Conversion: + job.mode |= MODE_CONV; + break; + case DetectorMode::Raw: + break; + case DetectorMode::PedestalG0: + job.mode |= MODE_PEDESTAL_G0; + break; + case DetectorMode::PedestalG1: + job.mode |= MODE_PEDESTAL_G1; + break; + case DetectorMode::PedestalG2: + job.mode |= MODE_PEDESTAL_G2; + break; + } + if (!x.IsPixelSigned()) job.mode |= MODE_UNSIGNED; if (x.GetPixelDepth() == 4) @@ -221,6 +245,12 @@ void FPGAAcquisitionDevice::Start(const DiffractionExperiment &experiment, uint3 if (!HW_IsIdle()) throw(JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, "Hardware action running prior to start of data acquisition")); + + if (experiment.IsUsingInternalPacketGen()) + SetDataSource(AcquisitionDeviceSource::FRAME_GENERATOR); + else + SetDataSource(default_data_source); + DataCollectionConfig cfg_in{}, cfg_out{}; FillActionRegister(experiment, cfg_in); @@ -256,8 +286,8 @@ void FPGAAcquisitionDevice::SetInternalGeneratorFrame() { } FPGAAcquisitionDevice::FPGAAcquisitionDevice(uint16_t data_stream) -: AcquisitionDevice(data_stream), -internal_pkt_gen_frame(RAW_MODULE_SIZE * MAX_MODULES_FPGA) { + : AcquisitionDevice(data_stream), + internal_pkt_gen_frame(RAW_MODULE_SIZE * MAX_MODULES_FPGA) { } void FPGAAcquisitionDevice::SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) { @@ -273,3 +303,42 @@ void FPGAAcquisitionDevice::SetSpotFinderParameters(int16_t count_threshold, dou SpotFinderParameters params{.count_threshold = count_threshold, .snr_threshold = to_fixed(snr_threshold, 2)}; HW_SetSpotFinderParameters(params); } + +AcquisitionDeviceSource FPGAAcquisitionDevice::GetDataSource() { + switch (HW_GetDataSource()) { + case STREAM_MERGE_SRC_100G: + return AcquisitionDeviceSource::MAC_100G; + case STREAM_MERGE_SRC_4x10G: + return AcquisitionDeviceSource::MAC_4x10G; + case STREAM_MERGE_SRC_FRAME_GEN: + return AcquisitionDeviceSource::FRAME_GENERATOR; + default: + return AcquisitionDeviceSource::NONE; + } +} + +void FPGAAcquisitionDevice::SetDefaultDataSource(AcquisitionDeviceSource id) { + default_data_source = id; + SetDataSource(id); +} + +void FPGAAcquisitionDevice::SetDataSource(AcquisitionDeviceSource id) { + switch (id) { + case AcquisitionDeviceSource::MAC_100G: + HW_SetDataSource(STREAM_MERGE_SRC_100G); + break; + case AcquisitionDeviceSource::MAC_4x10G: + HW_SetDataSource(STREAM_MERGE_SRC_4x10G); + break; + case AcquisitionDeviceSource::FRAME_GENERATOR: + HW_SetDataSource(STREAM_MERGE_SRC_FRAME_GEN); + break; + case AcquisitionDeviceSource::NONE: + HW_SetDataSource(STREAM_MERGE_SRC_NONE); + break; + } +} + +uint32_t FPGAAcquisitionDevice::GetExpectedDescriptorsPerModule() const { + return expected_descriptors_per_module; +} diff --git a/acquisition_device/FPGAAcquisitionDevice.h b/acquisition_device/FPGAAcquisitionDevice.h index 34cdc4c3..6f0e7946 100644 --- a/acquisition_device/FPGAAcquisitionDevice.h +++ b/acquisition_device/FPGAAcquisitionDevice.h @@ -9,6 +9,9 @@ class FPGAAcquisitionDevice : public AcquisitionDevice { uint16_t data_collection_id = 0; + uint32_t expected_descriptors_per_module = 1; + AcquisitionDeviceSource default_data_source = AcquisitionDeviceSource::MAC_100G; + virtual void FPGA_StartAction(const DiffractionExperiment &experiment) = 0; virtual void FPGA_EndAction() = 0; @@ -29,11 +32,14 @@ class FPGAAcquisitionDevice : public AcquisitionDevice { virtual void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) = 0; virtual void HW_LoadIntegrationMap(uint32_t modules) = 0; virtual void HW_LoadInternalGeneratorFrame(uint32_t modules) = 0; - virtual bool HW_ReadMailbox(uint32_t values[16]) = 0; + virtual bool HW_ReadMailbox(uint32_t *values) = 0; virtual bool HW_SendWorkRequest(uint32_t handle) = 0; virtual void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) = 0; + virtual void HW_SetDataSource(uint32_t val) = 0; + virtual uint32_t HW_GetDataSource() = 0; void StartSendingWorkRequests() override; void Start(const DiffractionExperiment &experiment, uint32_t flag) override; + void SetDataSource(AcquisitionDeviceSource id); // doesn't change default data protected: std::vector internal_pkt_gen_frame; explicit FPGAAcquisitionDevice(uint16_t data_stream); @@ -52,6 +58,9 @@ public: void SetInternalGeneratorFrame(); std::vector GetInternalGeneratorFrame() const override; void SetSpotFinderParameters(int16_t count_threshold, double snr_threshold) override; + AcquisitionDeviceSource GetDataSource() override; + void SetDefaultDataSource(AcquisitionDeviceSource id) override; + uint32_t GetExpectedDescriptorsPerModule() const override; }; #endif //JUNGFRAUJOCH_FPGAACQUISITIONDEVICE_H diff --git a/acquisition_device/HLSSimulatedDevice.cpp b/acquisition_device/HLSSimulatedDevice.cpp index 47005732..348078c3 100644 --- a/acquisition_device/HLSSimulatedDevice.cpp +++ b/acquisition_device/HLSSimulatedDevice.cpp @@ -35,7 +35,8 @@ HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buf datamover_out_hbm_1(Direction::Output, (char *) hbm.data()), datamover_in_hbm_0(Direction::Input, (char *) hbm.data()), datamover_in_hbm_1(Direction::Input, (char *) hbm.data()), - idle(true), hbm(hbm_if_size / 32 * hbm_if_count) { + idle(true), hbm(hbm_if_size / 32 * hbm_if_count), + dma_address_table(65536) { mac_addr = 0xCCAA11223344; ipv4_addr = 0x0132010A; @@ -43,6 +44,13 @@ HLSSimulatedDevice::HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buf MapBuffersStandard(in_frame_buffer_size_modules, (3 + 3 * 16) * max_modules + 2, numa_node); + + auto in_mem_location32 = (uint32_t *) dma_address_table.data(); + + for (int i = 0; i < buffer_device.size(); i++) { + in_mem_location32[2 * i ] = ((uint64_t) buffer_device[i]) & UINT32_MAX; + in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; + } } void HLSSimulatedDevice::CreateFinalPacket(const DiffractionExperiment& experiment) { @@ -137,7 +145,9 @@ void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experimen run_data_collection = 1; cancel_data_collection = 0; idle = false; + datamover_out.ClearCompletedDescriptors(); if (experiment.IsUsingInternalPacketGen()) { + data_source = STREAM_MERGE_SRC_FRAME_GEN; auto ret = frame_generator(din_frame_generator, hbm.data(), hbm.data(), @@ -154,6 +164,8 @@ void HLSSimulatedDevice::FPGA_StartAction(const DiffractionExperiment &experimen if (ret) throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, "Error running internal packet generator"); + } else { + data_source = STREAM_MERGE_SRC_100G; } action_thread = std::thread(&HLSSimulatedDevice::HLSMainThread, this ); } @@ -168,14 +180,11 @@ HLSSimulatedDevice::~HLSSimulatedDevice() { action_thread.join(); } -bool HLSSimulatedDevice::HW_ReadMailbox(uint32_t values[16]) { - if (completion_stream.size() < 16) - return false; - - for (int i = 0; i < 16; i++) - values[i] = completion_stream.read(); - - return true; +bool HLSSimulatedDevice::HW_ReadMailbox(uint32_t *values) { + ap_uint<32> tmp; + bool ret = completion_stream.read_nb(tmp); + values[0] = tmp; + return ret; } void HLSSimulatedDevice::Cancel() { @@ -188,13 +197,7 @@ bool HLSSimulatedDevice::HW_IsIdle() const { bool HLSSimulatedDevice::HW_SendWorkRequest(uint32_t handle) { - uint64_t address = (handle == UINT32_MAX) ? 0 : (uint64_t) buffer_device.at(handle); - uint32_t parity = (std::bitset<32>(handle).count() + std::bitset<64>(address).count()) % 2; - work_request_stream.write(handle); - work_request_stream.write(address >> 32); - work_request_stream.write(address & UINT32_MAX); - work_request_stream.write(parity); return true; } @@ -231,8 +234,10 @@ void HLSSimulatedDevice::HLSMainThread() { STREAM_512 converted_1; STREAM_512 converted_2; STREAM_512 converted_3; + STREAM_512 converted_3a; STREAM_512 converted_4; STREAM_512 converted_5; + STREAM_512 converted_5a; STREAM_576 converted_6; STREAM_512 converted_7; STREAM_512 converted_8; @@ -242,12 +247,12 @@ void HLSSimulatedDevice::HLSMainThread() { hls::stream addr2; hls::stream addr3; - hls::stream compl0, compl1, compl2, compl3, compl4, compl5, compl6, compl7; + hls::stream compl0, compl1, compl2, compl2a, compl3, compl4, compl5, compl6, compl7; hls::stream> hbm_handles; hls::stream> adu_histo_result; - hls::stream> integration_result_0; + hls::stream> integration_result_0; hls::stream> integration_result_1; hls::stream> spot_finder_result_0; @@ -256,10 +261,16 @@ void HLSSimulatedDevice::HLSMainThread() { hls::stream > udp_metadata; ap_uint<1> idle_data_collection; - ap_uint<8> err_reg; - - while ((!din_eth.empty()) || (!din_frame_generator.empty())) - stream_merge(din_eth, din_frame_generator, network0); + if (data_source == STREAM_MERGE_SRC_FRAME_GEN) { + while (!din_frame_generator.empty()) + stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); + } else if (data_source == STREAM_MERGE_SRC_100G) { + while (!din_eth.empty()) + stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); + } else if (data_source == STREAM_MERGE_SRC_4x10G) { + while (!din_eth_4x10G.empty()) + stream_merge(din_eth, din_eth_4x10G, din_frame_generator, network0, data_source); + } while(!network0.empty()) ethernet(network0, ip1, arp1, mac_addr, eth_packets, clear_counters); @@ -328,8 +339,24 @@ void HLSSimulatedDevice::HLSMainThread() { datamover_in_hbm_0.GetCtrlStream(), datamover_in_hbm_1.GetCtrlStream(), hbm_if_size);}); + hls_cores.emplace_back([&] { pedestal(converted_3, converted_3a, compl2, compl2a, + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm.data(), + hbm_if_size);}); + + // 3. Calculate histogram of ADU values - hls_cores.emplace_back([&] { adu_histo(converted_3, converted_4, adu_histo_result, compl2, compl3);}); + hls_cores.emplace_back([&] { adu_histo(converted_3a, converted_4, adu_histo_result, compl2a, compl3);}); // 4. Mask missing pixels hls_cores.emplace_back([&] { mask_missing(converted_4, converted_5, compl3, compl4);}); @@ -357,7 +384,7 @@ void HLSSimulatedDevice::HLSMainThread() { // 7. Integration of pixels hls_cores.emplace_back([&] { integration(stream_768_0, stream_768_1, integration_result_0, compl6, compl7, hbm.data(), hbm.data(), hbm.data(), hbm.data(), hbm_if_size);}); - hls_cores.emplace_back([&] { axis_128_to_512(integration_result_0, integration_result_1);}); + hls_cores.emplace_back([&] { axis_64_to_512(integration_result_0, integration_result_1);}); // 8. Spot finding hls_cores.emplace_back([&] { spot_finder(stream_768_1, stream_768_2, spot_finder_result_0, count_threshold, snr_threshold);}); @@ -371,7 +398,7 @@ void HLSSimulatedDevice::HLSMainThread() { host_writer(converted_7, adu_histo_result, integration_result_1, spot_finder_result_1, compl7, datamover_out.GetDataStream(), datamover_out.GetCtrlStream(), work_request_stream, completion_stream, - packets_processed, host_writer_idle, err_reg); }); + dma_address_table.data(), packets_processed, host_writer_idle);}); for (auto &i : hls_cores) i.join(); @@ -469,9 +496,6 @@ void HLSSimulatedDevice::HLSMainThread() { if (!datamover_in.GetDataStream().empty()) throw std::runtime_error("Datamover queue is not empty"); - if (err_reg != 0) - throw std::runtime_error("Error reg for frame_statistics not zero, val=" + std::to_string(err_reg)); - while (!datamover_out.IsIdle()) std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -495,13 +519,6 @@ void HLSSimulatedDevice::HW_LoadCalibration(uint32_t modules, uint32_t storage_c if (logger) logger->Info("Load calibration start"); - auto in_mem_location32 = (uint32_t *) calibration_addr_bram; - - for (int i = 0; i < modules * (3 + 3 * storage_cells); i++) { - in_mem_location32[2 * i ] = ((uint64_t) buffer_device[i]) & UINT32_MAX; - in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; - } - int ret = load_calibration(hbm.data(), hbm.data(), modules, @@ -510,7 +527,7 @@ void HLSSimulatedDevice::HW_LoadCalibration(uint32_t modules, uint32_t storage_c LOAD_CALIBRATION_DEST_CALIB, datamover_in.GetCtrlStream(), datamover_in.GetDataStream(), - calibration_addr_bram); + dma_address_table.data()); if (ret) throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, "Error in loading calibration"); @@ -525,13 +542,6 @@ void HLSSimulatedDevice::HW_LoadIntegrationMap(uint32_t modules) { if (logger) logger->Info("Load calibration start"); - auto in_mem_location32 = (uint32_t *) calibration_addr_bram; - - for (int i = 0; i < modules; i++) { - in_mem_location32[2 * i ] = ((uint64_t) buffer_device[i]) & UINT32_MAX; - in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; - } - int ret = load_calibration(hbm.data(), hbm.data(), modules, @@ -540,7 +550,7 @@ void HLSSimulatedDevice::HW_LoadIntegrationMap(uint32_t modules) { LOAD_CALIBRATION_DEST_INTEGRATION, datamover_in.GetCtrlStream(), datamover_in.GetDataStream(), - calibration_addr_bram); + dma_address_table.data()); if (ret) throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, @@ -557,13 +567,6 @@ void HLSSimulatedDevice::HW_LoadInternalGeneratorFrame(uint32_t modules) { if (logger) logger->Info("Load to frame generator"); - auto in_mem_location32 = (uint32_t *) calibration_addr_bram; - - for (int i = 0; i < modules; i++) { - in_mem_location32[2 * i ] = ((uint64_t) buffer_device[i]) & UINT32_MAX; - in_mem_location32[2 * i + 1] = ((uint64_t) buffer_device[i]) >> 32; - } - int ret = load_calibration(hbm.data(), hbm.data(), modules, @@ -572,7 +575,7 @@ void HLSSimulatedDevice::HW_LoadInternalGeneratorFrame(uint32_t modules) { LOAD_CALIBRATION_DEST_FRAME_GEN, datamover_in.GetCtrlStream(), datamover_in.GetDataStream(), - calibration_addr_bram); + dma_address_table.data()); if (ret) throw JFJochException(JFJochExceptionCategory::AcquisitionDeviceError, @@ -594,3 +597,11 @@ void HLSSimulatedDevice::HW_SetSpotFinderParameters(const SpotFinderParameters & count_threshold = params.count_threshold; snr_threshold = params.snr_threshold; } + +void HLSSimulatedDevice::HW_SetDataSource(uint32_t val) { + data_source = val; +} + +uint32_t HLSSimulatedDevice::HW_GetDataSource() { + return data_source; +} \ No newline at end of file diff --git a/acquisition_device/HLSSimulatedDevice.h b/acquisition_device/HLSSimulatedDevice.h index f41d6273..d6928eab 100644 --- a/acquisition_device/HLSSimulatedDevice.h +++ b/acquisition_device/HLSSimulatedDevice.h @@ -15,9 +15,11 @@ uint16_t checksum(const uint16_t *addr, size_t count); class HLSSimulatedDevice : public FPGAAcquisitionDevice { AXI_STREAM din_eth; + AXI_STREAM din_eth_4x10G; // Not used AXI_STREAM din_frame_generator; AXI_STREAM dout_eth; + ap_uint<2> data_source = STREAM_MERGE_SRC_100G; ap_int<16> count_threshold = INT16_MAX; ap_uint<8> snr_threshold = 0; @@ -25,7 +27,7 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { volatile bool idle; - constexpr static const size_t hbm_if_count = 22; + constexpr static const size_t hbm_if_count = 28; constexpr static const size_t hbm_if_size = 32*1024*1024LU; std::vector> hbm; @@ -44,7 +46,7 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { ap_uint<1> cancel_data_collection; volatile ap_uint<1> host_writer_idle; - uint64_t calibration_addr_bram[LOAD_CALIBRATION_BRAM_SIZE]; + std::vector dma_address_table; void HW_ReadActionRegister(DataCollectionConfig *job) override; void HW_WriteActionRegister(const DataCollectionConfig *job) override; @@ -52,13 +54,15 @@ class HLSSimulatedDevice : public FPGAAcquisitionDevice { void FPGA_StartAction(const DiffractionExperiment &experiment) override; void FPGA_EndAction() override; bool HW_IsIdle() const override; - bool HW_ReadMailbox(uint32_t values[16]) override; + bool HW_ReadMailbox(uint32_t *values) override; bool HW_SendWorkRequest(uint32_t handle) override; void HW_LoadCalibration(uint32_t modules, uint32_t storage_cells) override; void HW_LoadIntegrationMap(uint32_t modules) override; void HW_LoadInternalGeneratorFrame(uint32_t modules) override; void HW_GetStatus(DataCollectionStatus *status) const override; void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) override; + uint32_t HW_GetDataSource() override; + void HW_SetDataSource(uint32_t val) override; void HLSMainThread() ; public: HLSSimulatedDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, int16_t numa_node = -1); diff --git a/acquisition_device/MockAcquisitionDevice.cpp b/acquisition_device/MockAcquisitionDevice.cpp deleted file mode 100644 index 185bcc15..00000000 --- a/acquisition_device/MockAcquisitionDevice.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include "MockAcquisitionDevice.h" -#include "../common/JFJochException.h" -#include -#include "../jungfrau/JFConversionFloatingPoint.h" - -void MockAcquisitionDevice::Start(const DiffractionExperiment& experiment, uint32_t optional_flags) { - idle = false; - cancel = false; - completed_descriptors = 0; - - if (experiment.IsUsingInternalPacketGen()) { - if (experiment.GetDetectorMode() == DetectorMode::Conversion) { - for (auto &i: buffer_device) - memcpy(i, internal_pkt_gen_frame_conv.data(), RAW_MODULE_SIZE * sizeof(uint16_t)); - } else { - for (auto &i: buffer_device) - memcpy(i, internal_pkt_gen_frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t)); - } - measure = std::async(std::launch::async, &MockAcquisitionDevice::InternalPacketGeneratorThread, this, - experiment.GetModulesNum(data_stream), experiment.GetFrameNum()); - } else { - measure = std::async(std::launch::async, &MockAcquisitionDevice::MeasureThread, this); - } -} - -void MockAcquisitionDevice::Cancel() { - if (!idle) { - if (logger) - logger->Info("MockAcquisitionDevice cancelling " + std::to_string(data_stream)); - cancel = true; - } -} - -MockAcquisitionDevice::MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, - int16_t in_numa_node) -: AcquisitionDevice(data_stream) { - max_modules = MAX_MODULES_FPGA; - internal_pkt_gen_frame.resize(max_modules * RAW_MODULE_SIZE); - internal_pkt_gen_frame_conv.resize(max_modules * RAW_MODULE_SIZE); - MapBuffersStandard(in_frame_buffer_size_modules, 1, in_numa_node); - max_handle = in_frame_buffer_size_modules; - - for (int i = 0; i < max_modules * RAW_MODULE_SIZE; i++) { - internal_pkt_gen_frame[i] = i % 65536; - internal_pkt_gen_frame_conv[i] = i % 65536; - } -} - -void MockAcquisitionDevice::SendCompletion(uint32_t handle, uint16_t module_number, uint64_t frame_number) { - Completion c; - c.handle = handle; - c.module_number = module_number; - c.frame_number = frame_number - 1; - c.type = Completion::Type::Image; - c.packet_mask[0] = UINT64_MAX; - c.packet_mask[1] = UINT64_MAX; - c.packet_count = 128; - mock_completions.Put(c); -} - -void MockAcquisitionDevice::AddModule(uint64_t frame_number, uint16_t module_number, const uint16_t *data) { - - if (module_number >= max_modules) - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, - "Module number exceeding limit"); - - if (current_handle < max_handle) { - memcpy(buffer_device.at(current_handle), data, RAW_MODULE_SIZE * sizeof(uint16_t)); - SendCompletion(current_handle, module_number, frame_number); - current_handle++; - } else { - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No buffer handles left"); - } -} - -void MockAcquisitionDevice::MeasureThread() { - work_completion_queue.Put(Completion{.type = Completion::Type::Start}); - - while (!cancel) { - Completion c{}; - - if (mock_completions.Get(c)) { - if (c.type == Completion::Type::Image) { - work_completion_queue.Put(c); - completed_descriptors++; - } else - cancel = true; - } else - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } - - work_completion_queue.Put(Completion{ - .type = Completion::Type::End - }); - idle = true; -} - -void MockAcquisitionDevice::InternalPacketGeneratorThread(uint32_t nmodules, uint32_t nframes) { - work_completion_queue.Put(Completion{.type = Completion::Type::Start}); - uint32_t curr_module = 0; - uint32_t curr_frame = 0; - - while ((!cancel) && (curr_frame < nframes)) { - WorkRequest wr{}; - if (work_request_queue.Get(wr)) { - Completion c{}; - - c.handle = wr.handle; - c.module_number = curr_module; - c.frame_number = curr_frame; - c.type = Completion::Type::Image; - c.packet_mask[0] = UINT64_MAX; - c.packet_mask[1] = UINT64_MAX; - c.packet_count = 128; - work_completion_queue.Put(c); - completed_descriptors++; - curr_module++; - if (curr_module == nmodules) { - curr_module = 0; - curr_frame++; - } - } else - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } - - work_completion_queue.Put(Completion{ - .type = Completion::Type::End - }); - idle = true; -} - -void MockAcquisitionDevice::Terminate() { - mock_completions.Put(Completion{ - .type = Completion::Type::End - }); -} - -std::string MockAcquisitionDevice::GetMACAddress() const { - return "00:00:00:00:00:00"; -} - -std::string MockAcquisitionDevice::GetIPv4Address() const { - return "127.0.0.1"; -} - -void MockAcquisitionDevice::Finalize() { - if (measure.valid()) - measure.get(); -} - -void MockAcquisitionDevice::SetCustomInternalGeneratorFrame(const std::vector &v) { - if (v.size() != RAW_MODULE_SIZE) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Error in size of custom internal generator frame"); - for (int m = 0; m < max_modules ; m++) { - memcpy(internal_pkt_gen_frame.data() + RAW_MODULE_SIZE * m, - v.data(), - RAW_MODULE_SIZE); - memcpy(internal_pkt_gen_frame_conv.data() + RAW_MODULE_SIZE * m, - v.data(), - RAW_MODULE_SIZE); - } -} - -void MockAcquisitionDevice::InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) { - for (int m = 0; (m < experiment.GetModulesNum(data_stream)) && (m < max_modules); m++) { - JFConversionFloatingPoint conv; - conv.Setup(calib.GainCalibration(experiment.GetFirstModuleOfDataStream(data_stream) + m), - calib.Pedestal(experiment.GetFirstModuleOfDataStream(data_stream) + m,0), - calib.Pedestal(experiment.GetFirstModuleOfDataStream(data_stream) + m,1), - calib.Pedestal(experiment.GetFirstModuleOfDataStream(data_stream) + m,2), - experiment.GetPhotonEnergy_keV()); - - conv.ConvertModule((int16_t *) internal_pkt_gen_frame_conv.data() + m * RAW_MODULE_SIZE, - internal_pkt_gen_frame.data() + m * RAW_MODULE_SIZE); - } -} - -std::vector MockAcquisitionDevice::GetInternalGeneratorFrame() const { - return internal_pkt_gen_frame; -} - -uint32_t MockAcquisitionDevice::GetCompletedDescriptors() const { - return completed_descriptors * 2; -} diff --git a/acquisition_device/MockAcquisitionDevice.h b/acquisition_device/MockAcquisitionDevice.h deleted file mode 100644 index be01398b..00000000 --- a/acquisition_device/MockAcquisitionDevice.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#ifndef JUNGFRAUJOCH_MOCKACQUISITIONDEVICE_H -#define JUNGFRAUJOCH_MOCKACQUISITIONDEVICE_H - -#include "AcquisitionDevice.h" -#include "../common/ThreadSafeFIFO.h" -#include - -class MockAcquisitionDevice : public AcquisitionDevice { - uint32_t current_handle = 0; - uint32_t max_handle = 0; - std::atomic completed_descriptors = 0; - bool idle = true; - std::future measure; - volatile bool cancel = false; - std::vector internal_pkt_gen_frame; - std::vector internal_pkt_gen_frame_conv; - ThreadSafeFIFO mock_completions; - - void SendCompletion(uint32_t handle, uint16_t module_number, uint64_t frame_number); - void Start(const DiffractionExperiment& experiment, uint32_t optional_flags) override; - void MeasureThread(); - void InternalPacketGeneratorThread(uint32_t nmodules, uint32_t nframes); -public: - MockAcquisitionDevice(uint16_t data_stream, size_t in_frame_buffer_size_modules, - int16_t numa_node = -1); - void AddModule(uint64_t frame, uint16_t module_number, const uint16_t *data); - void Terminate(); - std::string GetMACAddress() const override; - std::string GetIPv4Address() const override; - void Cancel() override; - void Finalize() override; - - // Warning - internal packet generator in MockAcquisitionDevice has one limitation: - // it assumes the same calibration for all modules of the device - // For routine receiver tests one should always set one module per device - void SetCustomInternalGeneratorFrame(const std::vector &v); - std::vector GetInternalGeneratorFrame() const override; - void InitializeCalibration(const DiffractionExperiment &experiment, const JFCalibration &calib) override; - uint32_t GetCompletedDescriptors() const override; -}; - -#endif //JUNGFRAUJOCH_MOCKACQUISITIONDEVICE_H diff --git a/acquisition_device/PCIExpressDevice.cpp b/acquisition_device/PCIExpressDevice.cpp index 675ff3b1..5a8475f6 100644 --- a/acquisition_device/PCIExpressDevice.cpp +++ b/acquisition_device/PCIExpressDevice.cpp @@ -20,10 +20,6 @@ PCIExpressDevice::PCIExpressDevice(uint16_t data_stream, const std::string &devi uint32_t num_buf = GetNumKernelBuffers(); - if (num_buf < max_modules * (3 + 3 * 16)) - throw JFJochException(JFJochExceptionCategory::PCIeError, - "Need to increase number of host-device buffers"); - buffer_device.resize(num_buf, nullptr); try { for (int i = 0; i < num_buf; i++) @@ -138,3 +134,12 @@ DeviceStatus PCIExpressDevice::GetDeviceStatus() const { DataCollectionStatus PCIExpressDevice::GetDataCollectionStatus() const { return dev.GetDataCollectionStatus(); } + +uint32_t PCIExpressDevice::HW_GetDataSource() { + return dev.GetDataSource(); +} + +void PCIExpressDevice::HW_SetDataSource(uint32_t val) { + return dev.SetDataSource(val); +} + diff --git a/acquisition_device/PCIExpressDevice.h b/acquisition_device/PCIExpressDevice.h index c97b7bc4..d22abb39 100644 --- a/acquisition_device/PCIExpressDevice.h +++ b/acquisition_device/PCIExpressDevice.h @@ -9,7 +9,7 @@ class PCIExpressDevice : public FPGAAcquisitionDevice { JungfraujochDevice dev; - bool HW_ReadMailbox(uint32_t values[16]) override; + bool HW_ReadMailbox(uint32_t *values) override; bool HW_SendWorkRequest(uint32_t handle) override; void FPGA_StartAction(const DiffractionExperiment &experiment) override; bool HW_IsIdle() const final; @@ -19,6 +19,8 @@ class PCIExpressDevice : public FPGAAcquisitionDevice { void HW_LoadIntegrationMap(uint32_t modules) override; void HW_LoadInternalGeneratorFrame(uint32_t modules) override; void HW_SetSpotFinderParameters(const SpotFinderParameters ¶ms) override; + uint32_t HW_GetDataSource() override; + void HW_SetDataSource(uint32_t val) override; void FPGA_EndAction() override; uint32_t GetNumKernelBuffers() const; public: diff --git a/broker/JFJochBrokerParser.cpp b/broker/JFJochBrokerParser.cpp index b71faf16..5e3c54c7 100644 --- a/broker/JFJochBrokerParser.cpp +++ b/broker/JFJochBrokerParser.cpp @@ -249,9 +249,6 @@ void ParseAcquisitionDeviceGroup(const nlohmann::json &input, const std::string& if (dev_type == "pcie") { for (int i = 0; i < dev_count; i++) aq_devices.AddPCIeDevice("/dev/jfjoch" + std::to_string(i)); - } else if (dev_type == "mock") { - for (int i = 0; i < dev_count; i++) - aq_devices.AddMockDevice(buffer_size); } else if (dev_type == "hls") { if (dev_count > 1) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Device count must be 1 for HLS"); diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 5808c2fe..702f4991 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -51,7 +51,8 @@ ADD_LIBRARY( CommonFunctions STATIC ADUHistogram.cpp ADUHistogram.h RawToConvertedGeometryCore.h - Plot.h) + Plot.h + DeviceOutput.h) TARGET_LINK_LIBRARIES(CommonFunctions Compression FrameSerialize libzmq JFCalibration -lrt) diff --git a/common/Definitions.h b/common/Definitions.h index 29b9d649..48614287 100644 --- a/common/Definitions.h +++ b/common/Definitions.h @@ -46,35 +46,32 @@ #define DEFAULT_G1_FACTOR (-1.439) #define DEFAULT_G2_FACTOR (-0.1145) -// For FPGA -#define ACTION_TYPE 0x52324158 -#define RELEASE_LEVEL 0x0046 - -#define MODE_CONV 0x0001L -#define MODE_BITSHUFFLE_FPGA 0x0002L -#define MODE_ADD_MULTIPIXEL 0x0004L -#define MODE_MODULE_UPSIDE_DOWN 0x0008L -#define MODE_32BIT 0x0010L -#define MODE_UNSIGNED 0x0020L - #define TASK_NO_DATA_STREAM UINT16_MAX -#define PIXEL_OUT_SATURATION (INT16_MAX) -#define PIXEL_OUT_LOST (INT16_MIN) -#define PIXEL_OUT_0xFFFF (INT16_MIN) -#define PIXEL_OUT_G1_SATURATION (INT16_MIN) -#define PIXEL_OUT_GAINBIT_2 (INT16_MIN) +// For FPGA +#define ACTION_TYPE 0x52324158 +#define RELEASE_LEVEL 0x004A + +#define MODE_CONV 0x0001L +#define MODE_32BIT 0x0002L +#define MODE_UNSIGNED 0x0004L +#define MODE_PEDESTAL_G0 0x0010L +#define MODE_PEDESTAL_G1 0x0020L +#define MODE_PEDESTAL_G2 0x0040L + +#define STREAM_MERGE_SRC_NONE 0 +#define STREAM_MERGE_SRC_100G 1 +#define STREAM_MERGE_SRC_4x10G 2 +#define STREAM_MERGE_SRC_FRAME_GEN 3 + +#define PIXEL_OUT_LOST (INT16_MIN) -#define LOAD_CALIBRATION_BRAM_SIZE 1632 #define LOAD_CALIBRATION_DEST_CALIB 0 #define LOAD_CALIBRATION_DEST_INTEGRATION 1 #define LOAD_CALIBRATION_DEST_FRAME_GEN 2 -// FPGA register map - -#define HANDLE_START (UINT32_MAX - 1) -#define HANDLE_SKIP_FRAME (UINT32_MAX - 2) -#define HANDLE_END (UINT32_MAX ) +#define HANDLE_START (UINT16_MAX - 1) +#define HANDLE_END (UINT16_MAX ) #define INT_PKT_GEN_DEBUG 0x0 #define INT_PKT_GEN_BUNCHID 0xCACACACACA @@ -88,5 +85,4 @@ #define ADU_HISTO_BIN_WIDTH 32 #define ADU_HISTO_BIN_COUNT (65536/ ADU_HISTO_BIN_WIDTH) - #endif //DEFINITIONS_H diff --git a/common/DeviceOutput.h b/common/DeviceOutput.h new file mode 100644 index 00000000..3d7d87c1 --- /dev/null +++ b/common/DeviceOutput.h @@ -0,0 +1,49 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#ifndef JUNGFRAUJOCH_DEVICEOUTPUT_H +#define JUNGFRAUJOCH_DEVICEOUTPUT_H + +#include + +#pragma pack(push) +#pragma pack(1) + +struct IntegrationResult { + uint32_t count; + float sum; +}; + +struct SpotFindingResult { + char strong_pixel[RAW_MODULE_SIZE / 8]; + int32_t count_threshold; + int32_t snr_threshold; + int32_t strong_pixel_count; + int32_t reserved[13]; +}; + +struct ModuleStatistics { + uint64_t frame_number; + uint64_t timestamp; + uint64_t bunchid; + uint64_t reserved_1; + uint64_t reserved_2; + uint32_t exptime; + uint32_t debug; + uint32_t pedestal; + uint32_t packet_count; + uint32_t module_number; + uint32_t reserved_3; + uint64_t packet_mask[4]; // This is reserved for 256 packets/module +}; + +struct DeviceOutput { + int16_t pixels[RAW_MODULE_SIZE * 2]; // accommodate 32-bit output + SpotFindingResult spot_finding_result; + IntegrationResult integration_result[FPGA_INTEGRATION_BIN_COUNT]; + uint32_t adu_histogram[ADU_HISTO_BIN_COUNT]; + ModuleStatistics module_statistics; +}; + +#pragma pack(pop) + +#endif //JUNGFRAUJOCH_DEVICEOUTPUT_H diff --git a/common/NUMAHWPolicy.cpp b/common/NUMAHWPolicy.cpp index 6aa0433a..9c8dc0e4 100644 --- a/common/NUMAHWPolicy.cpp +++ b/common/NUMAHWPolicy.cpp @@ -20,6 +20,9 @@ NUMAHWPolicy::NUMAHWPolicy(const std::string &policy) : name(policy) { bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 2}); bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 0, .gpu = 1}); bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 1, .gpu = 3}); + } else if (policy == "n2g2_hbm") { + bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); + bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 1}); } else if (policy == "n2g4_hbm") { bindings.emplace_back(NUMABinding{.cpu_node = 0, .mem_node = 2, .gpu = 0}); bindings.emplace_back(NUMABinding{.cpu_node = 1, .mem_node = 3, .gpu = 2}); diff --git a/fpga/CMakeLists.txt b/fpga/CMakeLists.txt index a712172a..0a707e7d 100644 --- a/fpga/CMakeLists.txt +++ b/fpga/CMakeLists.txt @@ -21,7 +21,7 @@ ADD_SUBDIRECTORY(host_library) IF(VIVADO_HLS AND VIVADO) ADD_CUSTOM_COMMAND(OUTPUT action/hw/hdl/action_config.v COMMAND ${CMAKE_COMMAND} -E env SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} HLS_IP_DIR=${CMAKE_CURRENT_BINARY_DIR}/action/ip/hls bash ${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup_action.sh - DEPENDS hls hdl/action_config.v hdl/check_eth_busy.v hdl/gen_xdma_descriptor.v scripts/bd_pcie.tcl scripts/jfjoch.tcl scripts/network_stack.tcl scripts/hbm_u55c.tcl scripts/mac_100g_pcie.tcl scripts/pcie_dma.tcl scripts/setup_action.sh + DEPENDS hls hdl/action_config.v hdl/gen_xdma_descriptor.v scripts/bd_pcie.tcl scripts/jfjoch.tcl scripts/network_stack.tcl scripts/hbm_u55c.tcl scripts/mac_100g_pcie.tcl scripts/pcie_dma.tcl scripts/setup_action.sh ) ADD_CUSTOM_TARGET(action_pcie DEPENDS action/hw/hdl/action_config.v hls diff --git a/fpga/README.md b/fpga/README.md index c6c8adac..a4c497b2 100644 --- a/fpga/README.md +++ b/fpga/README.md @@ -34,9 +34,13 @@ The following procedures require having AMD (Xilinx) Vivado and Vitis HLS toolse Due to the nature of TCL scripts used to generate board designs Vivado version has to exactly match one provided above - specifically newer versions of Vivado will not work. -Additionally, a non-cost license for Ultrascale+ 100G core has to be separately requested from AMD/Xilinx, -see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html). +In additional to Intellectual Property (IP) cores included in Vivado, three additional licenses are necessary: +* Non-cost license for Ultrascale+ 100G core has to be requested from AMD/Xilinx website, see [Xilinx website](https://www.xilinx.com/products/intellectual-property/cmac_usplus.html). +* Auto-negotiation/link-training license for Ultrascale+ 100G core +* 10G/25G Subsystem for Ultrascale+ +PSI received a non-cost license from Xilinx University Program for the two latter cores. Therefore, usage of bitstreams +generated by PSI continuous integration pipeline is only allowed for non-commercial use. ### HLS compilation Make HLS routines: ``` @@ -99,6 +103,18 @@ Currently, this is best done by powering the server off and on again. More effic To test that FPGA board is working properly without access to a JUNGFRAU detector, you can use `jfjoch_action_test` tool. +## Card release number +To ensure compatibility of the card with driver and user application, each design is marked by **release number**. +This number is incremented after each change of functionality of the card or interface to communicate with the host. +This ensures consistency between the FPGA card, driver and user application. Changes within the design (e.g. size of FIFOs), +that are invisible to interactions with host do not require change in release number. + +To check release number, look for constant `RELEASE_NUMBER` in [common/Definitions.h](../common/Definitions.h) header file. +For FPGA design, release number is also included in the generated bitstream name. + +In case there is mismatch in release number between card and kernel driver, the latter will not create the character device and return error (check `dmesg`). + +It is recommended to use the same git commit hash for building the design and user application, though it is not strictly necessary in case care is made to have same release number. ## FPGA reference ### Frame generator @@ -145,55 +161,46 @@ FPGA setup can be done via 32-bit registers: | 0x010104 | 32 | Spot finder signal-to-noise ratio threshold (in fixed-point: 6 int. + 4 frac. bit format) | R/W | | | 0x010200 | 64 | MAC address of FPGA card | R/W | network byte order | | 0x010208 | 32 | IPv4 address of FPGA card | R/W | network byte order | -| 0x01020C | 32 | Number of detector modules | R/W | | +| 0x01020C | 32 | Number of detector modules (value minus one: 0 => 1 module, 1 => 2 modules, etc.) | R/W | | | 0x010210 | 32 | Data collection mode | R/W | | | | | Bit 0 - Conversion to photons | | | | | | Bit 16:31 - Data collection ID (carried with completions) | | | | 0x010214 | 32 | One over energy in keV (in fixed-point:12 int. + 24 frac. bit format) | R/W | | | 0x010218 | 32 | Number of frames expected in the data collection (defines termination condition) | R/W | | | 0x01021C | 32 | Number of storage cells | R/W | | -| 0x010220 | 32 | Summation on card (minus one: 0 => summation of 1, 1 => summation of 2, etc.) | R/W | | +| 0x010220 | 32 | Summation on card (value minus one: 0 => summation of 1, 1 => summation of 2, etc.) | R/W | | | 0x020000 - 0x02FFFF | | CMAC 100G | | See Xilinx PG203 for register map | | 0x030000 - 0x03FFFF | | AXI Mailbox for Work Request / Work Completion | | See Xilinx PG114 for register map | | 0x040000 - 0x04FFFF | | QuadSPI flash | | See Xilinx PG153 for register map | -| 0x060000 - 0x060FFF | 64 | Input calibration memory addresses block RAM | | | +| 0x050000 - 0x05FFFF | | Interrupt controller | | See Xilinx PG099 for register map | +| 0x060000 - 0x06FFFF | | Load calibration (HLS) | | | | 0x070000 - 0x07FFFF | | AXI Firewall | | See Xilinx PG293 for register map | -| 0x080000 - 0x08FFFF | | Frame generator | | | +| 0x080000 - 0x08FFFF | | Frame generator (HLS) | | | | 0x090000 - 0x09FFFF | | PCIe DMA control | | See Xilinx PG195 for register map | -| 0x0A0000 - 0x0AFFFF | | Transfer between UltraRAM buffer <-> HBM (HLS registers) | | | | 0x0C0000 - 0x0FFFFF | | Xilinx Card Management Solution Subsystem management subsystem | | See Xilinx PG348 for register map | -| 0x100000 - 0x1FFFFF | 16 | Internal packet generator frame | | | -| 0x200000 - 0x2FFFFF | | UltraRAM buffer for transfers to/from HBM | | | +| 0x100000 - 0x10FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x110000 - 0x11FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x120000 - 0x12FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x130000 - 0x13FFFF | | MAC 10G | | See Xilinx PG210 for register map | +| 0x200000 - 0x27FFFF | 64 | Address table: decodes handles used by load_calibration and host_writer to DMA addresses | | | ### AXI Mailbox AXI mailbox is used to send work request from host to action, and receive work completions. -Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114). -Messages are always multiple of 128-bit. +Messages are exchanged through AXI Mailbox IP from Xilinx (see Xilinx PG114). Work request has the following structure: | Bit start | Bit end | Meaning | |-----------|---------|----------------------------------------------------| -| 0 | 31 | Work request ID (handle) | -| 32 | 95 | Address (Virt: OpenCAPI, DMA: PCIe) | -| 96 | 127 | Includes parity bit, so bits 0-127 are even parity | +| 0 | 15 | Work request ID (handle) | Work completion has the following structure: -| Bit start | Bit end | Meaning | -|-----------|---------|--------------------------------------------------------------------| -| 0 | 31 | Work request ID (handle) | -| 32 | 39 | Module number | -| 40 | 40 | All packets for the module arrived OK | -| 41 | 41 | Trigger signal high | -| 42 | 62 | Reserved | -| 63 | 63 | Parity bit - bits 0-127 are even parity | -| 64 | 127 | Frame number | -| 128 | 191 | JF Timestamp | -| 192 | 255 | Bunch ID | -| 256 | 287 | Exptime | -| 288 | 319 | JF debug | -| 320 | 351 | Reserved | -| 352 | 383 | Data collection ID (16-bit) | -| 384 | 511 | Packet mask (1 bit per packet: 0 packet missing, 1 packet arrived) | +| Bit start | Bit end | Meaning | +|-----------|---------|----------------------------------| +| 0 | 15 | Work request ID (handle) | +| | | Special values: | +| | | 65534 - start of data collection | +| | | 65535 - end of data collection | +| 15 | 31 | Data collection ID | \ No newline at end of file diff --git a/fpga/hdl/action_config.v b/fpga/hdl/action_config.v index 147a42b0..fefd964d 100644 --- a/fpga/hdl/action_config.v +++ b/fpga/hdl/action_config.v @@ -61,12 +61,11 @@ `define ADDR_NSTORAGE_CELLS 16'h021C `define ADDR_NSUMMATION 16'h0220 +`define ADDR_DATA_SOURCE 16'h0224 module action_config #(parameter C_S_AXI_ADDR_WIDTH = 16, - parameter C_S_AXI_DATA_WIDTH = 32, - parameter MAX_MODULES_FPGA_PARAM = `MAX_MODULES_FPGA, - parameter DESIGN_NUMBER = 1'b0 + parameter C_S_AXI_DATA_WIDTH = 32 )( (* X_INTERFACE_INFO = "xilinx.com:signal:clock:1.0 clk CLK" *) (* X_INTERFACE_PARAMETER = "ASSOCIATED_BUSIF s_axi, ASSOCIATED_RESET resetn" *) @@ -101,6 +100,7 @@ module action_config output reg [4:0] nmodules , output reg [3:0] nstorage_cells , output reg [7:0] nsummation , + output reg [1:0] data_source , output wire [31:0] hbm_size_bytes , output reg [15:0] spot_finder_count_threshold, output reg [7:0] spot_finder_snr_threshold, @@ -115,6 +115,8 @@ module action_config input calib_addr_fifo_empty , input calib_addr_fifo_full , + input proc_fifo_empty , + input proc_fifo_full , input udp_fifo_empty , input udp_fifo_full , input c2h_data_fifo_empty , @@ -125,10 +127,6 @@ module action_config input h2c_data_fifo_full , input h2c_cmd_fifo_empty , input h2c_cmd_fifo_full , - input work_req_fifo_empty , - input work_req_fifo_full , - input work_compl_fifo_empty , - input work_compl_fifo_full , input last_data_fifo_empty , input last_data_fifo_full , input last_addr_fifo_empty , @@ -142,7 +140,6 @@ module action_config input hbm_handles_fifo_empty , input hbm_handles_fifo_full , input mailbox_interrupt_0 , - input mailbox_interrupt_1 , input [63:0] stalls_hbm , input [63:0] stalls_host , @@ -172,8 +169,6 @@ module action_config input [31:0] udp_err_len , input udp_err_len_valid , - input [7:0] host_writer_err , - input host_writer_err_valid , output reg clear_counters ); //------------------------Parameter---------------------- @@ -220,7 +215,6 @@ localparam reg [31:0] reg_udp_err_eth; reg [31:0] reg_fifo_status; - reg [7:0] reg_host_writer_err; reg reg_data_collection_idle; //------------------------Instantiation------------------ @@ -338,6 +332,9 @@ always @(posedge clk) begin `ADDR_NSUMMATION: begin rdata <= nsummation; end + `ADDR_DATA_SOURCE: begin + rdata <= data_source; + end `ADDR_ACTION_TYPE: begin rdata <= `ACTION_TYPE; end @@ -348,7 +345,7 @@ always @(posedge clk) begin rdata <= `GIT_SHA1; end `ADDR_MAX_MODULES_FPGA: begin - rdata <= MAX_MODULES_FPGA_PARAM; + rdata <= `MAX_MODULES_FPGA; end `ADDR_STALLS_HBM_HI: begin rdata <= reg_stalls_hbm[63:32]; @@ -459,10 +456,7 @@ always @(posedge clk) begin reg_ctrl[2] <= data_collection_cancel; reg_ctrl[3] <= clear_counters; reg_ctrl[4] <= host_writer_idle; - reg_ctrl[7] <= DESIGN_NUMBER; reg_ctrl[16] <= mailbox_interrupt_0; - reg_ctrl[17] <= mailbox_interrupt_1; - reg_ctrl[31:24] <= reg_host_writer_err; end end @@ -578,6 +572,15 @@ always @(posedge clk) begin end end +always @(posedge clk) begin + if (!resetn) + data_source <= 0; + else if (reg_data_collection_idle) begin + if (w_hs && waddr == `ADDR_DATA_SOURCE) + data_source <= (s_axi_WDATA[1:0] & wmask[1:0]) | (data_source & !wmask[1:0]); + end +end + always @(posedge clk) begin if (!resetn) spot_finder_snr_threshold <= 0; @@ -606,7 +609,6 @@ always @ (posedge clk) begin reg_packets_udp <= 0; reg_packets_icmp <= 0; reg_packets_sls <= 0; - reg_host_writer_err <= 0; end else begin @@ -636,8 +638,6 @@ always @ (posedge clk) begin reg_udp_err_len <= udp_err_len; if (udp_err_eth_valid) reg_udp_err_eth <= udp_err_eth; - if (host_writer_err_valid) - reg_host_writer_err <= host_writer_err; end end @@ -651,16 +651,14 @@ always @(posedge clk) begin reg_fifo_status[1] <= calib_data_fifo_full; reg_fifo_status[2] <= calib_addr_fifo_empty; reg_fifo_status[3] <= calib_addr_fifo_full; + reg_fifo_status[4] <= proc_fifo_empty; + reg_fifo_status[5] <= proc_fifo_full; reg_fifo_status[6] <= udp_fifo_empty; reg_fifo_status[7] <= udp_fifo_full; reg_fifo_status[8] <= c2h_data_fifo_empty; reg_fifo_status[9] <= c2h_data_fifo_full; reg_fifo_status[10] <= c2h_cmd_fifo_empty; reg_fifo_status[11] <= c2h_cmd_fifo_full; - reg_fifo_status[12] <= work_req_fifo_empty; - reg_fifo_status[13] <= work_req_fifo_full; - reg_fifo_status[14] <= work_compl_fifo_empty; - reg_fifo_status[15] <= work_compl_fifo_full; reg_fifo_status[16] <= last_data_fifo_empty; reg_fifo_status[17] <= last_data_fifo_full; reg_fifo_status[18] <= last_addr_fifo_empty; diff --git a/fpga/hdl/check_eth_busy.v b/fpga/hdl/check_eth_busy.v deleted file mode 100644 index 0f9abaa5..00000000 --- a/fpga/hdl/check_eth_busy.v +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -`timescale 1ns / 1ps - -module check_eth_busy( - output reg eth_busy, - (* X_INTERFACE_INFO = "xilinx.com:signal:clock:1.0 clk CLK" *) - (* X_INTERFACE_PARAMETER = "ASSOCIATED_BUSIF S_AXIS:M_AXIS" *) - input clk, - - (* X_INTERFACE_PARAMETER = "HAS_TLAST 1,HAS_TKEEP 1,HAS_TREADY 1,TUSER_WIDTH 1,TDATA_NUM_BYTES 64" *) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TDATA" *) - input [511:0] S_AXIS_tdata, // Transfer Data (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TKEEP" *) - input [63:0] S_AXIS_tkeep, // Transfer Null Byte Indicators (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TLAST" *) - input S_AXIS_tlast, // Packet Boundary Indicator (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TUSER" *) - input S_AXIS_tuser, // Transfer user sideband (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TVALID" *) - input S_AXIS_tvalid, // Transfer valid (required) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 S_AXIS TREADY" *) - output S_AXIS_tready, // Transfer ready (optional) - - (* X_INTERFACE_PARAMETER = "HAS_TLAST 1,HAS_TKEEP 1,HAS_TREADY 1,TUSER_WIDTH 1,TDATA_NUM_BYTES 64" *) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TDATA" *) - output [511:0] M_AXIS_tdata, // Transfer Data (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TKEEP" *) - output [63:0] M_AXIS_tkeep, // Transfer Null Byte Indicators (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TLAST" *) - output M_AXIS_tlast, // Packet Boundary Indicator (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TUSER" *) - output M_AXIS_tuser, // Transfer user sideband (optional) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TVALID" *) - output M_AXIS_tvalid, // Transfer valid (required) - (* X_INTERFACE_INFO = "xilinx.com:interface:axis:1.0 M_AXIS TREADY" *) - input M_AXIS_tready // Transfer ready (optional) - ); - always @(posedge clk) - eth_busy <= S_AXIS_tvalid; - assign M_AXIS_tdata = S_AXIS_tdata; - assign M_AXIS_tkeep = S_AXIS_tkeep; - assign M_AXIS_tlast = S_AXIS_tlast; - assign M_AXIS_tuser = S_AXIS_tuser; - assign M_AXIS_tvalid = S_AXIS_tvalid; - assign S_AXIS_tready = M_AXIS_tready; -endmodule diff --git a/fpga/hls/CMakeLists.txt b/fpga/hls/CMakeLists.txt index 4af30da7..423693a0 100644 --- a/fpga/hls/CMakeLists.txt +++ b/fpga/hls/CMakeLists.txt @@ -22,17 +22,30 @@ ADD_LIBRARY( HLSSimulation STATIC axis_broadcast.cpp adu_histo.cpp axis_helpers.cpp - hls_bitshuffle.cpp - add_multipixel.cpp - module_upside_down.cpp frame_summation.cpp frame_summation_reorder_compl.cpp + pedestal.cpp stream_24bit_conv.cpp) TARGET_INCLUDE_DIRECTORIES(HLSSimulation PUBLIC ../include) TARGET_LINK_LIBRARIES(HLSSimulation CommonFunctions) TARGET_COMPILE_DEFINITIONS(HLSSimulation PUBLIC -DJFJOCH_HLS_NOSYNTH) +ADD_EXECUTABLE(frame_summation_tb frame_summation_tb.cpp) +TARGET_LINK_LIBRARIES(frame_summation_tb HLSSimulation) + +ADD_EXECUTABLE(frame_summation_tb_2 frame_summation_tb_2.cpp) +TARGET_LINK_LIBRARIES(frame_summation_tb_2 HLSSimulation) + +ADD_EXECUTABLE(integration_tb integration_tb.cpp) +TARGET_LINK_LIBRARIES(integration_tb HLSSimulation) + +ADD_EXECUTABLE(spot_finder_tb spot_finder_tb.cpp) +TARGET_LINK_LIBRARIES(spot_finder_tb HLSSimulation) + +ADD_EXECUTABLE(adu_histo_tb adu_histo_tb.cpp) +TARGET_LINK_LIBRARIES(adu_histo_tb HLSSimulation) + IF(VIVADO_HLS) IF(NOT HLS_SOLUTION_NAME) @@ -53,7 +66,6 @@ ENDFUNCTION(MAKE_HLS_MODULE) MAKE_HLS_MODULE(data_collection_fsm data_collection_fsm.cpp "") MAKE_HLS_MODULE(timer_host timer.cpp "") MAKE_HLS_MODULE(jf_conversion jf_conversion.cpp "") -MAKE_HLS_MODULE(bitshuffle hls_bitshuffle.cpp "") MAKE_HLS_MODULE(load_calibration load_calibration.cpp "") MAKE_HLS_MODULE(host_writer host_writer.cpp "") MAKE_HLS_MODULE(icmp icmp.cpp "") @@ -67,15 +79,14 @@ MAKE_HLS_MODULE(stream_merge stream_merge.cpp "") MAKE_HLS_MODULE(load_from_hbm load_from_hbm.cpp "") MAKE_HLS_MODULE(save_to_hbm save_to_hbm.cpp "") MAKE_HLS_MODULE(mask_missing mask_missing.cpp "") -MAKE_HLS_MODULE(integration integration.cpp "") +MAKE_HLS_MODULE(integration integration.cpp integration_tb.cpp) MAKE_HLS_MODULE(spot_finder spot_finder.cpp spot_finder_tb.cpp) MAKE_HLS_MODULE(axis_broadcast axis_broadcast.cpp "") -MAKE_HLS_MODULE(axis_128_to_512 axis_helpers.cpp "") +MAKE_HLS_MODULE(axis_64_to_512 axis_helpers.cpp "") MAKE_HLS_MODULE(axis_32_to_512 axis_helpers.cpp "") -MAKE_HLS_MODULE(adu_histo adu_histo.cpp "") -MAKE_HLS_MODULE(add_multipixel add_multipixel.cpp add_multipixel_tb.cpp) -MAKE_HLS_MODULE(module_upside_down module_upside_down.cpp module_upside_down_tb.cpp) -MAKE_HLS_MODULE(frame_summation frame_summation.cpp frame_summation_tb.cpp) +MAKE_HLS_MODULE(adu_histo adu_histo.cpp adu_histo_tb.cpp) +MAKE_HLS_MODULE(pedestal pedestal.cpp "") +MAKE_HLS_MODULE(frame_summation frame_summation.cpp frame_summation_tb_2.cpp) MAKE_HLS_MODULE(frame_summation_reorder_compl frame_summation_reorder_compl.cpp "") MAKE_HLS_MODULE(stream_24bit_conv stream_24bit_conv.cpp stream_24bit_conv_tb.cpp) diff --git a/fpga/hls/add_multipixel.cpp b/fpga/hls/add_multipixel.cpp deleted file mode 100644 index 22feb1a1..00000000 --- a/fpga/hls/add_multipixel.cpp +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include "hls_jfjoch.h" - -struct add_multipixel_packet { - ap_uint<512> data; - ap_uint<1> user; - ap_uint<1> last; -}; - -ap_uint<512> divide_by_two(const ap_uint<512> &input) { -#pragma HLS PIPELINE II=1 - ap_int<16> val[32]; - unpack32(input, val); - for (int i = 0; i < 32; i++) { - if ((val[i] == INT16_MIN) || (val[i] == INT16_MAX)) - val[i] = val[i]; - else - val[i] = val[i] / 2; - } - return pack32(val); -} - -void divide_last_by_two(ap_uint<512> &input) { -#pragma HLS PIPELINE II=1 - ap_int<16> val[32]; - unpack32(input, val); - if ((val[31] == INT16_MIN) || (val[31] == INT16_MAX)) - val[31] = val[31]; - else - val[31] = val[31] / 2; - input = pack32(val); -} - - -void divide_first_by_two(ap_uint<512> &input) { -#pragma HLS PIPELINE II=1 - ap_int<16> val[32]; - unpack32(input, val); - if ((val[0] == INT16_MIN) || (val[0] == INT16_MAX)) - val[0] = val[0]; - else - val[0] = val[0] / 2; - input = pack32(val); -} - -void add_multipixel_in_stream(STREAM_512 &data_in, - hls::stream &data_out) { - packet_512_t packet_in; - data_in >> packet_in; - data_out << add_multipixel_packet{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last}; - data_in >> packet_in; - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - data_out << add_multipixel_packet{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last}; - data_in >> packet_in; - } - data_out << add_multipixel_packet{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last}; -} - -void add_multipixel_div(hls::stream &data_in, - hls::stream &data_out) { -#pragma HLS INTERFACE axis port=data_in -#pragma HLS INTERFACE axis port=data_out - add_multipixel_packet packet_in, packet_out; - data_in >> packet_in; - data_out << packet_in; - ap_uint<1> add_multipixel = ((ACT_REG_MODE(packet_in.data) & MODE_ADD_MULTIPIXEL)) ? 1 : 0; - - data_in >> packet_in; - if (!add_multipixel) { - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - data_out << packet_in; - data_in >> packet_in; - } - } else { - while (!packet_in.user) { -#pragma HLS PIPELINE II=32 - for (int i = 0; i < 32; i++) { - if ((i == 8 - 1) || (i == 2 * 8 - 1) || (i == 3 * 8 - 1)) - divide_last_by_two(packet_in.data); - - if ((i == 8) || (i == 2 * 8) || (i == 3 * 8)) - divide_first_by_two(packet_in.data); - data_out << packet_in; - data_in >> packet_in; - } - } - } - data_out << packet_in; -} - - -void add_multipixel_cols(hls::stream &data_in, - hls::stream &data_out) { -#pragma HLS INTERFACE axis port=data_in -#pragma HLS INTERFACE axis port=data_out - add_multipixel_packet packet_in, packet_out; - data_in >> packet_in; - data_out << packet_in; - ap_uint<1> add_multipixel = ((ACT_REG_MODE(packet_in.data) & MODE_ADD_MULTIPIXEL)) ? 1 : 0; - - data_in >> packet_in; - if (!add_multipixel) { - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - data_out << packet_in; - data_in >> packet_in; - } - } else { - while (!packet_in.user) { -#pragma HLS PIPELINE II=33 - - packet_out.last = 0; - packet_out.user = 0; - - ap_uint<512> save = packet_in.data; - data_in >> packet_in; - for (int i = 1; i < 8; i++) { - packet_out.data(511-16,0) = save(511,16); - packet_out.data(511,512-16) = packet_in.data(15, 0); - data_out << packet_out; - save = packet_in.data; - data_in >> packet_in; - } - packet_out = packet_in; - packet_out.data = save(511,16); - packet_out.data(511, 512-16) = packet_out.data(511-16, 512-32); - data_out << packet_out; - - - for (int k = 1; k < 4; k++) { - if (k == 1) { - save = packet_in.data(15, 0); - } else if (k == 2) { - save(31,16) = save(15,0); - } else if (k == 3) { - save(63,48) = save(47,32); - } - for (int i = 0; i < 8; i++) { - packet_out.data(511, (k * 2 - 1) * 16) = packet_in.data(511 - (k * 2 - 1) * 16, 0); - packet_out.data((k * 2 - 1) * 16 - 1, 0) = save; - save = packet_in.data(511, 512 - (2 * k - 1) * 16); - data_out << packet_out; - data_in >> packet_in; - } - if (k == 1) - save(47, 32) = packet_in.data(15,0); - else if (k == 2) - save(79, 64) = packet_in.data(15,0); - } - - packet_out.data = save; - packet_out.last = 0; - packet_out.user = 0; - data_out << packet_out; - } - } - data_out << packet_in; -} - -void add_multipixel_line(hls::stream &data_in, - hls::stream &data_out) { - add_multipixel_packet packet_in, packet_out; - data_in >> packet_in; - data_out << packet_in; - ap_uint<1> add_multipixel = ((ACT_REG_MODE(packet_in.data) & MODE_ADD_MULTIPIXEL)) ? 1 : 0; - - data_in >> packet_in; - if (!add_multipixel) { - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - data_out << packet_in; - data_in >> packet_in; - } - } else { - while (!packet_in.user) { - for (int i = 0; i < 255 * 33; i++) { -#pragma HLS PIPELINE II=1 - if (i >= 33) - data_out << packet_in; - data_in >> packet_in; - } - - for (int k = 0; k < 2; k++) { -#pragma HLS PIPELINE II=66 - ap_uint<512> packet[33]; - for (int i = 0; i < 33; i++) { - packet[i] = divide_by_two(packet_in.data); - data_out << add_multipixel_packet{.data = packet[i], .user = 0, .last = 0}; - data_in >> packet_in; - } - for (int i = 0; i < 33; i++) { - data_out << add_multipixel_packet{.data = packet[i], .user = 0, .last = 0}; - } - } - - for (int i = 0; i < 255 * 33; i++) { -#pragma HLS PIPELINE II=1 - if (i < 254*33) - data_out << packet_in; - data_in >> packet_in; - } - } - } - data_out << packet_in; -} - -void add_multipixel_output(hls::stream &data_in, - STREAM_512 &data_out) { - add_multipixel_packet packet_in; - packet_512_t packet_out; - data_in >> packet_in; - data_out << packet_512_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last}; - - ap_uint<1> add_multipixel = ((ACT_REG_MODE(packet_in.data) & MODE_ADD_MULTIPIXEL)) ? 1 : 0; - - data_in >> packet_in; - if (!add_multipixel) { - while (!packet_in.user) { - for (int i = 0; i < 512 * 32; i++) { -#pragma HLS PIPELINE II=1 - data_out << packet_512_t{.data = packet_in.data, .user = packet_in.user, .last = (i == 512 * 32 - 1)}; - data_in >> packet_in; - } - } - } else { - while (!packet_in.user) { - for (int i = 0; i < 64; i++) { - packet_out.user = 0; - packet_out.last = 0; - - for (int j = 0; j < 32; j++) { -#pragma HLS PIPELINE II=1 - data_out << packet_512_t{.data = packet_in.data, .user = packet_in.user, .last = 0}; - data_in >> packet_in; - } - ap_uint<512-64> save = 0; - - for (int k = 1; k < 8; k++) { -#pragma HLS PIPELINE OFF - for (int j = 0; j < 33; j++) { -#pragma HLS PIPELINE II=1 - if (j == 0) { - save(64 * k - 1, 64 * (k-1)) = packet_in.data(64, 0); - data_in >> packet_in; - } else { - packet_out.data(64 * k - 1, 0) = save(64 * k - 1, 0); - packet_out.data(511, 64 * k) = packet_in.data(511 - 64 * k, 0); - save(64 * k - 1, 0) = packet_in.data(511, 512 - 64 * k); - data_out << packet_out; - data_in >> packet_in; - } - } - } - packet_out.data(64 * 7, 0) = save; - packet_out.data(511, 512 - 64) = packet_in.data(63, 0); - data_out << packet_out; - data_in >> packet_in; - } - } - } - data_out << packet_512_t{.data = packet_in.data, .user = packet_in.user, .last = packet_in.last}; -} - -void add_multipixel(STREAM_512 &data_in, STREAM_512 &data_out) { -#pragma HLS INTERFACE axis port=data_in -#pragma HLS INTERFACE axis port=data_out -#pragma HLS DATAFLOW - hls::stream stream_0; - hls::stream stream_1; - hls::stream stream_2; - hls::stream stream_3; - -#ifndef JFJOCH_HLS_NOSYNTH - add_multipixel_in_stream(data_in, stream_0); - add_multipixel_div(stream_0, stream_1); - add_multipixel_cols(stream_1, stream_2); - add_multipixel_line(stream_2, stream_3); - add_multipixel_output(stream_3, data_out); -#else - std::vector multipix_cores; - - multipix_cores.emplace_back([&] {add_multipixel_in_stream(data_in, stream_0);}); - multipix_cores.emplace_back([&] {add_multipixel_div(stream_0, stream_1);}); - multipix_cores.emplace_back([&] {add_multipixel_cols(stream_1, stream_2);}); - multipix_cores.emplace_back([&] {add_multipixel_line(stream_2, stream_3);}); - multipix_cores.emplace_back([&] {add_multipixel_output(stream_3, data_out);}); - - for (auto &i : multipix_cores) - i.join(); -#endif -} diff --git a/fpga/hls/add_multipixel_tb.cpp b/fpga/hls/add_multipixel_tb.cpp deleted file mode 100644 index 6cfbefaa..00000000 --- a/fpga/hls/add_multipixel_tb.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include "hls_jfjoch.h" -#include "../../common/RawToConvertedGeometryCore.h" - -int main() { - - int ret = 0; - - STREAM_512 input; - STREAM_512 output; - - std::vector input_frame(RAW_MODULE_SIZE), input_frame_transformed(CONVERTED_MODULE_SIZE), - output_frame(257 * 64 * 32); - for (int i = 0; i < RAW_MODULE_SIZE; i++) - input_frame[i] = i % INT16_MAX; - - TransferModuleAdjustMultipixels(input_frame_transformed.data(), input_frame.data(), - CONVERTED_MODULE_COLS, INT16_MIN, INT16_MAX); - - auto input_frame_512 = (ap_uint<512>*) input_frame.data(); - auto output_frame_512 = (ap_uint<512>*) output_frame.data(); - - ap_uint<512> action_control = 0; - ACT_REG_MODE(action_control) = MODE_ADD_MULTIPIXEL; - - input << packet_512_t { .data = action_control, .user = 0 }; - for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) - input << packet_512_t { .data = input_frame_512[i], .user = 0 }; - - input << packet_512_t { .user = 1 }; - - add_multipixel(input, output); - - if (input.size() != 0) - ret = 1; - if (output.size() != (257 * 64) + 2) - ret = 1; - - output.read(); - for (int i = 0; i < 257 * 64 ; i++) - output_frame_512[i] = output.read().data; - output.read(); - - size_t diff = 0; - for (int line = 0; line < 512; line++) { - for (int col = 0; col < 1027; col++) { - if (output_frame[line * 1028 + col] != input_frame_transformed[(line + 1) * 1030 + (col+1)]) - diff++; - } - - } - - if (diff > 0) { - ret = 1; - std::cout << diff << std::endl; - } - if (ret != 0) { - printf("Test failed !!!\n"); - ret = 1; - } else { - printf("Test passed !\n"); - } - - return ret; -} diff --git a/fpga/hls/adu_histo.cpp b/fpga/hls/adu_histo.cpp index 6919cc03..c408b326 100644 --- a/fpga/hls/adu_histo.cpp +++ b/fpga/hls/adu_histo.cpp @@ -35,7 +35,10 @@ void adu_histo(STREAM_512 &data_in, axis_completion cmpl; s_axis_completion >> cmpl; while (!cmpl.last) { - for (int s = 0; s < sum+1; s++) { + ap_uint<9> sum_local = sum + 1; + if (cmpl.pedestal) + sum_local = 1; + for (int s = 0; s < sum_local; s++) { m_axis_completion << cmpl; for (int i = 0; i < RAW_MODULE_SIZE / (32 * 2); i++) { #pragma HLS PIPELINE II=2 @@ -51,7 +54,7 @@ void adu_histo(STREAM_512 &data_in, s_axis_completion >> cmpl; } for (int i = 0; i < ADU_HISTO_BIN_COUNT / 16; i++) { -#pragma HLS PIPELINE II=1 +#pragma HLS PIPELINE II=8 ap_uint<512> val = 0; for (int k = 0; k < 16; k++) { ap_uint<32> tmp = 0; diff --git a/fpga/hls/adu_histo_tb.cpp b/fpga/hls/adu_histo_tb.cpp new file mode 100644 index 00000000..90a90782 --- /dev/null +++ b/fpga/hls/adu_histo_tb.cpp @@ -0,0 +1,57 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#include "hls_jfjoch.h" + +int main() { + + size_t nframes = 4; + + int ret = 0; + + STREAM_512 input; + STREAM_512 output; + hls::stream> adu_result; + hls::stream compl_in; + hls::stream compl_out; + + input << packet_512_t{.user = 0}; + for (int f = 0; f < nframes; f++) { + compl_in << axis_completion{.frame_number = f, .module = 0, .last = 0}; + for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) + input << packet_512_t{.data = 0, .user = 0}; + } + + compl_in << axis_completion{.last = 1}; + input << packet_512_t{.user = 1}; + + adu_histo(input, output, adu_result, compl_in, compl_out); + + if (input.size() != 0) + ret = 1; + if (compl_in.size() != 0) + ret = 1; + + if (output.size() != nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) + ret = 1; + for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2; i++) + output.read(); + + if (adu_result.size() != nframes * (ADU_HISTO_BIN_COUNT / 16)) + ret = 1; + for (int i = 0; i < nframes * (ADU_HISTO_BIN_COUNT / 16); i++) + adu_result.read(); + + if (compl_out.size() != nframes + 1) + ret = 1; + for (int i = 0; i < nframes + 1; i++) + compl_out.read(); + + if (ret != 0) { + printf("Test failed !!!\n"); + ret = 1; + } else { + printf("Test passed !\n"); + } + + return ret; +} diff --git a/fpga/hls/axis_helpers.cpp b/fpga/hls/axis_helpers.cpp index c1879e3f..d693f8a6 100644 --- a/fpga/hls/axis_helpers.cpp +++ b/fpga/hls/axis_helpers.cpp @@ -2,23 +2,25 @@ #include "hls_jfjoch.h" -void axis_128_to_512(hls::stream> &data_in, +void axis_64_to_512(hls::stream> &data_in, hls::stream> &data_out) { #pragma HLS INTERFACE ap_ctrl_none port=return #pragma HLS INTERFACE axis register both port=data_in #pragma HLS INTERFACE axis register both port=data_out - ap_axiu<128,1,1,1> packet_128[4]; + ap_axiu<64,1,1,1> packet_64; + data_in >> packet_64; - data_in >> packet_128[0]; - - while (!packet_128[0].user) { -#pragma HLS PIPELINE II=4 - data_in >> packet_128[1]; - data_in >> packet_128[2]; - data_in >> packet_128[3]; - data_out << (packet_128[3].data,packet_128[2].data, packet_128[1].data,packet_128[0].data); - data_in >> packet_128[0]; + while (!packet_64.user) { +#pragma HLS PIPELINE II=8 + ap_uint<512> val = 0; + val(63,0) = packet_64.data; + for (int i = 1; i < 8; i++) { + data_in >> packet_64; + val(i * 64 + 63, i * 64) = packet_64.data; + } + data_out << val; + data_in >> packet_64; } } diff --git a/fpga/hls/datamover_model.h b/fpga/hls/datamover_model.h index 6a866e3a..b4778b8f 100644 --- a/fpga/hls/datamover_model.h +++ b/fpga/hls/datamover_model.h @@ -85,6 +85,10 @@ public: return descriptors_done; } + void ClearCompletedDescriptors() { + descriptors_done = 0; + } + hls::stream& GetCtrlStream() { return control; } hls::stream >& GetDataStream() { return data; } ~Datamover() { Stop(); } diff --git a/fpga/hls/ethernet.cpp b/fpga/hls/ethernet.cpp index 2cb7719a..cd3c0957 100644 --- a/fpga/hls/ethernet.cpp +++ b/fpga/hls/ethernet.cpp @@ -45,7 +45,8 @@ void ethernet(AXI_STREAM ð_in, state = FORWARD; dest = DEST_IP; internal_counter++; - } else if ((dest_mac == MAC_BROADCAST) && (ether_type == ETHER_ARP) && (packet_in.last)) { + } else if ((dest_mac == MAC_BROADCAST) && (ether_type == ETHER_ARP) && (packet_in.last) + && (packet_in.id == AXI_STREAM_ID_ENABLE_ARP_ICMP)) { state = FORWARD; dest = DEST_ARP; } diff --git a/fpga/hls/frame_summation.cpp b/fpga/hls/frame_summation.cpp index 5e0cedff..0cc66b7c 100644 --- a/fpga/hls/frame_summation.cpp +++ b/fpga/hls/frame_summation.cpp @@ -40,13 +40,14 @@ void frame_summation(STREAM_576 &data_in, STREAM_768 &data_out, data_out << packet_768_t{.data = packet_in.data, .user = 0, .last = 0}; data_in >> packet_in; - if (sum > 0) { - axis_completion cmpl, cmpl_out; - s_axis_completion >> cmpl; - while (!cmpl.last) { - later_frames: + + axis_completion cmpl, cmpl_out; + s_axis_completion >> cmpl; + while (!cmpl.last) { + later_frames: + if ((sum > 0) && (!cmpl.pedestal)) { cmpl_out = cmpl; - cmpl_out.frame_number = cmpl.frame_number / (sum+1); + cmpl_out.frame_number = cmpl.frame_number / (sum + 1); for (int s = 0; s <= sum; s++) { if (s > 0) { @@ -92,12 +93,7 @@ void frame_summation(STREAM_576 &data_in, STREAM_768 &data_out, } s_axis_completion >> cmpl; } - } - m_axis_completion << cmpl; - } else { - axis_completion cmpl; - s_axis_completion >> cmpl; - while (!cmpl.last) { + } else { m_axis_completion << cmpl; for (int i = 0; i < 16384; i++) { #pragma HLS PIPELINE II=1 @@ -109,7 +105,7 @@ void frame_summation(STREAM_576 &data_in, STREAM_768 &data_out, } s_axis_completion >> cmpl; } - m_axis_completion << cmpl; } + m_axis_completion << cmpl; data_out << packet_768_t{.data = packet_in.data, .user = 1, .last = 0}; -} +} \ No newline at end of file diff --git a/fpga/hls/frame_summation_tb.cpp b/fpga/hls/frame_summation_tb.cpp index ac4007ee..3387de6d 100644 --- a/fpga/hls/frame_summation_tb.cpp +++ b/fpga/hls/frame_summation_tb.cpp @@ -14,7 +14,7 @@ int test(size_t nframes, int16_t min, int16_t max) { std::vector input_frame(nframes * RAW_MODULE_SIZE); - STREAM_512 input; + STREAM_576 input; STREAM_768 output; hls::stream compl_in; hls::stream compl_out; @@ -38,14 +38,20 @@ int test(size_t nframes, int16_t min, int16_t max) { auto input_frame_512 = (ap_uint<512>*) input_frame.data(); - ap_uint<512> action_control = 0; + ap_uint<576> action_control = 0; ACT_REG_NSUMMATION(action_control) = nframes - 1; - input << packet_512_t { .data = action_control, .user = 0 }; - for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) - input << packet_512_t { .data = input_frame_512[i], .user = 0 }; + input << packet_576_t { .data = action_control, .user = 0 }; + for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) { + ap_int<16> tmp1[32]; + ap_int<18> tmp2[32]; + unpack32(input_frame_512[i], tmp1); + for (int j = 0; j < 32; j++) + tmp2[j] = tmp1[j]; + input << packet_576_t{.data = pack32(tmp2), .user = 0}; + } - input << packet_512_t { .user = 1 }; + input << packet_576_t { .user = 1 }; ap_uint<128> packet_mask; for (int i = 0; i < 128; i++) @@ -123,8 +129,8 @@ int main() { return 1; if (test(nframes, 0, INT16_MAX-1) != 0) return 1; - if (test(nframes, INT16_MIN, INT16_MAX) != 0) - return 1; + //if (test(nframes, INT16_MIN, INT16_MAX) != 0) + // return 1; return 0; } diff --git a/fpga/hls/frame_summation_tb_2.cpp b/fpga/hls/frame_summation_tb_2.cpp new file mode 100644 index 00000000..e90d227b --- /dev/null +++ b/fpga/hls/frame_summation_tb_2.cpp @@ -0,0 +1,98 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#include +#include "hls_jfjoch.h" + +int test(size_t nframes, int16_t min, int16_t max) { + + std::cout << "Nframes " << nframes << " min " << min << " max " << max << std::endl; + + std::vector input_frame(nframes * RAW_MODULE_SIZE); + + STREAM_576 input; + STREAM_768 output; + hls::stream compl_in; + hls::stream compl_out; + std::vector output_frame_ref_32(RAW_MODULE_SIZE, 0); + std::vector output_frame_32(RAW_MODULE_SIZE, 0); + + std::mt19937 g1(1387); + std::uniform_int_distribution dist(min, max); + + for (auto &i: input_frame) + i = dist(g1); + + for (int n = 0; n < nframes * RAW_MODULE_SIZE; n++) { + if ((input_frame[n] == INT16_MIN) || (output_frame_ref_32[n % RAW_MODULE_SIZE] == INT24_MIN)) + output_frame_ref_32[n % RAW_MODULE_SIZE] = INT24_MIN; + else if ((input_frame[n] == INT16_MAX) || (output_frame_ref_32[n % RAW_MODULE_SIZE] == INT24_MAX)) + output_frame_ref_32[n % RAW_MODULE_SIZE] = INT24_MAX; + else + output_frame_ref_32[n % RAW_MODULE_SIZE] += input_frame[n]; + } + + auto input_frame_512 = (ap_uint<512>*) input_frame.data(); + + ap_uint<576> action_control = 0; + ACT_REG_NSUMMATION(action_control) = 0; + + input << packet_576_t { .data = action_control, .user = 0 }; + for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) { + ap_int<16> tmp1[32]; + ap_int<18> tmp2[32]; + unpack32(input_frame_512[i], tmp1); + for (int j = 0; j < 32; j++) + tmp2[j] = tmp1[j]; + input << packet_576_t{.data = pack32(tmp2), .user = 0}; + } + + input << packet_576_t { .user = 1 }; + + ap_uint<128> packet_mask; + for (int i = 0; i < 128; i++) + packet_mask[i] = 1; + + for (int i = 0; i < nframes; i++) + compl_in << axis_completion{.packet_mask = packet_mask, .frame_number = 100 + i, .packet_count = 128, .last = 0}; + compl_in << axis_completion{.last = 1}; + + frame_summation(input, output, compl_in, compl_out); + + if (compl_in.size() != 0) { + std::cout << "compl_in should be empty: " << compl_in.size() << std::endl; + return 1; + } + + if (compl_out.size() != nframes + 1) { + std::cout << "compl_out should be size 2: " << compl_out.size() << std::endl; + return 1; + } + + if (input.size() != 0) + return 1; + + if (output.size() != nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) + return 1; + + output.read(); + for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 ; i++) + output.read(); + + output.read(); + + axis_completion cmpl; + for (int i = 0; i < nframes; i++) + compl_out.read(); + compl_out.read(); + + return 0; +} + +int main() { + size_t nframes = 12; + + if (test(nframes, 0, 5000) != 0) + return 1; + + return 0; +} diff --git a/fpga/hls/hls_bitshuffle.cpp b/fpga/hls/hls_bitshuffle.cpp deleted file mode 100644 index dfe9603e..00000000 --- a/fpga/hls/hls_bitshuffle.cpp +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#define AP_INT_MAX_W 2048 - -#include "hls_jfjoch.h" - -struct bshuf_packet { - ap_uint<512> data; - ap_uint<3> dest; - ap_uint<1> id; - ap_uint<1> last; - ap_uint<1> user; -}; - -typedef hls::stream BSHUF_STREAM; - -template ap_uint<16*N> bshuf16(const ap_uint<16*N> in) { -#pragma HLS INLINE - ap_uint<16*N> out; - for (int i = 0; i < N * 16; i++) - out[(i%16) * N + (i/16)] = in[i]; - return out; -} - -void bshuf256(STREAM_512 &data_in, BSHUF_STREAM &data_out) { -#pragma HLS INTERFACE ap_ctrl_none port=return -#pragma HLS INTERFACE axis port=data_in -#pragma HLS INTERFACE axis port=data_out - packet_512_t packet_in[4]; - bshuf_packet packet_out[4]; - - data_in >> packet_in[0]; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in[0].data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - packet_out[0].data = packet_in[0].data; - packet_out[0].id = packet_in[0].id; - packet_out[0].last = packet_in[0].last; - packet_out[0].user = packet_in[0].user; - packet_out[0].dest = 0; - - data_out << packet_out[0]; - - data_in >> packet_in[0]; - - ap_uint<2> counter = 0; - - if (shuffle) - bitshuffle_loop: - while (!packet_in[0].user) { -#pragma HLS PIPELINE II=4 - - for (int i = 1; i < 4; i++) - data_in >> packet_in[i]; - - ap_uint<2048> tmp_reg_in = (packet_in[3].data, packet_in[2].data, packet_in[1].data, packet_in[0].data); - ap_uint<2048> tmp_reg_out = bshuf16<128>(tmp_reg_in); - for (int i = 0; i < 4; i++) { - packet_out[i].data = tmp_reg_out(512*i+511,512*i); - packet_out[i].dest = counter; - packet_out[i].last = packet_in[i].last; - packet_out[i].user = packet_in[i].user; - packet_out[i].id = packet_in[i].id; - data_out << packet_out[i]; - } - counter++; - data_in >> packet_in[0]; - } - - forward_frames: - while (!packet_in[0].user) { -#pragma HLS PIPELINE II=1 - packet_out[0].dest = 0; - packet_out[0].data = packet_in[0].data; - packet_out[0].last = packet_in[0].last; - packet_out[0].user = packet_in[0].user; - packet_out[0].id = packet_in[0].id; - data_out << packet_out[0]; - data_in >> packet_in[0]; - } - packet_out[0].dest = 0; - packet_out[0].data = packet_in[0].data; - packet_out[0].last = packet_in[0].last; - packet_out[0].user = packet_in[0].user; - packet_out[0].id = packet_in[0].id; - data_out << packet_out[0]; - -} - -void bshuf1k_axis_split(BSHUF_STREAM &in, BSHUF_STREAM &out_0, BSHUF_STREAM &out_1, BSHUF_STREAM &out_2, BSHUF_STREAM &out_3) { - bshuf_packet packet_in; - in >> packet_in; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in.data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - out_0 << packet_in; - - in >> packet_in; - - forward_frames: - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - if (packet_in.dest == 1) - out_1 << packet_in; - else if (packet_in.dest == 2) - out_2 << packet_in; - else if (packet_in.dest == 3) - out_3 << packet_in; - else - out_0 << packet_in; - in >> packet_in; - } - out_0 << packet_in; -} - -void bshuf1k_axis_combine(BSHUF_STREAM &in_0, BSHUF_STREAM &in_1, BSHUF_STREAM &in_2, BSHUF_STREAM &in_3, - BSHUF_STREAM &out) { - bshuf_packet packet_in; - in_0 >> packet_in; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in.data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - out << packet_in; - - in_0 >> packet_in; - - if (shuffle) - bitshuffle_loop: - while (!packet_in.user) { -#pragma HLS PIPELINE II=4 - out << packet_in; - - in_1 >> packet_in; - out << packet_in; - - in_2 >> packet_in; - out << packet_in; - - in_3 >> packet_in; - out << packet_in; - - in_0 >> packet_in; - } - - forward_frames: - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - out << packet_in; - in_0 >> packet_in; - } - out << packet_in; -} - -void bshuf1k_shuffle(BSHUF_STREAM &in, BSHUF_STREAM &out) { - bshuf_packet packet_in[4]; - in >> packet_in[0]; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in[0].data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - out << packet_in[0]; - - in >> packet_in[0]; - - ap_uint<5> counter = 0; - - if (shuffle) - bitshuffle_loop: - while (!packet_in[0].user) { -#pragma HLS PIPELINE II=4 - - for (int i = 1; i < 4; i++) - in >> packet_in[i]; - - ap_uint<2048> tmp_reg_in = (packet_in[3].data, packet_in[2].data, packet_in[1].data, packet_in[0].data); - ap_uint<2048> tmp_reg_out; - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 4; j++) { - tmp_reg_out(512 * i + 128 * j + 127, 512*i + 128 * j) = packet_in[j].data(128*i+127, 128*i); - } - } - - for (int i = 0; i < 4; i++) { - packet_in[i].data = tmp_reg_out(512*i+511,512*i); - packet_in[i].dest = (counter / 4) % 4; - out << packet_in[i]; - } - counter += 1; - - in >> packet_in[0]; - } - - forward_frames: - while (!packet_in[0].user) { -#pragma HLS PIPELINE II=1 - packet_in[0].dest = 0; - out << packet_in[0]; - in >> packet_in[0]; - } - packet_in[0].dest = 0; - out << packet_in[0]; -} - -void bshuf4k_axis_split(BSHUF_STREAM &in, - BSHUF_STREAM &out_0, - BSHUF_STREAM &out_1, - BSHUF_STREAM &out_2, - BSHUF_STREAM &out_3) { - bshuf_packet packet_in; - in >> packet_in; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in.data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - out_0 << packet_in; - - in >> packet_in; - - forward_frames: - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - if (packet_in.dest == 1) - out_1 << packet_in; - else if (packet_in.dest == 2) - out_2 << packet_in; - else if (packet_in.dest == 3) - out_3 << packet_in; - else - out_0 << packet_in; - in >> packet_in; - } - out_0 << packet_in; -} - -void bshuf4k_axis_combine(BSHUF_STREAM &in_0, BSHUF_STREAM &in_1, BSHUF_STREAM &in_2, BSHUF_STREAM &in_3, - STREAM_512 &out) { - bshuf_packet packet_in; - packet_512_t packet_out; - - in_0 >> packet_in; - ap_uint<1> shuffle = (ACT_REG_MODE(packet_in.data) & MODE_BITSHUFFLE_FPGA) ? 1 : 0; - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - - in_0 >> packet_in; - - if (shuffle) - bitshuffle_loop: - while (!packet_in.user) { -#pragma HLS PIPELINE II=4 - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - - in_1 >> packet_in; - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - - in_2 >> packet_in; - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - - in_3 >> packet_in; - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - - in_0 >> packet_in; - } - - forward_frames: - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; - in_0 >> packet_in; - } - packet_out.data = packet_in.data; - packet_out.last = packet_in.last; - packet_out.user = packet_in.user; - packet_out.id = packet_in.id; - out << packet_out; -} - -void bitshuffle(STREAM_512 &data_in, STREAM_512 &data_out) { -#pragma HLS INTERFACE ap_ctrl_none port=return -#pragma HLS INTERFACE axis register both port=data_in -#pragma HLS INTERFACE axis register both port=data_out -#pragma HLS DATAFLOW - - BSHUF_STREAM fifo_0; -#pragma HLS STREAM variable=fifo_0 depth=3 -#pragma HLS RESOURCE variable=fifo_0 core=FIFO_SRL - BSHUF_STREAM fifo_1_0; -#pragma HLS STREAM variable=fifo_1_0 depth=8 -#pragma HLS RESOURCE variable=fifo_1_0 core=FIFO_SRL - BSHUF_STREAM fifo_1_1; -#pragma HLS STREAM variable=fifo_1_1 depth=8 -#pragma HLS RESOURCE variable=fifo_1_1 core=FIFO_SRL - BSHUF_STREAM fifo_1_2; -#pragma HLS STREAM variable=fifo_1_2 depth=8 -#pragma HLS RESOURCE variable=fifo_1_2 core=FIFO_SRL - BSHUF_STREAM fifo_1_3; -#pragma HLS STREAM variable=fifo_1_3 depth=8 -#pragma HLS RESOURCE variable=fifo_1_3 core=FIFO_SRL - BSHUF_STREAM fifo_2; -#pragma HLS STREAM variable=fifo_2 depth=3 -#pragma HLS RESOURCE variable=fifo_2 core=FIFO_SRL - BSHUF_STREAM fifo_3; -#pragma HLS STREAM variable=fifo_3 depth=3 -#pragma HLS RESOURCE variable=fifo_3 core=FIFO_SRL - BSHUF_STREAM fifo_4_0; -#pragma HLS STREAM variable=fifo_4_0 depth=32 - BSHUF_STREAM fifo_4_1; -#pragma HLS STREAM variable=fifo_4_1 depth=32 - BSHUF_STREAM fifo_4_2; -#pragma HLS STREAM variable=fifo_4_2 depth=32 - BSHUF_STREAM fifo_4_3; -#pragma HLS STREAM variable=fifo_4_3 depth=32 - -#ifndef JFJOCH_HLS_NOSYNTH - bshuf256(data_in, fifo_0); - bshuf1k_axis_split(fifo_0, fifo_1_0, fifo_1_1, fifo_1_2, fifo_1_3); - bshuf1k_axis_combine(fifo_1_0, fifo_1_1, fifo_1_2, fifo_1_3, fifo_2); - bshuf1k_shuffle(fifo_2, fifo_3); - bshuf4k_axis_split(fifo_3,fifo_4_0, fifo_4_1, fifo_4_2, fifo_4_3); - bshuf4k_axis_combine(fifo_4_0, fifo_4_1, fifo_4_2, fifo_4_3,data_out); -#else - std::vector bshuf_cores; - - bshuf_cores.emplace_back([&] { bshuf256(data_in, fifo_0);}); - bshuf_cores.emplace_back([&] {bshuf1k_axis_split(fifo_0, fifo_1_0, fifo_1_1, fifo_1_2, fifo_1_3);}); - bshuf_cores.emplace_back([&] {bshuf1k_axis_combine(fifo_1_0, fifo_1_1, fifo_1_2, fifo_1_3, fifo_2);}); - bshuf_cores.emplace_back([&] {bshuf1k_shuffle(fifo_2, fifo_3);}); - bshuf_cores.emplace_back([&] {bshuf4k_axis_split(fifo_3,fifo_4_0, fifo_4_1, fifo_4_2, fifo_4_3);}); - bshuf_cores.emplace_back([&] {bshuf4k_axis_combine(fifo_4_0, fifo_4_1, fifo_4_2, fifo_4_3,data_out);}); - - for (auto &i : bshuf_cores) - i.join(); -#endif - -} diff --git a/fpga/hls/hls_jfjoch.h b/fpga/hls/hls_jfjoch.h index e11c909c..0182cba7 100644 --- a/fpga/hls/hls_jfjoch.h +++ b/fpga/hls/hls_jfjoch.h @@ -23,6 +23,9 @@ #define INT24_MAX 8388607 #define INT24_MIN (-8388608) +#define AXI_STREAM_ID_ENABLE_ARP_ICMP 0 +#define AXI_STREAM_ID_DISABLE_ARP_ICMP 1 + typedef ap_ufixed<16,2, AP_RND_CONV> gainG0_t; typedef ap_ufixed<16,4, AP_RND_CONV> gainG1_t; typedef ap_ufixed<16,6, AP_RND_CONV> gainG2_t; @@ -87,16 +90,24 @@ struct axis_completion { ap_uint<5> module; ap_uint<1> last; ap_uint<1> ignore; + ap_uint<1> pedestal; +}; + +union float_uint32 { + float f; + uint32_t u; }; void setup_datamover (hls::stream &datamover_cmd_stream, uint64_t address, size_t bytes_to_write); -void stream_merge(AXI_STREAM &input_0, - AXI_STREAM &input_1, - AXI_STREAM &output); +void stream_merge(AXI_STREAM &input_100g, + AXI_STREAM &input_4x10g, + AXI_STREAM &input_frame_generator, + AXI_STREAM &data_out, + volatile ap_uint<2> &source); -void axis_128_to_512(hls::stream> &data_in, - hls::stream> &data_out); +void axis_64_to_512(hls::stream> &data_in, + hls::stream> &data_out); void axis_32_to_512(hls::stream> &data_in, hls::stream> &data_out); @@ -234,6 +245,23 @@ void adu_histo(STREAM_512 &data_in, hls::stream &s_axis_completion, hls::stream &m_axis_completion); +void pedestal(STREAM_512 &data_in, STREAM_512 &data_out, + hls::stream &s_axis_completion, + hls::stream &m_axis_completion, + ap_uint<256> *d_hbm_p0, + ap_uint<256> *d_hbm_p0_w, + ap_uint<256> *d_hbm_p1, + ap_uint<256> *d_hbm_p1_w, + ap_uint<256> *d_hbm_p2, + ap_uint<256> *d_hbm_p2_w, + ap_uint<256> *d_hbm_p3, + ap_uint<256> *d_hbm_p3_w, + ap_uint<256> *d_hbm_p4, + ap_uint<256> *d_hbm_p4_w, + ap_uint<256> *d_hbm_p5, + ap_uint<256> *d_hbm_p5_w, + ap_uint<32> hbm_size_bytes); + void jf_conversion(STREAM_512 &data_in, STREAM_576 &data_out, hls::stream &s_axis_completion, hls::stream &m_axis_completion, @@ -251,12 +279,9 @@ void spot_finder(STREAM_768 &data_in, volatile ap_int<16> &in_count_threshold, volatile ap_uint<8> &in_snr_threshold); -void bitshuffle(STREAM_512 &data_in, - STREAM_512 &data_out); - void integration(STREAM_768 &data_in, STREAM_768 &data_out, - hls::stream> &result_out, + hls::stream> &result_out, hls::stream &s_axis_completion, hls::stream &m_axis_completion, ap_uint<256> *d_hbm_p0, @@ -290,9 +315,9 @@ void host_writer(STREAM_512 &data_in, hls::stream &datamover_out_cmd, hls::stream > &s_axis_work_request, hls::stream > &m_axis_completion, + const uint64_t *dma_address_table, volatile uint64_t &packets_processed, - volatile ap_uint<1> &idle, - volatile ap_uint<8> &err_reg); + volatile ap_uint<1> &idle); void load_from_hbm(STREAM_512 &data_in, STREAM_512 &data_out, @@ -346,10 +371,7 @@ int load_calibration(ap_uint<256> *d_hbm_p0, ap_uint<8> destination, hls::stream &datamover_in_cmd, hls::stream > &host_memory_in, - uint64_t in_mem_location[(3 * 16 + 3) * MAX_MODULES_FPGA]) ; - -void add_multipixel(STREAM_512 &data_in, STREAM_512 &data_out); -void module_upside_down(STREAM_512 &data_in, STREAM_512 &data_out); + const uint64_t *handle_to_dma_address); void frame_summation(STREAM_576 &data_in, STREAM_768 &data_out, hls::stream &s_axis_completion, diff --git a/fpga/hls/host_writer.cpp b/fpga/hls/host_writer.cpp index 9df57e3f..f983cf72 100644 --- a/fpga/hls/host_writer.cpp +++ b/fpga/hls/host_writer.cpp @@ -1,76 +1,37 @@ // Copyright (2019-2023) Paul Scherrer Institute - #include "hls_jfjoch.h" +#include "../../common/DeviceOutput.h" -#ifndef __SYNTHESIS__ -#include -#endif +#define o(field) offsetof(ModuleStatistics, field) +#define sf(msg, field, s) msg(o(field)*8 + s - 1, o(field)*8) + +inline ap_uint<512> fill_module_info(axis_completion &cmpl) { + ap_uint<512> msg = 0; + sf(msg, frame_number, 64) = cmpl.frame_number; + sf(msg, timestamp, 64) = cmpl.timestamp; + sf(msg, bunchid, 64) = cmpl.bunchid; + sf(msg, exptime, 32) = cmpl.exptime; + sf(msg, debug, 32) = cmpl.debug; + sf(msg, pedestal, 32) = cmpl.pedestal; + sf(msg, packet_count, 32) = cmpl.packet_count; + sf(msg, module_number, 32) = cmpl.module; + return msg; +} inline void write_completion(hls::stream > &m_axis_completion, - const ap_uint<32> &handle, - const ap_uint<8> &module_number, - const ap_uint<64> &frame_num, - const ap_uint<128> &packet_mask, - const ap_uint<16> &packet_count, - const ap_uint<32> &debug, - const ap_uint<64> ×tamp, - const ap_uint<64> &bunchid, - const ap_uint<32> &exptime, - const ap_uint<32> &data_collection_id) { + const ap_uint<16> &handle, + const ap_uint<16> &data_collection_id) { #pragma HLS INLINE - ap_uint<1> all_packets_ok = packet_mask.and_reduce(); - ap_uint<1> any_packets_received = packet_mask.or_reduce(); - ap_uint<8> status = 0; - status[0] = all_packets_ok; - status[1] = any_packets_received; - ap_uint<128> tmp = (handle, packet_count, status, module_number, frame_num); - status[7] = tmp.xor_reduce(); // ensure completion has even parity - - if (handle != HANDLE_SKIP_FRAME) { - m_axis_completion << handle; - m_axis_completion << (packet_count, status, module_number); - m_axis_completion << frame_num(63, 32); - m_axis_completion << frame_num(31, 0); - - m_axis_completion << timestamp(63,32); - m_axis_completion << timestamp(31,0); - m_axis_completion << bunchid(63,32); - m_axis_completion << bunchid(31,0); - - m_axis_completion << exptime; - m_axis_completion << debug; - m_axis_completion << 0; - m_axis_completion << data_collection_id; - - m_axis_completion << packet_mask(127,96); - m_axis_completion << packet_mask( 95,64); - m_axis_completion << packet_mask( 63,32); - m_axis_completion << packet_mask( 31, 0); - } + m_axis_completion << (data_collection_id, handle); } -inline ap_uint<1> read_request(hls::stream > &s_axis_work_request, - ap_uint<32> &handle, - ap_uint<64> &address) { +void read_request(hls::stream > &s_axis_work_request, ap_uint<16> &handle) { #pragma HLS INLINE - ap_uint<32> tmp1, tmp2, tmp3, tmp4; - + ap_uint<32> tmp1; s_axis_work_request >> tmp1; - s_axis_work_request >> tmp2; - s_axis_work_request >> tmp3; - s_axis_work_request >> tmp4; - - handle = tmp1; - address = (tmp2, tmp3); - - ap_uint<128> tmp_all = (tmp1, tmp2, tmp3, tmp4); - - if (tmp_all.xor_reduce() != 0) - return 1; - else - return 0; + handle = tmp1(15, 0); } void host_writer(STREAM_512 &data_in, @@ -82,9 +43,9 @@ void host_writer(STREAM_512 &data_in, hls::stream &datamover_out_cmd, hls::stream > &s_axis_work_request, hls::stream > &m_axis_completion, + const uint64_t *dma_address_table, volatile uint64_t &packets_processed, - volatile ap_uint<1> &idle, - volatile ap_uint<8> &err_reg) { + volatile ap_uint<1> &idle) { #pragma HLS INTERFACE ap_ctrl_none port=return #pragma HLS INTERFACE register both axis port=data_in #pragma HLS INTERFACE register both axis port=adu_histo_in @@ -96,21 +57,16 @@ void host_writer(STREAM_512 &data_in, #pragma HLS INTERFACE register both axis port=m_axis_completion #pragma HLS INTERFACE register both axis port=s_axis_work_request #pragma HLS INTERFACE register ap_vld port=packets_processed -#pragma HLS INTERFACE register ap_vld port=err_reg #pragma HLS INTERFACE register ap_none port=idle - err_reg = 0; +#pragma HLS INTERFACE mode=m_axi port=dma_address_table bundle=dma_address_table depth=65536 offset=off \ + max_read_burst_length=2 max_write_burst_length=2 latency=10 num_write_outstanding=1 num_read_outstanding=1 + idle = 1; - ap_uint<32> req_handle; + ap_uint<16> req_handle; ap_uint<64> req_host_offset; - while (data_in.empty()) { -#pragma HLS PIPELINE II=4 - if (!s_axis_work_request.empty()) - read_request(s_axis_work_request, req_handle, req_host_offset); - } - packet_512_t packet; data_in >> packet; @@ -120,15 +76,15 @@ void host_writer(STREAM_512 &data_in, uint64_t internal_packets_processed = 0; packets_processed = internal_packets_processed; - write_completion(m_axis_completion, HANDLE_START, 0, 0, 0, 0, 0, 0, 0, 0, data_collection_id); + write_completion(m_axis_completion, HANDLE_START, data_collection_id); idle = 0; axis_completion cmpl; s_axis_completion >> cmpl; while (!cmpl.last) { - read_request(s_axis_work_request, req_handle, req_host_offset); - + read_request(s_axis_work_request, req_handle); + req_host_offset = dma_address_table[req_handle]; packet_512_t packet_out; packet_out.strb = UINT64_MAX; packet_out.keep = UINT64_MAX; @@ -142,58 +98,76 @@ void host_writer(STREAM_512 &data_in, pixel_depth = 4; else pixel_depth = 2; - setup_datamover(datamover_out_cmd, req_host_offset, RAW_MODULE_SIZE * pixel_depth); - for (int i = 0; i < RAW_MODULE_SIZE * pixel_depth / 64; i++) { -#pragma HLS PIPELINE II=1 - data_in >> packet; - packet_out.data = packet.data; - if (i == RAW_MODULE_SIZE * pixel_depth / 64 - 1) - packet_out.last = 1; - else - packet_out.last = 0; - host_memory_out << packet_out; - } + setup_datamover(datamover_out_cmd, req_host_offset, RAW_MODULE_SIZE * pixel_depth); + setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, spot_finding_result), + RAW_MODULE_SIZE * sizeof(uint16_t) / 16 + 64); + setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, integration_result), + (FPGA_INTEGRATION_BIN_COUNT / 8) * 64); + setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, adu_histogram), + ADU_HISTO_BIN_COUNT / 16 * 64); + setup_datamover(datamover_out_cmd, req_host_offset + offsetof(DeviceOutput, module_statistics), 2*64); + + if (pixel_depth == 2) { + for (int i = 0; i < RAW_MODULE_SIZE * 2 / 64; i++) { +#pragma HLS PIPELINE II=1 + data_in >> packet; + packet_out.data = packet.data; + if (i == RAW_MODULE_SIZE * pixel_depth / 64 - 1) + packet_out.last = 1; + else + packet_out.last = 0; + host_memory_out << packet_out; + } + } else { + for (int i = 0; i < RAW_MODULE_SIZE * 4 / 64; i++) { +#pragma HLS PIPELINE II=1 + data_in >> packet; + packet_out.data = packet.data; + if (i == RAW_MODULE_SIZE * pixel_depth / 64 - 1) + packet_out.last = 1; + else + packet_out.last = 0; + host_memory_out << packet_out; + } + } packet_out.last = 0; - setup_datamover(datamover_out_cmd, req_host_offset + 512 * 64 * 32 * sizeof(uint16_t), - RAW_MODULE_SIZE * sizeof(uint16_t) / 16 + 64 - + (FPGA_INTEGRATION_BIN_COUNT / 4) * 64 - + ADU_HISTO_BIN_COUNT / 16 * 64); - + // 1024 transfers x 512-bit for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / (64 * 16) + 1; i++) { #pragma HLS PIPELINE II=1 spot_finder_in >> packet_out.data; + packet_out.last = (i == RAW_MODULE_SIZE * sizeof(uint16_t) / (64 * 16) + 1 - 1); host_memory_out << packet_out; } - for (int i = 0; i < FPGA_INTEGRATION_BIN_COUNT / 4; i++) { + // 128 transfers x 512-bit + for (int i = 0; i < FPGA_INTEGRATION_BIN_COUNT / 8; i++) { #pragma HLS PIPELINE II=1 integration_in >> packet_out.data; + packet_out.last = (i == FPGA_INTEGRATION_BIN_COUNT / 8 - 1); host_memory_out << packet_out; } + // 128 transfers x 512-bit for (int i = 0; i < ADU_HISTO_BIN_COUNT / 16; i++) { #pragma HLS PIPELINE II=1 ap_uint<512> tmp; adu_histo_in >> packet_out.data; - if (i == ADU_HISTO_BIN_COUNT / 16 - 1) - packet_out.last = 1; + packet_out.last = (i == ADU_HISTO_BIN_COUNT / 16 - 1); host_memory_out << packet_out; } - write_completion(m_axis_completion, - req_handle, - cmpl.module, - cmpl.frame_number, - cmpl.packet_mask, - cmpl.packet_count, - cmpl.debug, - cmpl.timestamp, - cmpl.bunchid, - cmpl.exptime, - data_collection_id); + packet_out.data = fill_module_info(cmpl); + packet_out.last = 0; + host_memory_out << packet_out; + + packet_out.data = cmpl.packet_mask; + packet_out.last = 1; + host_memory_out << packet_out; + + write_completion(m_axis_completion, req_handle, data_collection_id); internal_packets_processed += cmpl.packet_count; packets_processed = internal_packets_processed; @@ -202,5 +176,5 @@ void host_writer(STREAM_512 &data_in, data_in >> packet; - write_completion(m_axis_completion, HANDLE_END, 0, 0, 0, 0, 0, 0, 0, 0, data_collection_id); + write_completion(m_axis_completion, HANDLE_END, data_collection_id); } diff --git a/fpga/hls/integration.cpp b/fpga/hls/integration.cpp index 9afff613..5dbda0ea 100644 --- a/fpga/hls/integration.cpp +++ b/fpga/hls/integration.cpp @@ -4,7 +4,7 @@ void integration(STREAM_768 &data_in, STREAM_768 &data_out, - hls::stream> &result_out, + hls::stream> &result_out, hls::stream &s_axis_completion, hls::stream &m_axis_completion, ap_uint<256> *d_hbm_p0, @@ -12,20 +12,19 @@ void integration(STREAM_768 &data_in, ap_uint<256> *d_hbm_p2, ap_uint<256> *d_hbm_p3, ap_uint<32> hbm_size_bytes) { -#pragma HLS INTERFACE ap_ctrl_none port=return #pragma HLS INTERFACE register both axis port=data_in #pragma HLS INTERFACE register both axis port=data_out #pragma HLS INTERFACE register both axis port=result_out #pragma HLS INTERFACE register both axis port=m_axis_completion #pragma HLS INTERFACE register both axis port=s_axis_completion #pragma HLS INTERFACE register ap_none port=hbm_size_bytes -#pragma HLS INTERFACE m_axi port=d_hbm_p0 bundle=d_hbm_p0 depth=512 offset=off \ +#pragma HLS INTERFACE m_axi port=d_hbm_p0 bundle=d_hbm_p0 depth=16384 offset=off \ max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8 -#pragma HLS INTERFACE m_axi port=d_hbm_p1 bundle=d_hbm_p1 depth=512 offset=off \ +#pragma HLS INTERFACE m_axi port=d_hbm_p1 bundle=d_hbm_p1 depth=16384 offset=off \ max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8 -#pragma HLS INTERFACE m_axi port=d_hbm_p2 bundle=d_hbm_p2 depth=512 offset=off \ +#pragma HLS INTERFACE m_axi port=d_hbm_p2 bundle=d_hbm_p2 depth=16384 offset=off \ max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8 -#pragma HLS INTERFACE m_axi port=d_hbm_p3 bundle=d_hbm_p3 depth=512 offset=off \ +#pragma HLS INTERFACE m_axi port=d_hbm_p3 bundle=d_hbm_p3 depth=16384 offset=off \ max_read_burst_length=16 max_write_burst_length=2 latency=120 num_write_outstanding=2 num_read_outstanding=8 ap_fixed<50,34, AP_RND_CONV> sum[64][FPGA_INTEGRATION_BIN_COUNT]; @@ -72,7 +71,7 @@ void integration(STREAM_768 &data_in, bins_0 = d_hbm_p0[offset_hbm_0 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k]; bins_1 = d_hbm_p1[offset_hbm_1 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k]; coeff_0 = d_hbm_p2[offset_hbm_2 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k]; - coeff_1 = d_hbm_p3[offset_hbm_2 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k]; + coeff_1 = d_hbm_p3[offset_hbm_3 + cmpl.module * RAW_MODULE_SIZE * sizeof(int16_t) / 64 + i * 2 + k]; unpack_2xhbm_to_32x16bit(bins_0, bins_1, in_bin); unpack_2xhbm_to_32x16bit(coeff_0, coeff_1, in_coeff); @@ -93,23 +92,26 @@ void integration(STREAM_768 &data_in, for (int i = 0; i < FPGA_INTEGRATION_BIN_COUNT; i++) { #pragma HLS PIPELINE II=1 - ap_axiu<128,1,1,1> res; + ap_axiu<64,1,1,1> res; ap_fixed<64, 40, AP_RND_CONV> main_sum = 0; - ap_int<22> main_count = 0; + ap_uint<22> main_count = 0; for (int j = 0; j < 64; j++) { main_sum += sum[j][i]; main_count += count[j][i]; sum[j][i] = 0; count[j][i] = 0; } - res.data(63, 0) = main_count; - for (int j = 0; j < 64; j++) { - res.data[64 + j] = main_sum[j]; - } + res.data(31, 0) = main_count; + + float_uint32 conv; + conv.f = main_sum.to_float(); + + res.data(63, 32) = conv.u; + res.dest = 0; res.id = 0; - res.keep = UINT32_MAX; - res.strb = UINT32_MAX; + res.keep = UINT8_MAX; + res.strb = UINT8_MAX; res.user = 0; res.last = ((i == FPGA_INTEGRATION_BIN_COUNT - 1) ? 1 : 0); result_out << res; @@ -117,7 +119,7 @@ void integration(STREAM_768 &data_in, } m_axis_completion << cmpl; - result_out << ap_axiu<128,1,1,1>{.user = 1}; + result_out << ap_axiu<64,1,1,1>{.user = 1}; data_in >> packet_in; data_out << packet_in; diff --git a/fpga/hls/integration_tb.cpp b/fpga/hls/integration_tb.cpp new file mode 100644 index 00000000..c8b24b8c --- /dev/null +++ b/fpga/hls/integration_tb.cpp @@ -0,0 +1,64 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#include "hls_jfjoch.h" +#include + +int main() { + + size_t nframes = 12; + + int ret = 0; + + STREAM_768 input; + STREAM_768 output; + hls::stream> integration_result; + hls::stream compl_in; + hls::stream compl_out; + + ap_uint<256> hbm_0[16384]; + ap_uint<256> hbm_1[16384]; + ap_uint<256> hbm_2[16384]; + ap_uint<256> hbm_3[16384]; + + input << packet_768_t{.user = 0}; + for (int f = 0; f < nframes; f++) { + compl_in << axis_completion{.frame_number = f, .module = 0, .last = 0}; + for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) + input << packet_768_t{.data = 0, .user = 0}; + } + + compl_in << axis_completion{.last = 1}; + input << packet_768_t{.user = 1}; + + integration(input, output, integration_result, compl_in, compl_out, hbm_0, hbm_1, hbm_2, hbm_3, 0); + + if (input.size() != 0) + ret = 1; + if (compl_in.size() != 0) + ret = 1; + + if (output.size() != nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) + ret = 1; + for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2; i++) + output.read(); + + if (integration_result.size() != nframes * FPGA_INTEGRATION_BIN_COUNT+ 1) + ret = 1; + for (int i = 0; i < nframes * FPGA_INTEGRATION_BIN_COUNT + 1; i++) + integration_result.read(); + + + if (compl_out.size() != nframes + 1) + ret = 1; + for (int i = 0; i < nframes + 1; i++) + compl_out.read(); + + if (ret != 0) { + printf("Test failed !!!\n"); + ret = 1; + } else { + printf("Test passed !\n"); + } + + return ret; +} diff --git a/fpga/hls/ipv4.cpp b/fpga/hls/ipv4.cpp index eab00a9e..e7840ae2 100644 --- a/fpga/hls/ipv4.cpp +++ b/fpga/hls/ipv4.cpp @@ -37,7 +37,7 @@ void ipv4(AXI_STREAM ð_in, if (ipv4_protocol == PROTOCOL_UDP) { state = FORWARD; dest = DEST_UDP; - } else if (ipv4_protocol == PROTOCOL_ICMP) { + } else if ((ipv4_protocol == PROTOCOL_ICMP) && (packet_in.id == AXI_STREAM_ID_ENABLE_ARP_ICMP)) { state = FORWARD; dest = DEST_ICMP; } diff --git a/fpga/hls/jf_conversion.cpp b/fpga/hls/jf_conversion.cpp index 0ab6839b..bb064df4 100644 --- a/fpga/hls/jf_conversion.cpp +++ b/fpga/hls/jf_conversion.cpp @@ -139,10 +139,12 @@ void jf_conversion(STREAM_512 &data_in, STREAM_576 &data_out, packet_out.user = packet_in.user; data_out << packet_out; + float_uint32 in_one_over_energy; + ap_uint<64> mode = ACT_REG_MODE(packet_in.data); ap_uint<1> conversion = (mode & MODE_CONV) ? 1 : 0; ap_uint<6> modules = ACT_REG_NMODULES(packet_in.data) + 1; - ap_uint<32> in_one_over_energy = ACT_REG_ONE_OVER_ENERGY(packet_in.data); + in_one_over_energy.u = ACT_REG_ONE_OVER_ENERGY(packet_in.data); ap_uint<5> storage_cells = ACT_REG_NSTORAGE_CELLS(packet_in.data); ap_uint<32> offset_hbm_0 = 0 * hbm_size_bytes / 32; @@ -158,9 +160,7 @@ void jf_conversion(STREAM_512 &data_in, STREAM_576 &data_out, ap_uint<32> offset_hbm_10 = 10 * hbm_size_bytes / 32; ap_uint<32> offset_hbm_11 = 11 * hbm_size_bytes / 32; - one_over_energy_t one_over_energy; - for (int i = 0; i < 32; i++) - one_over_energy[i] = in_one_over_energy[i]; + one_over_energy_t one_over_energy = in_one_over_energy.f; if (conversion) { axis_completion cmpl; diff --git a/fpga/hls/load_calibration.cpp b/fpga/hls/load_calibration.cpp index 7d31963c..698f3ec6 100644 --- a/fpga/hls/load_calibration.cpp +++ b/fpga/hls/load_calibration.cpp @@ -33,9 +33,8 @@ int load_calibration(ap_uint<256> *d_hbm_p0, ap_uint<8> destination, hls::stream &datamover_in_cmd, hls::stream > &host_memory_in, - uint64_t in_mem_location[(3 * 16 + 3) * MAX_MODULES_FPGA]) { + const uint64_t *dma_address_table) { #pragma HLS INTERFACE mode=s_axilite port=return -#pragma HLS INTERFACE mode=s_axilite port=in_mem_location #pragma HLS INTERFACE register both axis port=datamover_in_cmd #pragma HLS INTERFACE register both axis port=host_memory_in @@ -48,6 +47,8 @@ int load_calibration(ap_uint<256> *d_hbm_p0, max_read_burst_length=2 max_write_burst_length=16 latency=120 num_write_outstanding=8 num_read_outstanding=2 #pragma HLS INTERFACE mode=m_axi port=d_hbm_p1 bundle=d_hbm_p1 depth=512 offset=off \ max_read_burst_length=2 max_write_burst_length=16 latency=120 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE mode=m_axi port=dma_address_table bundle=dma_address_table depth=65536 offset=off \ + max_read_burst_length=2 max_write_burst_length=2 latency=10 num_write_outstanding=1 num_read_outstanding=1 if ((modules == 0) || (modules > MAX_MODULES_FPGA)) return 1; @@ -57,12 +58,12 @@ int load_calibration(ap_uint<256> *d_hbm_p0, return 1; for (int i = 0; i < 3 * (modules + storage_cells * modules); i++) { - if (in_mem_location[i] == 0) + if (dma_address_table[i] == 0) return 2; } for (int i = 0; i < 3 * (modules + storage_cells * modules); i++) - setup_datamover(datamover_in_cmd, in_mem_location[i], RAW_MODULE_SIZE * sizeof(int16_t)); + setup_datamover(datamover_in_cmd, dma_address_table[i], RAW_MODULE_SIZE * sizeof(int16_t)); for (int c = 0; c < 3; c++) { for (int m = 0; m < modules; m++) { @@ -86,12 +87,12 @@ int load_calibration(ap_uint<256> *d_hbm_p0, // load maps for (int i = 0; i < 2 * modules; i++) { - if (in_mem_location[i] == 0) + if (dma_address_table[i] == 0) return 2; } for (int i = 0; i < 2 * modules; i++) - setup_datamover(datamover_in_cmd, in_mem_location[i], RAW_MODULE_SIZE * sizeof(int16_t)); + setup_datamover(datamover_in_cmd, dma_address_table[i], RAW_MODULE_SIZE * sizeof(int16_t)); for (int m = 0; m < modules; m++) { #pragma HLS PIPELINE OFF @@ -108,12 +109,12 @@ int load_calibration(ap_uint<256> *d_hbm_p0, } } else if (destination == LOAD_CALIBRATION_DEST_FRAME_GEN) { for (int i = 0; i < modules; i++) { - if (in_mem_location[i] == 0) + if (dma_address_table[i] == 0) return 2; } for (int i = 0; i < modules; i++) - setup_datamover(datamover_in_cmd, in_mem_location[i], RAW_MODULE_SIZE * sizeof(int16_t)); + setup_datamover(datamover_in_cmd, dma_address_table[i], RAW_MODULE_SIZE * sizeof(int16_t)); for (int m = 0; m < modules; m++) { #pragma HLS PIPELINE OFF diff --git a/fpga/hls/module_upside_down.cpp b/fpga/hls/module_upside_down.cpp deleted file mode 100644 index 9a3b112e..00000000 --- a/fpga/hls/module_upside_down.cpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include "hls_jfjoch.h" - -#define upside(x) ((RAW_MODULE_LINES - 1 - i / 32) * 32 + i % 32) - -void module_upside_down(STREAM_512 &data_in, STREAM_512 &data_out) { -#pragma HLS INTERFACE axis register both port=data_in -#pragma HLS INTERFACE axis register both port=data_out - - ap_uint<512> memory_0[16384]; -#pragma HLS BIND_STORAGE variable=memory_0 type=ram_t2p impl=uram latency=3 - ap_uint<512> memory_1[16384]; -#pragma HLS BIND_STORAGE variable=memory_1 type=ram_t2p impl=uram latency=3 - - ap_uint<1> mem_0_full = 0; - ap_uint<1> mem_1_full = 0; - ap_uint<1> curr_mem = 0; - - packet_512_t packet_in, packet_out; - data_in >> packet_in; - ap_uint<1> reverse = ((ACT_REG_MODE(packet_in.data) & MODE_MODULE_UPSIDE_DOWN)) ? 1 : 0; - data_out << packet_in; - - data_in >> packet_in; - if (reverse) { - while (!packet_in.user) { - if (curr_mem == 0) { - for (int i = 0; i < 16384; i++) { -#pragma HLS PIPELINE II=1 - memory_0[i] = packet_in.data; - if (mem_1_full) { - packet_out.data = memory_1[upside(i)]; - data_out << packet_out; - } - data_in >> packet_in; - } - mem_0_full = 1; - mem_1_full = 0; - curr_mem = 1; - } else { - for (int i = 0; i < 16384; i++) { -#pragma HLS PIPELINE II=1 - memory_1[i] = packet_in.data; - if (mem_0_full) { - packet_out.data = memory_0[upside(i)]; - data_out << packet_out; - } - data_in >> packet_in; - } - mem_0_full = 0; - mem_1_full = 1; - curr_mem = 0; - } - } - drain_memory: - for (int i = 0; i < 16384; i++) { -#pragma HLS PIPELINE II=1 - if (mem_0_full) - packet_out.data = memory_0[upside(i)]; - else - packet_out.data = memory_1[upside(i)]; - data_out << packet_out; - } - - } else { - while (!packet_in.user) { -#pragma HLS PIPELINE II=1 - data_out << packet_in; - data_in >> packet_in; - } - } - data_out << packet_in; -} diff --git a/fpga/hls/module_upside_down_tb.cpp b/fpga/hls/module_upside_down_tb.cpp deleted file mode 100644 index f3b33ed7..00000000 --- a/fpga/hls/module_upside_down_tb.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include - -#include "hls_jfjoch.h" - -int main() { - - int ret = 0; - - STREAM_512 input; - STREAM_512 output; - - for (size_t nframes = 1; nframes < 5; nframes++) { - std::vector input_frame(nframes * RAW_MODULE_SIZE); - std::vector input_frame_transformed(nframes * RAW_MODULE_SIZE); - std::vector output_frame(nframes * RAW_MODULE_SIZE); - - std::mt19937 g1(1387); - std::uniform_int_distribution dist(0, 65535); - - for (int n = 0; n < nframes; n++) { - for (int line = 0; line < RAW_MODULE_LINES; line++) { - size_t line_transformed = RAW_MODULE_LINES - 1 - line; - for (int col = 0; col < RAW_MODULE_COLS; col++) { - uint16_t tmp = dist(g1); - input_frame[n * RAW_MODULE_SIZE + line * RAW_MODULE_COLS + col] = tmp; - input_frame_transformed[n * RAW_MODULE_SIZE + line_transformed * RAW_MODULE_COLS + col] = tmp; - } - } - } - - auto input_frame_512 = (ap_uint<512>*) input_frame.data(); - auto output_frame_512 = (ap_uint<512>*) output_frame.data(); - - ap_uint<512> action_control = 0; - ACT_REG_MODE(action_control) = MODE_MODULE_UPSIDE_DOWN; - - input << packet_512_t { .data = action_control, .user = 0 }; - for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) - input << packet_512_t { .data = input_frame_512[i], .user = 0 }; - - input << packet_512_t { .user = 1 }; - - module_upside_down(input, output); - - if (input.size() != 0) - ret = 1; - - if (output.size() != nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 + 2) - ret = 1; - - output.read(); - for (int i = 0; i < nframes * RAW_MODULE_SIZE * sizeof(uint16_t) / 64 ; i++) - output_frame_512[i] = output.read().data; - output.read(); - - if (output_frame != input_frame_transformed) { - std::cout << "Input and output don't match" << std::endl; - ret = 1; - } - } - if (ret != 0) { - printf("Test failed !!!\n"); - ret = 1; - } else { - printf("Test passed !\n"); - } - - return ret; -} diff --git a/fpga/hls/pedestal.cpp b/fpga/hls/pedestal.cpp new file mode 100644 index 00000000..13d4671a --- /dev/null +++ b/fpga/hls/pedestal.cpp @@ -0,0 +1,254 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#include "hls_jfjoch.h" + +#define PEDESTAL_G0_PRECISION 24 // 14-bit + 1-bit (fractional) + 7-bit (for 128 pixel window) + +typedef ap_ufixed pedestal_g0_t; +typedef ap_uint packed_pedestal_g0_t; + +void pack(packed_pedestal_g0_t& out, pedestal_g0_t in[32]) { +#pragma HLS INLINE + for (int i = 0; i < 32; i ++) { + for (int j = 0; j < PEDESTAL_G0_PRECISION; j ++) out[i * PEDESTAL_G0_PRECISION + j] = in[i][j]; + } +} + +inline void unpack(packed_pedestal_g0_t in, pedestal_g0_t out[32]) { +#pragma HLS INLINE + for (int i = 0; i < 32; i ++) { + for (int j = 0; j < PEDESTAL_G0_PRECISION; j ++) out[i][j] = in[i * PEDESTAL_G0_PRECISION + j]; + } +} + +ap_uint<512> pack_and_reduce(const packed_pedestal_g0_t &in) { +#pragma HLS INLINE + ap_uint<512> out; + pedestal_g0_t tmp_full[32]; + + unpack(in, tmp_full); + for (int i = 0; i < 32; i++) { + ap_ufixed<16,14, AP_RND_CONV> tmp1 = tmp_full[i]; // returns limited precision! + for (int j = 0; j < 16; j++) { + out[i*16+j] = tmp1[j]; + } + } + return out; +} + + +packed_pedestal_g0_t update_pedestal(ap_uint<512> data_in, + const packed_pedestal_g0_t packed_pedestal_in, + ap_uint<1> accumulate, ap_uint<8> mode) { +#pragma HLS INLINE + // Load current pedestal + pedestal_g0_t pedestal[32]; + unpack(packed_pedestal_in, pedestal); + + for (int j = 0; j < 32; j++) { + ap_uint<2> gain = data_in(16 * j + 15,16 * j + 14); + ap_uint<14> adu = data_in(16 * j + 13,16 * j); + + // Correct pedestal based on gain + if ((((gain == 0x0) && ((mode & MODE_PEDESTAL_G0) != 0)) || + ((gain == 0x1) && ((mode & MODE_PEDESTAL_G1) != 0)) || + ((gain == 0x3) && ((mode & MODE_PEDESTAL_G2) != 0))) + && (pedestal[j] < pedestal_g0_t(16383.25))) { + + if (accumulate) + pedestal[j] += pedestal_g0_t(adu) / PEDESTAL_WINDOW_SIZE; + else + pedestal[j] += ap_fixed(adu - pedestal[j]) / PEDESTAL_WINDOW_SIZE; + + } else + pedestal[j] = pedestal_g0_t(16383.5); + + } + packed_pedestal_g0_t packed_pedestal_out; + // Save pedestal + pack(packed_pedestal_out, pedestal); + return packed_pedestal_out; +} + +void pedestal(STREAM_512 &data_in, STREAM_512 &data_out, + hls::stream &s_axis_completion, + hls::stream &m_axis_completion, + ap_uint<256> *d_hbm_p0, + ap_uint<256> *d_hbm_p0_w, + ap_uint<256> *d_hbm_p1, + ap_uint<256> *d_hbm_p1_w, + ap_uint<256> *d_hbm_p2, + ap_uint<256> *d_hbm_p2_w, + ap_uint<256> *d_hbm_p3, + ap_uint<256> *d_hbm_p3_w, + ap_uint<256> *d_hbm_p4, + ap_uint<256> *d_hbm_p4_w, + ap_uint<256> *d_hbm_p5, + ap_uint<256> *d_hbm_p5_w, + ap_uint<32> hbm_size_bytes) { +#pragma HLS INTERFACE ap_ctrl_none port=return +#pragma HLS INTERFACE axis register both port=data_in +#pragma HLS INTERFACE axis register both port=data_out +#pragma HLS INTERFACE axis register both port=s_axis_completion +#pragma HLS INTERFACE axis register both port=m_axis_completion +#pragma HLS INTERFACE register ap_none port=hbm_size_bytes + +#pragma HLS INTERFACE m_axi port=d_hbm_p0 bundle=d_hbm_p0 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 +#pragma HLS INTERFACE m_axi port=d_hbm_p1 bundle=d_hbm_p1 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 +#pragma HLS INTERFACE m_axi port=d_hbm_p2 bundle=d_hbm_p2 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 +#pragma HLS INTERFACE m_axi port=d_hbm_p3 bundle=d_hbm_p3 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 +#pragma HLS INTERFACE m_axi port=d_hbm_p4 bundle=d_hbm_p4 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 +#pragma HLS INTERFACE m_axi port=d_hbm_p5 bundle=d_hbm_p5 depth=512 offset=off \ + max_read_burst_length=16 max_write_burst_length=2 latency=130 num_write_outstanding=2 num_read_outstanding=16 + +#pragma HLS INTERFACE m_axi port=d_hbm_p0_w bundle=d_hbm_p0_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE m_axi port=d_hbm_p1_w bundle=d_hbm_p1_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE m_axi port=d_hbm_p2_w bundle=d_hbm_p2_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE m_axi port=d_hbm_p3_w bundle=d_hbm_p3_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE m_axi port=d_hbm_p4_w bundle=d_hbm_p4_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 +#pragma HLS INTERFACE m_axi port=d_hbm_p5_w bundle=d_hbm_p5_w depth=512 offset=off \ + max_read_burst_length=2 max_write_burst_length=16 latency=130 num_write_outstanding=8 num_read_outstanding=2 + + uint64_t frame_count[MAX_MODULES_FPGA*16]; + for (int i = 0; i < MAX_MODULES_FPGA*16; i++) + frame_count[i] = 0; + + packet_512_t packet; + data_in >> packet; + ap_uint<8> conversion_mode = ACT_REG_MODE(packet.data); + ap_uint<8> nmodules = ACT_REG_NMODULES(packet.data) + 1; + ap_uint<5> nstoragecells = ACT_REG_NSTORAGE_CELLS(packet.data); + data_out << packet; + + ap_uint<1> pedestal_mode = ((conversion_mode & MODE_PEDESTAL_G0) != 0) + || ((conversion_mode & MODE_PEDESTAL_G1) != 0) + || ((conversion_mode & MODE_PEDESTAL_G2) != 0); + + ap_uint<32> offset_hbm_0 = 22 * hbm_size_bytes / 32; + ap_uint<32> offset_hbm_1 = 23 * hbm_size_bytes / 32; + ap_uint<32> offset_hbm_2 = 24 * hbm_size_bytes / 32; + ap_uint<32> offset_hbm_3 = 25 * hbm_size_bytes / 32; + ap_uint<32> offset_hbm_4 = 26 * hbm_size_bytes / 32; + ap_uint<32> offset_hbm_5 = 27 * hbm_size_bytes / 32; + + if (pedestal_mode) { + clean_hbm: + for (int i = 0; i < nmodules * nstoragecells * RAW_MODULE_SIZE * sizeof(uint16_t) / 128; i++) { +#pragma HLS PIPELINE II=1 + d_hbm_p0_w[offset_hbm_0 + i] = 0; + d_hbm_p1_w[offset_hbm_1 + i] = 0; + d_hbm_p2_w[offset_hbm_2 + i] = 0; + d_hbm_p3_w[offset_hbm_3 + i] = 0; + d_hbm_p4_w[offset_hbm_4 + i] = 0; + d_hbm_p5_w[offset_hbm_5 + i] = 0; + } + } + + axis_completion cmpl; + s_axis_completion >> cmpl; + while (!cmpl.last) { + m_axis_completion << cmpl; + if ((cmpl.packet_count == 128) && pedestal_mode) { + ap_int<5> storage_cell = ((nstoragecells > 1) ? ap_int<5>((cmpl.frame_number % nstoragecells)) : ap_int<5>(0)); + + ap_uint<1> accumulate = (frame_count[nmodules * storage_cell + cmpl.module] < PEDESTAL_WINDOW_SIZE); + frame_count[nmodules * storage_cell + cmpl.module]++; + + size_t offset_local = (storage_cell * nmodules + cmpl.module) * (RAW_MODULE_SIZE * sizeof(uint16_t) / 128); + process_data: + for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 128; i++) { +#pragma HLS PIPELINE II=2 + data_in >> packet; + data_out << packet; + + packed_pedestal_g0_t packed_pedestal_in_0, packed_pedestal_out_0; + + packed_pedestal_in_0(255, 0) = d_hbm_p0[offset_hbm_0 + offset_local + i]; + packed_pedestal_in_0(511, 256) = d_hbm_p1[offset_hbm_1 + offset_local + i]; + packed_pedestal_in_0(767, 512) = d_hbm_p2[offset_hbm_2 + offset_local + i]; + + packed_pedestal_out_0 = update_pedestal(packet.data, packed_pedestal_in_0, accumulate, conversion_mode); + + d_hbm_p0_w[offset_hbm_0 + offset_local + i] = packed_pedestal_out_0(255, 0); + d_hbm_p1_w[offset_hbm_1 + offset_local + i] = packed_pedestal_out_0(511, 256); + d_hbm_p2_w[offset_hbm_2 + offset_local + i] = packed_pedestal_out_0(767, 512); + + data_in >> packet; + data_out << packet; + + packed_pedestal_in_0(255, 0) = d_hbm_p3[offset_hbm_3 + offset_local + i]; + packed_pedestal_in_0(511, 256) = d_hbm_p4[offset_hbm_4 + offset_local + i]; + packed_pedestal_in_0(767, 512) = d_hbm_p5[offset_hbm_5 + offset_local + i]; + + packed_pedestal_out_0 = update_pedestal(packet.data, packed_pedestal_in_0, accumulate, conversion_mode); + + d_hbm_p3_w[offset_hbm_3 + offset_local + i] = packed_pedestal_out_0(255, 0); + d_hbm_p4_w[offset_hbm_4 + offset_local + i] = packed_pedestal_out_0(511, 256); + d_hbm_p5_w[offset_hbm_5 + offset_local + i] = packed_pedestal_out_0(767, 512); + } + } else { + for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) { +#pragma HLS PIPELINE II=1 + data_in >> packet; + data_out << packet; + } + } + s_axis_completion >> cmpl; + } + + if (pedestal_mode) { + for (int s = 0; s < nstoragecells; s++) { + for (int m = 0; m < nmodules; m++) { + if (frame_count[s * nmodules + m] > 0) { + axis_completion cmpl_pedestal; + cmpl_pedestal.last = 0; + cmpl_pedestal.frame_number = s; + cmpl_pedestal.module = m; + cmpl_pedestal.packet_mask(63, 0) = UINT64_MAX; + cmpl_pedestal.packet_mask(127, 64) = UINT64_MAX; + cmpl_pedestal.packet_count = frame_count[s * nmodules + m]; + cmpl_pedestal.pedestal = 1; + + m_axis_completion << cmpl_pedestal; + save_frames: + for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 128; i++) { +#pragma HLS PIPELINE II=2 + packet_512_t packet_out; + packed_pedestal_g0_t packed_pedestal_in_0; + size_t offset_local = (s * nmodules + m) * (RAW_MODULE_SIZE * sizeof(uint16_t) / 128) + i; + packed_pedestal_in_0(255, 0) = d_hbm_p0[offset_hbm_0 + offset_local]; + packed_pedestal_in_0(511, 256) = d_hbm_p1[offset_hbm_1 + offset_local]; + packed_pedestal_in_0(767, 512) = d_hbm_p2[offset_hbm_2 + offset_local]; + packet_out.data = pack_and_reduce(packed_pedestal_in_0); + packet_out.user = 0; + packet_out.last = 0; + data_out << packet_out; + + packed_pedestal_in_0(255, 0) = d_hbm_p3[offset_hbm_3 + offset_local]; + packed_pedestal_in_0(511, 256) = d_hbm_p4[offset_hbm_4 + offset_local]; + packed_pedestal_in_0(767, 512) = d_hbm_p5[offset_hbm_5 + offset_local]; + packet_out.data = pack_and_reduce(packed_pedestal_in_0); + packet_out.user = 0; + packet_out.last = (i == RAW_MODULE_SIZE * sizeof(uint16_t) / 128 - 1); + data_out << packet_out; + } + } + } + } + } + + m_axis_completion << cmpl; + + data_in >> packet; + data_out << packet; +} diff --git a/fpga/hls/save_to_hbm.cpp b/fpga/hls/save_to_hbm.cpp index ff3ba96f..ff0c04c0 100644 --- a/fpga/hls/save_to_hbm.cpp +++ b/fpga/hls/save_to_hbm.cpp @@ -66,7 +66,7 @@ void save_to_hbm(STREAM_512 &data_in, cmpl[id].ignore = 0; cmpl[id].packet_mask = ap_uint<128>(1) << eth_packet; cmpl[id].packet_count = 1; - + cmpl[id].pedestal = 0; curr_handle = s_axis_free_handles.read(); cmpl[id].handle = curr_handle; } else { diff --git a/fpga/hls/spot_finder.cpp b/fpga/hls/spot_finder.cpp index 54d994ef..b5f96f46 100644 --- a/fpga/hls/spot_finder.cpp +++ b/fpga/hls/spot_finder.cpp @@ -277,6 +277,5 @@ void spot_finder(STREAM_768 &data_in, in_snr_threshold);}); for (auto &i : spot_finder_cores) i.join(); - #endif } diff --git a/fpga/hls/stream_merge.cpp b/fpga/hls/stream_merge.cpp index 688b90b3..802c2f88 100644 --- a/fpga/hls/stream_merge.cpp +++ b/fpga/hls/stream_merge.cpp @@ -2,44 +2,43 @@ #include "hls_jfjoch.h" -void stream_merge(AXI_STREAM &input_0, - AXI_STREAM &input_1, - AXI_STREAM &output) { +void stream_merge(AXI_STREAM &input_100g, + AXI_STREAM &input_4x10g, + AXI_STREAM &input_frame_generator, + AXI_STREAM &data_out, + volatile ap_uint<2> &source) { #pragma HLS INTERFACE ap_ctrl_none port=return -#pragma HLS INTERFACE axis register both port=input_0 -#pragma HLS INTERFACE axis register both port=input_1 -#pragma HLS INTERFACE axis register both port=output - +#pragma HLS INTERFACE axis register both port=input_100g +#pragma HLS INTERFACE axis register both port=input_4x10g +#pragma HLS INTERFACE axis register both port=input_frame_generator +#pragma HLS INTERFACE axis register both port=data_out +#pragma HLS INTERFACE ap_none register port=source #pragma HLS PIPELINE II=1 style=flp - enum state {ARBITRATE, FORWARD}; - static state state = ARBITRATE; - static ap_uint<1> select_input = 0; - packet_512_t packet_in; -#pragma HLS RESET variable=state - switch (state) { - case ARBITRATE: - if (input_0.read_nb(packet_in)) { - select_input = 0; - if (!packet_in.last) - state = FORWARD; - output.write(packet_in); - } else if (input_1.read_nb(packet_in)) { - select_input = 1; - if (!packet_in.last) - state = FORWARD; - output.write(packet_in); - } - break; - case FORWARD: - if (select_input == 0) { - input_0.read(packet_in); - } else - input_1.read(packet_in); - output.write(packet_in); + static ap_uint<2> data_source_local = STREAM_MERGE_SRC_NONE; + ap_uint<2> data_source_local_read = source; + if (data_source_local != data_source_local_read) { + data_source_local = data_source_local_read; + data_out.write(packet_512_t{.user = 1, .last = 1}); + } else { + packet_512_t packet_in_frame_gen, packet_in_eth_100g, packet_in_eth_4x10g; - if (packet_in.last) - state = ARBITRATE; - break; + bool rcv_frame_gen = input_frame_generator.read_nb(packet_in_frame_gen); + bool rcv_eth_100g = input_100g.read_nb(packet_in_eth_100g); + bool rcv_eth_4x10g = input_4x10g.read_nb(packet_in_eth_4x10g); + + if (data_source_local == STREAM_MERGE_SRC_100G) { + packet_in_eth_100g.id = AXI_STREAM_ID_ENABLE_ARP_ICMP; + if (rcv_eth_100g) + data_out.write(packet_in_eth_100g); + } else if (data_source_local == STREAM_MERGE_SRC_4x10G) { + packet_in_eth_4x10g.id = AXI_STREAM_ID_DISABLE_ARP_ICMP; + if (rcv_eth_4x10g) + data_out.write(packet_in_eth_4x10g); + } else if (data_source_local == STREAM_MERGE_SRC_FRAME_GEN) { + packet_in_frame_gen.id = AXI_STREAM_ID_DISABLE_ARP_ICMP; + if (rcv_frame_gen) + data_out.write(packet_in_frame_gen); + } } } diff --git a/fpga/host_library/JungfraujochDevice.cpp b/fpga/host_library/JungfraujochDevice.cpp index 483e6b18..e83b4ada 100644 --- a/fpga/host_library/JungfraujochDevice.cpp +++ b/fpga/host_library/JungfraujochDevice.cpp @@ -11,9 +11,14 @@ JungfraujochDevice::JungfraujochDevice(const std::string &device_name, bool write_access) { fd = open(device_name.c_str(), write_access ? O_RDWR : O_RDONLY); - if (fd == -1) - throw JFJochException(JFJochExceptionCategory::PCIeError, "Cannot open device"); - + if (fd == -1) { + if (errno == EBUSY) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Device already opened"); + else if (errno == EACCES) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Permission denied"); + else + throw JFJochException(JFJochExceptionCategory::PCIeError, "Cannot open device"); + } auto action_type = GetDataCollectionStatus().action_type; if (action_type != ACTION_TYPE) throw JFJochException(JFJochExceptionCategory::PCIeError, "Wrong device type"); @@ -209,3 +214,18 @@ void JungfraujochDevice::SetSpotFinderParameters(const SpotFinderParameters& par if (ioctl(fd, IOCTL_JFJOCH_SPOT_FINDER_PAR, ¶ms) != 0) throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed settings spot finder parameters", errno); } + +uint32_t JungfraujochDevice::GetDataSource() { + uint32_t tmp; + if (ioctl(fd, IOCTL_JFJOCH_GET_DATA_SOURCE, &tmp) != 0) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed getting data source", errno); + return tmp; +} + +void JungfraujochDevice::SetDataSource(uint32_t id) { + if (id >= 4) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Data source is 2-bit variable", errno); + + if (ioctl(fd, IOCTL_JFJOCH_SET_DATA_SOURCE, &id) != 0) + throw JFJochException(JFJochExceptionCategory::PCIeError, "Failed setting data source", errno); +} \ No newline at end of file diff --git a/fpga/host_library/JungfraujochDevice.h b/fpga/host_library/JungfraujochDevice.h index bd9b38bb..d9e9eebf 100644 --- a/fpga/host_library/JungfraujochDevice.h +++ b/fpga/host_library/JungfraujochDevice.h @@ -94,6 +94,10 @@ public: // Allocate id kernel buffer ( id must be less than GetBufferCount() ) // buffer has to be unmapped using munmap uint16_t *MapKernelBuffer(uint32_t id); + + // Select data source on the FPGA - which network connector is used + void SetDataSource(uint32_t id); + uint32_t GetDataSource(); }; #endif //JUNGFRAUJOCH_JUNGFRAUJOCHDEVICE_H diff --git a/fpga/host_library/jfjoch_pcie_status.cpp b/fpga/host_library/jfjoch_pcie_status.cpp index 97b39800..a71ecdd1 100644 --- a/fpga/host_library/jfjoch_pcie_status.cpp +++ b/fpga/host_library/jfjoch_pcie_status.cpp @@ -26,16 +26,15 @@ void FIFO_check(uint32_t fifo_register) { fifo_map["UDP"] = FIFO_check(fifo_register, 6, 7); fifo_map["Conversion input (data)"] = FIFO_check(fifo_register, 0, 1); fifo_map["Conversion input (cmd)"] = FIFO_check(fifo_register, 2, 3); - fifo_map["Work Request"] = FIFO_check(fifo_register, 12, 13); - fifo_map["Work Completion"] = FIFO_check(fifo_register, 14, 15); + fifo_map["Processing FIFO (rad int -> spot finding)"] = FIFO_check(fifo_register, 4, 5); fifo_map["Writer input (data)"] = FIFO_check(fifo_register, 16, 17); fifo_map["Writer input (cmd)"] = FIFO_check(fifo_register, 18, 19); fifo_map["C2H (data)"] = FIFO_check(fifo_register, 8, 9); fifo_map["C2H (cmd)"] = FIFO_check(fifo_register, 10, 11); fifo_map["H2C (data)"] = FIFO_check(fifo_register, 20, 21); fifo_map["H2C (cmd)"] = FIFO_check(fifo_register, 22, 23); - fifo_map["HBM handles"] = FIFO_check(fifo_register, 28, 29); - fifo_map["HBM completion"] = FIFO_check(fifo_register, 30, 31); + fifo_map["HBM handles"] = FIFO_check(fifo_register, 30, 31); + fifo_map["HBM completion"] = FIFO_check(fifo_register, 28, 29); std::cout << "FIFO status" << std::endl; for (const auto &[x,y]: fifo_map) @@ -70,11 +69,16 @@ int main(int argc, char **argv) { std::cout << "HBM temperature #1 " << fpga_env_data.hbm_1_temp_C << std::endl; std::cout << "HBM size (MiB) " << fpga_status.hbm_size_bytes / static_cast(1024 * 1024) << std::endl; std::cout << "Data collection idle " << device.IsIdle() << std::endl; - std::cout << "Host writer idle " << (fpga_status.ctrl_reg & (1<<4)) << std::endl; - std::cout << "Data collection cancel " << (fpga_status.ctrl_reg & (1<<2)) << std::endl; - + std::cout << "Host writer idle " << ((fpga_status.ctrl_reg & (1<<4)) ? 1 : 0) << std::endl; + std::cout << "Data collection cancel " << ((fpga_status.ctrl_reg & (1<<2)) ? 1 : 0) << std::endl; + std::cout << "Mailbox status " << std::hex << fpga_env_data.mailbox_status_reg << std::dec << std::endl; + std::cout << "Mailbox error " << std::hex << fpga_env_data.mailbox_err_reg << std::dec << std::endl; + std::cout << "Mailbox interrupt status " << std::hex << fpga_env_data.mailbox_interrupt_status << std::dec << std::endl; + std::cout << "Mailbox interrupt 0 " << ((fpga_status.ctrl_reg & (1<<16)) ? 1 : 0) << std::endl; + std::cout << "Mailbox interrupt 1 " << ((fpga_status.ctrl_reg & (1<<17)) ? 1 : 0) << std::endl; + std::cout << "Data source " << device.GetDataSource() << std::endl; std::cout << "Full status register " << std::bitset<32>(fpga_status.ctrl_reg) << std::endl; - + std::cout << "Work completion FIFO avail " << fpga_env_data.work_compl_fifo_avail << std::endl; std::cout << "AXI-STREAM beats encountered " << std::endl; std::cout << " - before HBM cache " << fpga_status.pipeline_beats_hbm << std::endl; std::cout << " - before processing " << fpga_status.pipeline_beats_proc << std::endl; @@ -106,6 +110,14 @@ int main(int argc, char **argv) { std::cout << " - err. len. " << fpga_status.udp_err_len << std::endl; std::cout << std::endl; + std::cout << "User interrupts mask: " << std::bitset<32>(fpga_env_data.pcie_user_interrupt_mask) + << " pending: " << std::bitset<32>(fpga_env_data.pcie_user_interrupt_pending) + << " request: " << std::bitset<32>(fpga_env_data.pcie_user_interrupt_request) << std::endl; + + std::cout << "DMA interrupts mask: " << std::bitset<32>(fpga_env_data.pcie_dma_interrupt_mask) + << " pending: " << std::bitset<32>(fpga_env_data.pcie_dma_interrupt_pending) + << " request: " << std::bitset<32>(fpga_env_data.pcie_dma_interrupt_request) << std::endl; + std::cout << "H2C descriptors:" << fpga_env_data.pcie_h2c_descriptors << " beats: " << fpga_env_data.pcie_h2c_beats << " status:" << fpga_env_data.pcie_h2c_status << std::endl; std::cout << "C2H descriptors:" << fpga_env_data.pcie_c2h_descriptors << " beats: " << fpga_env_data.pcie_c2h_beats @@ -117,4 +129,4 @@ int main(int argc, char **argv) { std::cout << std::endl; } -} \ No newline at end of file +} diff --git a/fpga/pcie_driver/ActionConfig.h b/fpga/pcie_driver/ActionConfig.h index b67c0bde..8fd6c87a 100644 --- a/fpga/pcie_driver/ActionConfig.h +++ b/fpga/pcie_driver/ActionConfig.h @@ -54,6 +54,8 @@ struct DataCollectionStatus { struct DeviceStatus { uint32_t mailbox_status_reg; uint32_t mailbox_err_reg; + uint32_t mailbox_interrupt_status; + uint32_t fpga_temp_C; uint32_t fpga_pcie_12V_I_mA; @@ -69,9 +71,17 @@ struct DeviceStatus { uint32_t pcie_h2c_status; uint32_t pcie_c2h_status; + uint32_t pcie_user_interrupt_mask; + uint32_t pcie_user_interrupt_request; + uint32_t pcie_user_interrupt_pending; + uint32_t pcie_dma_interrupt_mask; + uint32_t pcie_dma_interrupt_request; + uint32_t pcie_dma_interrupt_pending; + uint32_t hbm_0_temp_C; uint32_t hbm_1_temp_C; + uint32_t work_compl_fifo_avail; bool ethernet_aligned; }; diff --git a/fpga/pcie_driver/CMakeLists.txt b/fpga/pcie_driver/CMakeLists.txt index 3fa0d939..d37abd2a 100644 --- a/fpga/pcie_driver/CMakeLists.txt +++ b/fpga/pcie_driver/CMakeLists.txt @@ -3,7 +3,8 @@ ADD_LIBRARY(jfjoch_kernel_module STATIC jfjoch_ioctl.h jfjoch_drv.c jfjoch_drv.h jfjoch_ioctl.h jfjoch_pcie_setup.c jfjoch_memory.c jfjoch_ioctl.c jfjoch_function.c jfjoch_miscdev.c -jfjoch_sysfs.c) +jfjoch_sysfs.c + jfjoch_int.c) EXECUTE_PROCESS(COMMAND uname -r OUTPUT_VARIABLE KERNEL_RELEASE diff --git a/fpga/pcie_driver/Makefile b/fpga/pcie_driver/Makefile index a54c9ce7..44eeed72 100644 --- a/fpga/pcie_driver/Makefile +++ b/fpga/pcie_driver/Makefile @@ -1,7 +1,7 @@ obj-m += jfjoch.o cflags-m=-std=c99 -jfjoch-y := jfjoch_drv.o jfjoch_ioctl.o jfjoch_memory.o jfjoch_pcie_setup.o jfjoch_function.o jfjoch_miscdev.o jfjoch_sysfs.o +jfjoch-y := jfjoch_drv.o jfjoch_ioctl.o jfjoch_memory.o jfjoch_pcie_setup.o jfjoch_function.o jfjoch_miscdev.o jfjoch_sysfs.o jfjoch_int.o all: make -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) modules diff --git a/fpga/pcie_driver/README.md b/fpga/pcie_driver/README.md index 05677d0a..14e70595 100644 --- a/fpga/pcie_driver/README.md +++ b/fpga/pcie_driver/README.md @@ -18,13 +18,64 @@ After installing the kernel driver, it should be possible to insert it into the modprobe jfjoch ``` +## DKMS +To avoid problems with updating the kernel, it is possible to use DKMS to autobuild Jungfraujoch kernel +module, when new kernel is installed. This first requires to install DKMS - for RHEL it is available via EPEL repository: +``` +sudo dnf install dkms +``` +Then use script provided in the driver directory to copy driver code to DKMS directory: +``` +./install_dkms.sh +``` +If upgrading the driver, please first remove current driver from DKMS system: +``` +dkms remove jfjoch -v 0.1 --all +``` + +## Driver parameters +Currently, there is one driver parameter `nbuffers`, that defines count of exchange buffers (see below). +This can be adjusted in the modprobe operation, for example: +``` +modprobe jfjoch nbuffers=1024 +``` + +## Exchange buffers +The parameter defines number of buffers used to exchange data between card and host application. +Each buffer can hold one detector module (1024x512) in 16-bit or 32-bit mode + associated processing results and metadata. +These buffers are used by both card-to-host and host-to-card operations. + +Buffers use special allocation, as they are continuous in physical address space, which helps the FPGA card to transfer all +data associated with detector module in two DMA transfers (one data, one metadata). +Useful buffer size is a bit more than 2 MiB, but given that kernel allocates physical memory in power of two, **4 MiB** is safe number for one buffer size. +Buffer can be mapped into user space, but performing `mmap` system call on the `/dev/jfjoch` character device. + +Buffer count can be adjusted by setting `nbuffers` parameter. There are two considerations for setting optimal value: +1. For card-to-host transfers, minimal value is roughly +` * `, +this way each thread can have enough data for operation. Default thread count for Jungfraujoch receiver is 64. +2. For host-to-card transfers, full detector calibration has to fit into memory and one buffer accommodates one calibration set for one module. +So minimal count is ` * (3 + 3 * )`. + +Based on both rules, optimal number is 512 buffers (2 GiB), though this can be adjusted for particular system and configuration. + +## Known problems +To avoid inconsistent behavior, this driver won't load if release number differs between the kernel driver and FPGA card. +See relevant section in the [FPGA readme](../README.md#card-release-number). + +It is recommended to use the same git commit hash for building the design and user application. + ## CMake file While CMake file is present in the driver directory, it is only for the purpose of proper detection of the files in CLion IDE. It is not made for actual compilation of the kernel driver and should not be used for that purpose. -## Character device -For each FPGA device a character device is created called /dev/jfjoch<number of device>. +## Character device access +For each FPGA device a character device is created called `/dev/jfjoch`. When device is opened two operations are possible: -mmap() to map device buffers +mmap() to map exchange buffers ioctl() to communicate with the cards Interfacing should be done through the JungfraujochDevice class in `fpga/host_library` directory. + +## Sysfs access +Certain performance counters can be read through sysfs mechanism in the kernel. +One needs to `cat` files in `/sys/class/misc/jfjoch`. \ No newline at end of file diff --git a/fpga/pcie_driver/dkms.conf b/fpga/pcie_driver/dkms.conf new file mode 100644 index 00000000..f1e04bfb --- /dev/null +++ b/fpga/pcie_driver/dkms.conf @@ -0,0 +1,10 @@ +PACKAGE_NAME=jfjoch +PACKAGE_VERSION=0.1 + +DEST_MODULE_LOCATION=/extra +BUILT_MODULE_NAME=jfjoch +BUILT_MODULE_LOCATION=src/ + +MAKE="'make' -C src/ all" +CLEAN="'make' -C src/ clean" +AUTOINSTALL="yes" \ No newline at end of file diff --git a/fpga/pcie_driver/install_dkms.sh b/fpga/pcie_driver/install_dkms.sh new file mode 100755 index 00000000..0d4379b3 --- /dev/null +++ b/fpga/pcie_driver/install_dkms.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Copyright (2019-2023) Paul Scherrer Institute +VERSION=0.1 + +mkdir -p /usr/src/jfjoch-${VERSION}/src +cp dkms.conf /usr/src/jfjoch-${VERSION} +cp *.c *.h Makefile ../../common/Definitions.h /usr/src/jfjoch-0.1/src +sed -i "s,../../common/Definitions.h,Definitions.h," /usr/src/jfjoch-0.1/src/jfjoch_drv.h + +dkms add -m jfjoch -v ${VERSION} +dkms install -m jfjoch -v ${VERSION} diff --git a/fpga/pcie_driver/jfjoch_drv.c b/fpga/pcie_driver/jfjoch_drv.c index 9a38e29e..2a217855 100644 --- a/fpga/pcie_driver/jfjoch_drv.c +++ b/fpga/pcie_driver/jfjoch_drv.c @@ -31,22 +31,27 @@ static int jfjoch_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id struct jfjoch_drvdata *drvdata; struct DataCollectionStatus status; - if ((nbuffer < 64) || (nbuffer > 65536)) { - dev_err(dev, "nbuffer parameter must be in range 64-65536\n"); - err = -EINVAL; - goto ret_with_err; + if ((nbuffer < 64) || (nbuffer > MAX_FPGA_BUFFER)) { + dev_err(dev, "nbuffer parameter must be in range 64-%d\n", MAX_FPGA_BUFFER); + return -EINVAL; } // Setup private data structure drvdata = kzalloc(sizeof(struct jfjoch_drvdata), GFP_KERNEL); if (drvdata == NULL) { dev_err(dev, "Failed to allocate driver data\n"); - err = -ENOMEM; - goto ret_with_err; + return -ENOMEM; } + pci_set_drvdata(pdev, drvdata); drvdata->pdev = pdev; + INIT_KFIFO(drvdata->work_compl); + mutex_init(&drvdata->work_compl_read_mutex); + init_waitqueue_head(&drvdata->work_compl_wait_queue); + + spin_lock_init(&drvdata->work_request_submit_spinlock); + // Enable device err = pci_enable_device(pdev); if (err) { @@ -71,10 +76,14 @@ static int jfjoch_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id if (err) goto deregister_misc; + err = jfjoch_setup_pcie_interrupt(pdev); + if (err) + goto clear_master; + // Allocate memory err = jfjoch_alloc_phys_continous_buf(pdev); if (err) - goto deregister_misc; + goto free_interrupts; jfjoch_get_status(drvdata, &status); drvdata->git_sha1 = status.git_sha1; @@ -86,13 +95,19 @@ static int jfjoch_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id err = jfjoch_register_sysfs(drvdata); if (err) - goto unregister_sysfs; + goto free_phys_continous_buf; dev_info(drvdata->miscdev.this_device, "Jungfraujoch FPGA loaded with FW build: %x", drvdata->git_sha1); return 0; - unregister_sysfs: - jfjoch_unregister_sysfs(drvdata); + free_phys_continous_buf: + jfjoch_free_phys_continous_buf(pdev); + + free_interrupts: + jfjoch_free_pcie_interrupt(pdev); + + clear_master: + pci_clear_master(pdev); deregister_misc: misc_deregister(&drvdata->miscdev); @@ -107,7 +122,6 @@ static int jfjoch_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id dealloc_private_data: kfree(drvdata); - ret_with_err: return err; } @@ -126,6 +140,7 @@ static void jfjoch_pci_remove(struct pci_dev *pdev) { return; pci_clear_master(pdev); + jfjoch_free_pcie_interrupt(pdev); jfjoch_unregister_sysfs(drvdata); jfjoch_unregister_misc_dev(pdev); diff --git a/fpga/pcie_driver/jfjoch_drv.h b/fpga/pcie_driver/jfjoch_drv.h index 807c893c..b01a1591 100644 --- a/fpga/pcie_driver/jfjoch_drv.h +++ b/fpga/pcie_driver/jfjoch_drv.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "../../common/Definitions.h" #include "ActionConfig.h" @@ -33,6 +35,12 @@ #define XDMA_CTRL_POLL_MODE_WB (1UL << 26) #define XDMA_CTRL_STM_MODE_WB (1UL << 27) +#define ADDR_XDMA_USER_IE_MASK 0x2004 +#define ADDR_XDMA_DMA_IE_MASK 0x2010 +#define ADDR_XDMA_USER_IR 0x2040 +#define ADDR_XDMA_DMA_IR 0x2044 +#define ADDR_XDMA_USER_IP 0x2048 +#define ADDR_XDMA_DMA_IP 0x204C // Offset for BAR #0 for action configuration #define ACTION_CONFIG_OFFSET (0x010000) @@ -42,6 +50,7 @@ #define CMAC_OFFSET (0x020000) #define PCIE_OFFSET (0x090000) #define FRAME_GEN_OFFSET (0x080000) +#define ADDRESS_TABLE_OFFSET (0x200000) // Action config #define ADDR_CTRL_REGISTER 0x0000 @@ -78,6 +87,7 @@ #define ADDR_ONE_OVER_ENERGY 0x0214 #define ADDR_NFRAMES 0x0218 #define ADDR_NSTORAGE_CELLS 0x021C +#define ADDR_DATA_SOURCE 0x0224 #define ADDR_SPOT_FINDER_THRESHOLD 0x0100 #define ADDR_SPOT_FINDER_SNR 0x0104 @@ -85,9 +95,14 @@ #define ADDR_MAILBOX_WRDATA 0x00 #define ADDR_MAILBOX_RDDATA 0x08 #define ADDR_MAILBOX_STATUS 0x10 +#define ADDR_MAILBOX_ERR 0x14 #define ADDR_MAILBOX_SIT 0x18 #define ADDR_MAILBOX_RIT 0x1C +#define ADDR_MAILBOX_IS 0x20 +#define ADDR_MAILBOX_IE 0x24 +#define ADDR_MAILBOX_CTRL 0x2C +#define MAILBOX_INTERRUPT_RIT (1 << 1) #define MAILBOX_EMPTY (1 << 0) #define MAILBOX_FULL (1 << 1) #define MAILBOX_STA (1 << 2) @@ -100,7 +115,6 @@ #define ADDR_LOAD_CALIBRATION_MOD (LOAD_CALIBRATION_OFFSET | 0x000018) #define ADDR_LOAD_CALIBRATION_SC (LOAD_CALIBRATION_OFFSET | 0x000020) #define ADDR_LOAD_CALIBRATION_DEST (LOAD_CALIBRATION_OFFSET | 0x000028) -#define ADDR_LOAD_CALIBRATION_MEM (LOAD_CALIBRATION_OFFSET | 0x004000) #define ADDR_FRAME_GEN_CTRL (FRAME_GEN_OFFSET | 0x000000) #define ADDR_FRAME_GEN_RETURN (FRAME_GEN_OFFSET | 0x000010) @@ -128,6 +142,10 @@ #define ADDR_CMS_HBM_TEMP2_INS_REG 0x0282BC // in C #define ADDR_CMS_HOST_STATUS2_REG 0x02830C +#define MAX_FPGA_BUFFER 2048 + +extern int nbuffer; + struct jfjoch_buf { dma_addr_t dma_address; void *kernel_address; @@ -142,6 +160,15 @@ struct jfjoch_drvdata { char name[256]; u32 max_modules; u32 git_sha1; + + spinlock_t file_write_open_count_spinlock; + int file_write_open_count; + + spinlock_t work_request_submit_spinlock; + + DECLARE_KFIFO(work_compl, u32, MAX_FPGA_BUFFER); + struct mutex work_compl_read_mutex; + wait_queue_head_t work_compl_wait_queue; }; int jfjoch_register_misc_dev(struct pci_dev *pdev); @@ -153,6 +180,8 @@ void jfjoch_free_phys_continous_buf(struct pci_dev *pdev); int jfjoch_setup_pcie_for_dma(struct pci_dev *pdev); int jfjoch_map_cfg_bar(struct pci_dev *pdev); int jfjoch_check_version(struct pci_dev *pdev); +int jfjoch_setup_pcie_interrupt(struct pci_dev *pdev); +void jfjoch_free_pcie_interrupt(struct pci_dev *pdev); long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int jfjoch_cdev_mmap(struct file *file, struct vm_area_struct *vma); @@ -181,6 +210,8 @@ int jfjoch_load_integration_map(struct jfjoch_drvdata *drvdata, struct DataColle int jfjoch_run_frame_gen(struct jfjoch_drvdata *drvdata, struct FrameGeneratorConfig *config); u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata); void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params); +u32 jfjoch_get_data_source(struct jfjoch_drvdata *drvdata); +void jfjoch_set_data_source(struct jfjoch_drvdata *drvdata, const u32 *val); u64 jfjoch_read_mac_addr(struct jfjoch_drvdata *drvdata); @@ -206,6 +237,4 @@ static const struct file_operations jfjoch_cdev_fileops = { .llseek = no_llseek }; -extern int nbuffer; - #endif //JUNGFRAUJOCH_JFJOCH_DRV_H diff --git a/fpga/pcie_driver/jfjoch_function.c b/fpga/pcie_driver/jfjoch_function.c index 330afef2..aba8cf36 100644 --- a/fpga/pcie_driver/jfjoch_function.c +++ b/fpga/pcie_driver/jfjoch_function.c @@ -5,12 +5,6 @@ #include #include -DEFINE_MUTEX(set_config_mutex); -DEFINE_MUTEX(set_mac_mutex); - -DEFINE_MUTEX(send_wr_mutex); -DEFINE_MUTEX(read_wc_mutex); - void jfjoch_write_register(struct jfjoch_drvdata *drvdata, uint32_t addr, uint32_t val) { iowrite32(val, drvdata->bar0 + addr); } @@ -28,11 +22,6 @@ void jfjoch_start(struct jfjoch_drvdata *drvdata) { // Run C2H iowrite32(JFJOCH_DMA_SETTINGS, drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x04); - // Set Mailbox FIFOs, so interrupt threshold is 16 messages - // => This way it ensures that one can always execute read/write operation on the FIFO - iowrite32(255-16, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_SIT); - iowrite32(15 , drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_RIT); - // Write Start value to action config register iowrite32(0x1, drvdata->bar0 + ACTION_CONFIG_OFFSET); } @@ -51,58 +40,44 @@ void jfjoch_cancel(struct jfjoch_drvdata *drvdata) { } int jfjoch_send_wr(struct jfjoch_drvdata *drvdata, u32 handle) { - u32 sta; - dma_addr_t addr = 0; - u32 parity = 0; + u32 full; - if (handle != 0xFFFFFFFFLU) { - if (handle >= nbuffer) - return -EFAULT; - addr = drvdata->bufs[handle].dma_address; - } + if (handle >= nbuffer) + return -EFAULT; - parity = (hweight32(handle) + hweight64(addr)) % 2; + spin_lock(&drvdata->work_request_submit_spinlock); - mutex_lock(&send_wr_mutex); - - sta = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_STA; - if (!sta) { - mutex_unlock(&send_wr_mutex); + full = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_FULL; + if (full) { + spin_unlock(&drvdata->work_request_submit_spinlock); return -EAGAIN; } iowrite32(handle, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA); - iowrite32(PCI_DMA_H(addr), drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA); - iowrite32(PCI_DMA_L(addr), drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA); - iowrite32(parity, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_WRDATA); - mutex_unlock(&send_wr_mutex); + spin_unlock(&drvdata->work_request_submit_spinlock); return 0; } int jfjoch_read_wc(struct jfjoch_drvdata *drvdata, u32 *output) { - u32 rta; - int i; - mutex_lock(&read_wc_mutex); + int ret, tmp; + mutex_lock(&drvdata->work_compl_read_mutex); - rta = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_RTA; - if (!rta) { - mutex_unlock(&read_wc_mutex); + ret = wait_event_interruptible_timeout(drvdata->work_compl_wait_queue, !kfifo_is_empty(&drvdata->work_compl), HZ); + if (ret >= 0) + tmp = kfifo_get(&drvdata->work_compl, output); + + mutex_unlock(&drvdata->work_compl_read_mutex); + if (ret < 0) + return ret; + else if (tmp == 0) return -EAGAIN; - } - - for (i = 0; i < 16; i++) - output[i] = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_RDDATA); - - mutex_unlock(&read_wc_mutex); - - return 0; + else + return 0; } void jfjoch_set_config(struct jfjoch_drvdata *drvdata, const struct DataCollectionConfig *config) { - mutex_lock(&set_config_mutex); memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_NMODULES, config, sizeof(struct DataCollectionConfig)); - mutex_unlock(&set_config_mutex); } void jfjoch_get_config(struct jfjoch_drvdata *drvdata, struct DataCollectionConfig *config) { @@ -114,9 +89,7 @@ void jfjoch_get_status(struct jfjoch_drvdata *drvdata, struct DataCollectionStat } void jfjoch_set_mac_addr(struct jfjoch_drvdata *drvdata, u64 *mac_addr) { - mutex_lock(&set_mac_mutex); memcpy_toio((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_MAC_ADDR_LO, mac_addr, sizeof(uint64_t)); - mutex_unlock(&set_mac_mutex); } void jfjoch_get_mac_addr(struct jfjoch_drvdata *drvdata, u64 *mac_addr) { @@ -124,16 +97,13 @@ void jfjoch_get_mac_addr(struct jfjoch_drvdata *drvdata, u64 *mac_addr) { } void jfjoch_set_ipv4_addr(struct jfjoch_drvdata *drvdata, const u32 *addr) { - mutex_lock(&set_mac_mutex); iowrite32(*addr, (drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_IPV4_ADDR); - mutex_unlock(&set_mac_mutex); } void jfjoch_get_ipv4_addr(struct jfjoch_drvdata *drvdata, u32 *addr) { *addr = ioread32((drvdata->bar0) + ACTION_CONFIG_OFFSET + ADDR_IPV4_ADDR); } - u64 jfjoch_read_mac_addr(struct jfjoch_drvdata *drvdata) { struct device *const dev = &drvdata->pdev->dev; u32 tmp, host_msg_offset, msg_len, opcode; @@ -273,8 +243,9 @@ void jfjoch_setup_network(struct jfjoch_drvdata *drvdata) { } void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *env_params) { - env_params->mailbox_status_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + 0x10); - env_params->mailbox_err_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + 0x14); + env_params->mailbox_status_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS); + env_params->mailbox_err_reg = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_ERR); + env_params->mailbox_interrupt_status = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_IS); env_params->fpga_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_FPGA_TEMP_INS_REG); @@ -291,12 +262,21 @@ void jfjoch_get_env_data(struct jfjoch_drvdata *drvdata, struct DeviceStatus *en env_params->pcie_c2h_beats = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0xCC); env_params->pcie_c2h_status = ioread32(drvdata->bar0 + PCIE_OFFSET + (1<<12) + 0x40); + env_params->pcie_user_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IE_MASK); + env_params->pcie_dma_interrupt_mask = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IE_MASK); + env_params->pcie_user_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IP); + env_params->pcie_dma_interrupt_pending = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IP); + env_params->pcie_user_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IR); + env_params->pcie_dma_interrupt_request = ioread32(drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_DMA_IR); + env_params->hbm_0_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP1_INS_REG); env_params->hbm_1_temp_C = ioread32(drvdata->bar0 + CMS_OFFSET + ADDR_CMS_HBM_TEMP2_INS_REG); // Somehow it is better to ask twice env_params->ethernet_aligned = ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204) & 0x2; env_params->ethernet_aligned = ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204) & 0x2; + + env_params->work_compl_fifo_avail = kfifo_avail(&drvdata->work_compl); } void jfjoch_clr_net_counters(struct jfjoch_drvdata *drvdata) { @@ -320,12 +300,6 @@ int jfjoch_load_from_host(struct jfjoch_drvdata *drvdata, struct DataCollectionC return -EINVAL; } - for (i = 0; i < cell_count; i++) { - u64 addr = drvdata->bufs[i].dma_address; - iowrite32(PCI_DMA_L(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + i * 2 * 4); - iowrite32(PCI_DMA_H(addr), drvdata->bar0 + ADDR_LOAD_CALIBRATION_MEM + (i * 2 + 1) * 4); - } - // Start DMA // Clear counters and RUN H2C iowrite32((1 << 1), drvdata->bar0 + PCIE_OFFSET + (0<<12) + 0xC0); @@ -410,4 +384,12 @@ u32 jfjoch_get_c2h_descriptors(struct jfjoch_drvdata *drvdata) { void jfjoch_set_spot_finder_parameters(struct jfjoch_drvdata *drvdata, struct SpotFinderParameters *params) { iowrite32(params->count_threshold, drvdata->bar0 + ADDR_SPOT_FINDER_THRESHOLD); iowrite32(params->snr_threshold, drvdata->bar0 + ADDR_SPOT_FINDER_SNR); -} \ No newline at end of file +} + +u32 jfjoch_get_data_source(struct jfjoch_drvdata *drvdata) { + return ioread32(drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_DATA_SOURCE); +} + +void jfjoch_set_data_source(struct jfjoch_drvdata *drvdata, const u32 *val) { + iowrite32(*val, drvdata->bar0 + ACTION_CONFIG_OFFSET + ADDR_DATA_SOURCE); +} diff --git a/fpga/pcie_driver/jfjoch_int.c b/fpga/pcie_driver/jfjoch_int.c new file mode 100644 index 00000000..09e171fe --- /dev/null +++ b/fpga/pcie_driver/jfjoch_int.c @@ -0,0 +1,53 @@ +// Copyright (2019-2023) Paul Scherrer Institute + +#include "jfjoch_drv.h" + +static irqreturn_t jfjoch_irq_compl(int irq, void *data) { + struct jfjoch_drvdata *drvdata = data; + u32 empty; + // For interrupt handler, it is guaranteed that same interrupt is masked + // Kfifo doesn't need synchronization for writers if single writer + // No need of spinlock + + // acknowledge interrupt + iowrite32(MAILBOX_INTERRUPT_RIT, drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_IS); + + empty = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_EMPTY; + while (!empty) { + kfifo_put(&drvdata->work_compl, ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_RDDATA)); + empty = ioread32(drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_STATUS) & MAILBOX_EMPTY; + } + wake_up_interruptible_all(&drvdata->work_compl_wait_queue); + return IRQ_HANDLED; +} + +int jfjoch_setup_pcie_interrupt(struct pci_dev *pdev) { + int ret; + struct jfjoch_drvdata *drvdata = pci_get_drvdata(pdev); + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX); + if (ret != 1) + return -ENOSPC; + ret = pci_request_irq(pdev, 0, jfjoch_irq_compl, NULL, drvdata, "Jungraujoch work completion ready" ); + if (ret) { + pci_free_irq_vectors(pdev); + return ret; + } + + // Set Mailbox receive interrupt threshold to 0 (i.e. interrupt if receive queue elements > 0) + iowrite32(0 , drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_RIT); + // Set Mailbox receive interrupt to enable + iowrite32(MAILBOX_INTERRUPT_RIT , drvdata->bar0 + MAILBOX_OFFSET + ADDR_MAILBOX_IE); + + // Enable user interrupt lines 1 and 2 in XDMA core + iowrite32((1<<1), drvdata->bar0 + PCIE_OFFSET + ADDR_XDMA_USER_IE_MASK); + + return 0; +} + +void jfjoch_free_pcie_interrupt(struct pci_dev *pdev) { + struct jfjoch_drvdata *drvdata = pci_get_drvdata(pdev); + + pci_free_irq(pdev, 0, drvdata); + pci_free_irq_vectors(pdev); +} \ No newline at end of file diff --git a/fpga/pcie_driver/jfjoch_ioctl.c b/fpga/pcie_driver/jfjoch_ioctl.c index 02a92076..83140819 100644 --- a/fpga/pcie_driver/jfjoch_ioctl.c +++ b/fpga/pcie_driver/jfjoch_ioctl.c @@ -77,7 +77,7 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { err = jfjoch_read_wc(drvdata,exchange); if (err) return err; - if (copy_to_user((char *) arg, exchange, 16 * sizeof(u32)) != 0) + if (copy_to_user((char *) arg, exchange, sizeof(u32)) != 0) return -EFAULT; return 0; case IOCTL_JFJOCH_BUF_COUNT: @@ -103,6 +103,18 @@ long jfjoch_cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { return -EFAULT; jfjoch_set_ipv4_addr(drvdata, (u32 *) exchange); return 0; + case IOCTL_JFJOCH_SET_DATA_SOURCE: + if (!(file->f_mode & FMODE_WRITE)) + return -EACCES; + if (copy_from_user(exchange, (char *) arg, sizeof(u32)) != 0) + return -EFAULT; + jfjoch_set_data_source(drvdata, (u32 *) exchange); + return 0; + case IOCTL_JFJOCH_GET_DATA_SOURCE: + exchange[0] = jfjoch_get_data_source(drvdata); + if (copy_to_user((char *) arg, exchange, sizeof(u32)) != 0) + return -EFAULT; + return 0; case IOCTL_JFJOCH_GET_IPV4: jfjoch_get_ipv4_addr(drvdata, (u32 *) exchange); if (copy_to_user((char *) arg, exchange, sizeof(u32)) != 0) diff --git a/fpga/pcie_driver/jfjoch_ioctl.h b/fpga/pcie_driver/jfjoch_ioctl.h index f86bb2d4..f10bf707 100644 --- a/fpga/pcie_driver/jfjoch_ioctl.h +++ b/fpga/pcie_driver/jfjoch_ioctl.h @@ -18,7 +18,7 @@ #define IOCTL_JFJOCH_READ_CONFIG _IOR(IOCTL_JFJOCH_MAGIC, 2, struct DataCollectionConfig) #define IOCTL_JFJOCH_SET_CONFIG _IOW(IOCTL_JFJOCH_MAGIC, 3, struct DataCollectionConfig) #define IOCTL_JFJOCH_CANCEL _IO (IOCTL_JFJOCH_MAGIC, 4) -#define IOCTL_JFJOCH_READ_WC_MBOX _IOR(IOCTL_JFJOCH_MAGIC, 5, uint32_t[16]) +#define IOCTL_JFJOCH_READ_WC_MBOX _IOR(IOCTL_JFJOCH_MAGIC, 5, uint32_t) #define IOCTL_JFJOCH_SEND_WR _IOW(IOCTL_JFJOCH_MAGIC, 6, uint32_t) #define IOCTL_JFJOCH_BUF_COUNT _IOR(IOCTL_JFJOCH_MAGIC, 7, uint32_t) #define IOCTL_JFJOCH_SET_MAC _IOW(IOCTL_JFJOCH_MAGIC, 8, uint64_t) @@ -40,5 +40,7 @@ #define IOCTL_JFJOCH_WRITE_REGISTER _IOW(IOCTL_JFJOCH_MAGIC, 25, struct RegisterConfig ) #define IOCTL_JFJOCH_READ_REGISTER _IOWR(IOCTL_JFJOCH_MAGIC, 26, struct RegisterConfig ) #define IOCTL_JFJOCH_SPOT_FINDER_PAR _IOW(IOCTL_JFJOCH_MAGIC, 27, struct SpotFinderParameters) +#define IOCTL_JFJOCH_SET_DATA_SOURCE _IOW(IOCTL_JFJOCH_MAGIC, 28, uint32_t) +#define IOCTL_JFJOCH_GET_DATA_SOURCE _IOR(IOCTL_JFJOCH_MAGIC, 28, uint32_t) #endif //JUNGFRAUJOCH_JFJOCH_IOCTL_H diff --git a/fpga/pcie_driver/jfjoch_memory.c b/fpga/pcie_driver/jfjoch_memory.c index 7b6192dd..82240f30 100644 --- a/fpga/pcie_driver/jfjoch_memory.c +++ b/fpga/pcie_driver/jfjoch_memory.c @@ -3,9 +3,11 @@ #include "jfjoch_drv.h" #include +#include +#include -int nbuffer = 8192; -module_param(nbuffer, int, 0600); +int nbuffer = 512; +module_param(nbuffer, int, 0); int jfjoch_alloc_phys_continous_buf(struct pci_dev *pdev) { struct jfjoch_drvdata *drvdata = pci_get_drvdata(pdev); @@ -22,15 +24,20 @@ int jfjoch_alloc_phys_continous_buf(struct pci_dev *pdev) { for (i = 0; i < nbuffer; i++) { dma_addr_t bus_addr; - void *tmp = pci_zalloc_consistent(pdev,FPGA_BUFFER_LOCATION_SIZE,&bus_addr); + void *tmp = dma_alloc_coherent(&pdev->dev, FPGA_BUFFER_LOCATION_SIZE, &bus_addr, GFP_KERNEL); + if (tmp == NULL) { dev_err(dev, "Failed to allocate %d PCI consistent buffer\n", i); jfjoch_free_phys_continous_buf(pdev); return -ENOMEM; } + + // Save DMA address to address table on FPGA + iowrite32(PCI_DMA_L(bus_addr), drvdata->bar0 + ADDRESS_TABLE_OFFSET + i * 2 * 4); + iowrite32(PCI_DMA_H(bus_addr), drvdata->bar0 + ADDRESS_TABLE_OFFSET + (i * 2 + 1) * 4); + drvdata->bufs[i].kernel_address = tmp; drvdata->bufs[i].dma_address = bus_addr; - drvdata->nbuf++; } @@ -42,10 +49,10 @@ void jfjoch_free_phys_continous_buf(struct pci_dev *pdev) { struct jfjoch_drvdata *drvdata = pci_get_drvdata(pdev); for (i = 0; i < drvdata->nbuf; i++) { if (drvdata->bufs[i].kernel_address != NULL) { - pci_free_consistent(pdev, - FPGA_BUFFER_LOCATION_SIZE, - drvdata->bufs[i].kernel_address, - drvdata->bufs[i].dma_address); + dma_free_coherent(&pdev->dev, + FPGA_BUFFER_LOCATION_SIZE, + drvdata->bufs[i].kernel_address, + drvdata->bufs[i].dma_address); } } kfree(drvdata->bufs); @@ -60,7 +67,12 @@ int jfjoch_cdev_mmap(struct file *file, struct vm_area_struct *vma) { return -EINVAL; } + // In newer kernel use: +#if LINUX_VERSION_CODE <= KERNEL_VERSION(6,3,0) vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_LOCKED; +#else + vm_flags_set(vma, (VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_LOCKED)); +#endif // Don't allow region to be expanded (i.e. via mremap() in user mode), to be copied on fork, or dumped, or swapped // pgoff determines which buffer to use diff --git a/fpga/pcie_driver/jfjoch_miscdev.c b/fpga/pcie_driver/jfjoch_miscdev.c index 32b77598..cc8d86ba 100644 --- a/fpga/pcie_driver/jfjoch_miscdev.c +++ b/fpga/pcie_driver/jfjoch_miscdev.c @@ -2,6 +2,7 @@ #include "jfjoch_drv.h" #include +#include static int device_index = 0; @@ -9,6 +10,9 @@ int jfjoch_register_misc_dev(struct pci_dev *pdev) { int err; struct jfjoch_drvdata *drvdata = pci_get_drvdata(pdev); + drvdata->file_write_open_count = 0; + spin_lock_init(&drvdata->file_write_open_count_spinlock); + snprintf( drvdata->name, 255, "jfjoch%d", device_index++); drvdata->miscdev.parent = &pdev->dev; drvdata->miscdev.mode = 0660; @@ -28,12 +32,29 @@ void jfjoch_unregister_misc_dev(struct pci_dev *pdev) { } int jfjoch_cdev_open(struct inode *inode, struct file *file) { + struct jfjoch_drvdata *drvdata = container_of(file->private_data, struct jfjoch_drvdata, miscdev); + + bool ok = true; + if (file->f_mode & FMODE_WRITE) { + unsigned long flags; + spin_lock_irqsave(&drvdata->file_write_open_count_spinlock, flags); + ok = (drvdata->file_write_open_count == 0); + drvdata->file_write_open_count = 1; + spin_unlock_irqrestore(&drvdata->file_write_open_count_spinlock, flags); + } + if (!ok) + return -EBUSY; return 0; } int jfjoch_cdev_release(struct inode *inode, struct file *file) { struct jfjoch_drvdata *drvdata = container_of(file->private_data, struct jfjoch_drvdata, miscdev); - if (file->f_mode & FMODE_WRITE) + if (file->f_mode & FMODE_WRITE) { + unsigned long flags; jfjoch_cancel(drvdata); + spin_lock_irqsave(&drvdata->file_write_open_count_spinlock, flags); + drvdata->file_write_open_count = 0; + spin_unlock_irqrestore(&drvdata->file_write_open_count_spinlock, flags); + } return 0; } diff --git a/fpga/pcie_driver/jfjoch_pcie_setup.c b/fpga/pcie_driver/jfjoch_pcie_setup.c index b9fd599a..50422e33 100644 --- a/fpga/pcie_driver/jfjoch_pcie_setup.c +++ b/fpga/pcie_driver/jfjoch_pcie_setup.c @@ -6,13 +6,13 @@ int jfjoch_setup_pcie_for_dma(struct pci_dev *pdev) { struct device *const dev = &pdev->dev; // Set DMA masks - int err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + int err = dma_set_mask(dev, DMA_BIT_MASK(64)); if (err) { dev_err(dev, "Failed to set 64-bit DMA mask (%d)\n", err); return err; } - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (err) { dev_err(dev, "Failed to set 64-bit DMA consistent mask (%d)\n", err); return err; @@ -63,4 +63,4 @@ int jfjoch_check_version(struct pci_dev *pdev) { return -EINVAL; } return 0; -} \ No newline at end of file +} diff --git a/fpga/pcie_driver/jfjoch_sysfs.c b/fpga/pcie_driver/jfjoch_sysfs.c index 24dc906d..7d8f6a8b 100644 --- a/fpga/pcie_driver/jfjoch_sysfs.c +++ b/fpga/pcie_driver/jfjoch_sysfs.c @@ -65,10 +65,11 @@ static ssize_t eth_aligned_show(struct device *dev, char *buf) { struct miscdevice* miscdev = dev_get_drvdata(dev); struct jfjoch_drvdata* drvdata = container_of(miscdev, struct jfjoch_drvdata, miscdev); + unsigned int val; // Need to ask twice to get recent answer ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204); - unsigned int val = ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204) & 0x2; + val = ioread32(drvdata->bar0 + CMAC_OFFSET + 0x0204) & 0x2; return scnprintf(buf, PAGE_SIZE, "%d", val); } @@ -151,11 +152,11 @@ static struct device_attribute device_attrs[] = { }; int jfjoch_register_sysfs(struct jfjoch_drvdata *drvdata) { + int i, err; + if (!drvdata) return -EINVAL; - int i, err; - for (i = 0; i < ARRAY_SIZE(device_attrs); i++) { err = device_create_file(drvdata->miscdev.this_device, &device_attrs[i]); if (err) { diff --git a/fpga/scripts/bd_pcie.tcl b/fpga/scripts/bd_pcie.tcl index a4f0992a..bb2fb937 100644 --- a/fpga/scripts/bd_pcie.tcl +++ b/fpga/scripts/bd_pcie.tcl @@ -41,7 +41,7 @@ if { [string first $scripts_vivado_version $current_vivado_version] == -1 } { # The design that will be created by this Tcl script contains the following # module references: -# gen_xdma_descriptor, action_config, check_eth_busy, resetn_sync +# gen_xdma_descriptor, action_config # Please add the sources of those modules before sourcing this Tcl script. @@ -138,38 +138,40 @@ xilinx.com:ip:cms_subsystem:4.0\ xilinx.com:ip:xlconstant:1.1\ xilinx.com:ip:proc_sys_reset:5.0\ xilinx.com:ip:smartconnect:1.0\ +psi.ch:hls:stream_merge:1.0\ xilinx.com:ip:xlconcat:2.1\ xilinx.com:ip:axi_protocol_converter:2.1\ xilinx.com:ip:axi_register_slice:2.1\ xilinx.com:ip:hbm:1.0\ xilinx.com:ip:util_vector_logic:2.0\ -psi.ch:hls:add_multipixel:1.0\ -psi.ch:hls:adu_histo:1.0\ -xilinx.com:ip:axi_datamover:5.1\ -psi.ch:hls:axis_128_to_512:1.0\ -psi.ch:hls:axis_32_to_512:1.0\ xilinx.com:ip:axis_data_fifo:2.0\ xilinx.com:ip:axis_register_slice:1.1\ -psi.ch:hls:bitshuffle:1.0\ psi.ch:hls:data_collection_fsm:1.0\ psi.ch:hls:frame_generator:1.0\ psi.ch:hls:host_writer:1.0\ -psi.ch:hls:integration:1.0\ -psi.ch:hls:jf_conversion:1.0\ psi.ch:hls:load_calibration:1.0\ -psi.ch:hls:load_from_hbm:1.0\ xilinx.com:ip:mailbox:2.1\ -psi.ch:hls:mask_missing:1.0\ -psi.ch:hls:module_upside_down:1.0\ -psi.ch:hls:save_to_hbm:1.0\ -psi.ch:hls:spot_finder:1.0\ -psi.ch:hls:stream_merge:1.0\ psi.ch:hls:timer_host:1.0\ xilinx.com:ip:cmac_usplus:3.1\ -xilinx.com:ip:axi_firewall:1.2\ xilinx.com:ip:axis_clock_converter:1.1\ +xilinx.com:ip:axis_dwidth_converter:1.1\ +xilinx.com:ip:xxv_ethernet:4.1\ +xilinx.com:ip:axi_firewall:1.2\ xilinx.com:ip:util_ds_buf:2.2\ xilinx.com:ip:xdma:4.1\ +xilinx.com:ip:axi_datamover:5.1\ +psi.ch:hls:frame_summation_reorder_compl:1.0\ +psi.ch:hls:load_from_hbm:1.0\ +psi.ch:hls:save_to_hbm:1.0\ +psi.ch:hls:adu_histo:1.0\ +psi.ch:hls:axis_64_to_512:1.0\ +psi.ch:hls:axis_32_to_512:1.0\ +psi.ch:hls:frame_summation:1.0\ +psi.ch:hls:integration:1.0\ +psi.ch:hls:jf_conversion:1.0\ +psi.ch:hls:mask_missing:1.0\ +psi.ch:hls:spot_finder:1.0\ +psi.ch:hls:stream_24bit_conv:1.0\ psi.ch:hls:arp:1.0\ xilinx.com:ip:axis_switch:1.1\ psi.ch:hls:ethernet:1.0\ @@ -202,9 +204,8 @@ psi.ch:hls:udp:1.0\ set bCheckModules 1 if { $bCheckModules == 1 } { set list_check_mods "\ -gen_xdma_descriptor\ action_config\ -check_eth_busy\ +gen_xdma_descriptor\ " set list_mods_missing "" @@ -276,6 +277,13 @@ proc create_root_design { parentCell } { CONFIG.FREQ_HZ {161132812} \ ] $qsfp0_ref + set qsfp1 [ create_bd_intf_port -mode Master -vlnv xilinx.com:interface:gt_rtl:1.0 qsfp1 ] + + set qsfp1_ref [ create_bd_intf_port -mode Slave -vlnv xilinx.com:interface:diff_clock_rtl:1.0 qsfp1_ref ] + set_property -dict [ list \ + CONFIG.FREQ_HZ {161132812} \ + ] $qsfp1_ref + set ref100 [ create_bd_intf_port -mode Slave -vlnv xilinx.com:interface:diff_clock_rtl:1.0 ref100 ] set_property -dict [ list \ CONFIG.FREQ_HZ {100000000} \ @@ -287,7 +295,6 @@ proc create_root_design { parentCell } { # Create ports set hbm_cattrip [ create_bd_port -dir O -from 0 -to 0 hbm_cattrip ] set pcie_perstn [ create_bd_port -dir I -type rst pcie_perstn ] - set qsfp0_led_busy [ create_bd_port -dir O -from 0 -to 0 qsfp0_led_busy ] set qsfp0_led_conn [ create_bd_port -dir O -from 0 -to 0 qsfp0_led_conn ] set satellite_gpio_0 [ create_bd_port -dir I -from 3 -to 0 -type intr satellite_gpio_0 ] set_property -dict [ list \ @@ -348,6 +355,9 @@ proc create_root_design { parentCell } { # Create instance: mac_100g create_hier_cell_mac_100g [current_bd_instance .] mac_100g + # Create instance: mac_4x10g + create_hier_cell_mac_4x10g [current_bd_instance .] mac_4x10g + # Create instance: one, and set properties set one [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 one ] @@ -370,12 +380,21 @@ proc create_root_design { parentCell } { set smartconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_0 ] set_property -dict [ list \ CONFIG.NUM_CLKS {4} \ - CONFIG.NUM_MI {6} \ + CONFIG.NUM_MI {7} \ CONFIG.NUM_SI {1} \ ] $smartconnect_0 + # Create instance: xlconcat_intc, and set properties + set xlconcat_intc [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconcat:2.1 xlconcat_intc ] + set_property -dict [ list \ + CONFIG.NUM_PORTS {2} \ + ] $xlconcat_intc + # Create instance: xlconcat_irq, and set properties set xlconcat_irq [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconcat:2.1 xlconcat_irq ] + set_property -dict [ list \ + CONFIG.NUM_PORTS {2} \ + ] $xlconcat_irq # Create instance: zero, and set properties set zero [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 zero ] @@ -386,6 +405,7 @@ proc create_root_design { parentCell } { # Create interface connections connect_bd_intf_net -intf_net S_AXIS_100G_1 [get_bd_intf_pins jungfraujoch/eth_out] [get_bd_intf_pins mac_100g/s_axis_eth_out] connect_bd_intf_net -intf_net cms_subsystem_0_satellite_uart [get_bd_intf_ports satellite_uart_0] [get_bd_intf_pins cms_subsystem_0/satellite_uart] + connect_bd_intf_net -intf_net eth_4x10g_in_1 [get_bd_intf_pins jungfraujoch/eth_4x10g_in] [get_bd_intf_pins mac_4x10g/m_axis] connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p0 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_0] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p0] connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p1 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_1] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p1] connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p2 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_2] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p2] @@ -408,45 +428,55 @@ proc create_root_design { parentCell } { connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p19 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_19] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p19] connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p20 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_20] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p20] connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p21 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_21] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p21] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p22 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_22] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p22] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p23 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_23] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p23] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p24 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_24] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p24] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p25 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_25] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p25] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p26 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_26] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p26] + connect_bd_intf_net -intf_net jungfraujoch_m_axi_d_hbm_p27 [get_bd_intf_pins hbm_infrastructure/s_axi_hbm_27] [get_bd_intf_pins jungfraujoch/m_axi_d_hbm_p27] connect_bd_intf_net -intf_net jungfraujoch_m_axis_c2h_data [get_bd_intf_pins jungfraujoch/m_axis_c2h_data] [get_bd_intf_pins pcie_dma_0/s_axis_c2h_data] connect_bd_intf_net -intf_net jungfraujoch_m_axis_c2h_datamover_cmd [get_bd_intf_pins jungfraujoch/m_axis_c2h_datamover_cmd] [get_bd_intf_pins pcie_dma_0/s_axis_c2h_cmd] connect_bd_intf_net -intf_net jungfraujoch_m_axis_h2c_datamover_cmd [get_bd_intf_pins jungfraujoch/m_axis_h2c_datamover_cmd] [get_bd_intf_pins pcie_dma_0/s_axis_h2c_cmd] - connect_bd_intf_net -intf_net mac_100g_1_M_AXIS_100G [get_bd_intf_pins jungfraujoch/eth_in] [get_bd_intf_pins mac_100g/m_axis_eth_in] connect_bd_intf_net -intf_net mac_100g_1_qsfp0 [get_bd_intf_ports qsfp0] [get_bd_intf_pins mac_100g/qsfp] + connect_bd_intf_net -intf_net mac_100g_m_axis_eth_in [get_bd_intf_pins jungfraujoch/eth_100g_in] [get_bd_intf_pins mac_100g/m_axis_eth_in] + connect_bd_intf_net -intf_net mac_4x10g_qsfp1 [get_bd_intf_ports qsfp1] [get_bd_intf_pins mac_4x10g/qsfp1] connect_bd_intf_net -intf_net pcie0_ref_1 [get_bd_intf_ports pcie0_ref] [get_bd_intf_pins pcie_dma_0/pcie_refclk] connect_bd_intf_net -intf_net pcie_dma_0_M_AXI [get_bd_intf_pins pcie_dma_0/m_axi_ctrl] [get_bd_intf_pins smartconnect_0/S00_AXI] connect_bd_intf_net -intf_net pcie_dma_0_pcie0_mgt [get_bd_intf_ports pcie0_mgt] [get_bd_intf_pins pcie_dma_0/pcie_mgt] connect_bd_intf_net -intf_net qsfp0_ref_1 [get_bd_intf_ports qsfp0_ref] [get_bd_intf_pins mac_100g/qsfp_ref] + connect_bd_intf_net -intf_net qsfp1_ref_1 [get_bd_intf_ports qsfp1_ref] [get_bd_intf_pins mac_4x10g/qsfp_ref] connect_bd_intf_net -intf_net ref100_1 [get_bd_intf_ports ref100] [get_bd_intf_pins clk_wiz_0/CLK_IN1_D] connect_bd_intf_net -intf_net s_axi_1 [get_bd_intf_pins jungfraujoch/s_axi] [get_bd_intf_pins smartconnect_0/M00_AXI] connect_bd_intf_net -intf_net s_axi_2 [get_bd_intf_pins mac_100g/s_axi] [get_bd_intf_pins smartconnect_0/M02_AXI] connect_bd_intf_net -intf_net s_axis_h2c_data_1 [get_bd_intf_pins jungfraujoch/s_axis_h2c_data] [get_bd_intf_pins pcie_dma_0/m_axis_h2c_data] - connect_bd_intf_net -intf_net smartconnect_0_M01_AXI [get_bd_intf_pins smartconnect_0/M01_AXI] [get_bd_intf_pins cms_subsystem_0/s_axi_ctrl] + connect_bd_intf_net -intf_net smartconnect_0_M01_AXI [get_bd_intf_pins cms_subsystem_0/s_axi_ctrl] [get_bd_intf_pins smartconnect_0/M01_AXI] connect_bd_intf_net -intf_net smartconnect_0_M03_AXI [get_bd_intf_pins axi_quad_spi_0/AXI_LITE] [get_bd_intf_pins smartconnect_0/M03_AXI] connect_bd_intf_net -intf_net smartconnect_0_M04_AXI [get_bd_intf_pins axi_intc_0/s_axi] [get_bd_intf_pins smartconnect_0/M04_AXI] connect_bd_intf_net -intf_net smartconnect_0_M05_AXI [get_bd_intf_pins pcie_dma_0/s_axi_dma_ctrl] [get_bd_intf_pins smartconnect_0/M05_AXI] + connect_bd_intf_net -intf_net smartconnect_0_M06_AXI [get_bd_intf_pins mac_4x10g/s_axi] [get_bd_intf_pins smartconnect_0/M06_AXI] # Create port connections connect_bd_net -net axi_clk_1 [get_bd_pins pcie_dma_0/axi_aclk] [get_bd_pins proc_sys_reset_pcie_0/slowest_sync_clk] [get_bd_pins smartconnect_0/aclk3] - connect_bd_net -net axi_quad_spi_0_ip2intc_irpt [get_bd_pins axi_quad_spi_0/ip2intc_irpt] [get_bd_pins xlconcat_irq/In0] - connect_bd_net -net cms_subsystem_0_interrupt_host [get_bd_pins cms_subsystem_0/interrupt_host] [get_bd_pins xlconcat_irq/In1] + connect_bd_net -net axi_intc_0_irq [get_bd_pins axi_intc_0/irq] [get_bd_pins xlconcat_irq/In0] + connect_bd_net -net axi_quad_spi_0_ip2intc_irpt [get_bd_pins axi_quad_spi_0/ip2intc_irpt] [get_bd_pins xlconcat_intc/In0] + connect_bd_net -net cms_subsystem_0_interrupt_host [get_bd_pins cms_subsystem_0/interrupt_host] [get_bd_pins xlconcat_intc/In1] connect_bd_net -net hbm_infrastructure_hbm_temp_trip_1 [get_bd_ports hbm_cattrip] [get_bd_pins cms_subsystem_0/interrupt_hbm_cattrip] [get_bd_pins hbm_infrastructure/hbm_cattrip] connect_bd_net -net hbm_infrastructure_hbm_temperature_0 [get_bd_pins cms_subsystem_0/hbm_temp_1] [get_bd_pins hbm_infrastructure/hbm_temperature_0] connect_bd_net -net hbm_infrastructure_hbm_temperature_1 [get_bd_pins cms_subsystem_0/hbm_temp_2] [get_bd_pins hbm_infrastructure/hbm_temperature_1] - connect_bd_net -net mac_100g_eth_busy_n [get_bd_ports qsfp0_led_busy] [get_bd_pins mac_100g/eth_busy_n] + connect_bd_net -net jungfraujoch_Interrupt_0 [get_bd_pins jungfraujoch/Interrupt_0] [get_bd_pins xlconcat_irq/In1] connect_bd_net -net mac_100g_stat_rx_aligned_n [get_bd_ports qsfp0_led_conn] [get_bd_pins mac_100g/stat_rx_aligned_n] - connect_bd_net -net net_refclk50 [get_bd_pins axi_intc_0/s_axi_aclk] [get_bd_pins axi_intc_1/s_axi_aclk] [get_bd_pins axi_quad_spi_0/s_axi_aclk] [get_bd_pins clk_wiz_0/clk_out1] [get_bd_pins cms_subsystem_0/aclk_ctrl] [get_bd_pins proc_sys_reset_refclk/slowest_sync_clk] [get_bd_pins proc_sys_reset_refclk1/slowest_sync_clk] [get_bd_pins smartconnect_0/aclk2] + connect_bd_net -net net_refclk50 [get_bd_pins axi_intc_0/s_axi_aclk] [get_bd_pins axi_quad_spi_0/s_axi_aclk] [get_bd_pins clk_wiz_0/clk_out1] [get_bd_pins cms_subsystem_0/aclk_ctrl] [get_bd_pins mac_4x10g/refclk50] [get_bd_pins proc_sys_reset_refclk/slowest_sync_clk] [get_bd_pins smartconnect_0/aclk2] connect_bd_net -net net_refclk100 [get_bd_pins axi_quad_spi_0/ext_spi_clk] [get_bd_pins clk_wiz_0/clk_out2] [get_bd_pins hbm_infrastructure/refclk100] [get_bd_pins mac_100g/refclk100] [get_bd_pins smartconnect_0/aclk1] - connect_bd_net -net net_refclk200 [get_bd_pins clk_wiz_0/clk_out3] [get_bd_pins hbm_infrastructure/axi_clk] [get_bd_pins jungfraujoch/axi_clk] [get_bd_pins mac_100g/axiclk] [get_bd_pins pcie_dma_0/refclk200] [get_bd_pins smartconnect_0/aclk] - connect_bd_net -net one_dout [get_bd_pins one/dout] [get_bd_pins cms_subsystem_0/aresetn_ctrl] [get_bd_pins proc_sys_reset_pcie_0/dcm_locked] [get_bd_pins proc_sys_reset_refclk/dcm_locked] + connect_bd_net -net net_refclk200 [get_bd_pins clk_wiz_0/clk_out3] [get_bd_pins hbm_infrastructure/axi_clk] [get_bd_pins jungfraujoch/axi_clk] [get_bd_pins mac_100g/axiclk] [get_bd_pins mac_4x10g/m_axis_aclk] [get_bd_pins pcie_dma_0/refclk200] [get_bd_pins smartconnect_0/aclk] + connect_bd_net -net one_dout [get_bd_pins cms_subsystem_0/aresetn_ctrl] [get_bd_pins one/dout] [get_bd_pins proc_sys_reset_pcie_0/dcm_locked] [get_bd_pins proc_sys_reset_refclk/dcm_locked] connect_bd_net -net pcie_dma_0_axi_aresetn [get_bd_pins pcie_dma_0/axi_aresetn] [get_bd_pins proc_sys_reset_pcie_0/ext_reset_in] [get_bd_pins proc_sys_reset_refclk/ext_reset_in] [get_bd_pins smartconnect_0/aresetn] connect_bd_net -net pcie_perstn_1 [get_bd_ports pcie_perstn] [get_bd_pins pcie_dma_0/pcie_perstn] connect_bd_net -net proc_sys_reset_pcie_0_interconnect_aresetn [get_bd_pins pcie_dma_0/axi_clk_resetn] [get_bd_pins proc_sys_reset_pcie_0/interconnect_aresetn] - connect_bd_net -net proc_sys_reset_refclk_peripheral_aresetn [get_bd_pins axi_intc_0/s_axi_aresetn] [get_bd_pins axi_quad_spi_0/s_axi_aresetn] [get_bd_pins hbm_infrastructure/axi_resetn] [get_bd_pins jungfraujoch/ap_rst_n] [get_bd_pins mac_100g/ap_rst_n] [get_bd_pins proc_sys_reset_refclk/peripheral_aresetn] + connect_bd_net -net proc_sys_reset_refclk_peripheral_aresetn [get_bd_pins axi_intc_0/s_axi_aresetn] [get_bd_pins axi_quad_spi_0/s_axi_aresetn] [get_bd_pins hbm_infrastructure/axi_resetn] [get_bd_pins jungfraujoch/ap_rst_n] [get_bd_pins mac_100g/ap_rst_n] [get_bd_pins mac_4x10g/axi_aresetn] [get_bd_pins proc_sys_reset_refclk/peripheral_aresetn] connect_bd_net -net resetn_1 [get_bd_pins jungfraujoch/axi_rst_n] [get_bd_pins mac_100g/resetn] [get_bd_pins pcie_dma_0/refclk200_resetn] [get_bd_pins proc_sys_reset_refclk/interconnect_aresetn] connect_bd_net -net satellite_gpio_0_1 [get_bd_ports satellite_gpio_0] [get_bd_pins cms_subsystem_0/satellite_gpio] - connect_bd_net -net usr_irq_req_1 [get_bd_pins axi_intc_0/irq] [get_bd_pins pcie_dma_0/usr_irq_req] - connect_bd_net -net xlconcat_irq_dout [get_bd_pins axi_intc_0/intr] [get_bd_pins axi_intc_1/intr] [get_bd_pins xlconcat_irq/dout] + connect_bd_net -net xlconcat_0_dout [get_bd_pins pcie_dma_0/usr_irq_req] [get_bd_pins xlconcat_irq/dout] + connect_bd_net -net xlconcat_irq_dout [get_bd_pins axi_intc_0/intr] [get_bd_pins xlconcat_intc/dout] connect_bd_net -net zero_dout [get_bd_pins axi_quad_spi_0/usrcclkts] [get_bd_pins zero/dout] # Create address segments @@ -460,14 +490,14 @@ proc create_root_design { parentCell } { assign_bd_address -offset 0x00080000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs jungfraujoch/frame_generator_0/s_axi_control/Reg] -force assign_bd_address -offset 0x00090000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs pcie_dma_0/xdma_0/S_AXI_LITE/CTL0] -force assign_bd_address -offset 0x000C0000 -range 0x00040000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs cms_subsystem_0/s_axi_ctrl/Mem] -force - + assign_bd_address -offset 0x00100000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs mac_4x10g/xxv_ethernet_0/s_axi_0/Reg] -force + assign_bd_address -offset 0x00110000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs mac_4x10g/xxv_ethernet_0/s_axi_1/Reg] -force + assign_bd_address -offset 0x00120000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs mac_4x10g/xxv_ethernet_0/s_axi_2/Reg] -force + assign_bd_address -offset 0x00130000 -range 0x00010000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs mac_4x10g/xxv_ethernet_0/s_axi_3/Reg] -force + assign_bd_address -offset 0x00200000 -range 0x00080000 -target_address_space [get_bd_addr_spaces pcie_dma_0/xdma_0/M_AXI_LITE] [get_bd_addr_segs jungfraujoch/axi_bram_ctrl_calibration_addr_0/S_AXI/Mem0] -force + assign_bd_address -offset 0x00000000 -range 0x00080000 -target_address_space [get_bd_addr_spaces jungfraujoch/load_calibration_0/Data_m_axi_handle_to_dma_address] [get_bd_addr_segs jungfraujoch/axi_bram_ctrl_calibration_addr_1/S_AXI/Mem0] -force assign_bd_address - set_property -dict [list \ - CONFIG.MAX_MODULES_FPGA_PARAM {0x00000010} \ - CONFIG.DESIGN_NUMBER {0} \ - ] [get_bd_cells jungfraujoch/action_config_0] - # Restore current instance current_bd_instance $oldCurInst diff --git a/fpga/scripts/build_pcie_design.tcl b/fpga/scripts/build_pcie_design.tcl index 7d76051a..f71992f8 100644 --- a/fpga/scripts/build_pcie_design.tcl +++ b/fpga/scripts/build_pcie_design.tcl @@ -20,7 +20,6 @@ set source_set [get_filesets sources_1] set hdl_files [list \ [file normalize "action/hw/hdl/action_config.v"] \ - [file normalize "action/hw/hdl/check_eth_busy.v"] \ [file normalize "action/hw/hdl/gen_xdma_descriptor.v"] \ ] @@ -50,6 +49,7 @@ source $origin_dir/hbm_cache.tcl source $origin_dir/image_processing.tcl source $origin_dir/pcie_dma.tcl source $origin_dir/mac_100g_pcie.tcl +source $origin_dir/mac_4x10g.tcl source $origin_dir/bd_pcie.tcl >> build_pcie.log make_wrapper -files [get_files "vivado/jfjoch_pcie.srcs/sources_1/bd/jfjoch_pcie/jfjoch_pcie.bd"] -top >> make_wrapper.log diff --git a/fpga/scripts/hbm_u55c.tcl b/fpga/scripts/hbm_u55c.tcl index f4ae6067..ea85c6c3 100644 --- a/fpga/scripts/hbm_u55c.tcl +++ b/fpga/scripts/hbm_u55c.tcl @@ -217,9 +217,6 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } { CONFIG.USER_MC9_LOOKAHEAD_SBRF {true} \ CONFIG.USER_MC9_REF_TEMP_COMP {false} \ CONFIG.USER_MC9_TRAFFIC_OPTION {Linear} \ - CONFIG.USER_MC_ENABLE_11 {FALSE} \ - CONFIG.USER_MC_ENABLE_12 {FALSE} \ - CONFIG.USER_MC_ENABLE_13 {FALSE} \ CONFIG.USER_MC_ENABLE_14 {FALSE} \ CONFIG.USER_MC_ENABLE_15 {FALSE} \ CONFIG.USER_MC_ENABLE_APB_01 {FALSE} \ @@ -232,12 +229,6 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } { CONFIG.USER_PHY_ENABLE_13 {TRUE} \ CONFIG.USER_PHY_ENABLE_14 {TRUE} \ CONFIG.USER_PHY_ENABLE_15 {TRUE} \ - CONFIG.USER_SAXI_22 {false} \ - CONFIG.USER_SAXI_23 {false} \ - CONFIG.USER_SAXI_24 {false} \ - CONFIG.USER_SAXI_25 {false} \ - CONFIG.USER_SAXI_26 {false} \ - CONFIG.USER_SAXI_27 {false} \ CONFIG.USER_SAXI_28 {false} \ CONFIG.USER_SAXI_29 {false} \ CONFIG.USER_SAXI_30 {false} \ @@ -278,7 +269,7 @@ proc create_hier_cell_hbm_infrastructure { parentCell nameHier } { connect_bd_net [get_bd_pins axi_resetn] [get_bd_pins hbm/APB_0_PRESET_N] [get_bd_pins hbm/APB_1_PRESET_N] connect_bd_net [get_bd_pins refclk100] [get_bd_pins hbm/APB_0_PCLK] [get_bd_pins hbm/APB_1_PCLK] [get_bd_pins hbm/HBM_REF_CLK_0] [get_bd_pins hbm/HBM_REF_CLK_1] - for {set i 0} {$i < 22} {incr i} { + for {set i 0} {$i < 28} {incr i} { create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:aximm_rtl:1.0 s_axi_hbm_$i set cell [create_bd_cell -type ip -vlnv {xilinx.com:ip:axi_register_slice:*} axi_register_slice_$i ] diff --git a/fpga/scripts/image_processing.tcl b/fpga/scripts/image_processing.tcl index a5f4b212..9a8366cb 100644 --- a/fpga/scripts/image_processing.tcl +++ b/fpga/scripts/image_processing.tcl @@ -73,6 +73,18 @@ proc create_hier_cell_image_processing { parentCell nameHier } { create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p19 + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p22 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p23 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p24 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p25 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p26 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p27 + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis_completion create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 result_out @@ -89,22 +101,38 @@ proc create_hier_cell_image_processing { parentCell nameHier } { create_bd_pin -dir I -from 31 -to 0 -type data hbm_size_bytes create_bd_pin -dir I -from 15 -to 0 -type data in_count_threshold create_bd_pin -dir I -from 7 -to 0 -type data in_snr_threshold + create_bd_pin -dir O proc_fifo_empty + create_bd_pin -dir O proc_fifo_full # Create instance: adu_histo_0, and set properties set adu_histo_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:adu_histo:1.0 adu_histo_0 ] - # Create instance: axis_128_to_512_0, and set properties - set axis_128_to_512_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:axis_128_to_512:1.0 axis_128_to_512_0 ] - # Create instance: axis_32_to_512_0, and set properties set axis_32_to_512_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:axis_32_to_512:1.0 axis_32_to_512_0 ] + # Create instance: axis_64_to_512_0, and set properties + set axis_64_to_512_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:axis_64_to_512:1.0 axis_64_to_512_0 ] + + # Create instance: axis_compl_fifo_0, and set properties + set axis_compl_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_0 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {16} \ + CONFIG.HAS_AEMPTY {0} \ + CONFIG.HAS_AFULL {0} \ + ] $axis_compl_fifo_0 + + # Create instance: axis_compl_fifo_1, and set properties + set axis_compl_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_1 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {16} \ + CONFIG.HAS_AEMPTY {0} \ + CONFIG.HAS_AFULL {0} \ + ] $axis_compl_fifo_1 + # Create instance: axis_compl_fifo_2, and set properties set axis_compl_fifo_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_2 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {16} \ - CONFIG.HAS_AEMPTY {0} \ - CONFIG.HAS_AFULL {0} \ ] $axis_compl_fifo_2 # Create instance: axis_compl_fifo_3, and set properties @@ -119,12 +147,6 @@ proc create_hier_cell_image_processing { parentCell nameHier } { CONFIG.FIFO_DEPTH {16} \ ] $axis_compl_fifo_4 - # Create instance: axis_compl_fifo_5, and set properties - set axis_compl_fifo_5 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_5 ] - set_property -dict [ list \ - CONFIG.FIFO_DEPTH {16} \ - ] $axis_compl_fifo_5 - # Create instance: axis_data_fifo_0, and set properties set axis_data_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_0 ] set_property -dict [ list \ @@ -133,12 +155,21 @@ proc create_hier_cell_image_processing { parentCell nameHier } { # Create instance: axis_data_fifo_1, and set properties set axis_data_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_1 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {256} \ + ] $axis_data_fifo_1 # Create instance: axis_data_fifo_2, and set properties set axis_data_fifo_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_2 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {256} \ + ] $axis_data_fifo_2 # Create instance: axis_data_fifo_3, and set properties set axis_data_fifo_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_3 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {256} \ + ] $axis_data_fifo_3 # Create instance: axis_data_fifo_4, and set properties set axis_data_fifo_4 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_4 ] @@ -156,24 +187,26 @@ proc create_hier_cell_image_processing { parentCell nameHier } { set axis_data_fifo_6 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_6 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {256} \ + CONFIG.HAS_AEMPTY {1} \ + CONFIG.HAS_AFULL {1} \ ] $axis_data_fifo_6 + # Create instance: axis_data_fifo_7, and set properties + set axis_data_fifo_7 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_7 ] + set_property -dict [ list \ + CONFIG.FIFO_DEPTH {256} \ + ] $axis_data_fifo_7 + # Create instance: axis_integration_result_fifo_0, and set properties set axis_integration_result_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_integration_result_fifo_0 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {256} \ - CONFIG.FIFO_MEMORY_TYPE {auto} \ - CONFIG.HAS_AEMPTY {0} \ - CONFIG.HAS_AFULL {0} \ ] $axis_integration_result_fifo_0 # Create instance: axis_integration_result_fifo_1, and set properties set axis_integration_result_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_integration_result_fifo_1 ] set_property -dict [ list \ - CONFIG.FIFO_DEPTH {256} \ - CONFIG.FIFO_MEMORY_TYPE {auto} \ - CONFIG.HAS_AEMPTY {0} \ - CONFIG.HAS_AFULL {0} \ + CONFIG.FIFO_DEPTH {512} \ ] $axis_integration_result_fifo_1 # Create instance: axis_register_slice_data_1, and set properties @@ -197,14 +230,14 @@ proc create_hier_cell_image_processing { parentCell nameHier } { # Create instance: axis_spot_finder_fifo_0, and set properties set axis_spot_finder_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_spot_finder_fifo_0 ] set_property -dict [ list \ - CONFIG.FIFO_DEPTH {32768} \ - CONFIG.FIFO_MEMORY_TYPE {ultra} \ + CONFIG.FIFO_DEPTH {32} \ ] $axis_spot_finder_fifo_0 # Create instance: axis_spot_finder_fifo_1, and set properties set axis_spot_finder_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_spot_finder_fifo_1 ] set_property -dict [ list \ - CONFIG.FIFO_DEPTH {16} \ + CONFIG.FIFO_DEPTH {2048} \ + CONFIG.FIFO_MEMORY_TYPE {ultra} \ ] $axis_spot_finder_fifo_1 # Create instance: frame_summation_0, and set properties @@ -222,6 +255,27 @@ proc create_hier_cell_image_processing { parentCell nameHier } { # Create instance: one, and set properties set one [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 one ] + # Create instance: pedestal_0, and set properties + set pedestal_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:pedestal:1.0 pedestal_0 ] + + # Create instance: smartconnect_0, and set properties + set smartconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_0 ] + + # Create instance: smartconnect_1, and set properties + set smartconnect_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_1 ] + + # Create instance: smartconnect_2, and set properties + set smartconnect_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_2 ] + + # Create instance: smartconnect_3, and set properties + set smartconnect_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_3 ] + + # Create instance: smartconnect_4, and set properties + set smartconnect_4 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_4 ] + + # Create instance: smartconnect_5, and set properties + set smartconnect_5 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_5 ] + # Create instance: spot_finder_0, and set properties set spot_finder_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:spot_finder:1.0 spot_finder_0 ] @@ -239,56 +293,80 @@ proc create_hier_cell_image_processing { parentCell nameHier } { connect_bd_intf_net -intf_net Conn12 [get_bd_intf_pins m_axi_d_hbm_p3] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p3] connect_bd_intf_net -intf_net Conn13 [get_bd_intf_pins m_axi_d_hbm_p4] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p4] connect_bd_intf_net -intf_net Conn15 [get_bd_intf_pins m_axi_d_hbm_p5] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p5] - connect_bd_intf_net -intf_net Conn17 [get_bd_intf_pins s_axis_completion] [get_bd_intf_pins adu_histo_0/s_axis_completion] connect_bd_intf_net -intf_net Conn19 [get_bd_intf_pins result_out] [get_bd_intf_pins adu_histo_0/result_out] connect_bd_intf_net -intf_net Conn21 [get_bd_intf_pins m_axi_d_hbm_p6] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p6] connect_bd_intf_net -intf_net Conn23 [get_bd_intf_pins m_axi_d_hbm_p7] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p7] - connect_bd_intf_net -intf_net Conn25 [get_bd_intf_pins data_in] [get_bd_intf_pins axis_data_fifo_0/S_AXIS] connect_bd_intf_net -intf_net Conn26 [get_bd_intf_pins m_axi_d_hbm_p8] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p8] connect_bd_intf_net -intf_net Conn27 [get_bd_intf_pins m_axi_d_hbm_p9] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p9] connect_bd_intf_net -intf_net Conn28 [get_bd_intf_pins m_axi_d_hbm_p10] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p10] connect_bd_intf_net -intf_net Conn29 [get_bd_intf_pins m_axi_d_hbm_p11] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p11] connect_bd_intf_net -intf_net Conn30 [get_bd_intf_pins m_axi_d_hbm_p2] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p2] connect_bd_intf_net -intf_net Conn31 [get_bd_intf_pins m_axi_d_hbm_p1] [get_bd_intf_pins jf_conversion_0/m_axi_d_hbm_p1] - connect_bd_intf_net -intf_net adu_histo_0_data_out [get_bd_intf_pins adu_histo_0/data_out] [get_bd_intf_pins axis_data_fifo_1/S_AXIS] - connect_bd_intf_net -intf_net adu_histo_0_m_axis_completion [get_bd_intf_pins adu_histo_0/m_axis_completion] [get_bd_intf_pins axis_compl_fifo_2/S_AXIS] - connect_bd_intf_net -intf_net axis_128_to_512_0_data_out [get_bd_intf_pins axis_128_to_512_0/data_out] [get_bd_intf_pins axis_integration_result_fifo_1/S_AXIS] + connect_bd_intf_net -intf_net adu_histo_0_data_out [get_bd_intf_pins adu_histo_0/data_out] [get_bd_intf_pins axis_data_fifo_2/S_AXIS] + connect_bd_intf_net -intf_net adu_histo_0_m_axis_completion [get_bd_intf_pins adu_histo_0/m_axis_completion] [get_bd_intf_pins axis_compl_fifo_1/S_AXIS] connect_bd_intf_net -intf_net axis_32_to_512_0_data_out [get_bd_intf_pins axis_32_to_512_0/data_out] [get_bd_intf_pins axis_spot_finder_fifo_1/S_AXIS] - connect_bd_intf_net -intf_net axis_compl_fifo_2_M_AXIS [get_bd_intf_pins axis_compl_fifo_2/M_AXIS] [get_bd_intf_pins mask_missing_0/s_axis_completion] - connect_bd_intf_net -intf_net axis_compl_fifo_3_M_AXIS [get_bd_intf_pins axis_compl_fifo_3/M_AXIS] [get_bd_intf_pins jf_conversion_0/s_axis_completion] - connect_bd_intf_net -intf_net axis_compl_fifo_4_M_AXIS [get_bd_intf_pins axis_compl_fifo_4/M_AXIS] [get_bd_intf_pins frame_summation_0/s_axis_completion] - connect_bd_intf_net -intf_net axis_compl_fifo_5_M_AXIS [get_bd_intf_pins axis_compl_fifo_5/M_AXIS] [get_bd_intf_pins integration_0/s_axis_completion] - connect_bd_intf_net -intf_net axis_data_fifo_3_M_AXIS [get_bd_intf_pins adu_histo_0/data_in] [get_bd_intf_pins axis_data_fifo_0/M_AXIS] - connect_bd_intf_net -intf_net axis_data_fifo_4_M_AXIS [get_bd_intf_pins axis_data_fifo_1/M_AXIS] [get_bd_intf_pins mask_missing_0/data_in] - connect_bd_intf_net -intf_net axis_data_fifo_5_M_AXIS [get_bd_intf_pins axis_data_fifo_2/M_AXIS] [get_bd_intf_pins jf_conversion_0/data_in] - connect_bd_intf_net -intf_net axis_data_fifo_5_M_AXIS1 [get_bd_intf_pins axis_data_fifo_5/M_AXIS] [get_bd_intf_pins spot_finder_0/data_in] - connect_bd_intf_net -intf_net axis_data_fifo_6_M_AXIS [get_bd_intf_pins axis_data_fifo_3/M_AXIS] [get_bd_intf_pins axis_register_slice_data_1/S_AXIS] - connect_bd_intf_net -intf_net axis_data_fifo_6_M_AXIS1 [get_bd_intf_pins axis_data_fifo_6/M_AXIS] [get_bd_intf_pins stream_24bit_conv_0/data_in] - connect_bd_intf_net -intf_net axis_data_fifo_7_M_AXIS [get_bd_intf_pins axis_data_fifo_4/M_AXIS] [get_bd_intf_pins integration_0/data_in] - connect_bd_intf_net -intf_net axis_integration_result_fifo_0_M_AXIS [get_bd_intf_pins axis_128_to_512_0/data_in] [get_bd_intf_pins axis_integration_result_fifo_0/M_AXIS] + connect_bd_intf_net -intf_net axis_64_to_512_0_data_out [get_bd_intf_pins axis_64_to_512_0/data_out] [get_bd_intf_pins axis_integration_result_fifo_1/S_AXIS] + connect_bd_intf_net -intf_net axis_compl_fifo_0_M_AXIS [get_bd_intf_pins adu_histo_0/s_axis_completion] [get_bd_intf_pins axis_compl_fifo_0/M_AXIS] + connect_bd_intf_net -intf_net axis_compl_fifo_1_M_AXIS [get_bd_intf_pins axis_compl_fifo_2/M_AXIS] [get_bd_intf_pins jf_conversion_0/s_axis_completion] + connect_bd_intf_net -intf_net axis_compl_fifo_2_M_AXIS [get_bd_intf_pins axis_compl_fifo_1/M_AXIS] [get_bd_intf_pins mask_missing_0/s_axis_completion] + connect_bd_intf_net -intf_net axis_compl_fifo_4_M_AXIS [get_bd_intf_pins axis_compl_fifo_3/M_AXIS] [get_bd_intf_pins frame_summation_0/s_axis_completion] + connect_bd_intf_net -intf_net axis_compl_fifo_5_M_AXIS [get_bd_intf_pins axis_compl_fifo_4/M_AXIS] [get_bd_intf_pins integration_0/s_axis_completion] + connect_bd_intf_net -intf_net axis_data_fifo_0_M_AXIS [get_bd_intf_pins axis_data_fifo_0/M_AXIS] [get_bd_intf_pins pedestal_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_3_M_AXIS [get_bd_intf_pins adu_histo_0/data_in] [get_bd_intf_pins axis_data_fifo_1/M_AXIS] + connect_bd_intf_net -intf_net axis_data_fifo_3_M_AXIS1 [get_bd_intf_pins axis_data_fifo_3/M_AXIS] [get_bd_intf_pins jf_conversion_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_4_M_AXIS [get_bd_intf_pins axis_data_fifo_2/M_AXIS] [get_bd_intf_pins mask_missing_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_5_M_AXIS1 [get_bd_intf_pins axis_data_fifo_6/M_AXIS] [get_bd_intf_pins spot_finder_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_6_M_AXIS [get_bd_intf_pins axis_data_fifo_4/M_AXIS] [get_bd_intf_pins axis_register_slice_data_1/S_AXIS] + connect_bd_intf_net -intf_net axis_data_fifo_6_M_AXIS1 [get_bd_intf_pins axis_data_fifo_7/M_AXIS] [get_bd_intf_pins stream_24bit_conv_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_7_M_AXIS [get_bd_intf_pins axis_data_fifo_5/M_AXIS] [get_bd_intf_pins integration_0/data_in] + connect_bd_intf_net -intf_net axis_integration_result_fifo_0_M_AXIS [get_bd_intf_pins axis_64_to_512_0/data_in] [get_bd_intf_pins axis_integration_result_fifo_0/M_AXIS] connect_bd_intf_net -intf_net axis_register_slice_data_1_M_AXIS [get_bd_intf_pins axis_register_slice_data_1/M_AXIS] [get_bd_intf_pins frame_summation_0/data_in] - connect_bd_intf_net -intf_net axis_register_slice_data_2_M_AXIS [get_bd_intf_pins axis_data_fifo_4/S_AXIS] [get_bd_intf_pins axis_register_slice_data_2/M_AXIS] + connect_bd_intf_net -intf_net axis_register_slice_data_2_M_AXIS [get_bd_intf_pins axis_data_fifo_5/S_AXIS] [get_bd_intf_pins axis_register_slice_data_2/M_AXIS] connect_bd_intf_net -intf_net axis_register_slice_data_3_M_AXIS [get_bd_intf_pins data_out] [get_bd_intf_pins axis_register_slice_data_3/M_AXIS] connect_bd_intf_net -intf_net axis_spot_finder_fifo_0_M_AXIS [get_bd_intf_pins axis_32_to_512_0/data_in] [get_bd_intf_pins axis_spot_finder_fifo_0/M_AXIS] + connect_bd_intf_net -intf_net data_in_1 [get_bd_intf_pins data_in] [get_bd_intf_pins axis_data_fifo_0/S_AXIS] connect_bd_intf_net -intf_net frame_summation_0_data_out [get_bd_intf_pins axis_register_slice_data_2/S_AXIS] [get_bd_intf_pins frame_summation_0/data_out] - connect_bd_intf_net -intf_net frame_summation_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_5/S_AXIS] [get_bd_intf_pins frame_summation_0/m_axis_completion] - connect_bd_intf_net -intf_net integration_0_data_out [get_bd_intf_pins axis_data_fifo_5/S_AXIS] [get_bd_intf_pins integration_0/data_out] + connect_bd_intf_net -intf_net frame_summation_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_4/S_AXIS] [get_bd_intf_pins frame_summation_0/m_axis_completion] + connect_bd_intf_net -intf_net integration_0_data_out [get_bd_intf_pins axis_data_fifo_6/S_AXIS] [get_bd_intf_pins integration_0/data_out] connect_bd_intf_net -intf_net integration_0_m_axis_completion [get_bd_intf_pins m_axis_completion] [get_bd_intf_pins integration_0/m_axis_completion] connect_bd_intf_net -intf_net integration_0_result_out [get_bd_intf_pins axis_integration_result_fifo_0/S_AXIS] [get_bd_intf_pins integration_0/result_out] - connect_bd_intf_net -intf_net jf_conversion_0_data_out [get_bd_intf_pins axis_data_fifo_3/S_AXIS] [get_bd_intf_pins jf_conversion_0/data_out] - connect_bd_intf_net -intf_net jf_conversion_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_4/S_AXIS] [get_bd_intf_pins jf_conversion_0/m_axis_completion] - connect_bd_intf_net -intf_net mask_missing_0_data_out [get_bd_intf_pins axis_data_fifo_2/S_AXIS] [get_bd_intf_pins mask_missing_0/data_out] - connect_bd_intf_net -intf_net mask_missing_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_3/S_AXIS] [get_bd_intf_pins mask_missing_0/m_axis_completion] - connect_bd_intf_net -intf_net spot_finder_0_data_out [get_bd_intf_pins axis_data_fifo_6/S_AXIS] [get_bd_intf_pins spot_finder_0/data_out] + connect_bd_intf_net -intf_net jf_conversion_0_data_out [get_bd_intf_pins axis_data_fifo_4/S_AXIS] [get_bd_intf_pins jf_conversion_0/data_out] + connect_bd_intf_net -intf_net jf_conversion_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_3/S_AXIS] [get_bd_intf_pins jf_conversion_0/m_axis_completion] + connect_bd_intf_net -intf_net mask_missing_0_data_out [get_bd_intf_pins axis_data_fifo_3/S_AXIS] [get_bd_intf_pins mask_missing_0/data_out] + connect_bd_intf_net -intf_net mask_missing_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_2/S_AXIS] [get_bd_intf_pins mask_missing_0/m_axis_completion] + connect_bd_intf_net -intf_net pedestal_0_data_out [get_bd_intf_pins axis_data_fifo_1/S_AXIS] [get_bd_intf_pins pedestal_0/data_out] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p0 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p0] [get_bd_intf_pins smartconnect_0/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p0_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p0_w] [get_bd_intf_pins smartconnect_0/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p1 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p1] [get_bd_intf_pins smartconnect_1/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p1_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p1_w] [get_bd_intf_pins smartconnect_1/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p2 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p2] [get_bd_intf_pins smartconnect_2/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p2_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p2_w] [get_bd_intf_pins smartconnect_2/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p3 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p3] [get_bd_intf_pins smartconnect_3/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p3_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p3_w] [get_bd_intf_pins smartconnect_3/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p4 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p4] [get_bd_intf_pins smartconnect_4/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p4_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p4_w] [get_bd_intf_pins smartconnect_4/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p5 [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p5] [get_bd_intf_pins smartconnect_5/S00_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axi_d_hbm_p5_w [get_bd_intf_pins pedestal_0/m_axi_d_hbm_p5_w] [get_bd_intf_pins smartconnect_5/S01_AXI] + connect_bd_intf_net -intf_net pedestal_0_m_axis_completion [get_bd_intf_pins axis_compl_fifo_0/S_AXIS] [get_bd_intf_pins pedestal_0/m_axis_completion] + connect_bd_intf_net -intf_net s_axis_completion_1 [get_bd_intf_pins s_axis_completion] [get_bd_intf_pins pedestal_0/s_axis_completion] + connect_bd_intf_net -intf_net smartconnect_0_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p22] [get_bd_intf_pins smartconnect_0/M00_AXI] + connect_bd_intf_net -intf_net smartconnect_1_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p23] [get_bd_intf_pins smartconnect_1/M00_AXI] + connect_bd_intf_net -intf_net smartconnect_2_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p24] [get_bd_intf_pins smartconnect_2/M00_AXI] + connect_bd_intf_net -intf_net smartconnect_3_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p25] [get_bd_intf_pins smartconnect_3/M00_AXI] + connect_bd_intf_net -intf_net smartconnect_4_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p26] [get_bd_intf_pins smartconnect_4/M00_AXI] + connect_bd_intf_net -intf_net smartconnect_5_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p27] [get_bd_intf_pins smartconnect_5/M00_AXI] + connect_bd_intf_net -intf_net spot_finder_0_data_out [get_bd_intf_pins axis_data_fifo_7/S_AXIS] [get_bd_intf_pins spot_finder_0/data_out] connect_bd_intf_net -intf_net spot_finder_0_strong_pixel_out [get_bd_intf_pins axis_spot_finder_fifo_0/S_AXIS] [get_bd_intf_pins spot_finder_0/strong_pixel_out] connect_bd_intf_net -intf_net stream_24bit_conv_0_data_out [get_bd_intf_pins axis_register_slice_data_3/S_AXIS] [get_bd_intf_pins stream_24bit_conv_0/data_out] # Create port connections - connect_bd_net -net ap_rst_n_1 [get_bd_pins ap_rst_n] [get_bd_pins adu_histo_0/ap_rst_n] [get_bd_pins axis_128_to_512_0/ap_rst_n] [get_bd_pins axis_32_to_512_0/ap_rst_n] [get_bd_pins frame_summation_0/ap_rst_n] [get_bd_pins integration_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins mask_missing_0/ap_rst_n] [get_bd_pins spot_finder_0/ap_rst_n] [get_bd_pins stream_24bit_conv_0/ap_rst_n] - connect_bd_net -net ap_start_1 [get_bd_pins frame_summation_0/ap_start] [get_bd_pins one/dout] [get_bd_pins spot_finder_0/ap_start] [get_bd_pins stream_24bit_conv_0/ap_start] - connect_bd_net -net axi_clk_1 [get_bd_pins axi_clk] [get_bd_pins adu_histo_0/ap_clk] [get_bd_pins axis_128_to_512_0/ap_clk] [get_bd_pins axis_32_to_512_0/ap_clk] [get_bd_pins axis_compl_fifo_2/s_axis_aclk] [get_bd_pins axis_compl_fifo_3/s_axis_aclk] [get_bd_pins axis_compl_fifo_4/s_axis_aclk] [get_bd_pins axis_compl_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_6/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_1/s_axis_aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_2/aclk] [get_bd_pins axis_register_slice_data_3/aclk] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aclk] [get_bd_pins frame_summation_0/ap_clk] [get_bd_pins integration_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins mask_missing_0/ap_clk] [get_bd_pins spot_finder_0/ap_clk] [get_bd_pins stream_24bit_conv_0/ap_clk] - connect_bd_net -net axi_rst_n_1 [get_bd_pins axi_rst_n] [get_bd_pins axis_compl_fifo_2/s_axis_aresetn] [get_bd_pins axis_compl_fifo_3/s_axis_aresetn] [get_bd_pins axis_compl_fifo_4/s_axis_aresetn] [get_bd_pins axis_compl_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_6/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_1/s_axis_aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_2/aresetn] [get_bd_pins axis_register_slice_data_3/aresetn] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aresetn] - connect_bd_net -net hbm_size_bytes_1 [get_bd_pins hbm_size_bytes] [get_bd_pins integration_0/hbm_size_bytes] [get_bd_pins jf_conversion_0/hbm_size_bytes] + connect_bd_net -net ap_rst_n_1 [get_bd_pins ap_rst_n] [get_bd_pins adu_histo_0/ap_rst_n] [get_bd_pins axis_32_to_512_0/ap_rst_n] [get_bd_pins axis_64_to_512_0/ap_rst_n] [get_bd_pins frame_summation_0/ap_rst_n] [get_bd_pins integration_0/ap_rst_n] [get_bd_pins jf_conversion_0/ap_rst_n] [get_bd_pins mask_missing_0/ap_rst_n] [get_bd_pins pedestal_0/ap_rst_n] [get_bd_pins spot_finder_0/ap_rst_n] [get_bd_pins stream_24bit_conv_0/ap_rst_n] + connect_bd_net -net ap_start_1 [get_bd_pins frame_summation_0/ap_start] [get_bd_pins one/dout] [get_bd_pins spot_finder_0/ap_start] [get_bd_pins stream_24bit_conv_0/ap_start] [get_bd_pins integration_0/ap_start] + connect_bd_net -net axi_clk_1 [get_bd_pins axi_clk] [get_bd_pins adu_histo_0/ap_clk] [get_bd_pins axis_32_to_512_0/ap_clk] [get_bd_pins axis_64_to_512_0/ap_clk] [get_bd_pins axis_compl_fifo_0/s_axis_aclk] [get_bd_pins axis_compl_fifo_1/s_axis_aclk] [get_bd_pins axis_compl_fifo_2/s_axis_aclk] [get_bd_pins axis_compl_fifo_3/s_axis_aclk] [get_bd_pins axis_compl_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_4/s_axis_aclk] [get_bd_pins axis_data_fifo_5/s_axis_aclk] [get_bd_pins axis_data_fifo_6/s_axis_aclk] [get_bd_pins axis_data_fifo_7/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_0/s_axis_aclk] [get_bd_pins axis_integration_result_fifo_1/s_axis_aclk] [get_bd_pins axis_register_slice_data_1/aclk] [get_bd_pins axis_register_slice_data_2/aclk] [get_bd_pins axis_register_slice_data_3/aclk] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aclk] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aclk] [get_bd_pins frame_summation_0/ap_clk] [get_bd_pins integration_0/ap_clk] [get_bd_pins jf_conversion_0/ap_clk] [get_bd_pins mask_missing_0/ap_clk] [get_bd_pins pedestal_0/ap_clk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins smartconnect_1/aclk] [get_bd_pins smartconnect_2/aclk] [get_bd_pins smartconnect_3/aclk] [get_bd_pins smartconnect_4/aclk] [get_bd_pins smartconnect_5/aclk] [get_bd_pins spot_finder_0/ap_clk] [get_bd_pins stream_24bit_conv_0/ap_clk] + connect_bd_net -net axi_rst_n_1 [get_bd_pins axi_rst_n] [get_bd_pins axis_compl_fifo_0/s_axis_aresetn] [get_bd_pins axis_compl_fifo_1/s_axis_aresetn] [get_bd_pins axis_compl_fifo_2/s_axis_aresetn] [get_bd_pins axis_compl_fifo_3/s_axis_aresetn] [get_bd_pins axis_compl_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_4/s_axis_aresetn] [get_bd_pins axis_data_fifo_5/s_axis_aresetn] [get_bd_pins axis_data_fifo_6/s_axis_aresetn] [get_bd_pins axis_data_fifo_7/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_0/s_axis_aresetn] [get_bd_pins axis_integration_result_fifo_1/s_axis_aresetn] [get_bd_pins axis_register_slice_data_1/aresetn] [get_bd_pins axis_register_slice_data_2/aresetn] [get_bd_pins axis_register_slice_data_3/aresetn] [get_bd_pins axis_spot_finder_fifo_0/s_axis_aresetn] [get_bd_pins axis_spot_finder_fifo_1/s_axis_aresetn] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins smartconnect_1/aresetn] [get_bd_pins smartconnect_2/aresetn] [get_bd_pins smartconnect_3/aresetn] [get_bd_pins smartconnect_4/aresetn] [get_bd_pins smartconnect_5/aresetn] + connect_bd_net -net axis_data_fifo_6_almost_empty [get_bd_pins proc_fifo_empty] [get_bd_pins axis_data_fifo_6/almost_empty] + connect_bd_net -net axis_data_fifo_6_almost_full [get_bd_pins proc_fifo_full] [get_bd_pins axis_data_fifo_6/almost_full] + connect_bd_net -net hbm_size_bytes_1 [get_bd_pins hbm_size_bytes] [get_bd_pins integration_0/hbm_size_bytes] [get_bd_pins jf_conversion_0/hbm_size_bytes] [get_bd_pins pedestal_0/hbm_size_bytes] connect_bd_net -net in_count_threshold_1 [get_bd_pins in_count_threshold] [get_bd_pins spot_finder_0/in_count_threshold] connect_bd_net -net in_snr_threshold_1 [get_bd_pins in_snr_threshold] [get_bd_pins spot_finder_0/in_snr_threshold] diff --git a/fpga/scripts/jfjoch.tcl b/fpga/scripts/jfjoch.tcl index 68ad21e9..a49ecbf8 100644 --- a/fpga/scripts/jfjoch.tcl +++ b/fpga/scripts/jfjoch.tcl @@ -35,7 +35,9 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { current_bd_instance $hier_obj # Create interface pins - create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:axis_rtl:1.0 eth_in + create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:axis_rtl:1.0 eth_100g_in + + create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:axis_rtl:1.0 eth_4x10g_in create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 eth_out @@ -83,6 +85,18 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p21 + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p22 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p23 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p24 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p25 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p26 + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:aximm_rtl:1.0 m_axi_d_hbm_p27 + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis_c2h_data create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis_c2h_datamover_cmd @@ -95,6 +109,7 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { # Create pins + create_bd_pin -dir O -type intr Interrupt_0 create_bd_pin -dir I -type rst ap_rst_n create_bd_pin -dir I -type clk axi_clk create_bd_pin -dir I -type rst axi_rst_n @@ -109,10 +124,24 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { catch {common::send_gid_msg -ssname BD::TCL -id 2096 -severity "ERROR" "Unable to referenced block <$block_name>. Please add the files for ${block_name}'s definition into the project."} return 1 } - set_property -dict [ list \ - CONFIG.DESIGN_NUMBER {0} \ - CONFIG.MAX_MODULES_FPGA_PARAM {0x00000010} \ - ] $action_config_0 + + # Create instance: axi_bram_ctrl_calibration_addr_0, and set properties + set axi_bram_ctrl_calibration_addr_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axi_bram_ctrl:4.1 axi_bram_ctrl_calibration_addr_0 ] + set_property -dict [ list \ + CONFIG.PROTOCOL {AXI4LITE} \ + CONFIG.READ_LATENCY {4} \ + CONFIG.SINGLE_PORT_BRAM {1} \ + CONFIG.USE_ECC {0} \ + ] $axi_bram_ctrl_calibration_addr_0 + + # Create instance: axi_bram_ctrl_calibration_addr_1, and set properties + set axi_bram_ctrl_calibration_addr_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axi_bram_ctrl:4.1 axi_bram_ctrl_calibration_addr_1 ] + set_property -dict [ list \ + CONFIG.DATA_WIDTH {32} \ + CONFIG.PROTOCOL {AXI4} \ + CONFIG.READ_LATENCY {4} \ + CONFIG.SINGLE_PORT_BRAM {1} \ + ] $axi_bram_ctrl_calibration_addr_1 # Create instance: axis_addr_fifo_0, and set properties set axis_addr_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_addr_fifo_0 ] @@ -129,19 +158,19 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { CONFIG.FIFO_DEPTH {1024} \ ] $axis_adu_histo_result_fifo - # Create instance: axis_compl_fifo_1, and set properties - set axis_compl_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_1 ] + # Create instance: axis_compl_fifo_0, and set properties + set axis_compl_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_0 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {16} \ CONFIG.HAS_AEMPTY {1} \ CONFIG.HAS_AFULL {1} \ - ] $axis_compl_fifo_1 + ] $axis_compl_fifo_0 - # Create instance: axis_compl_fifo_6, and set properties - set axis_compl_fifo_6 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_6 ] + # Create instance: axis_compl_fifo_1, and set properties + set axis_compl_fifo_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_compl_fifo_1 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {16} \ - ] $axis_compl_fifo_6 + ] $axis_compl_fifo_1 # Create instance: axis_data_fifo_0, and set properties set axis_data_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_0 ] @@ -159,20 +188,20 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { CONFIG.HAS_AFULL {1} \ ] $axis_data_fifo_1 - # Create instance: axis_data_fifo_11, and set properties - set axis_data_fifo_11 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_11 ] + # Create instance: axis_data_fifo_2, and set properties + set axis_data_fifo_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_2 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {256} \ - ] $axis_data_fifo_11 + ] $axis_data_fifo_2 - # Create instance: axis_data_fifo_12, and set properties - set axis_data_fifo_12 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_12 ] + # Create instance: axis_data_fifo_3, and set properties + set axis_data_fifo_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_3 ] set_property -dict [ list \ CONFIG.FIFO_DEPTH {32768} \ CONFIG.FIFO_MEMORY_TYPE {ultra} \ CONFIG.HAS_AEMPTY {1} \ CONFIG.HAS_AFULL {1} \ - ] $axis_data_fifo_12 + ] $axis_data_fifo_3 # Create instance: axis_data_fifo_c2h_cmd, and set properties set axis_data_fifo_c2h_cmd [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_c2h_cmd ] @@ -269,28 +298,29 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { CONFIG.HAS_AFULL {1} \ ] $axis_udp_fifo_0 - # Create instance: axis_work_completion_fifo_0, and set properties - set axis_work_completion_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_work_completion_fifo_0 ] - set_property -dict [ list \ - CONFIG.FIFO_DEPTH {8192} \ - CONFIG.FIFO_MEMORY_TYPE {ultra} \ - CONFIG.HAS_AEMPTY {1} \ - CONFIG.HAS_AFULL {1} \ - ] $axis_work_completion_fifo_0 - - # Create instance: axis_work_request_fifo_0, and set properties - set axis_work_request_fifo_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_work_request_fifo_0 ] - set_property -dict [ list \ - CONFIG.FIFO_DEPTH {1024} \ - CONFIG.FIFO_MEMORY_TYPE {auto} \ - CONFIG.HAS_AEMPTY {1} \ - CONFIG.HAS_AFULL {1} \ - CONFIG.HAS_PROG_FULL {0} \ - ] $axis_work_request_fifo_0 - # Create instance: data_collection_fsm_0, and set properties set data_collection_fsm_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:data_collection_fsm:1.0 data_collection_fsm_0 ] + # Create instance: dma_address_table, and set properties + set dma_address_table [ create_bd_cell -type ip -vlnv xilinx.com:ip:blk_mem_gen:8.4 dma_address_table ] + set_property -dict [ list \ + CONFIG.Assume_Synchronous_Clk {true} \ + CONFIG.EN_SAFETY_CKT {false} \ + CONFIG.Enable_B {Use_ENB_Pin} \ + CONFIG.Memory_Type {True_Dual_Port_RAM} \ + CONFIG.Operating_Mode_A {NO_CHANGE} \ + CONFIG.Operating_Mode_B {NO_CHANGE} \ + CONFIG.PRIM_type_to_Implement {URAM} \ + CONFIG.Port_A_Write_Rate {50} \ + CONFIG.Port_B_Clock {100} \ + CONFIG.Port_B_Enable_Rate {100} \ + CONFIG.Port_B_Write_Rate {50} \ + CONFIG.READ_LATENCY_A {4} \ + CONFIG.READ_LATENCY_B {4} \ + CONFIG.Use_Byte_Write_Enable {true} \ + CONFIG.Use_RSTB_Pin {true} \ + ] $dma_address_table + # Create instance: frame_generator_0, and set properties set frame_generator_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:frame_generator:1.0 frame_generator_0 ] @@ -311,7 +341,7 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { set_property -dict [ list \ CONFIG.C_IMPL_STYLE {1} \ CONFIG.C_INTERCONNECT_PORT_1 {4} \ - CONFIG.C_MAILBOX_DEPTH {256} \ + CONFIG.C_MAILBOX_DEPTH {8192} \ ] $mailbox_0 # Create instance: network_stack @@ -321,7 +351,7 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { set smartconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_0 ] set_property -dict [ list \ CONFIG.NUM_CLKS {1} \ - CONFIG.NUM_MI {4} \ + CONFIG.NUM_MI {5} \ CONFIG.NUM_SI {1} \ ] $smartconnect_0 @@ -331,6 +361,12 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { # Create instance: smartconnect_2, and set properties set smartconnect_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_2 ] + # Create instance: smartconnect_3, and set properties + set smartconnect_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_3 ] + set_property -dict [ list \ + CONFIG.NUM_SI {2} \ + ] $smartconnect_3 + # Create instance: stream_merge_0, and set properties set stream_merge_0 [ create_bd_cell -type ip -vlnv psi.ch:hls:stream_merge:1.0 stream_merge_0 ] @@ -346,20 +382,22 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { # Create interface connections connect_bd_intf_net -intf_net Conn2 [get_bd_intf_pins eth_out] [get_bd_intf_pins network_stack/M00_AXIS] connect_bd_intf_net -intf_net S_AXIS_1 [get_bd_intf_pins s_axis_h2c_data] [get_bd_intf_pins axis_data_fifo_h2c_data/S_AXIS] + connect_bd_intf_net -intf_net axi_bram_ctrl_calibration_addr_0_BRAM_PORTA [get_bd_intf_pins axi_bram_ctrl_calibration_addr_0/BRAM_PORTA] [get_bd_intf_pins dma_address_table/BRAM_PORTA] + connect_bd_intf_net -intf_net axi_bram_ctrl_calibration_addr_1_BRAM_PORTA [get_bd_intf_pins axi_bram_ctrl_calibration_addr_1/BRAM_PORTA] [get_bd_intf_pins dma_address_table/BRAM_PORTB] connect_bd_intf_net -intf_net axis_addr_fifo_0_M_AXIS [get_bd_intf_pins axis_addr_fifo_0/M_AXIS] [get_bd_intf_pins axis_register_slice_addr_0/S_AXIS] connect_bd_intf_net -intf_net axis_adu_histo_result_fifo_M_AXIS [get_bd_intf_pins axis_adu_histo_result_fifo/M_AXIS] [get_bd_intf_pins host_writer_0/adu_histo_in] - connect_bd_intf_net -intf_net axis_compl_fifo_1_M_AXIS [get_bd_intf_pins axis_compl_fifo_1/M_AXIS] [get_bd_intf_pins image_processing/s_axis_completion] - connect_bd_intf_net -intf_net axis_compl_fifo_6_M_AXIS [get_bd_intf_pins axis_compl_fifo_6/M_AXIS] [get_bd_intf_pins host_writer_0/s_axis_completion] + connect_bd_intf_net -intf_net axis_compl_fifo_0_M_AXIS [get_bd_intf_pins axis_compl_fifo_0/M_AXIS] [get_bd_intf_pins image_processing/s_axis_completion] + connect_bd_intf_net -intf_net axis_compl_fifo_1_M_AXIS [get_bd_intf_pins axis_compl_fifo_1/M_AXIS] [get_bd_intf_pins host_writer_0/s_axis_completion] connect_bd_intf_net -intf_net axis_data_fifo_0_M_AXIS [get_bd_intf_pins axis_data_fifo_0/M_AXIS] [get_bd_intf_pins timer_hbm/data_in] - connect_bd_intf_net -intf_net axis_data_fifo_10_M_AXIS1 [get_bd_intf_pins axis_data_fifo_11/M_AXIS] [get_bd_intf_pins timer_host/data_in] connect_bd_intf_net -intf_net axis_data_fifo_1_M_AXIS [get_bd_intf_pins axis_data_fifo_1/M_AXIS] [get_bd_intf_pins axis_register_slice_data_0/S_AXIS] - connect_bd_intf_net -intf_net axis_data_fifo_9_M_AXIS [get_bd_intf_pins axis_data_fifo_12/M_AXIS] [get_bd_intf_pins host_writer_0/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_2_M_AXIS [get_bd_intf_pins axis_data_fifo_2/M_AXIS] [get_bd_intf_pins timer_host/data_in] + connect_bd_intf_net -intf_net axis_data_fifo_3_M_AXIS [get_bd_intf_pins axis_data_fifo_3/M_AXIS] [get_bd_intf_pins host_writer_0/data_in] connect_bd_intf_net -intf_net axis_data_fifo_c2h_cmd_M_AXIS [get_bd_intf_pins m_axis_c2h_datamover_cmd] [get_bd_intf_pins axis_data_fifo_c2h_cmd/M_AXIS] connect_bd_intf_net -intf_net axis_data_fifo_c2h_data_M_AXIS [get_bd_intf_pins m_axis_c2h_data] [get_bd_intf_pins axis_data_fifo_c2h_data/M_AXIS] connect_bd_intf_net -intf_net axis_data_fifo_h2c_cmd_M_AXIS [get_bd_intf_pins m_axis_h2c_datamover_cmd] [get_bd_intf_pins axis_data_fifo_h2c_cmd/M_AXIS] connect_bd_intf_net -intf_net axis_data_fifo_h2c_data_M_AXIS [get_bd_intf_pins axis_data_fifo_h2c_data/M_AXIS] [get_bd_intf_pins axis_register_slice_data_in_0/S_AXIS] connect_bd_intf_net -intf_net axis_eth_in_fifo_M_AXIS [get_bd_intf_pins axis_eth_in_fifo/M_AXIS] [get_bd_intf_pins network_stack/eth_in] - connect_bd_intf_net -intf_net axis_frame_generator_fifo_0_M_AXIS [get_bd_intf_pins axis_frame_generator_fifo_0/M_AXIS] [get_bd_intf_pins stream_merge_0/input_0] + connect_bd_intf_net -intf_net axis_frame_generator_fifo_0_M_AXIS [get_bd_intf_pins axis_frame_generator_fifo_0/M_AXIS] [get_bd_intf_pins stream_merge_0/input_frame_generator] connect_bd_intf_net -intf_net axis_register_slice_addr_0_M_AXIS [get_bd_intf_pins axis_register_slice_addr_0/M_AXIS] [get_bd_intf_pins hbm_cache/addr_in] connect_bd_intf_net -intf_net axis_register_slice_data_0_M_AXIS [get_bd_intf_pins axis_register_slice_data_0/M_AXIS] [get_bd_intf_pins hbm_cache/data_in] connect_bd_intf_net -intf_net axis_register_slice_data_in_0_M_AXIS1 [get_bd_intf_pins axis_register_slice_data_in_0/M_AXIS] [get_bd_intf_pins load_calibration_0/host_memory_in] @@ -367,11 +405,12 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_intf_net -intf_net axis_register_slice_udp_M_AXIS [get_bd_intf_pins axis_register_slice_udp/M_AXIS] [get_bd_intf_pins data_collection_fsm_0/eth_in] connect_bd_intf_net -intf_net axis_udp_addr_fifo_0_M_AXIS [get_bd_intf_pins axis_udp_addr_fifo_0/M_AXIS] [get_bd_intf_pins data_collection_fsm_0/addr_in] connect_bd_intf_net -intf_net axis_udp_fifo_0_M_AXIS [get_bd_intf_pins axis_register_slice_udp/S_AXIS] [get_bd_intf_pins axis_udp_fifo_0/M_AXIS] - connect_bd_intf_net -intf_net axis_work_completion_fifo_0_M_AXIS [get_bd_intf_pins axis_work_completion_fifo_0/M_AXIS] [get_bd_intf_pins mailbox_0/S1_AXIS] - connect_bd_intf_net -intf_net axis_work_request_fifo_0_M_AXIS [get_bd_intf_pins axis_work_request_fifo_0/M_AXIS] [get_bd_intf_pins host_writer_0/s_axis_work_request] + connect_bd_intf_net -intf_net host_writer_0_m_axis_completion [get_bd_intf_pins host_writer_0/m_axis_completion] [get_bd_intf_pins mailbox_0/S1_AXIS] + connect_bd_intf_net -intf_net mailbox_0_M1_AXIS [get_bd_intf_pins mailbox_0/M1_AXIS] [get_bd_intf_pins host_writer_0/s_axis_work_request] connect_bd_intf_net -intf_net data_collection_fsm_0_addr_out [get_bd_intf_pins axis_addr_fifo_0/S_AXIS] [get_bd_intf_pins data_collection_fsm_0/addr_out] connect_bd_intf_net -intf_net data_collection_fsm_0_data_out [get_bd_intf_pins axis_data_fifo_0/S_AXIS] [get_bd_intf_pins data_collection_fsm_0/data_out] - connect_bd_intf_net -intf_net eth_in_1 [get_bd_intf_pins eth_in] [get_bd_intf_pins stream_merge_0/input_1] + connect_bd_intf_net -intf_net eth_100g_in_1 [get_bd_intf_pins eth_100g_in] [get_bd_intf_pins stream_merge_0/input_100g] + connect_bd_intf_net -intf_net eth_4x10g_in_1 [get_bd_intf_pins eth_4x10g_in] [get_bd_intf_pins stream_merge_0/input_4x10g] connect_bd_intf_net -intf_net frame_generator_0_data_out [get_bd_intf_pins axis_frame_generator_fifo_0/S_AXIS] [get_bd_intf_pins frame_generator_0/data_out] connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p0 [get_bd_intf_pins m_axi_d_hbm_p20] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p0] connect_bd_intf_net -intf_net frame_generator_0_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p21] [get_bd_intf_pins frame_generator_0/m_axi_d_hbm_p1] @@ -380,13 +419,13 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_intf_net -intf_net hbm_cache_m_axi_d_hbm_p13 [get_bd_intf_pins m_axi_d_hbm_p13] [get_bd_intf_pins hbm_cache/m_axi_d_hbm_p13] connect_bd_intf_net -intf_net hbm_cache_m_axi_d_hbm_p14 [get_bd_intf_pins m_axi_d_hbm_p14] [get_bd_intf_pins hbm_cache/m_axi_d_hbm_p14] connect_bd_intf_net -intf_net hbm_cache_m_axi_d_hbm_p15 [get_bd_intf_pins m_axi_d_hbm_p15] [get_bd_intf_pins hbm_cache/m_axi_d_hbm_p15] - connect_bd_intf_net -intf_net hbm_cache_m_axis_completion [get_bd_intf_pins axis_compl_fifo_1/S_AXIS] [get_bd_intf_pins hbm_cache/m_axis_completion] + connect_bd_intf_net -intf_net hbm_cache_m_axis_completion [get_bd_intf_pins axis_compl_fifo_0/S_AXIS] [get_bd_intf_pins hbm_cache/m_axis_completion] connect_bd_intf_net -intf_net host_writer_0_datamover_out_cmd [get_bd_intf_pins axis_data_fifo_c2h_cmd/S_AXIS] [get_bd_intf_pins host_writer_0/datamover_out_cmd] connect_bd_intf_net -intf_net host_writer_0_host_memory_out [get_bd_intf_pins axis_register_slice_host_mem/S_AXIS] [get_bd_intf_pins host_writer_0/host_memory_out] - connect_bd_intf_net -intf_net host_writer_0_m_axis_completion [get_bd_intf_pins axis_work_completion_fifo_0/S_AXIS] [get_bd_intf_pins host_writer_0/m_axis_completion] + connect_bd_intf_net -intf_net host_writer_0_m_axi_dma_address_table [get_bd_intf_pins host_writer_0/m_axi_dma_address_table] [get_bd_intf_pins smartconnect_3/S01_AXI] connect_bd_intf_net -intf_net image_processing_M_AXIS [get_bd_intf_pins host_writer_0/spot_finder_in] [get_bd_intf_pins image_processing/spot_finder_out] connect_bd_intf_net -intf_net image_processing_M_AXIS1 [get_bd_intf_pins host_writer_0/integration_in] [get_bd_intf_pins image_processing/integration_result_out] - connect_bd_intf_net -intf_net image_processing_data_out6 [get_bd_intf_pins axis_data_fifo_11/S_AXIS] [get_bd_intf_pins image_processing/data_out] + connect_bd_intf_net -intf_net image_processing_data_out6 [get_bd_intf_pins axis_data_fifo_2/S_AXIS] [get_bd_intf_pins image_processing/data_out] connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p0 [get_bd_intf_pins image_processing/m_axi_d_hbm_p0] [get_bd_intf_pins smartconnect_1/S00_AXI] connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p1 [get_bd_intf_pins m_axi_d_hbm_p1] [get_bd_intf_pins image_processing/m_axi_d_hbm_p1] connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p2 [get_bd_intf_pins image_processing/m_axi_d_hbm_p2] [get_bd_intf_pins smartconnect_2/S00_AXI] @@ -403,12 +442,18 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p17 [get_bd_intf_pins m_axi_d_hbm_p17] [get_bd_intf_pins image_processing/m_axi_d_hbm_p17] connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p18 [get_bd_intf_pins m_axi_d_hbm_p18] [get_bd_intf_pins image_processing/m_axi_d_hbm_p18] connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p19 [get_bd_intf_pins m_axi_d_hbm_p19] [get_bd_intf_pins image_processing/m_axi_d_hbm_p19] - connect_bd_intf_net -intf_net image_processing_m_axis_completion3 [get_bd_intf_pins axis_compl_fifo_6/S_AXIS] [get_bd_intf_pins image_processing/m_axis_completion] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p22 [get_bd_intf_pins m_axi_d_hbm_p22] [get_bd_intf_pins image_processing/m_axi_d_hbm_p22] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p23 [get_bd_intf_pins m_axi_d_hbm_p23] [get_bd_intf_pins image_processing/m_axi_d_hbm_p23] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p24 [get_bd_intf_pins m_axi_d_hbm_p24] [get_bd_intf_pins image_processing/m_axi_d_hbm_p24] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p25 [get_bd_intf_pins m_axi_d_hbm_p25] [get_bd_intf_pins image_processing/m_axi_d_hbm_p25] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p26 [get_bd_intf_pins m_axi_d_hbm_p26] [get_bd_intf_pins image_processing/m_axi_d_hbm_p26] + connect_bd_intf_net -intf_net image_processing_m_axi_d_hbm_p27 [get_bd_intf_pins m_axi_d_hbm_p27] [get_bd_intf_pins image_processing/m_axi_d_hbm_p27] + connect_bd_intf_net -intf_net image_processing_m_axis_completion3 [get_bd_intf_pins axis_compl_fifo_1/S_AXIS] [get_bd_intf_pins image_processing/m_axis_completion] connect_bd_intf_net -intf_net image_processing_result_out [get_bd_intf_pins axis_adu_histo_result_fifo/S_AXIS] [get_bd_intf_pins image_processing/result_out] connect_bd_intf_net -intf_net load_calibration_0_datamover_in_cmd [get_bd_intf_pins axis_data_fifo_h2c_cmd/S_AXIS] [get_bd_intf_pins load_calibration_0/datamover_in_cmd] connect_bd_intf_net -intf_net load_calibration_0_m_axi_d_hbm_p0 [get_bd_intf_pins load_calibration_0/m_axi_d_hbm_p0] [get_bd_intf_pins smartconnect_1/S01_AXI] connect_bd_intf_net -intf_net load_calibration_0_m_axi_d_hbm_p1 [get_bd_intf_pins load_calibration_0/m_axi_d_hbm_p1] [get_bd_intf_pins smartconnect_2/S01_AXI] - connect_bd_intf_net -intf_net mailbox_0_M1_AXIS [get_bd_intf_pins axis_work_request_fifo_0/S_AXIS] [get_bd_intf_pins mailbox_0/M1_AXIS] + connect_bd_intf_net -intf_net load_calibration_0_m_axi_dma_address_table [get_bd_intf_pins load_calibration_0/m_axi_dma_address_table] [get_bd_intf_pins smartconnect_3/S00_AXI] connect_bd_intf_net -intf_net network_stack_udp_addr_out [get_bd_intf_pins axis_udp_addr_fifo_0/S_AXIS] [get_bd_intf_pins network_stack/udp_addr_out] connect_bd_intf_net -intf_net network_stack_udp_out [get_bd_intf_pins axis_udp_fifo_0/S_AXIS] [get_bd_intf_pins network_stack/udp_out] connect_bd_intf_net -intf_net s_axi_1 [get_bd_intf_pins s_axi] [get_bd_intf_pins smartconnect_0/S00_AXI] @@ -416,18 +461,21 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_intf_net -intf_net smartconnect_0_M01_AXI [get_bd_intf_pins mailbox_0/S0_AXI] [get_bd_intf_pins smartconnect_0/M01_AXI] connect_bd_intf_net -intf_net smartconnect_0_M02_AXI [get_bd_intf_pins load_calibration_0/s_axi_control] [get_bd_intf_pins smartconnect_0/M02_AXI] connect_bd_intf_net -intf_net smartconnect_0_M04_AXI [get_bd_intf_pins frame_generator_0/s_axi_control] [get_bd_intf_pins smartconnect_0/M03_AXI] + connect_bd_intf_net -intf_net smartconnect_0_M04_AXI1 [get_bd_intf_pins axi_bram_ctrl_calibration_addr_0/S_AXI] [get_bd_intf_pins smartconnect_0/M04_AXI] connect_bd_intf_net -intf_net smartconnect_1_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p0] [get_bd_intf_pins smartconnect_1/M00_AXI] connect_bd_intf_net -intf_net smartconnect_2_M00_AXI [get_bd_intf_pins m_axi_d_hbm_p2] [get_bd_intf_pins smartconnect_2/M00_AXI] - connect_bd_intf_net -intf_net stream_merge_0_output_r [get_bd_intf_pins axis_eth_in_fifo/S_AXIS] [get_bd_intf_pins stream_merge_0/output_r] + connect_bd_intf_net -intf_net smartconnect_3_M00_AXI [get_bd_intf_pins axi_bram_ctrl_calibration_addr_1/S_AXI] [get_bd_intf_pins smartconnect_3/M00_AXI] + connect_bd_intf_net -intf_net stream_merge_0_data_out [get_bd_intf_pins axis_eth_in_fifo/S_AXIS] [get_bd_intf_pins stream_merge_0/data_out] connect_bd_intf_net -intf_net timer_hbm_data_out [get_bd_intf_pins axis_data_fifo_1/S_AXIS] [get_bd_intf_pins timer_hbm/data_out] connect_bd_intf_net -intf_net timer_host_0_data_out [get_bd_intf_pins image_processing/data_in] [get_bd_intf_pins timer_proc/data_out] - connect_bd_intf_net -intf_net timer_host_data_out [get_bd_intf_pins axis_data_fifo_12/S_AXIS] [get_bd_intf_pins timer_host/data_out] + connect_bd_intf_net -intf_net timer_host_data_out [get_bd_intf_pins axis_data_fifo_3/S_AXIS] [get_bd_intf_pins timer_host/data_out] # Create port connections connect_bd_net -net action_config_0_clear_counters [get_bd_pins action_config_0/clear_counters] [get_bd_pins network_stack/clear_counters] connect_bd_net -net action_config_0_data_collection_cancel [get_bd_pins action_config_0/data_collection_cancel] [get_bd_pins data_collection_fsm_0/in_cancel] connect_bd_net -net action_config_0_data_collection_fsm_start [get_bd_pins action_config_0/data_collection_start] [get_bd_pins data_collection_fsm_0/in_run] [get_bd_pins network_stack/data_collection_start] connect_bd_net -net action_config_0_data_collection_mode [get_bd_pins action_config_0/data_collection_mode] [get_bd_pins data_collection_fsm_0/mode] + connect_bd_net -net action_config_0_data_source [get_bd_pins action_config_0/data_source] [get_bd_pins stream_merge_0/source] connect_bd_net -net action_config_0_fpga_ipv4_addr [get_bd_pins action_config_0/fpga_ipv4_addr] [get_bd_pins frame_generator_0/src_ipv4_addr] [get_bd_pins network_stack/fpga_ipv4_addr] connect_bd_net -net action_config_0_fpga_mac_addr [get_bd_pins action_config_0/fpga_mac_addr] [get_bd_pins frame_generator_0/src_mac_addr] [get_bd_pins network_stack/fpga_mac_addr] connect_bd_net -net action_config_0_frames_per_trigger [get_bd_pins action_config_0/nframes] [get_bd_pins data_collection_fsm_0/nframes] @@ -438,13 +486,13 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_net -net action_config_0_one_over_energy [get_bd_pins action_config_0/one_over_energy] [get_bd_pins data_collection_fsm_0/one_over_energy] connect_bd_net -net action_config_0_spot_finder_count_threshold [get_bd_pins action_config_0/spot_finder_count_threshold] [get_bd_pins image_processing/in_count_threshold] connect_bd_net -net action_config_0_spot_finder_snr_threshold [get_bd_pins action_config_0/spot_finder_snr_threshold] [get_bd_pins image_processing/in_snr_threshold] - connect_bd_net -net ap_clk_1 [get_bd_pins axi_clk] [get_bd_pins action_config_0/clk] [get_bd_pins axis_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_adu_histo_result_fifo/s_axis_aclk] [get_bd_pins axis_compl_fifo_1/s_axis_aclk] [get_bd_pins axis_compl_fifo_6/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_11/s_axis_aclk] [get_bd_pins axis_data_fifo_12/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aclk] [get_bd_pins axis_eth_in_fifo/s_axis_aclk] [get_bd_pins axis_frame_generator_fifo_0/s_axis_aclk] [get_bd_pins axis_register_slice_addr_0/aclk] [get_bd_pins axis_register_slice_data_0/aclk] [get_bd_pins axis_register_slice_data_in_0/aclk] [get_bd_pins axis_register_slice_host_mem/aclk] [get_bd_pins axis_register_slice_udp/aclk] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_udp_fifo_0/s_axis_aclk] [get_bd_pins axis_work_completion_fifo_0/s_axis_aclk] [get_bd_pins axis_work_request_fifo_0/s_axis_aclk] [get_bd_pins data_collection_fsm_0/ap_clk] [get_bd_pins frame_generator_0/ap_clk] [get_bd_pins hbm_cache/axi_clk] [get_bd_pins host_writer_0/ap_clk] [get_bd_pins image_processing/axi_clk] [get_bd_pins load_calibration_0/ap_clk] [get_bd_pins mailbox_0/M1_AXIS_ACLK] [get_bd_pins mailbox_0/S0_AXI_ACLK] [get_bd_pins mailbox_0/S1_AXIS_ACLK] [get_bd_pins network_stack/axiclk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins smartconnect_1/aclk] [get_bd_pins smartconnect_2/aclk] [get_bd_pins stream_merge_0/ap_clk] [get_bd_pins timer_hbm/ap_clk] [get_bd_pins timer_host/ap_clk] [get_bd_pins timer_proc/ap_clk] + connect_bd_net -net ap_clk_1 [get_bd_pins axi_clk] [get_bd_pins action_config_0/clk] [get_bd_pins axi_bram_ctrl_calibration_addr_0/s_axi_aclk] [get_bd_pins axi_bram_ctrl_calibration_addr_1/s_axi_aclk] [get_bd_pins axis_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_adu_histo_result_fifo/s_axis_aclk] [get_bd_pins axis_compl_fifo_0/s_axis_aclk] [get_bd_pins axis_compl_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_0/s_axis_aclk] [get_bd_pins axis_data_fifo_1/s_axis_aclk] [get_bd_pins axis_data_fifo_2/s_axis_aclk] [get_bd_pins axis_data_fifo_3/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aclk] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aclk] [get_bd_pins axis_eth_in_fifo/s_axis_aclk] [get_bd_pins axis_frame_generator_fifo_0/s_axis_aclk] [get_bd_pins axis_register_slice_addr_0/aclk] [get_bd_pins axis_register_slice_data_0/aclk] [get_bd_pins axis_register_slice_data_in_0/aclk] [get_bd_pins axis_register_slice_host_mem/aclk] [get_bd_pins axis_register_slice_udp/aclk] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aclk] [get_bd_pins axis_udp_fifo_0/s_axis_aclk] [get_bd_pins data_collection_fsm_0/ap_clk] [get_bd_pins frame_generator_0/ap_clk] [get_bd_pins hbm_cache/axi_clk] [get_bd_pins host_writer_0/ap_clk] [get_bd_pins image_processing/axi_clk] [get_bd_pins load_calibration_0/ap_clk] [get_bd_pins mailbox_0/M1_AXIS_ACLK] [get_bd_pins mailbox_0/S0_AXI_ACLK] [get_bd_pins mailbox_0/S1_AXIS_ACLK] [get_bd_pins network_stack/axiclk] [get_bd_pins smartconnect_0/aclk] [get_bd_pins smartconnect_1/aclk] [get_bd_pins smartconnect_2/aclk] [get_bd_pins smartconnect_3/aclk] [get_bd_pins stream_merge_0/ap_clk] [get_bd_pins timer_hbm/ap_clk] [get_bd_pins timer_host/ap_clk] [get_bd_pins timer_proc/ap_clk] connect_bd_net -net axis_addr_fifo_0_almost_empty [get_bd_pins action_config_0/calib_addr_fifo_empty] [get_bd_pins axis_addr_fifo_0/almost_empty] connect_bd_net -net axis_addr_fifo_0_almost_full [get_bd_pins action_config_0/calib_addr_fifo_full] [get_bd_pins axis_addr_fifo_0/almost_full] - connect_bd_net -net axis_compl_fifo_1_almost_empty [get_bd_pins action_config_0/last_addr_fifo_empty] [get_bd_pins axis_compl_fifo_1/almost_empty] - connect_bd_net -net axis_compl_fifo_1_almost_full [get_bd_pins action_config_0/last_addr_fifo_full] [get_bd_pins axis_compl_fifo_1/almost_full] - connect_bd_net -net axis_data_fifo_10_almost_empty [get_bd_pins action_config_0/last_data_fifo_empty] [get_bd_pins axis_data_fifo_12/almost_empty] - connect_bd_net -net axis_data_fifo_10_almost_full [get_bd_pins action_config_0/last_data_fifo_full] [get_bd_pins axis_data_fifo_12/almost_full] + connect_bd_net -net axis_compl_fifo_1_almost_empty [get_bd_pins action_config_0/last_addr_fifo_empty] [get_bd_pins axis_compl_fifo_0/almost_empty] + connect_bd_net -net axis_compl_fifo_1_almost_full [get_bd_pins action_config_0/last_addr_fifo_full] [get_bd_pins axis_compl_fifo_0/almost_full] + connect_bd_net -net axis_data_fifo_10_almost_empty [get_bd_pins action_config_0/last_data_fifo_empty] [get_bd_pins axis_data_fifo_3/almost_empty] + connect_bd_net -net axis_data_fifo_10_almost_full [get_bd_pins action_config_0/last_data_fifo_full] [get_bd_pins axis_data_fifo_3/almost_full] connect_bd_net -net axis_data_fifo_4_almost_empty [get_bd_pins action_config_0/calib_data_fifo_empty] [get_bd_pins axis_data_fifo_1/almost_empty] connect_bd_net -net axis_data_fifo_4_almost_full [get_bd_pins action_config_0/calib_data_fifo_full] [get_bd_pins axis_data_fifo_1/almost_full] connect_bd_net -net axis_data_fifo_c2h_cmd_almost_empty [get_bd_pins action_config_0/c2h_cmd_fifo_empty] [get_bd_pins axis_data_fifo_c2h_cmd/almost_empty] @@ -461,22 +509,17 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_net -net axis_frame_generator_fifo_0_almost_full [get_bd_pins action_config_0/frame_generator_fifo_full] [get_bd_pins axis_frame_generator_fifo_0/almost_full] connect_bd_net -net axis_udp_fifo_0_almost_empty [get_bd_pins action_config_0/udp_fifo_empty] [get_bd_pins axis_udp_fifo_0/almost_empty] connect_bd_net -net axis_udp_fifo_0_almost_full [get_bd_pins action_config_0/udp_fifo_full] [get_bd_pins axis_udp_fifo_0/almost_full] - connect_bd_net -net axis_work_completion_fifo_0_almost_empty [get_bd_pins action_config_0/work_compl_fifo_empty] [get_bd_pins axis_work_completion_fifo_0/almost_empty] - connect_bd_net -net axis_work_completion_fifo_0_almost_full [get_bd_pins action_config_0/work_compl_fifo_full] [get_bd_pins axis_work_completion_fifo_0/almost_full] - connect_bd_net -net axis_work_request_fifo_0_almost_empty [get_bd_pins action_config_0/work_req_fifo_empty] [get_bd_pins axis_work_request_fifo_0/almost_empty] - connect_bd_net -net axis_work_request_fifo_0_almost_full [get_bd_pins action_config_0/work_req_fifo_full] [get_bd_pins axis_work_request_fifo_0/almost_full] connect_bd_net -net data_collection_fsm_0_out_idle_V [get_bd_pins action_config_0/data_collection_idle] [get_bd_pins data_collection_fsm_0/out_idle] connect_bd_net -net hbm_cache_almost_empty [get_bd_pins action_config_0/hbm_handles_fifo_empty] [get_bd_pins hbm_cache/hbm_handle_fifo_empty] connect_bd_net -net hbm_cache_almost_empty1 [get_bd_pins action_config_0/hbm_compl_fifo_empty] [get_bd_pins hbm_cache/compl_fifo_empty] connect_bd_net -net hbm_cache_almost_full [get_bd_pins action_config_0/hbm_handles_fifo_full] [get_bd_pins hbm_cache/hbm_handle_fifo_full] connect_bd_net -net hbm_cache_almost_full1 [get_bd_pins action_config_0/hbm_compl_fifo_full] [get_bd_pins hbm_cache/compl_fifo_full] - connect_bd_net -net host_writer_0_err_reg [get_bd_pins action_config_0/host_writer_err] [get_bd_pins host_writer_0/err_reg] - connect_bd_net -net host_writer_0_err_reg_ap_vld [get_bd_pins action_config_0/host_writer_err_valid] [get_bd_pins host_writer_0/err_reg_ap_vld] connect_bd_net -net host_writer_0_idle [get_bd_pins action_config_0/host_writer_idle] [get_bd_pins host_writer_0/idle] connect_bd_net -net host_writer_0_packets_processed [get_bd_pins action_config_0/packets_processed] [get_bd_pins host_writer_0/packets_processed] connect_bd_net -net host_writer_0_packets_processed_ap_vld [get_bd_pins action_config_0/packets_processed_valid] [get_bd_pins host_writer_0/packets_processed_ap_vld] - connect_bd_net -net mailbox_0_Interrupt_0 [get_bd_pins action_config_0/mailbox_interrupt_0] [get_bd_pins mailbox_0/Interrupt_0] - connect_bd_net -net mailbox_0_Interrupt_1 [get_bd_pins action_config_0/mailbox_interrupt_1] [get_bd_pins mailbox_0/Interrupt_1] + connect_bd_net -net image_processing_almost_empty [get_bd_pins action_config_0/proc_fifo_empty] [get_bd_pins image_processing/proc_fifo_empty] + connect_bd_net -net image_processing_almost_full [get_bd_pins action_config_0/proc_fifo_full] [get_bd_pins image_processing/proc_fifo_full] + connect_bd_net -net mailbox_0_Interrupt_0 [get_bd_pins Interrupt_0] [get_bd_pins action_config_0/mailbox_interrupt_0] [get_bd_pins mailbox_0/Interrupt_0] connect_bd_net -net network_stack_counter [get_bd_pins action_config_0/packets_eth] [get_bd_pins network_stack/packets_eth] connect_bd_net -net network_stack_counter_ap_vld [get_bd_pins action_config_0/packets_eth_valid] [get_bd_pins network_stack/packets_eth_ap_vld] connect_bd_net -net network_stack_counter_eth_error [get_bd_pins action_config_0/udp_err_eth] [get_bd_pins network_stack/counter_eth_error] @@ -489,21 +532,21 @@ proc create_hier_cell_jungfraujoch { parentCell nameHier } { connect_bd_net -net network_stack_packets_sls_ap_vld [get_bd_pins action_config_0/packets_sls_valid] [get_bd_pins network_stack/packets_sls_ap_vld] connect_bd_net -net network_stack_packets_udp [get_bd_pins action_config_0/packets_udp] [get_bd_pins network_stack/packets_udp] connect_bd_net -net network_stack_packets_udp_ap_vld [get_bd_pins action_config_0/packets_udp_valid] [get_bd_pins network_stack/packets_udp_ap_vld] - connect_bd_net -net reset_axi [get_bd_pins axi_rst_n] [get_bd_pins action_config_0/resetn] [get_bd_pins axis_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_adu_histo_result_fifo/s_axis_aresetn] [get_bd_pins axis_compl_fifo_1/s_axis_aresetn] [get_bd_pins axis_compl_fifo_6/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_11/s_axis_aresetn] [get_bd_pins axis_data_fifo_12/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aresetn] [get_bd_pins axis_eth_in_fifo/s_axis_aresetn] [get_bd_pins axis_frame_generator_fifo_0/s_axis_aresetn] [get_bd_pins axis_register_slice_addr_0/aresetn] [get_bd_pins axis_register_slice_data_0/aresetn] [get_bd_pins axis_register_slice_data_in_0/aresetn] [get_bd_pins axis_register_slice_host_mem/aresetn] [get_bd_pins axis_register_slice_udp/aresetn] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_udp_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_completion_fifo_0/s_axis_aresetn] [get_bd_pins axis_work_request_fifo_0/s_axis_aresetn] [get_bd_pins hbm_cache/axi_rst_n] [get_bd_pins image_processing/axi_rst_n] [get_bd_pins network_stack/resetn] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins smartconnect_1/aresetn] [get_bd_pins smartconnect_2/aresetn] - connect_bd_net -net reset_hls [get_bd_pins ap_rst_n] [get_bd_pins data_collection_fsm_0/ap_rst_n] [get_bd_pins frame_generator_0/ap_rst_n] [get_bd_pins hbm_cache/ap_rst_n] [get_bd_pins host_writer_0/ap_rst_n] [get_bd_pins image_processing/ap_rst_n] [get_bd_pins load_calibration_0/ap_rst_n] [get_bd_pins mailbox_0/S0_AXI_ARESETN] [get_bd_pins network_stack/ap_rst_n] [get_bd_pins stream_merge_0/ap_rst_n] [get_bd_pins timer_hbm/ap_rst_n] [get_bd_pins timer_host/ap_rst_n] [get_bd_pins timer_proc/ap_rst_n] - connect_bd_net -net timer_hbm_stalls [get_bd_pins action_config_0/stalls_hbm] [get_bd_pins timer_hbm/stalls] - connect_bd_net -net timer_hbm_stalls_ap_vld [get_bd_pins action_config_0/stalls_hbm_valid] [get_bd_pins timer_hbm/stalls_ap_vld] - connect_bd_net -net timer_host_stalls [get_bd_pins action_config_0/stalls_host] [get_bd_pins timer_host/stalls] - connect_bd_net -net timer_host_stalls_ap_vld [get_bd_pins action_config_0/stalls_host_valid] [get_bd_pins timer_host/stalls_ap_vld] - connect_bd_net -net timer_proc_stalls [get_bd_pins action_config_0/stalls_proc] [get_bd_pins timer_proc/stalls] - connect_bd_net -net timer_proc_stalls_ap_vld [get_bd_pins action_config_0/stalls_proc_valid] [get_bd_pins timer_proc/stalls_ap_vld] + connect_bd_net -net reset_axi [get_bd_pins axi_rst_n] [get_bd_pins action_config_0/resetn] [get_bd_pins axis_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_adu_histo_result_fifo/s_axis_aresetn] [get_bd_pins axis_compl_fifo_0/s_axis_aresetn] [get_bd_pins axis_compl_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_0/s_axis_aresetn] [get_bd_pins axis_data_fifo_1/s_axis_aresetn] [get_bd_pins axis_data_fifo_2/s_axis_aresetn] [get_bd_pins axis_data_fifo_3/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_c2h_data/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_cmd/s_axis_aresetn] [get_bd_pins axis_data_fifo_h2c_data/s_axis_aresetn] [get_bd_pins axis_eth_in_fifo/s_axis_aresetn] [get_bd_pins axis_frame_generator_fifo_0/s_axis_aresetn] [get_bd_pins axis_register_slice_addr_0/aresetn] [get_bd_pins axis_register_slice_data_0/aresetn] [get_bd_pins axis_register_slice_data_in_0/aresetn] [get_bd_pins axis_register_slice_host_mem/aresetn] [get_bd_pins axis_register_slice_udp/aresetn] [get_bd_pins axis_udp_addr_fifo_0/s_axis_aresetn] [get_bd_pins axis_udp_fifo_0/s_axis_aresetn] [get_bd_pins hbm_cache/axi_rst_n] [get_bd_pins image_processing/axi_rst_n] [get_bd_pins network_stack/resetn] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins smartconnect_1/aresetn] [get_bd_pins smartconnect_2/aresetn] [get_bd_pins smartconnect_3/aresetn] + connect_bd_net -net reset_hls [get_bd_pins ap_rst_n] [get_bd_pins axi_bram_ctrl_calibration_addr_0/s_axi_aresetn] [get_bd_pins axi_bram_ctrl_calibration_addr_1/s_axi_aresetn] [get_bd_pins data_collection_fsm_0/ap_rst_n] [get_bd_pins frame_generator_0/ap_rst_n] [get_bd_pins hbm_cache/ap_rst_n] [get_bd_pins host_writer_0/ap_rst_n] [get_bd_pins image_processing/ap_rst_n] [get_bd_pins load_calibration_0/ap_rst_n] [get_bd_pins mailbox_0/S0_AXI_ARESETN] [get_bd_pins network_stack/ap_rst_n] [get_bd_pins stream_merge_0/ap_rst_n] [get_bd_pins timer_hbm/ap_rst_n] [get_bd_pins timer_host/ap_rst_n] [get_bd_pins timer_proc/ap_rst_n] connect_bd_net -net timer_hbm_beats [get_bd_pins action_config_0/beats_hbm] [get_bd_pins timer_hbm/beats] connect_bd_net -net timer_hbm_beats_ap_vld [get_bd_pins action_config_0/beats_hbm_valid] [get_bd_pins timer_hbm/beats_ap_vld] + connect_bd_net -net timer_hbm_stalls [get_bd_pins action_config_0/stalls_hbm] [get_bd_pins timer_hbm/stalls] + connect_bd_net -net timer_hbm_stalls_ap_vld [get_bd_pins action_config_0/stalls_hbm_valid] [get_bd_pins timer_hbm/stalls_ap_vld] connect_bd_net -net timer_host_beats [get_bd_pins action_config_0/beats_host] [get_bd_pins timer_host/beats] connect_bd_net -net timer_host_beats_ap_vld [get_bd_pins action_config_0/beats_host_valid] [get_bd_pins timer_host/beats_ap_vld] + connect_bd_net -net timer_host_stalls [get_bd_pins action_config_0/stalls_host] [get_bd_pins timer_host/stalls] + connect_bd_net -net timer_host_stalls_ap_vld [get_bd_pins action_config_0/stalls_host_valid] [get_bd_pins timer_host/stalls_ap_vld] connect_bd_net -net timer_proc_beats [get_bd_pins action_config_0/beats_proc] [get_bd_pins timer_proc/beats] connect_bd_net -net timer_proc_beats_ap_vld [get_bd_pins action_config_0/beats_proc_valid] [get_bd_pins timer_proc/beats_ap_vld] - + connect_bd_net -net timer_proc_stalls [get_bd_pins action_config_0/stalls_proc] [get_bd_pins timer_proc/stalls] + connect_bd_net -net timer_proc_stalls_ap_vld [get_bd_pins action_config_0/stalls_proc_valid] [get_bd_pins timer_proc/stalls_ap_vld] + # Restore current instance current_bd_instance $oldCurInst } diff --git a/fpga/scripts/mac_100g_pcie.tcl b/fpga/scripts/mac_100g_pcie.tcl index 8a6f64c5..27d65111 100644 --- a/fpga/scripts/mac_100g_pcie.tcl +++ b/fpga/scripts/mac_100g_pcie.tcl @@ -49,10 +49,9 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { # Create pins create_bd_pin -dir I -type rst ap_rst_n create_bd_pin -dir I -type clk axiclk - create_bd_pin -dir O eth_busy_n create_bd_pin -dir I -type clk refclk100 create_bd_pin -dir I -type rst resetn - create_bd_pin -dir O stat_rx_aligned_n + create_bd_pin -dir O -from 0 -to 0 stat_rx_aligned_n # Create instance: axis_data_fifo_rx_0, and set properties set axis_data_fifo_rx_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_rx_0 ] @@ -64,9 +63,9 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { # Create instance: axis_data_fifo_rx_1, and set properties set axis_data_fifo_rx_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_data_fifo:2.0 axis_data_fifo_rx_1 ] set_property -dict [ list \ - CONFIG.FIFO_DEPTH {16384} \ - CONFIG.FIFO_MEMORY_TYPE {ultra} \ CONFIG.ENABLE_ECC {1} \ + CONFIG.FIFO_DEPTH {2048} \ + CONFIG.FIFO_MEMORY_TYPE {ultra} \ ] $axis_data_fifo_rx_1 # Create instance: axis_data_fifo_tx, and set properties @@ -95,17 +94,6 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { CONFIG.REG_CONFIG {16} \ ] $axis_register_slice_tx - # Create instance: check_eth_busy_0, and set properties - set block_name check_eth_busy - set block_cell_name check_eth_busy_0 - if { [catch {set check_eth_busy_0 [create_bd_cell -type module -reference $block_name $block_cell_name] } errmsg] } { - catch {common::send_gid_msg -ssname BD::TCL -id 2095 -severity "ERROR" "Unable to add referenced block <$block_name>. Please add the files for ${block_name}'s definition into the project."} - return 1 - } elseif { $check_eth_busy_0 eq "" } { - catch {common::send_gid_msg -ssname BD::TCL -id 2096 -severity "ERROR" "Unable to referenced block <$block_name>. Please add the files for ${block_name}'s definition into the project."} - return 1 - } - # Create instance: cmac_usplus_0, and set properties set cmac_usplus_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:cmac_usplus:3.1 cmac_usplus_0 ] set_property -dict [ list \ @@ -116,10 +104,11 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { CONFIG.GT_DRP_CLK {100.00} \ CONFIG.GT_GROUP_SELECT {X0Y24~X0Y27} \ CONFIG.GT_REF_CLK_FREQ {161.1328125} \ + CONFIG.INCLUDE_AUTO_NEG_LT_LOGIC {1} \ CONFIG.INCLUDE_RS_FEC {1} \ CONFIG.INCLUDE_STATISTICS_COUNTERS {1} \ CONFIG.NUM_LANES {4x25} \ - CONFIG.RX_EQ_MODE {AUTO} \ + CONFIG.RX_EQ_MODE {LPM} \ CONFIG.RX_FLOW_CONTROL {0} \ CONFIG.TX_FLOW_CONTROL {0} \ CONFIG.USER_INTERFACE {AXIS} \ @@ -149,14 +138,6 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { CONFIG.LOGO_FILE {data/sym_notgate.png} \ ] $util_vector_logic_2 - # Create instance: util_vector_logic_3, and set properties - set util_vector_logic_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_3 ] - set_property -dict [ list \ - CONFIG.C_OPERATION {not} \ - CONFIG.C_SIZE {1} \ - CONFIG.LOGO_FILE {data/sym_notgate.png} \ - ] $util_vector_logic_3 - # Create instance: zero_0, and set properties set zero_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 zero_0 ] set_property -dict [ list \ @@ -171,8 +152,7 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { connect_bd_intf_net -intf_net axis_register_slice_rx_0_M_AXIS [get_bd_intf_pins axis_data_fifo_rx_1/S_AXIS] [get_bd_intf_pins axis_register_slice_rx_0/M_AXIS] connect_bd_intf_net -intf_net axis_register_slice_rx_1_M_AXIS [get_bd_intf_pins m_axis_eth_in] [get_bd_intf_pins axis_register_slice_rx_1/M_AXIS] connect_bd_intf_net -intf_net axis_register_slice_tx_M_AXIS [get_bd_intf_pins axis_data_fifo_tx/S_AXIS] [get_bd_intf_pins axis_register_slice_tx/M_AXIS] - connect_bd_intf_net -intf_net check_eth_busy_0_M_AXIS [get_bd_intf_pins axis_data_fifo_rx_0/S_AXIS] [get_bd_intf_pins check_eth_busy_0/M_AXIS] - connect_bd_intf_net -intf_net cmac_usplus_0_axis_rx [get_bd_intf_pins check_eth_busy_0/S_AXIS] [get_bd_intf_pins cmac_usplus_0/axis_rx] + connect_bd_intf_net -intf_net cmac_usplus_0_axis_rx [get_bd_intf_pins axis_data_fifo_rx_0/S_AXIS] [get_bd_intf_pins cmac_usplus_0/axis_rx] connect_bd_intf_net -intf_net gt_ref_clk_0_1 [get_bd_intf_pins qsfp_ref] [get_bd_intf_pins cmac_usplus_0/gt_ref_clk] connect_bd_intf_net -intf_net s_axi_1 [get_bd_intf_pins s_axi] [get_bd_intf_pins cmac_usplus_0/s_axi] connect_bd_intf_net -intf_net s_axis_eth_out_1 [get_bd_intf_pins s_axis_eth_out] [get_bd_intf_pins axis_register_slice_tx/S_AXIS] @@ -180,8 +160,7 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { # Create port connections connect_bd_net -net ap_rst_n_1 [get_bd_pins ap_rst_n] [get_bd_pins util_vector_logic_0/Op1] connect_bd_net -net axi_clk_net [get_bd_pins axiclk] [get_bd_pins axis_data_fifo_rx_0/m_axis_aclk] [get_bd_pins axis_data_fifo_rx_1/s_axis_aclk] [get_bd_pins axis_data_fifo_tx/s_axis_aclk] [get_bd_pins axis_register_slice_rx_0/aclk] [get_bd_pins axis_register_slice_rx_1/aclk] [get_bd_pins axis_register_slice_tx/aclk] - connect_bd_net -net check_eth_busy_0_eth_busy [get_bd_pins check_eth_busy_0/eth_busy] [get_bd_pins util_vector_logic_3/Op1] - connect_bd_net -net cmac_usplus_0_gt_txusrclk2 [get_bd_pins axis_data_fifo_rx_0/s_axis_aclk] [get_bd_pins axis_data_fifo_tx/m_axis_aclk] [get_bd_pins check_eth_busy_0/clk] [get_bd_pins cmac_usplus_0/gt_txusrclk2] [get_bd_pins cmac_usplus_0/rx_clk] + connect_bd_net -net cmac_usplus_0_gt_txusrclk2 [get_bd_pins axis_data_fifo_rx_0/s_axis_aclk] [get_bd_pins axis_data_fifo_tx/m_axis_aclk] [get_bd_pins cmac_usplus_0/gt_txusrclk2] [get_bd_pins cmac_usplus_0/rx_clk] connect_bd_net -net cmac_usplus_0_stat_rx_aligned [get_bd_pins cmac_usplus_0/stat_rx_aligned] [get_bd_pins util_vector_logic_2/Op1] connect_bd_net -net cmac_usplus_0_usr_rx_reset [get_bd_pins cmac_usplus_0/usr_rx_reset] [get_bd_pins util_vector_logic_1/Op1] connect_bd_net -net init_clk_1 [get_bd_pins refclk100] [get_bd_pins cmac_usplus_0/init_clk] [get_bd_pins cmac_usplus_0/s_axi_aclk] @@ -189,7 +168,6 @@ proc create_hier_cell_mac_100g { parentCell nameHier } { connect_bd_net -net util_vector_logic_0_Res [get_bd_pins cmac_usplus_0/s_axi_sreset] [get_bd_pins cmac_usplus_0/sys_reset] [get_bd_pins util_vector_logic_0/Res] connect_bd_net -net util_vector_logic_1_Res [get_bd_pins axis_data_fifo_rx_0/s_axis_aresetn] [get_bd_pins util_vector_logic_1/Res] connect_bd_net -net util_vector_logic_2_Res [get_bd_pins stat_rx_aligned_n] [get_bd_pins util_vector_logic_2/Res] - connect_bd_net -net util_vector_logic_3_Res [get_bd_pins eth_busy_n] [get_bd_pins util_vector_logic_3/Res] connect_bd_net -net zero_0_dout [get_bd_pins cmac_usplus_0/core_drp_reset] [get_bd_pins cmac_usplus_0/core_rx_reset] [get_bd_pins cmac_usplus_0/core_tx_reset] [get_bd_pins cmac_usplus_0/drp_clk] [get_bd_pins zero_0/dout] # Restore current instance diff --git a/fpga/scripts/mac_4x10g.tcl b/fpga/scripts/mac_4x10g.tcl new file mode 100644 index 00000000..95b8b561 --- /dev/null +++ b/fpga/scripts/mac_4x10g.tcl @@ -0,0 +1,214 @@ +## Copyright (2019-2023) Paul Scherrer Institute + +# Hierarchical cell: mac_4x10g +proc create_hier_cell_mac_4x10g { parentCell nameHier } { + + variable script_folder + + if { $parentCell eq "" || $nameHier eq "" } { + catch {common::send_gid_msg -ssname BD::TCL -id 2092 -severity "ERROR" "create_hier_cell_mac_4x10g() - Empty argument(s)!"} + return + } + + # Get object for parentCell + set parentObj [get_bd_cells $parentCell] + if { $parentObj == "" } { + catch {common::send_gid_msg -ssname BD::TCL -id 2090 -severity "ERROR" "Unable to find parent cell <$parentCell>!"} + return + } + + # Make sure parentObj is hier blk + set parentType [get_property TYPE $parentObj] + if { $parentType ne "hier" } { + catch {common::send_gid_msg -ssname BD::TCL -id 2091 -severity "ERROR" "Parent <$parentObj> has TYPE = <$parentType>. Expected to be ."} + return + } + + # Save current instance; Restore later + set oldCurInst [current_bd_instance .] + + # Set parent object as current + current_bd_instance $parentObj + + # Create cell and set as current instance + set hier_obj [create_bd_cell -type hier $nameHier] + current_bd_instance $hier_obj + + # Create interface pins + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:axis_rtl:1.0 m_axis + + create_bd_intf_pin -mode Master -vlnv xilinx.com:interface:gt_rtl:1.0 qsfp1 + + create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:diff_clock_rtl:1.0 qsfp_ref + + create_bd_intf_pin -mode Slave -vlnv xilinx.com:interface:aximm_rtl:1.0 s_axi + + + # Create pins + create_bd_pin -dir I -type rst axi_aresetn + create_bd_pin -dir I -type clk m_axis_aclk + create_bd_pin -dir I -type clk refclk50 + + # Create instance: axis_dwidth_converter_0, and set properties + set axis_dwidth_converter_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_dwidth_converter:1.1 axis_dwidth_converter_0 ] + set_property -dict [ list \ + CONFIG.M_TDATA_NUM_BYTES {64} \ + ] $axis_dwidth_converter_0 + + # Create instance: axis_dwidth_converter_1, and set properties + set axis_dwidth_converter_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_dwidth_converter:1.1 axis_dwidth_converter_1 ] + set_property -dict [ list \ + CONFIG.M_TDATA_NUM_BYTES {64} \ + ] $axis_dwidth_converter_1 + + # Create instance: axis_dwidth_converter_2, and set properties + set axis_dwidth_converter_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_dwidth_converter:1.1 axis_dwidth_converter_2 ] + set_property -dict [ list \ + CONFIG.M_TDATA_NUM_BYTES {64} \ + ] $axis_dwidth_converter_2 + + # Create instance: axis_dwidth_converter_3, and set properties + set axis_dwidth_converter_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_dwidth_converter:1.1 axis_dwidth_converter_3 ] + set_property -dict [ list \ + CONFIG.M_TDATA_NUM_BYTES {64} \ + ] $axis_dwidth_converter_3 + + # Create instance: axis_interconnect_0, and set properties + set axis_interconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:axis_interconnect:2.1 axis_interconnect_0 ] + set_property -dict [ list \ + CONFIG.ARB_ALGORITHM {0} \ + CONFIG.ARB_ON_TLAST {1} \ + CONFIG.ENABLE_ADVANCED_OPTIONS {0} \ + CONFIG.M00_FIFO_DEPTH {256} \ + CONFIG.M00_HAS_REGSLICE {1} \ + CONFIG.NUM_MI {1} \ + CONFIG.NUM_SI {4} \ + CONFIG.S00_FIFO_DEPTH {512} \ + CONFIG.S00_FIFO_MODE {1} \ + CONFIG.S01_FIFO_DEPTH {512} \ + CONFIG.S01_FIFO_MODE {1} \ + CONFIG.S02_FIFO_DEPTH {512} \ + CONFIG.S02_FIFO_MODE {1} \ + CONFIG.S03_FIFO_DEPTH {512} \ + CONFIG.S03_FIFO_MODE {1} \ + ] $axis_interconnect_0 + + # Create instance: smartconnect_0, and set properties + set smartconnect_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:smartconnect:1.0 smartconnect_0 ] + set_property -dict [ list \ + CONFIG.NUM_MI {4} \ + CONFIG.NUM_SI {1} \ + ] $smartconnect_0 + + # Create instance: util_vector_logic_0, and set properties + set util_vector_logic_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_0 ] + set_property -dict [ list \ + CONFIG.C_OPERATION {not} \ + CONFIG.C_SIZE {1} \ + CONFIG.LOGO_FILE {data/sym_notgate.png} \ + ] $util_vector_logic_0 + + # Create instance: util_vector_logic_1, and set properties + set util_vector_logic_1 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_1 ] + set_property -dict [ list \ + CONFIG.C_OPERATION {not} \ + CONFIG.C_SIZE {1} \ + CONFIG.LOGO_FILE {data/sym_notgate.png} \ + ] $util_vector_logic_1 + + # Create instance: util_vector_logic_2, and set properties + set util_vector_logic_2 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_2 ] + set_property -dict [ list \ + CONFIG.C_OPERATION {not} \ + CONFIG.C_SIZE {1} \ + CONFIG.LOGO_FILE {data/sym_notgate.png} \ + ] $util_vector_logic_2 + + # Create instance: util_vector_logic_3, and set properties + set util_vector_logic_3 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_3 ] + set_property -dict [ list \ + CONFIG.C_OPERATION {not} \ + CONFIG.C_SIZE {1} \ + CONFIG.LOGO_FILE {data/sym_notgate.png} \ + ] $util_vector_logic_3 + + # Create instance: util_vector_logic_4, and set properties + set util_vector_logic_4 [ create_bd_cell -type ip -vlnv xilinx.com:ip:util_vector_logic:2.0 util_vector_logic_4 ] + set_property -dict [ list \ + CONFIG.C_OPERATION {not} \ + CONFIG.C_SIZE {1} \ + CONFIG.LOGO_FILE {data/sym_notgate.png} \ + ] $util_vector_logic_4 + + # Create instance: xlconstant_0, and set properties + set xlconstant_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 xlconstant_0 ] + set_property -dict [ list \ + CONFIG.CONST_VAL {b101} \ + CONFIG.CONST_WIDTH {3} \ + ] $xlconstant_0 + + # Create instance: xxv_ethernet_0, and set properties + set xxv_ethernet_0 [ create_bd_cell -type ip -vlnv xilinx.com:ip:xxv_ethernet:4.1 xxv_ethernet_0 ] + set_property -dict [ list \ + CONFIG.BASE_R_KR {BASE-R} \ + CONFIG.ENABLE_GT_BOARD_INTERFACE {1} \ + CONFIG.ENABLE_PIPELINE_REG {1} \ + CONFIG.GT_GROUP_SELECT {Quad_X0Y7} \ + CONFIG.GT_REF_CLK_FREQ {161.1328125} \ + CONFIG.INCLUDE_AXI4_INTERFACE {1} \ + CONFIG.INCLUDE_STATISTICS_COUNTERS {1} \ + CONFIG.LANE1_GT_LOC {X0Y28} \ + CONFIG.LANE2_GT_LOC {X0Y29} \ + CONFIG.LANE3_GT_LOC {X0Y30} \ + CONFIG.LANE4_GT_LOC {X0Y31} \ + CONFIG.LINE_RATE {10} \ + CONFIG.NUM_OF_CORES {4} \ + ] $xxv_ethernet_0 + + # Create instance: zero, and set properties + set zero [ create_bd_cell -type ip -vlnv xilinx.com:ip:xlconstant:1.1 zero ] + set_property -dict [ list \ + CONFIG.CONST_VAL {0} \ + ] $zero + + # Create interface connections + connect_bd_intf_net -intf_net Conn1 [get_bd_intf_pins qsfp_ref] [get_bd_intf_pins xxv_ethernet_0/gt_ref_clk] + connect_bd_intf_net -intf_net Conn2 [get_bd_intf_pins qsfp1] [get_bd_intf_pins xxv_ethernet_0/gt_serial_port] + connect_bd_intf_net -intf_net Conn3 [get_bd_intf_pins s_axi] [get_bd_intf_pins smartconnect_0/S00_AXI] + connect_bd_intf_net -intf_net S01_AXIS_1 [get_bd_intf_pins axis_dwidth_converter_1/M_AXIS] [get_bd_intf_pins axis_interconnect_0/S01_AXIS] + connect_bd_intf_net -intf_net axis_dwidth_converter_0_M_AXIS [get_bd_intf_pins axis_dwidth_converter_0/M_AXIS] [get_bd_intf_pins axis_interconnect_0/S00_AXIS] + connect_bd_intf_net -intf_net axis_dwidth_converter_2_M_AXIS [get_bd_intf_pins axis_dwidth_converter_2/M_AXIS] [get_bd_intf_pins axis_interconnect_0/S02_AXIS] + connect_bd_intf_net -intf_net axis_dwidth_converter_3_M_AXIS [get_bd_intf_pins axis_dwidth_converter_3/M_AXIS] [get_bd_intf_pins axis_interconnect_0/S03_AXIS] + connect_bd_intf_net -intf_net axis_interconnect_0_M00_AXIS [get_bd_intf_pins m_axis] [get_bd_intf_pins axis_interconnect_0/M00_AXIS] + connect_bd_intf_net -intf_net smartconnect_0_M00_AXI [get_bd_intf_pins smartconnect_0/M00_AXI] [get_bd_intf_pins xxv_ethernet_0/s_axi_0] + connect_bd_intf_net -intf_net smartconnect_0_M01_AXI [get_bd_intf_pins smartconnect_0/M01_AXI] [get_bd_intf_pins xxv_ethernet_0/s_axi_1] + connect_bd_intf_net -intf_net smartconnect_0_M02_AXI [get_bd_intf_pins smartconnect_0/M02_AXI] [get_bd_intf_pins xxv_ethernet_0/s_axi_2] + connect_bd_intf_net -intf_net smartconnect_0_M03_AXI [get_bd_intf_pins smartconnect_0/M03_AXI] [get_bd_intf_pins xxv_ethernet_0/s_axi_3] + connect_bd_intf_net -intf_net xxv_ethernet_0_axis_rx_0 [get_bd_intf_pins axis_dwidth_converter_0/S_AXIS] [get_bd_intf_pins xxv_ethernet_0/axis_rx_0] + connect_bd_intf_net -intf_net xxv_ethernet_0_axis_rx_1 [get_bd_intf_pins axis_dwidth_converter_1/S_AXIS] [get_bd_intf_pins xxv_ethernet_0/axis_rx_1] + connect_bd_intf_net -intf_net xxv_ethernet_0_axis_rx_2 [get_bd_intf_pins axis_dwidth_converter_2/S_AXIS] [get_bd_intf_pins xxv_ethernet_0/axis_rx_2] + connect_bd_intf_net -intf_net xxv_ethernet_0_axis_rx_3 [get_bd_intf_pins axis_dwidth_converter_3/S_AXIS] [get_bd_intf_pins xxv_ethernet_0/axis_rx_3] + + # Create port connections + connect_bd_net -net dclk_1 [get_bd_pins refclk50] [get_bd_pins smartconnect_0/aclk] [get_bd_pins xxv_ethernet_0/dclk] [get_bd_pins xxv_ethernet_0/s_axi_aclk_0] [get_bd_pins xxv_ethernet_0/s_axi_aclk_1] [get_bd_pins xxv_ethernet_0/s_axi_aclk_2] [get_bd_pins xxv_ethernet_0/s_axi_aclk_3] + connect_bd_net -net m_axis_aclk_1 [get_bd_pins m_axis_aclk] [get_bd_pins axis_interconnect_0/ACLK] [get_bd_pins axis_interconnect_0/M00_AXIS_ACLK] + connect_bd_net -net s_axi_aresetn_0_1 [get_bd_pins axi_aresetn] [get_bd_pins axis_interconnect_0/ARESETN] [get_bd_pins axis_interconnect_0/M00_AXIS_ARESETN] [get_bd_pins smartconnect_0/aresetn] [get_bd_pins util_vector_logic_4/Op1] [get_bd_pins xxv_ethernet_0/s_axi_aresetn_0] [get_bd_pins xxv_ethernet_0/s_axi_aresetn_1] [get_bd_pins xxv_ethernet_0/s_axi_aresetn_2] [get_bd_pins xxv_ethernet_0/s_axi_aresetn_3] + connect_bd_net -net util_vector_logic_0_Res [get_bd_pins util_vector_logic_4/Res] [get_bd_pins xxv_ethernet_0/sys_reset] + connect_bd_net -net util_vector_logic_0_Res1 [get_bd_pins axis_dwidth_converter_0/aresetn] [get_bd_pins axis_interconnect_0/S00_AXIS_ARESETN] [get_bd_pins util_vector_logic_0/Res] + connect_bd_net -net util_vector_logic_1_Res [get_bd_pins axis_dwidth_converter_1/aresetn] [get_bd_pins axis_interconnect_0/S01_AXIS_ARESETN] [get_bd_pins util_vector_logic_1/Res] + connect_bd_net -net util_vector_logic_2_Res [get_bd_pins axis_dwidth_converter_2/aresetn] [get_bd_pins axis_interconnect_0/S02_AXIS_ARESETN] [get_bd_pins util_vector_logic_2/Res] + connect_bd_net -net util_vector_logic_3_Res [get_bd_pins axis_dwidth_converter_3/aresetn] [get_bd_pins axis_interconnect_0/S03_AXIS_ARESETN] [get_bd_pins util_vector_logic_3/Res] + connect_bd_net -net xlconstant_0_dout [get_bd_pins xlconstant_0/dout] [get_bd_pins xxv_ethernet_0/rxoutclksel_in_0] [get_bd_pins xxv_ethernet_0/rxoutclksel_in_1] [get_bd_pins xxv_ethernet_0/rxoutclksel_in_2] [get_bd_pins xxv_ethernet_0/rxoutclksel_in_3] [get_bd_pins xxv_ethernet_0/txoutclksel_in_0] [get_bd_pins xxv_ethernet_0/txoutclksel_in_1] [get_bd_pins xxv_ethernet_0/txoutclksel_in_2] [get_bd_pins xxv_ethernet_0/txoutclksel_in_3] + connect_bd_net -net xxv_ethernet_0_tx_clk_out_0 [get_bd_pins axis_dwidth_converter_0/aclk] [get_bd_pins axis_interconnect_0/S00_AXIS_ACLK] [get_bd_pins xxv_ethernet_0/rx_core_clk_0] [get_bd_pins xxv_ethernet_0/tx_clk_out_0] + connect_bd_net -net xxv_ethernet_0_tx_clk_out_1 [get_bd_pins axis_dwidth_converter_1/aclk] [get_bd_pins axis_interconnect_0/S01_AXIS_ACLK] [get_bd_pins xxv_ethernet_0/rx_core_clk_1] [get_bd_pins xxv_ethernet_0/tx_clk_out_1] + connect_bd_net -net xxv_ethernet_0_tx_clk_out_2 [get_bd_pins axis_dwidth_converter_2/aclk] [get_bd_pins axis_interconnect_0/S02_AXIS_ACLK] [get_bd_pins xxv_ethernet_0/rx_core_clk_2] [get_bd_pins xxv_ethernet_0/tx_clk_out_2] + connect_bd_net -net xxv_ethernet_0_tx_clk_out_3 [get_bd_pins axis_dwidth_converter_3/aclk] [get_bd_pins axis_interconnect_0/S03_AXIS_ACLK] [get_bd_pins xxv_ethernet_0/rx_core_clk_3] [get_bd_pins xxv_ethernet_0/tx_clk_out_3] + connect_bd_net -net xxv_ethernet_0_user_rx_reset_0 [get_bd_pins util_vector_logic_0/Op1] [get_bd_pins xxv_ethernet_0/user_rx_reset_0] + connect_bd_net -net xxv_ethernet_0_user_rx_reset_1 [get_bd_pins util_vector_logic_1/Op1] [get_bd_pins xxv_ethernet_0/user_rx_reset_1] + connect_bd_net -net xxv_ethernet_0_user_rx_reset_2 [get_bd_pins util_vector_logic_2/Op1] [get_bd_pins xxv_ethernet_0/user_rx_reset_2] + connect_bd_net -net xxv_ethernet_0_user_rx_reset_3 [get_bd_pins util_vector_logic_3/Op1] [get_bd_pins xxv_ethernet_0/user_rx_reset_3] + connect_bd_net -net zero_dout [get_bd_pins xxv_ethernet_0/gtwiz_reset_rx_datapath_0] [get_bd_pins xxv_ethernet_0/gtwiz_reset_rx_datapath_1] [get_bd_pins xxv_ethernet_0/gtwiz_reset_rx_datapath_2] [get_bd_pins xxv_ethernet_0/gtwiz_reset_rx_datapath_3] [get_bd_pins xxv_ethernet_0/gtwiz_reset_tx_datapath_0] [get_bd_pins xxv_ethernet_0/gtwiz_reset_tx_datapath_1] [get_bd_pins xxv_ethernet_0/gtwiz_reset_tx_datapath_2] [get_bd_pins xxv_ethernet_0/gtwiz_reset_tx_datapath_3] [get_bd_pins xxv_ethernet_0/qpllreset_in_0] [get_bd_pins xxv_ethernet_0/rx_reset_0] [get_bd_pins xxv_ethernet_0/rx_reset_1] [get_bd_pins xxv_ethernet_0/rx_reset_2] [get_bd_pins xxv_ethernet_0/rx_reset_3] [get_bd_pins xxv_ethernet_0/tx_reset_0] [get_bd_pins xxv_ethernet_0/tx_reset_1] [get_bd_pins xxv_ethernet_0/tx_reset_2] [get_bd_pins xxv_ethernet_0/tx_reset_3] [get_bd_pins zero/dout] + + # Restore current instance + current_bd_instance $oldCurInst +} \ No newline at end of file diff --git a/fpga/scripts/pcie_dma.tcl b/fpga/scripts/pcie_dma.tcl index ad976b87..cd0a7fad 100644 --- a/fpga/scripts/pcie_dma.tcl +++ b/fpga/scripts/pcie_dma.tcl @@ -175,6 +175,7 @@ proc create_hier_cell_pcie_dma_0 { parentCell nameHier } { CONFIG.vendor_id {10EE} \ CONFIG.xdma_axi_intf_mm {AXI_Stream} \ CONFIG.xdma_axilite_slave {true} \ + CONFIG.xdma_num_usr_irq {2} \ CONFIG.xdma_wnum_chnl {1} \ ] $xdma_0 diff --git a/fpga/xdc/pcie_timing.xdc b/fpga/xdc/pcie_timing.xdc index 81497bd6..baa4bc06 100644 --- a/fpga/xdc/pcie_timing.xdc +++ b/fpga/xdc/pcie_timing.xdc @@ -1,19 +1,22 @@ # PCIe reset set_false_path -from [get_ports pcie_perstn] -# Reset synchronizers -set_false_path -to [get_pins jfjoch_pcie_i/*/resetn_sync_0/*/q?_reg/CLR] - # From 100G example design set_false_path -to [get_pins -leaf -of_objects [get_cells -hier *cdc_to* -filter {is_sequential}] -filter {NAME=~*cmac_cdc*/*/D}] set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -datapath_only 3.103 set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -datapath_only 3.103 -set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -to [get_clocks refclk100] -datapath_only 3.103 -set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks refclk100] -datapath_only 3.103 -set_max_delay -from [get_clocks refclk100] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -datapath_only 10.000 -set_max_delay -from [get_clocks refclk100] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -datapath_only 10.000 +set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -to [get_clocks clk_out2_jfjoch_pcie_clk_wiz_0_0] -datapath_only 3.103 +set_max_delay -from [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks clk_out2_jfjoch_pcie_clk_wiz_0_0] -datapath_only 3.103 +set_max_delay -from [get_clocks clk_out2_jfjoch_pcie_clk_wiz_0_0] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/RXOUTCLK}]] -datapath_only 10.000 +set_max_delay -from [get_clocks clk_out2_jfjoch_pcie_clk_wiz_0_0] -to [get_clocks -of_objects [get_pins -hierarchical -filter {NAME =~ */mac_100g*/*/channel_inst/*_CHANNEL_PRIM_INST/TXOUTCLK}]] -datapath_only 10.000 create_waiver -type CDC -id {CDC-11} -user "cmac" -tags "10930"\ -desc "The CDC-11 warning is waived as fan-out is expected for this stat signal"\ -to [get_pins -of [get_cells -hier -filter {name =~ */*stat_rx_aligned/*cdc_to*}] -filter {name =~ *D}] + +# From 10G/25G example design +set_false_path -to [get_cells -hierarchical -filter {NAME =~ */i_*_axi_if_top/*/i_*_syncer/*meta_reg*}] +set_false_path -to [get_cells -hierarchical -filter {NAME =~ */i_*_SYNC*/*stretch_reg*}] + +set_false_path -to [get_cells -hierarchical -filter {NAME=~ */i*syncer/*d2_cdc_to*}] \ No newline at end of file diff --git a/image_analysis/RadialIntegrationProfile.cpp b/image_analysis/RadialIntegrationProfile.cpp index 70330273..97a9a068 100644 --- a/image_analysis/RadialIntegrationProfile.cpp +++ b/image_analysis/RadialIntegrationProfile.cpp @@ -3,11 +3,11 @@ #include "RadialIntegrationProfile.h" #include "../common/JFJochException.h" -inline float sum_to_count(int64_t sum, uint64_t count) { +inline float sum_to_count(float sum, uint64_t count) { if (count == 0) return 0.0f; else - return static_cast(static_cast(sum) / (static_cast(count * (1LU<<24)))); + return sum / (static_cast(count)); } RadialIntegrationProfile::RadialIntegrationProfile(RadialIntegrationMapping &mapping, @@ -17,7 +17,7 @@ RadialIntegrationProfile::RadialIntegrationProfile(RadialIntegrationMapping &map count(mapping.GetBinNumber(), 0){ } -void RadialIntegrationProfile::Add(const std::vector &in_sum, const std::vector &in_count) { +void RadialIntegrationProfile::Add(const std::vector &in_sum, const std::vector &in_count) { std::unique_lock ul(m); if ((in_sum.size() == sum.size()) && (in_count.size() == count.size())) { @@ -48,7 +48,7 @@ Plot RadialIntegrationProfile::GetPlot() const { float RadialIntegrationProfile::GetMeanValueOfBins(uint16_t min_bin, uint16_t max_bin) const { std::unique_lock ul(m); - int64_t ret_sum = 0; + float ret_sum = 0; uint64_t ret_count = 0; for (int i = std::min((uint16_t)sum.size(),min_bin); diff --git a/image_analysis/RadialIntegrationProfile.h b/image_analysis/RadialIntegrationProfile.h index 7ff91d66..39033f11 100644 --- a/image_analysis/RadialIntegrationProfile.h +++ b/image_analysis/RadialIntegrationProfile.h @@ -12,13 +12,13 @@ class RadialIntegrationProfile { mutable std::mutex m; - std::vector sum; + std::vector sum; std::vector count; std::vector bin_to_q; public: explicit RadialIntegrationProfile(RadialIntegrationMapping &mapping, const DiffractionExperiment &experiment); void Add(const DeviceOutput &result); - void Add(const std::vector &sum, const std::vector &count); + void Add(const std::vector &sum, const std::vector &count); std::vector GetResult() const; float GetMeanValueOfBins(uint16_t min_bin, uint16_t max_bin) const; Plot GetPlot() const; diff --git a/jungfrau/JFModulePedestal.cpp b/jungfrau/JFModulePedestal.cpp index bac8cab6..1337e6d7 100644 --- a/jungfrau/JFModulePedestal.cpp +++ b/jungfrau/JFModulePedestal.cpp @@ -5,80 +5,73 @@ JFModulePedestal::JFModulePedestal() : JFModulePedestal(0) {} -JFModulePedestal::JFModulePedestal(uint16_t default_value) { - data = std::make_unique(); - for (auto & i : data->pedestal) +JFModulePedestal::JFModulePedestal(uint16_t default_value) : pedestal(RAW_MODULE_SIZE), pedestal_mask(RAW_MODULE_SIZE) { + for (auto & i : pedestal) i = default_value; - for (auto & i : data->pedestal_mask) + for (auto & i : pedestal_mask) i = 0; - data->collection_time = 0; - data->frames = -1; + collection_time = 0; + frames = -1; } JFModulePedestal::JFModulePedestal(int default_value) : JFModulePedestal(static_cast(default_value)){} -JFModulePedestal::JFModulePedestal(const JFModulePedestal &other) { - data = std::make_unique(); - *data = *other.data; -} - -JFModulePedestal& JFModulePedestal::operator=(const JFModulePedestal& other) { - *data = *other.data; - return *this; -} - double JFModulePedestal::Mean() const { uint64_t ret = 0; - for (auto & i : data->pedestal) + for (auto & i : pedestal) ret += i; return static_cast(ret) / static_cast(RAW_MODULE_SIZE); } size_t JFModulePedestal::CountMaskedPixels() const { size_t ret = 0; - for (auto i : data->pedestal_mask) { + for (auto i : pedestal_mask) { if (i != 0) ret++; } return ret; } void JFModulePedestal::SetCollectionTime(time_t input) { - data->collection_time = input; + collection_time = input; } time_t JFModulePedestal::GetCollectionTime() const { - return data->collection_time; + return collection_time; } void JFModulePedestal::SetFrameCount(int64_t input) { - data->frames = input; + frames = input; } int64_t JFModulePedestal::GetFrameCount() const { - return data->frames; + return frames; } const uint16_t *JFModulePedestal::GetPedestal() const { - return data->pedestal; + return pedestal.data(); } const uint8_t *JFModulePedestal::GetPedestalMask() const { - return data->pedestal_mask; + return pedestal_mask.data(); } uint16_t *JFModulePedestal::GetPedestal() { - return data->pedestal; + return pedestal.data(); } uint8_t *JFModulePedestal::GetPedestalMask() { - return data->pedestal_mask; + return pedestal_mask.data(); } -JFModulePedestal::JFModulePedestal(const JFModulePedestalData *other) { - *data = *other; -} - -const JFModulePedestalData *JFModulePedestal::Serialized() const { - return data.get(); +void JFModulePedestal::ImportFPGAPedestal(const DeviceOutput *output) { + auto pedestal_array = (const uint16_t*) output->pixels; + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + pedestal[i] = (pedestal_array[i] + 2) / 4; + if (pedestal_array[i] >= 65534) + pedestal_mask[i] = 2; + else + pedestal_mask[i] = 0; + } + frames = output->module_statistics.packet_count; } diff --git a/jungfrau/JFModulePedestal.h b/jungfrau/JFModulePedestal.h index 690fe744..f7677f8f 100644 --- a/jungfrau/JFModulePedestal.h +++ b/jungfrau/JFModulePedestal.h @@ -10,31 +10,24 @@ #include #include "../common/Definitions.h" - -struct JFModulePedestalData { - uint16_t pedestal[RAW_MODULE_SIZE]; - uint8_t pedestal_mask[RAW_MODULE_SIZE]; - time_t collection_time; - int64_t frames; -}; +#include "../common/DeviceOutput.h" class JFModulePedestal { - std::unique_ptr data; + std::vector pedestal; + std::vector pedestal_mask; + time_t collection_time; + int64_t frames; public: JFModulePedestal(); explicit JFModulePedestal(uint16_t default_value); explicit JFModulePedestal(int default_value); - JFModulePedestal(const JFModulePedestal& other); - explicit JFModulePedestal(const JFModulePedestalData *other); - JFModulePedestal& operator=(const JFModulePedestal& other); - [[nodiscard]] const JFModulePedestalData* Serialized() const; - - [[nodiscard]] double Mean() const; + [[nodiscard]] double Mean() const; + void ImportFPGAPedestal(const DeviceOutput *output); template void LoadPedestal(const std::vector &vector) { for (int i = 0; i < RAW_MODULE_SIZE; i++) { - data->pedestal[i] = std::lround(vector[i]); + pedestal[i] = std::lround(vector[i]); } } diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index f66ffccd..0085f3a9 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -98,35 +98,8 @@ JFJochReceiver::JFJochReceiver(const DiffractionExperiment& in_experiment, || (experiment.GetDetectorMode() == DetectorMode::PedestalG1) || (experiment.GetDetectorMode() == DetectorMode::PedestalG2)) { - if (experiment.GetImageNum() > 0) { - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Saving and calculating pedestal is not supported for the time being"); - } - - if (experiment.GetDetectorMode() == DetectorMode::PedestalG0) { - pedestal_result.resize(experiment.GetModulesNum() * experiment.GetStorageCellNumber()); - for (int s = 0; s < experiment.GetStorageCellNumber(); s++) { - for (int d = 0; d < ndatastreams; d++) { - for (int m = 0; m < experiment.GetModulesNum(d); m++) { - auto handle = std::async(std::launch::async, &JFJochReceiver::MeasurePedestalThread, this, - d, m, - s); - frame_transformation_futures.emplace_back(std::move(handle)); - } - } - } - } else { - pedestal_result.resize(experiment.GetModulesNum()); - for (int d = 0; d < ndatastreams; d++) { - for (int m = 0; m < experiment.GetModulesNum(d); m++) { - auto handle = std::async(std::launch::async, &JFJochReceiver::MeasurePedestalThread, this, d, m, - 0); - frame_transformation_futures.emplace_back(std::move(handle)); - } - } - } - - logger.Info("Pedestal threads ready"); + if (experiment.GetImageNum() == 0) + pedestal_mode = true; } if (experiment.GetImageNum() > 0) { @@ -259,69 +232,42 @@ void JFJochReceiver::AcquireThread(uint16_t data_stream) { try { logger.Debug("Device thread {} wait for FPGA action complete", data_stream); - acquisition_device[data_stream].WaitForActionComplete(); + acquisition_device[data_stream].WaitForActionComplete(pedestal_mode); } catch (const JFJochException &e) { Cancel(e); } logger.Info("Device thread {} done", data_stream); } -void JFJochReceiver::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell) { - try { - NUMAHWPolicy::RunOnNode(acquisition_device[data_stream].GetNUMANode()); - } catch (const JFJochException &e) { - logger.Error("HW bind error {}", e.what()); - } - - JFPedestalCalc pedestal_calc(experiment); - - bool storage_cell_G1G2 = (experiment.GetStorageCellNumber() > 1) - && ((experiment.GetDetectorMode() == DetectorMode::PedestalG1) - || (experiment.GetDetectorMode() == DetectorMode::PedestalG2)); - size_t staring_frame; - size_t frame_stride; - - size_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number; +void JFJochReceiver::RetrievePedestal() { if (experiment.GetDetectorMode() == DetectorMode::PedestalG0) { - staring_frame = storage_cell; - frame_stride = experiment.GetStorageCellNumber(); - offset += experiment.GetModulesNum() * storage_cell; - } else { - staring_frame = 0; - frame_stride = 1; - } - - uint32_t storage_cell_header = UINT32_MAX; - logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} starting", - data_stream, module_number, storage_cell); - try { - for (size_t frame = staring_frame; frame < experiment.GetFrameNum(); frame += frame_stride) { - // Frame will be processed only if one already collects frame+2 - acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number); - if (!storage_cell_G1G2 || (frame % 2 == 1)) { - // Partial packets will bring more problems, than benefit - if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number)) { - pedestal_calc.AnalyzeImage((uint16_t *) acquisition_device[data_stream].GetDeviceOutput(frame, module_number)->pixels); + pedestal_result.resize(experiment.GetModulesNum() * experiment.GetStorageCellNumber()); + for (int s = 0; s < experiment.GetStorageCellNumber(); s++) { + for (int d = 0; d < ndatastreams; d++) { + for (int m = 0; m < experiment.GetModulesNum(d); m++) { + size_t offset = experiment.GetModulesNum() * s + experiment.GetFirstModuleOfDataStream(d) + m; + JFModulePedestal pedestal; + pedestal.ImportFPGAPedestal(acquisition_device[d].GetDeviceOutputPedestal(s, m)); + pedestal_result[offset] = pedestal; + pedestal_result[offset].SetCollectionTime(start_time.time_since_epoch().count() / 1e9); } - auto tmp = acquisition_device[data_stream].Counters().GetCompletion(frame, module_number).debug; - storage_cell_header = (tmp >> 8) & 0xF; } - - acquisition_device[data_stream].FrameBufferRelease(frame, module_number); - - UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number)); - UpdateMaxImage(frame); } - - if (experiment.GetDetectorMode() == DetectorMode::PedestalG0) - pedestal_calc.Export(pedestal_result[offset], 2); - else - pedestal_calc.Export(pedestal_result[offset]); - pedestal_result[offset].SetFrameCount(experiment.GetFrameNum()); - pedestal_result[offset].SetCollectionTime(start_time.time_since_epoch().count() / 1e9); - } catch (const JFJochException &e) { Cancel(e); } - logger.Info("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done", - data_stream, module_number, storage_cell, storage_cell_header); + } else { + pedestal_result.resize(experiment.GetModulesNum()); + for (int d = 0; d < ndatastreams; d++) { + for (int m = 0; m < experiment.GetModulesNum(d); m++) { + size_t offset = experiment.GetFirstModuleOfDataStream(d) + m; + JFModulePedestal pedestal; + if (experiment.GetStorageCellNumber() == 2) + pedestal.ImportFPGAPedestal(acquisition_device[d].GetDeviceOutputPedestal(1, m)); + else + pedestal.ImportFPGAPedestal(acquisition_device[d].GetDeviceOutputPedestal(0, m)); + pedestal_result[offset] = pedestal; + pedestal_result[offset].SetCollectionTime(start_time.time_since_epoch().count() / 1e9); + } + } + } } void JFJochReceiver::FrameTransformationThread() { @@ -376,27 +322,27 @@ void JFJochReceiver::FrameTransformationThread() { acquisition_device[d].Counters().WaitForFrame(image_number + 1, m); const int16_t *src; if (acquisition_device[d].Counters().IsAnyPacketCollected(image_number, m)) { - src = acquisition_device[d].GetDeviceOutput(image_number, m)->pixels; + const DeviceOutput* output = acquisition_device[d].GetDeviceOutput(image_number, m); + src = output->pixels; if (!send_image) { - Completion c = acquisition_device[d].Counters().GetCompletion(image_number, m); // the information is for first module/frame that was collected in full - message.bunch_id = c.bunchid; - message.jf_info = c.debug; + message.bunch_id = output->module_statistics.bunchid; + message.jf_info = output->module_statistics.debug; message.storage_cell = (message.jf_info >> 8) & 0xF; - message.timestamp = c.timestamp; - message.exptime = c.exptime; + message.timestamp = output->module_statistics.timestamp; + message.exptime = output->module_statistics.exptime; } send_image = true; size_t module_abs_number = experiment.GetFirstModuleOfDataStream(d) + m; - adu_histogram_module[module_abs_number].Add(*acquisition_device[d].GetDeviceOutput(image_number, m)); - adu_histogram_total.Add(*acquisition_device[d].GetDeviceOutput(image_number, m)); + adu_histogram_module[module_abs_number].Add(*output); + adu_histogram_total.Add(*output); if (rad_int_profile_image) - rad_int_profile_image->Add(*acquisition_device[d].GetDeviceOutput(image_number, m)); + rad_int_profile_image->Add(*output); if (find_spots) - strong_pixel_set.ReadFPGAOutput(experiment, *acquisition_device[d].GetDeviceOutput(image_number, m), m); + strong_pixel_set.ReadFPGAOutput(experiment, *output, m); } else src = acquisition_device[d].GetErrorFrameBuffer(); @@ -612,8 +558,10 @@ void JFJochReceiver::FinalizeMeasurement() { logger.Info("Disconnected from writers"); } - for (int d = 0; d < ndatastreams; d++) - acquisition_device[d].Cancel(); + if (!pedestal_mode) { + for (int d = 0; d < ndatastreams; d++) + acquisition_device[d].Cancel(); + } end_time = std::chrono::system_clock::now(); @@ -623,6 +571,8 @@ void JFJochReceiver::FinalizeMeasurement() { for (int i = 0; i < send_buffer_count; i++) send_buffer_avail.GetBlocking(); + RetrievePedestal(); + logger.Info("Devices stopped"); logger.Info("Receiving data done"); } diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 3d8e2406..70e3f7a5 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -112,6 +112,7 @@ class JFJochReceiver { Histogram indexing_solution_per_file; // TODO: Histogram a,b,c,alpha,beta,gamma + bool pedestal_mode = false; bool find_spots = false; int64_t max_image_number_sent = 0; @@ -129,11 +130,10 @@ class JFJochReceiver { void AcquireThread(uint16_t data_stream); void FrameTransformationThread(); - void MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell); void Cancel(const JFJochException &e); void FinalizeMeasurement(); DataProcessingSettings GetDataProcessingSettings(); - + void RetrievePedestal(); void UpdateMaxImage(int64_t image_number); void UpdateMaxDelay(int64_t delay); public: diff --git a/receiver/jfjoch_action_test.cpp b/receiver/jfjoch_action_test.cpp index f1e21f90..dbe08f1b 100644 --- a/receiver/jfjoch_action_test.cpp +++ b/receiver/jfjoch_action_test.cpp @@ -3,18 +3,14 @@ #include #include "../acquisition_device/PCIExpressDevice.h" -#include "../acquisition_device/MockAcquisitionDevice.h" #include "JFJochReceiverTest.h" #include "../tests/FPGAUnitTest.h" void print_usage(Logger &logger) { logger.Info("Usage ./jfjoch_action_test {} "); logger.Info("Options:"); - logger.Info(" -M use mock device"); logger.Info(" -R raw"); logger.Info(" -v verbose"); - logger.Info(" -H mock aq. dev. with HBM (DL380 with Intel MAX only)"); - logger.Info(" -D mock aq. dev. with DDR (2 NUMA node machines only)"); logger.Info(" -S number of summed frames"); logger.Info(" -I use 32-bit integer"); logger.Info(" -s number of data streams (acquisition devices)"); @@ -34,13 +30,9 @@ int main(int argc, char **argv) { uint16_t nmodules = 1; uint16_t nsummation = 1; size_t nimages = 2; - uint64_t processing_period = 20; uint16_t nthreads = 64; - bool use_mock_device = false; bool verbose = false; std::string numa_policy_name; - bool use_hbm_for_aq_dev = false; - bool use_ddr_for_aq_dev = false; bool raw_data = false; bool force_32bit = false; @@ -50,11 +42,8 @@ int main(int argc, char **argv) { } int opt; - while ((opt = getopt(argc, argv, "s:i:m:p:N:P:MvHDRIS:")) != -1) { + while ((opt = getopt(argc, argv, "s:i:m:p:N:P:vRIS:")) != -1) { switch (opt) { - case 'M': - use_mock_device = true; - break; case 'i': nimages = atol(optarg); break; @@ -67,9 +56,6 @@ int main(int argc, char **argv) { case 's': nstreams = atol(optarg); break; - case 'p': - processing_period = atol(optarg); - break; case 'N': nthreads = atol(optarg); break; @@ -79,12 +65,6 @@ int main(int argc, char **argv) { case 'P': numa_policy_name = std::string(optarg); break; - case 'H': - use_hbm_for_aq_dev = true; - break; - case 'D': - use_ddr_for_aq_dev = true; - break; case 'R': raw_data = true; break; @@ -132,42 +112,18 @@ int main(int argc, char **argv) { std::vector input(RAW_MODULE_SIZE, 0); LoadBinaryFile(image_path, input.data(), RAW_MODULE_SIZE); - if (use_mock_device) { - if (nmodules > 1) { - logger.Warning("Conversion results might be wrong with more than 1 module per stream"); - } + if (nstreams > dev_name.size()) { + logger.Error("Only {} data streams allowed on this platform", dev_name.size()); + exit(EXIT_FAILURE); + } - for (int i = 0; i < nstreams; i++) { - int16_t numa_node = -1; - - if (use_hbm_for_aq_dev) - numa_node = 2 + (i % 2); - else if (use_ddr_for_aq_dev) - numa_node = i % 2; - - if (numa_node != -1) - logger.Info("Pinning stream {} to NUMA node {}", i, numa_node); - - auto tmp = std::make_unique(i, 1024, numa_node); - tmp->SetCustomInternalGeneratorFrame(input); - tmp->EnableLogging(&logger); - aq_devices.Add(std::move(tmp)); - } - - } else { - if (nstreams > dev_name.size()) { - logger.Error("Only {} data streams allowed on this platform", dev_name.size()); - exit(EXIT_FAILURE); - } - - for (int i = 0; i < nstreams; i++) { - auto tmp = std::make_unique(i, dev_name[i]); - tmp->SetInternalGeneratorFrameForAllModules(input); - tmp->EnableLogging(&logger); - tmp->SetDefaultMAC(); - tmp->SetIPv4Address((i << 24) + 0x010a0a0a); - aq_devices.Add(std::move(tmp)); - } + for (int i = 0; i < nstreams; i++) { + auto tmp = std::make_unique(i, dev_name[i]); + tmp->SetInternalGeneratorFrameForAllModules(input); + tmp->EnableLogging(&logger); + tmp->SetDefaultMAC(); + tmp->SetIPv4Address((i << 24) + 0x010a0a0a); + aq_devices.Add(std::move(tmp)); } volatile bool done = false; @@ -183,20 +139,18 @@ int main(int argc, char **argv) { done = true; }); - if (!use_mock_device) { - while (!done) { - for (int i = 0; i < nstreams; i++) { - auto coll_status = aq_devices[i].GetDataCollectionStatus(); - auto dev_status = aq_devices[i].GetDeviceStatus(); - double power = (dev_status.fpga_pcie_12V_I_mA * dev_status.fpga_pcie_12V_V_mV - + dev_status.fpga_pcie_3p3V_I_mA * dev_status.fpga_pcie_3p3V_V_mV) / (1000.0 * 1000.0); - logger.Info("Device {}: Slowest packet: {:8d} Power: {:5.1f} W FPGA Temp: {:d} degC HBM Temp: {:d}/{:d} degC Stalls: {:15d}/{:15d}", - i, aq_devices[i].Counters().GetSlowestFrameNumber(), power, - dev_status.fpga_temp_C, dev_status.hbm_0_temp_C, dev_status.hbm_1_temp_C, - coll_status.pipeline_stalls_hbm, coll_status.pipeline_stalls_host); - } - std::this_thread::sleep_for(std::chrono::seconds(1)); + while (!done) { + for (int i = 0; i < nstreams; i++) { + auto coll_status = aq_devices[i].GetDataCollectionStatus(); + auto dev_status = aq_devices[i].GetDeviceStatus(); + double power_3p3v = (dev_status.fpga_pcie_3p3V_I_mA * dev_status.fpga_pcie_3p3V_V_mV) / (1000.0 * 1000.0); + double power_12v = (dev_status.fpga_pcie_12V_I_mA * dev_status.fpga_pcie_12V_V_mV) / (1000.0 * 1000.0); + logger.Info("Device {}: Slowest packet: {:8d} Power: {:5.1f}+{:5.1f} W FPGA Temp: {:d} degC HBM Temp: {:d}/{:d} degC Stalls: {:15d}/{:15d}/{:15d}", + i, aq_devices[i].Counters().GetSlowestFrameNumber(), power_12v, power_3p3v, + dev_status.fpga_temp_C, dev_status.hbm_0_temp_C, dev_status.hbm_1_temp_C, + coll_status.pipeline_stalls_hbm, coll_status.pipeline_stalls_proc, coll_status.pipeline_stalls_host); } + std::this_thread::sleep_for(std::chrono::seconds(1)); } run_thread.join(); @@ -209,23 +163,22 @@ int main(int argc, char **argv) { logger.Info("Receiving time: {} s", receiving_time); logger.Info("Frame rate: {} Hz", static_cast(nimages)/receiving_time); logger.Info("Total throughput: {:.2f} GB/s", - static_cast(nimages*x.GetModulesNum()*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9)); + static_cast(nsummation * nimages*x.GetModulesNum()*RAW_MODULE_SIZE*sizeof(uint16_t)) / (receiving_time * 1e9)); - if (!use_mock_device) { - logger.Info(""); - for (int i = 0; i < nstreams; i++) { - auto coll_status = aq_devices[i].GetDataCollectionStatus(); - auto stalls_hbm = coll_status.pipeline_stalls_hbm; - auto stalls_host = coll_status.pipeline_stalls_host; + logger.Info(""); + for (int i = 0; i < nstreams; i++) { + auto coll_status = aq_devices[i].GetDataCollectionStatus(); + auto stalls_hbm = coll_status.pipeline_stalls_hbm; + auto stalls_host = coll_status.pipeline_stalls_host; - uint64_t throughput_MBs = nimages * x.GetModulesNum(i) * RAW_MODULE_SIZE * sizeof(uint16_t) * clock_MHz / - (nimages * x.GetModulesNum(i) * 128 * 128 + stalls_hbm); - double performance = static_cast(throughput_MBs) / 1000; + uint64_t throughput_MBs = nimages * nsummation * x.GetModulesNum(i) * RAW_MODULE_SIZE * sizeof(uint16_t) * clock_MHz / + (nimages * nsummation * x.GetModulesNum(i) * 128 * 128 + stalls_hbm); + double performance = static_cast(throughput_MBs) / 1000; - logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm, - stalls_host, performance); - } + logger.Info("Device {}: stalls HBM: {} stalls host: {} est. performance: {:.2f} GB/s", i, stalls_hbm, + stalls_host, performance); } + if (ret) { logger.Info(""); logger.Info("Test properly executed! (check stall values manually)"); diff --git a/tests/AcquisitionCountersTest.cpp b/tests/AcquisitionCountersTest.cpp index 8b43608b..1e1a1511 100644 --- a/tests/AcquisitionCountersTest.cpp +++ b/tests/AcquisitionCountersTest.cpp @@ -22,9 +22,6 @@ TEST_CASE("AcquisitionCountersTest","[AcquisitionDeviceCounters]") { c.frame_number = 32; c.module_number = 1; c.handle = 17; - c.bunchid = 1ul<<56; - c.debug = 1u<<31; - c.timestamp = 456; counters.UpdateCounters(&c); REQUIRE(counters.GetSlowestFrameNumber() == 0); @@ -36,9 +33,6 @@ TEST_CASE("AcquisitionCountersTest","[AcquisitionDeviceCounters]") { REQUIRE(counters.CalculateDelay(33, 1) == -1); REQUIRE(counters.GetBufferHandle(32, 1) == 17); REQUIRE(counters.GetBufferHandle(32, 0) == AcquisitionCounters::HandleNotFound); - REQUIRE(counters.GetCompletion(32, 1).bunchid == c.bunchid); - REQUIRE(counters.GetCompletion(32, 1).debug == c.debug); - REQUIRE(counters.GetCompletion(32, 1).timestamp == c.timestamp); c.frame_number = 15; c.module_number = 0; @@ -58,6 +52,20 @@ TEST_CASE("AcquisitionCountersTest","[AcquisitionDeviceCounters]") { REQUIRE(counters.CalculateDelay(31, 1) == 1); REQUIRE(counters.CalculateDelay(33, 1) == -1); + c.frame_number = 7; + c.module_number = 1; + c.handle = 11; + c.pedestal = true; + counters.UpdateCounters(&c); + + REQUIRE(counters.GetPedestalBufferHandle(7, 1) == 11); + REQUIRE(counters.GetBufferHandle(7, 1) == AcquisitionCounters::HandleNotFound); + + c.frame_number = 16; + c.module_number = 1; + c.pedestal = true; + REQUIRE_THROWS(counters.UpdateCounters(&c)); + counters.SetAcquisitionFinished(); REQUIRE(counters.GetFastestFrameNumber() == 32); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9f7eca5a..91ec0e17 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -13,9 +13,8 @@ ADD_EXECUTABLE(CatchTest CoordTest.cpp JFJochStateMachineTest.cpp JFJochReceiverIntegrationTest.cpp AcquisitionCountersTest.cpp - JFJochFullIntegrationTest.cpp IndexingUnitTest.cpp - MockAcquisitionDeviceTest.cpp ThreadSafeFIFOSetTest.cpp + ThreadSafeFIFOSetTest.cpp FPGANetworkTest.cpp ZMQPreviewPublisherTest.cpp JFPedestalTest.cpp @@ -24,9 +23,8 @@ ADD_EXECUTABLE(CatchTest StatusVectorTest.cpp CBORTest.cpp ../tests/stream2.h ../tests/stream2.c DetectorGeometryTest.cpp JFJochBrokerParserTest.cpp DetectorSetupTest.cpp DiffractionGeometryTest.cpp - FPGAHLSBitshuffleTest.cpp FPGASpotFindingUnitTest.cpp - FPGAAddMultipixelTest.cpp) +) target_link_libraries(CatchTest JFJochBroker JFJochReceiver JFJochWriter ImageAnalysis CommonFunctions HLSSimulation) target_include_directories(CatchTest PRIVATE .) diff --git a/tests/FPGAAddMultipixelTest.cpp b/tests/FPGAAddMultipixelTest.cpp deleted file mode 100644 index 7b7790f4..00000000 --- a/tests/FPGAAddMultipixelTest.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include -#include - -#include "../fpga/hls/hls_jfjoch.h" -#include "../common/RawToConvertedGeometryCore.h" - -TEST_CASE("FPGA_add_multipixel_core","[FPGA][SpotFinder]") { - STREAM_512 input; - STREAM_512 output; - - std::vector input_frame(RAW_MODULE_SIZE), input_frame_transformed(CONVERTED_MODULE_SIZE), - output_frame(257 * 64 * 32); - for (int i = 0; i < RAW_MODULE_SIZE; i++) - input_frame[i] = i % INT16_MAX; - - TransferModuleAdjustMultipixels(input_frame_transformed.data(), input_frame.data(), - CONVERTED_MODULE_COLS, INT16_MIN, INT16_MAX); - - auto input_frame_512 = (ap_uint<512>*) input_frame.data(); - auto output_frame_512 = (ap_uint<512>*) output_frame.data(); - - ap_uint<512> action_control = 0; - ACT_REG_MODE(action_control) = MODE_ADD_MULTIPIXEL; - - input << packet_512_t { .data = action_control, .user = 0 }; - for (int i = 0; i < RAW_MODULE_SIZE * sizeof(uint16_t) / 64; i++) - input << packet_512_t { .data = input_frame_512[i], .user = 0 }; - - input << packet_512_t { .user = 1 }; - - add_multipixel(input, output); - - REQUIRE(input.size() == 0); - REQUIRE(output.size() == (257 * 64) + 2); - - output.read(); - for (int i = 0; i < 257 * 64 ; i++) - output_frame_512[i] = output.read().data; - output.read(); - - size_t diff = 0; - for (int line = 0; line < 512; line++) { - for (int col = 0; col < 1027; col++) { - if (output_frame[line * 1028 + col] != input_frame_transformed[(line + 1) * 1030 + (col+1)]) - diff++; - } - } - REQUIRE(diff == 0); -} diff --git a/tests/FPGAHLSBitshuffleTest.cpp b/tests/FPGAHLSBitshuffleTest.cpp deleted file mode 100644 index 66671d01..00000000 --- a/tests/FPGAHLSBitshuffleTest.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include -#include - -#include "../fpga/hls/hls_jfjoch.h" - -TEST_CASE("HLS_bitshuffle") { - STREAM_512 data_in; - STREAM_512 data_out; - - packet_512_t packet_in, packet_out; - - ACT_REG_MODE(packet_in.data) = MODE_BITSHUFFLE_FPGA; - data_in << packet_in; - packet_in.user = 0; - packet_in.last = 0; - - ap_int<16> image[4096]; - - for (int i = 0; i < 4096; i++) - image[i] = i; - - for (int i = 0; i < 128; i++) { - packet_in.data = pack32(image + i * 32); - data_in << packet_in; - } - - packet_in.user = 1; - data_in << packet_in; - - bitshuffle(data_in, data_out); - - REQUIRE(data_out.size() == 130); - data_out >> packet_out; - - ap_int<16> image_out_shuf[128*32]; - ap_int<16> image_out[128*32]; - for (int i = 0; i < 128; i++) { - data_out >> packet_out; - unpack32(packet_out.data, image_out_shuf + 32 * i); - } - bshuf_bitunshuffle(image_out_shuf, image_out, 128*32, 2, 2048); - - int diff = 0; - for (int i = 0; i < 128*32; i++) { - if (image_out[i] != image[i]) diff++; - } - REQUIRE(diff == 0); -} - diff --git a/tests/FPGAIntegrationTest.cpp b/tests/FPGAIntegrationTest.cpp index 8622c2bd..e13fa770 100644 --- a/tests/FPGAIntegrationTest.cpp +++ b/tests/FPGAIntegrationTest.cpp @@ -3,7 +3,6 @@ #include #include -#include #include "../jungfrau/JFPedestalCalc.h" #include "../acquisition_device/HLSSimulatedDevice.h" #include "FPGAUnitTest.h" @@ -35,7 +34,8 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator", "[FPGA][Full]") { REQUIRE(imageBuf[i] == i % 65536); } } - REQUIRE(test.GetCompletedDescriptors() == 2 * (4 + DELAY_FRAMES_STOP_AND_QUIT - 1) * nmodules); + REQUIRE(test.GetExpectedDescriptorsPerModule() == 5); + REQUIRE(test.GetCompletedDescriptors() == 5 * (4 + DELAY_FRAMES_STOP_AND_QUIT - 1) * nmodules); } TEST_CASE("HLS_C_Simulation_internal_packet_generator_custom_frame", "[FPGA][Full]") { @@ -66,9 +66,9 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_custom_frame", "[FPGA][Ful for (int image = 0; image < nframes; image++) { for (int m = 0; m < nmodules; m++) { - REQUIRE(test.Counters().GetCompletion(image, m).timestamp == INT_PKT_GEN_EXPTTIME * image); - REQUIRE(test.Counters().GetCompletion(image, m).bunchid == INT_PKT_GEN_BUNCHID + image); - REQUIRE(test.Counters().GetCompletion(image, m).exptime == INT_PKT_GEN_EXPTTIME); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.timestamp == INT_PKT_GEN_EXPTTIME * image); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.bunchid == INT_PKT_GEN_BUNCHID + image); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.exptime == INT_PKT_GEN_EXPTTIME); auto imageBuf = (uint16_t *) test.GetDeviceOutput(image, m)->pixels; for (int i = 0; i < RAW_MODULE_SIZE; i++) { @@ -340,6 +340,7 @@ TEST_CASE("HLS_C_Simulation_check_single_packet", "[FPGA][Full]") { } TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") { + Logger logger("HLS_C_Simulation_check_convert_full_range"); std::vector data(RAW_MODULE_SIZE); std::vector gain(3 * RAW_MODULE_SIZE); @@ -359,8 +360,8 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") { auto gain_from_file = GainCalibrationFromTestFile(); - for (const auto energy : energy_values) { + logger.Info("Trying with {} keV", energy); x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(1).PhotonEnergy_keV(energy); REQUIRE(x.GetPhotonEnergy_keV() == Approx(energy)); @@ -381,7 +382,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") { REQUIRE_NOTHROW(test.OutputStream().read()); REQUIRE(test.OutputStream().size() == 0); - REQUIRE(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); + CHECK(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); double mean_error = CheckConversion(x, c_in, data.data(), test.GetDeviceOutput(0,0)->pixels); @@ -390,6 +391,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range", "[FPGA][Full]") { } TEST_CASE("HLS_C_Simulation_check_convert_full_range_I32", "[FPGA][Full]") { + Logger logger("HLS_C_Simulation_check_convert_full_range_I32"); std::vector data(RAW_MODULE_SIZE); std::vector gain(3 * RAW_MODULE_SIZE); @@ -410,6 +412,8 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_I32", "[FPGA][Full]") { auto gain_from_file = GainCalibrationFromTestFile(); for (const auto energy : energy_values) { + logger.Info("Trying with {} keV", energy); + x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(1).PhotonEnergy_keV(energy).FPGAOutputMode(FPGAPixelOutput::Int32); REQUIRE(x.GetPhotonEnergy_keV() == Approx(energy)); @@ -430,7 +434,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_I32", "[FPGA][Full]") { REQUIRE_NOTHROW(test.OutputStream().read()); REQUIRE(test.OutputStream().size() == 0); - REQUIRE(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); + CHECK(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); double mean_error = CheckConversion(x, c_in, data.data(), (int32_t *) test.GetDeviceOutput(0,0)->pixels); @@ -439,6 +443,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_I32", "[FPGA][Full]") { } TEST_CASE("HLS_C_Simulation_check_convert_full_range_sum4", "[FPGA][Full]") { + Logger logger("HLS_C_Simulation_check_convert_full_range_sum4"); std::vector data(RAW_MODULE_SIZE); std::vector gain(3 * RAW_MODULE_SIZE); @@ -461,6 +466,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_sum4", "[FPGA][Full]") { size_t nsummation = 4; for (const auto energy : energy_values) { + logger.Info("Trying with {} keV", energy); x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(1).PhotonEnergy_keV(energy).Summation(nsummation); REQUIRE(x.GetPhotonEnergy_keV() == Approx(energy)); @@ -482,7 +488,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_sum4", "[FPGA][Full]") { REQUIRE_NOTHROW(test.OutputStream().read()); REQUIRE(test.OutputStream().size() == 0); - REQUIRE(test.GetBytesReceived() == 128 * nsummation * JUNGFRAU_PACKET_SIZE_BYTES); + CHECK(test.GetBytesReceived() == 128 * nsummation * JUNGFRAU_PACKET_SIZE_BYTES); double mean_error = CheckConversion(x, c_in, data.data(), (int32_t *) test.GetDeviceOutput(0,0)->pixels); @@ -491,6 +497,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_sum4", "[FPGA][Full]") { } TEST_CASE("HLS_C_Simulation_check_convert_full_range_U16", "[FPGA][Full]") { + Logger logger("HLS_C_Simulation_check_convert_full_range_U16"); std::vector data(RAW_MODULE_SIZE); std::vector gain(3 * RAW_MODULE_SIZE); @@ -511,6 +518,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_U16", "[FPGA][Full]") { auto gain_from_file = GainCalibrationFromTestFile(); for (const auto energy : energy_values) { + logger.Info("Trying with {} keV", energy); x.PedestalG0Frames(0).NumTriggers(1).ImagesPerTrigger(1).PhotonEnergy_keV(energy).FPGAOutputMode(FPGAPixelOutput::Uint16); REQUIRE(x.GetPhotonEnergy_keV() == Approx(energy)); @@ -531,7 +539,7 @@ TEST_CASE("HLS_C_Simulation_check_convert_full_range_U16", "[FPGA][Full]") { REQUIRE_NOTHROW(test.OutputStream().read()); REQUIRE(test.OutputStream().size() == 0); - REQUIRE(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); + CHECK(test.GetBytesReceived() == 128 * JUNGFRAU_PACKET_SIZE_BYTES); double mean_error = CheckConversion(x, c_in, data.data(), (uint16_t *) test.GetDeviceOutput(0,0)->pixels); @@ -1257,23 +1265,23 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_integration", "[FPGA][Full REQUIRE(memcmp(imageBuf, frame.data(), RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); auto integration_result = test.GetDeviceOutput(0, 0)->integration_result; - CHECK(integration_result[0].sum == 32754LU * (RAW_MODULE_SIZE / 2) * (1LU<<24)); + CHECK(integration_result[0].sum == Approx(32754LU * (RAW_MODULE_SIZE / 2))); CHECK(integration_result[0].count == RAW_MODULE_SIZE / 2); CHECK(integration_result[1].sum == 0); CHECK(integration_result[1].count == 0); - CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].sum == 32754LU * (RAW_MODULE_SIZE / 2 - 1)* (1LU<<24)); + CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].sum == Approx(32754LU * (RAW_MODULE_SIZE / 2 - 1))); CHECK(integration_result[FPGA_INTEGRATION_BIN_COUNT - 1].count == RAW_MODULE_SIZE / 2 - 1); integration_result = test.GetDeviceOutput(0, 1)->integration_result; - CHECK(integration_result[54].sum == 32754LU * (RAW_MODULE_SIZE) * (1LU<<24)); + CHECK(integration_result[54].sum == Approx(32754LU * (RAW_MODULE_SIZE))); CHECK(integration_result[54].count == RAW_MODULE_SIZE); integration_result = test.GetDeviceOutput(0, 2)->integration_result; - CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE) * (1LU<<24)); + CHECK(integration_result[54].sum == Approx(32754 * (RAW_MODULE_SIZE))); CHECK(integration_result[54].count == RAW_MODULE_SIZE); integration_result = test.GetDeviceOutput(0, 3)->integration_result; - CHECK(integration_result[54].sum == 32754 * (RAW_MODULE_SIZE) * (1LU<<24)); + CHECK(integration_result[54].sum == Approx(32754 * (RAW_MODULE_SIZE))); CHECK(integration_result[54].count == RAW_MODULE_SIZE); } @@ -1393,9 +1401,13 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_32bit", "[FPGA][Full]") { for (int image = 0; image < nframes; image++) { for (int m = 0; m < nmodules; m++) { - REQUIRE(test.Counters().GetCompletion(image, m).timestamp == INT_PKT_GEN_EXPTTIME * image); - REQUIRE(test.Counters().GetCompletion(image, m).bunchid == INT_PKT_GEN_BUNCHID + image); - REQUIRE(test.Counters().GetCompletion(image, m).exptime == INT_PKT_GEN_EXPTTIME); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.frame_number == image); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.packet_count == 128); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.packet_mask[0] == UINT64_MAX); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.packet_mask[1] == UINT64_MAX); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.timestamp == INT_PKT_GEN_EXPTTIME * image); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.bunchid == INT_PKT_GEN_BUNCHID + image); + CHECK(test.GetDeviceOutput(image, m)->module_statistics.exptime == INT_PKT_GEN_EXPTTIME); auto imageBuf = (int32_t *) test.GetDeviceOutput(image, m)->pixels; for (int i = 0; i < RAW_MODULE_SIZE; i++) { @@ -1435,9 +1447,9 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_summation", "[FPGA][Full]" auto test_frame_signed = (uint16_t *) test_frame.data(); for (int image = 0; image < nframes; image++) { for (int m = 0; m < nmodules; m++) { - REQUIRE(test.Counters().GetCompletion(image, m).timestamp == INT_PKT_GEN_EXPTTIME * image * nsummation); - REQUIRE(test.Counters().GetCompletion(image, m).bunchid == INT_PKT_GEN_BUNCHID + image * nsummation); - REQUIRE(test.Counters().GetCompletion(image, m).exptime == INT_PKT_GEN_EXPTTIME); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.timestamp == INT_PKT_GEN_EXPTTIME * image * nsummation); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.bunchid == INT_PKT_GEN_BUNCHID + image * nsummation); + REQUIRE(test.GetDeviceOutput(image, m)->module_statistics.exptime == INT_PKT_GEN_EXPTTIME); auto imageBuf = (int32_t *) test.GetDeviceOutput(image, m)->pixels; for (int i = 0; i < RAW_MODULE_SIZE; i++) { @@ -1446,3 +1458,86 @@ TEST_CASE("HLS_C_Simulation_internal_packet_generator_summation", "[FPGA][Full]" } } } + +#define NFRAMES_PEDESTAL_CHECK 256 + +TEST_CASE("HLS_C_Simulation_check_convert_pedestal", "[FPGA][Full]") { + DiffractionExperiment x(DetectorGeometry(4)); + HLSSimulatedDevice test(0, NFRAMES_PEDESTAL_CHECK*4+8); + + for (int gain = 2; gain >= 0; gain--) { + x.ImagesPerTrigger(1).PedestalG0Frames(500).PedestalG1Frames(500).PedestalG2Frames(500).NumTriggers(1); + + std::vector data(NFRAMES_PEDESTAL_CHECK * RAW_MODULE_SIZE); + + double mean; + double stddev; + + if (gain == 0) { + x.Mode(DetectorMode::PedestalG0); + mean = 1000.0; + stddev = 50.0; + } else if (gain == 1) { + x.Mode(DetectorMode::PedestalG1); + mean = 13000.0; + stddev = 40.0; + } else { // gain == 2 + x.Mode(DetectorMode::PedestalG2); + mean = 14500.0; + stddev = 20.0; + } + + // Predictable random number generator + std::mt19937 g1(1984+gain); + std::normal_distribution distribution(mean, stddev); + + for (int i = 0; i < NFRAMES_PEDESTAL_CHECK; i++) { + for (int j = 0; j < RAW_MODULE_SIZE; j++) { + double number = distribution(g1); + if (number < 20) number = 20; + if (number > 16300) number = 16300; + data[i * RAW_MODULE_SIZE + j] = number + ((gain == 1)?0x4000:0) + ((gain == 2)? 0xc000:0); + } + } + + data[0 + 4096 * 45] = 0x4000 + 1; // gain_bit always 1 + data[1 + 4096 * 3] = 0x8000 + 1; // gain_bit always 2 + data[2 + 4096 * 100] = 0xc000 + 1; // gain_bit always 3 + + test.CreatePackets(x, 1, NFRAMES_PEDESTAL_CHECK, 0, data.data(), false); + + test.CreateFinalPacket(x); + + REQUIRE_NOTHROW(test.StartAction(x)); + REQUIRE_NOTHROW(test.WaitForActionComplete()); + + REQUIRE_NOTHROW(test.OutputStream().read()); + REQUIRE(test.OutputStream().size() == 0); + + REQUIRE(test.GetBytesReceived() == NFRAMES_PEDESTAL_CHECK * 128 * JUNGFRAU_PACKET_SIZE_BYTES); + + + REQUIRE(test.Counters().GetPedestalBufferHandle(0, 1) == AcquisitionCounters::HandleNotFound); + REQUIRE(test.Counters().GetPedestalBufferHandle(0, 0) != AcquisitionCounters::HandleNotFound); + + auto pedestal_buffer_0 = test.GetDeviceOutputPedestal(0, 0); + REQUIRE(pedestal_buffer_0->module_statistics.packet_count == NFRAMES_PEDESTAL_CHECK); + + JFPedestalCalc reference(x); + for (int i = 0; i < NFRAMES_PEDESTAL_CHECK; i++) + reference.AnalyzeImage(data.data() + i * RAW_MODULE_SIZE); + + JFModulePedestal reference_pedestal, fpga_pedestal; + reference.Export(reference_pedestal); + fpga_pedestal.ImportFPGAPedestal(pedestal_buffer_0); + + float error = 0; + for (int i = 0; i < RAW_MODULE_SIZE; i++) { + float diff = reference_pedestal.GetPedestal()[i] - fpga_pedestal.GetPedestal()[i]; + error += diff * diff; + } + error = sqrtf(error / RAW_MODULE_SIZE); + std::cout << "Pedestal G" << gain << " error: " << error << std::endl; + REQUIRE(error < 0.5); + } +} diff --git a/tests/JFJochFullIntegrationTest.cpp b/tests/JFJochFullIntegrationTest.cpp deleted file mode 100644 index 6910c4d0..00000000 --- a/tests/JFJochFullIntegrationTest.cpp +++ /dev/null @@ -1,1155 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include - -#include "../grpc/gRPCServer_Template.h" -#include "../broker/JFJochStateMachine.h" -#include "../writer/StreamWriter.h" -#include "FPGAUnitTest.h" -#include "../acquisition_device/MockAcquisitionDevice.h" -#include "../common/ZMQImagePusher.h" - -using namespace std::literals::chrono_literals; - -TEST_CASE("JFJochIntegrationTest_ZMQ", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ"); - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - - REQUIRE(!state_machine.GetMeasurementStatistics().has_value()); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - services.Receiver(&fpga_receiver); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.images_per_trigger = 5; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test"; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 2; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - BrokerStatus status; - status = state_machine.GetStatus(); - REQUIRE(status.progress == Approx(0.0)); - REQUIRE(status.broker_state == JFJochState::Measuring); - - for (int i = 0; i < ndatastream; i++) { - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - dynamic_cast(aq_devices[i]).AddModule(image_num, m, image.data()); - } - dynamic_cast(aq_devices[i]).Terminate(); - } - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - status = state_machine.GetStatus(); - REQUIRE(status.broker_state == JFJochState::Idle); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); - REQUIRE(statistics.max_image_number_sent == 4); - REQUIRE(!statistics.cancelled); - REQUIRE(statistics.file_prefix == "integration_test"); - REQUIRE(statistics.detector_width == 2068); - REQUIRE(statistics.detector_height == 2164); - REQUIRE(statistics.detector_pixel_depth == 2); - writer_future.get(); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_save_calibration", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_save_calibration"); - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - - REQUIRE(!state_machine.GetMeasurementStatistics().has_value()); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - services.Receiver(&fpga_receiver); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE);; - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.images_per_trigger = 5; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test_with_calibration"; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 2; - setup.save_calibration = true; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - BrokerStatus status; - status = state_machine.GetStatus(); - REQUIRE(status.progress == Approx(0.0)); - REQUIRE(status.broker_state == JFJochState::Measuring); - - for (int i = 0; i < ndatastream; i++) { - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - dynamic_cast(aq_devices[i]).AddModule(image_num, m, image.data()); - } - dynamic_cast(aq_devices[i]).Terminate(); - } - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - status = state_machine.GetStatus(); - REQUIRE(status.broker_state == JFJochState::Idle); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); - REQUIRE(statistics.max_image_number_sent == 4); - REQUIRE(!statistics.cancelled); - REQUIRE(statistics.file_prefix == "integration_test_with_calibration"); - REQUIRE(statistics.detector_width == 2068); - REQUIRE(statistics.detector_height == 2164); - REQUIRE(statistics.detector_pixel_depth == 2); - REQUIRE_NOTHROW(writer_future.get()); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_2DataStreams_4Devices", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_2DataStreams_4Devices"); - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < 4; i++) { - auto test = std::make_unique(i, 256); - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - - REQUIRE(!state_machine.GetMeasurementStatistics().has_value()); - - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - services.Receiver(&fpga_receiver); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.images_per_trigger = 5; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test"; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 2; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - BrokerStatus status; - status = state_machine.GetStatus(); - REQUIRE(status.progress == Approx(0.0)); - REQUIRE(status.broker_state == JFJochState::Measuring); - - for (int i = 0; i < ndatastream; i++) { - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - dynamic_cast(aq_devices[i]).AddModule(image_num, m, image.data()); - } - dynamic_cast(aq_devices[i]).Terminate(); - } - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - status = state_machine.GetStatus(); - REQUIRE(status.broker_state == JFJochState::Idle); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); - REQUIRE(statistics.max_image_number_sent == 4); - REQUIRE(!statistics.cancelled); - REQUIRE(statistics.file_prefix == "integration_test"); - REQUIRE(statistics.detector_width == 2068); - REQUIRE(statistics.detector_height == 2164); - REQUIRE(statistics.detector_pixel_depth == 2); - - REQUIRE_NOTHROW(writer_future.get()); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_RAW", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ"); - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - - REQUIRE(!state_machine.GetMeasurementStatistics().has_value()); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().Mode(DetectorMode::Conversion); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - DetectorSettings detector_settings{}; - detector_settings.frame_time_us = 500; - detector_settings.collect_raw_data = true; - detector_settings.storage_cell_count = 1; - REQUIRE_NOTHROW(state_machine.SetDetectorSettings(detector_settings)); - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_raw_test"; - setup.images_per_trigger = 5; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 5; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - BrokerStatus status; - status = state_machine.GetStatus(); - REQUIRE(status.progress == Approx(0.0)); - REQUIRE(status.broker_state == JFJochState::Measuring); - - for (int i = 0; i < ndatastream; i++) { - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - dynamic_cast(aq_devices[i]).AddModule(image_num, m, image.data()); - } - dynamic_cast(aq_devices[i]).Terminate(); - } - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - status = state_machine.GetStatus(); - REQUIRE(status.broker_state == JFJochState::Idle); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); - REQUIRE(statistics.file_prefix == "integration_raw_test"); - REQUIRE(statistics.detector_width == 1024); - REQUIRE(statistics.detector_height == 8 * 512); - REQUIRE(statistics.detector_pixel_depth == 2); - - REQUIRE_NOTHROW(writer_future.get()); -} - - -TEST_CASE("JFJochIntegrationTest_ZMQ_3Writers", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_3Writers"); - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 37; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#0", "inproc://#1", "inproc://#2"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer_0(zmq_context, logger, "inproc://#0"); - auto writer_0_future = writer_0.RunFuture(); - - StreamWriter writer_1(zmq_context, logger, "inproc://#1"); - auto writer_1_future = writer_1.RunFuture(); - - StreamWriter writer_2(zmq_context, logger, "inproc://#2"); - auto writer_2_future = writer_2.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test_3writers"; - setup.images_per_trigger = nimages; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 5; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - auto status = state_machine.GetStatus(); - REQUIRE(status.progress == Approx(0.0)); - REQUIRE(status.broker_state == JFJochState::Measuring); - - for (int i = 0; i < ndatastream; i++) { - for (int image_num = 1; image_num <= nimages; image_num++) { - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) - dynamic_cast(aq_devices[i]).AddModule(image_num, m, image.data()); - } - dynamic_cast(aq_devices[i]).Terminate(); - } - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - status = state_machine.GetStatus(); - REQUIRE(status.broker_state == JFJochState::Idle); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == nimages); - - REQUIRE_NOTHROW(writer_0_future.get()); - REQUIRE_NOTHROW(writer_1_future.get()); - REQUIRE_NOTHROW(writer_2_future.get()); -} - -TEST_CASE("JFJochIntegrationTest_Cancel", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_Cancel"); - - ZMQContext zmq_context; - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - test->EnableLogging(&logger); - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int frame = 1; frame <= nimages; frame++) - test->AddModule(frame, m, image.data()); - } - aq_devices.Add(std::move(test)); - } - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test"; - setup.images_per_trigger = 2 * nimages; - setup.photon_energy_keV = 12.4; - setup.data_file_count = 1; - setup.save_calibration = false; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - logger.Info("Cancelling"); - REQUIRE_NOTHROW(state_machine.Cancel()); - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 0.5); - REQUIRE(statistics.images_collected == 5); - REQUIRE(statistics.max_image_number_sent == 4); - REQUIRE(statistics.cancelled); - - REQUIRE_NOTHROW(writer_future.get()); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_with_preview"); - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0).PreviewPeriod( - 5ms); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - test->AddModule(image_num, m, image.data()); - } - test->Terminate(); - aq_devices.Add(std::move(test)); - } - ZMQContext zmq_context; - - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - ZMQPreviewPublisher preview(zmq_context, "inproc://#2"); - fpga_receiver.PreviewPublisher(&preview).NumThreads(1); - - ZMQSocket rcv_preview_socket(zmq_context, ZMQSocketType::Sub); - REQUIRE_NOTHROW(rcv_preview_socket.Connect("inproc://#2")); - rcv_preview_socket.SubscribeAll(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 100; - setup.file_prefix = "integration_test_preview"; - setup.images_per_trigger = 5; - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 123.0; - setup.data_file_count = 1; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - std::string s; - - JFJochFrameDeserializer deserializer; - - // Pixel mask - REQUIRE(rcv_preview_socket.Receive(s, false) > 0); - deserializer.Process((uint8_t *) s.data(), s.size()); - REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START); - auto start_msg = deserializer.GetStartMessage(); - REQUIRE(start_msg.image_size_x == state_machine.NotThreadSafe_Experiment().GetXPixelsNum()); - REQUIRE(start_msg.image_size_y == state_machine.NotThreadSafe_Experiment().GetYPixelsNum()); - - // First frame - REQUIRE(rcv_preview_socket.Receive(s, false) > 0); - - deserializer.Process((uint8_t *) s.data(), s.size()); - REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); - auto image_msg = deserializer.GetDataMessage(); - - REQUIRE(image_msg.number == 0); - - // Check no more frames waiting - REQUIRE(rcv_preview_socket.Receive(s, false) == -1); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); - - REQUIRE_NOTHROW(writer_future.get()); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_with_preview_no_writer", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_with_preview_no_writer"); - - RegisterHDF5Filter(); - - int64_t nimages = 5; - int64_t ndatastream = 2; - int64_t nmodules = 4; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0).PreviewPeriod( - 5ms); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - AcquisitionDeviceGroup aq_devices; - - for (int i = 0; i < ndatastream; i++) { - auto test = std::make_unique(i, 256); - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(i); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - test->AddModule(image_num, m, image.data()); - } - test->Terminate(); - aq_devices.Add(std::move(test)); - } - ZMQContext zmq_context; - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - ZMQPreviewPublisher preview(zmq_context, "inproc://#2"); - fpga_receiver.PreviewPublisher(&preview).NumThreads(1); - - ZMQSocket rcv_preview_socket(zmq_context, ZMQSocketType::Sub); - REQUIRE_NOTHROW(rcv_preview_socket.Connect("inproc://#2")); - rcv_preview_socket.SubscribeAll(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 100; - setup.images_per_trigger = 5; - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 123.0; - setup.beam_y_pxl = 878.0; - setup.data_file_count = 1; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - std::string s; - JFJochFrameDeserializer deserializer; - - // Pixel mask - REQUIRE(rcv_preview_socket.Receive(s, false) > 0); - deserializer.Process((uint8_t *) s.data(), s.size()); - REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START); - auto start_msg = deserializer.GetStartMessage(); - REQUIRE(start_msg.image_size_x == state_machine.NotThreadSafe_Experiment().GetXPixelsNum()); - REQUIRE(start_msg.image_size_y == state_machine.NotThreadSafe_Experiment().GetYPixelsNum()); - - // First frame - REQUIRE(rcv_preview_socket.Receive(s, false) > 0); - - deserializer.Process((uint8_t *) s.data(), s.size()); - REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); - auto image_msg = deserializer.GetDataMessage(); - - REQUIRE(image_msg.number == 0); - - // Check no more frames waiting - REQUIRE(rcv_preview_socket.Receive(s, false) == -1); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 5); -} - -TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index"); - - RegisterHDF5Filter(); - - size_t nimages = 1; - int64_t ndatastream = 1; - int64_t nmodules = 8; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - HDF5File data("../../tests/test_data/compression_benchmark.h5", false, false, false); - HDF5DataSet dataset(data, "/entry/data/data"); - HDF5DataSpace file_space(dataset); - - std::vector image_conv(nimages * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - std::vector start = {0, 0, 0}; - std::vector file_size = {nimages, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - dataset.ReadVector(image_conv, start, file_size); - - std::vector image_raw_geom( - nimages * state_machine.NotThreadSafe_Experiment().GetModulesNum() * RAW_MODULE_SIZE * sizeof(int16_t)); - for (int i = 0; i < nimages; i++) - ConvertedToRawGeometry(state_machine.NotThreadSafe_Experiment(), - image_raw_geom.data() + - i * RAW_MODULE_SIZE * state_machine.NotThreadSafe_Experiment().GetModulesNum(), - image_conv.data() + i * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - - AcquisitionDeviceGroup aq_devices; - - auto test = std::make_unique(0, 256); - - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(0); m++) { - test->AddModule(1, m, (uint16_t *) (image_raw_geom.data() + m * RAW_MODULE_SIZE)); - } - test->Terminate(); - - aq_devices.Add(std::move(test)); - - ZMQContext zmq_context; - - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 75; - setup.file_prefix = "spot_finding_test"; - setup.images_per_trigger = nimages; - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 1090; - setup.beam_y_pxl = 1136; - setup.unit_cell = UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha = 90.0, .beta = 90.0, .gamma = 90.0}; - setup.data_file_count = 1; - - DataProcessingSettings settings{}; - settings.signal_to_noise_threshold = 4; - settings.photon_count_threshold = 5; - settings.min_pix_per_spot = 3; - settings.max_pix_per_spot = 200; - settings.low_resolution_limit = 80.0; - settings.high_resolution_limit = 2.5; - - REQUIRE_NOTHROW(state_machine.SetDataProcessingSettings(settings)); - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == 1); - - REQUIRE_NOTHROW(writer_future.get()); -} -/* -TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index"); - - RegisterHDF5Filter(); - - size_t nimages = 24; - int64_t ndatastream = 1; - int64_t nmodules = 8; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0) - .SpotFindingPeriod(10ms); - services.Writer("unix:writer_test", "inproc://#1").Receiver(&fpga_receiver); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - HDF5File data("../../tests/test_data/compression_benchmark.h5", false, false, false); - HDF5DataSet dataset(data, "/entry/data/data"); - HDF5DataSpace file_space(dataset); - - std::vector image_conv(nimages * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - std::vector start = {0, 0, 0}; - std::vector file_size = {nimages, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - dataset.ReadVector(image_conv, start, file_size); - - std::vector image_raw_geom( - nimages * state_machine.NotThreadSafe_Experiment().GetModulesNum() * RAW_MODULE_SIZE * sizeof(int16_t)); - for (int i = 0; i < nimages; i++) - ConvertedToRawGeometry(state_machine.NotThreadSafe_Experiment(), - image_raw_geom.data() + - i * RAW_MODULE_SIZE * state_machine.NotThreadSafe_Experiment().GetModulesNum(), - image_conv.data() + i * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - - AcquisitionDeviceGroup aq_devices; - - auto *test = new MockAcquisitionDevice(0, 512); - - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(0); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - test->AddModule(image_num, m, (uint16_t *) (image_raw_geom.data() - + (m + (image_num - 1) * - state_machine.NotThreadSafe_Experiment().GetModulesNum( - 0)) - * RAW_MODULE_SIZE)); - } - test->Terminate(); - - aq_devices.Add(std::move(test)); - - ZMQContext zmq_context; - - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(tmp_devices, logger, pusher); - JFJochWriterService writer(zmq_context, logger);; - - auto writer_server = gRPCServer("unix:writer_test", writer); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 75; - setup.file_prefix = "spot_finding_test"; - setup.images_per_trigger = nimages; - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 1090; - setup.beam_y_pxl = 1136; - setup.mutable_unit_cell()->set_a(36.9); - setup.mutable_unit_cell()->set_b(78.95); - setup.mutable_unit_cell()->set_c(78.95); - setup.mutable_unit_cell()->set_alpha(90.0); - setup.mutable_unit_cell()->set_beta(90.0); - setup.mutable_unit_cell()->set_gamma(90.0); - setup.data_file_count = 1; - - JFJochProtoBuf::DataProcessingSettings settings; - settings.signal_to_noise_threshold = 4; - settings.photon_count_threshold = 5; - settings.min_pix_per_spot = 3; - settings.max_pix_per_spot = 200; - settings.low_resolution_limit = 80.0; - settings.high_resolution_limit = 2.5; - settings.local_bkg_size = 5; - - REQUIRE_NOTHROW(state_machine.SetDataProcessingSettings(settings)); - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.has_indexing_rate); - REQUIRE(statistics.indexing_rate == 1.0); - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == nimages); - - - writer_server->Shutdown(); -}*/ - - -TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_rad_int", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_rad_int"); - - RegisterHDF5Filter(); - - size_t nimages = 24; - int64_t ndatastream = 1; - int64_t nmodules = 8; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0); - state_machine.NotThreadSafe_Experiment().LowQForRadialInt_recipA(0.5).HighQForRadialInt_recipA(3.5) - .QSpacingForRadialInt_recipA(1.0); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - HDF5File data("../../tests/test_data/compression_benchmark.h5", false, false, false); - HDF5DataSet dataset(data, "/entry/data/data"); - HDF5DataSpace file_space(dataset); - - std::vector image_conv(nimages * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - std::vector start = {0, 0, 0}; - std::vector file_size = {nimages, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - dataset.ReadVector(image_conv, start, file_size); - - std::vector image_raw_geom( - nimages * state_machine.NotThreadSafe_Experiment().GetModulesNum() * RAW_MODULE_SIZE * sizeof(int16_t)); - for (int i = 0; i < nimages; i++) - ConvertedToRawGeometry(state_machine.NotThreadSafe_Experiment(), - image_raw_geom.data() + - i * RAW_MODULE_SIZE * state_machine.NotThreadSafe_Experiment().GetModulesNum(), - image_conv.data() + i * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - - AcquisitionDeviceGroup aq_devices; - - auto test = std::make_unique(0, 512); - - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(0); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - test->AddModule(image_num, m, (uint16_t *) (image_raw_geom.data() - + (m + (image_num - 1) * - state_machine.NotThreadSafe_Experiment().GetModulesNum( - 0)) - * RAW_MODULE_SIZE)); - } - test->Terminate(); - - aq_devices.Add(std::move(test)); - - ZMQContext zmq_context; - - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(aq_devices, logger, pusher); - services.Receiver(&fpga_receiver); - - StreamWriter writer(zmq_context, logger, "inproc://#1"); - auto writer_future = writer.RunFuture(); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 75; - setup.file_prefix = "spot_finding_test"; - setup.images_per_trigger = nimages; - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 1090; - setup.beam_y_pxl = 1136; - setup.unit_cell = UnitCell{.a = 36.9, .b = 78.95, .c = 78.95, .alpha = 90.0, .beta = 90.0, .gamma = 90.0}; - setup.data_file_count = 4; - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - auto rad_int = state_machine.GetRadialIntegrationProfiles(); - REQUIRE(rad_int.profiles.size() == 4+1); - auto &plot_map = rad_int.profiles; - - CHECK(plot_map[1].title == "file0"); - CHECK(plot_map[1].plot.x.size() == 3); - CHECK(plot_map[1].plot.y.size() == 3); - - CHECK(plot_map[4].title == "file3"); - REQUIRE(plot_map[4].plot.x.size() == 3); - CHECK(plot_map[4].plot.x[0] == Approx(1.0)); - CHECK(plot_map[4].plot.y.size() == 3); - - CHECK(plot_map[0].title == "dataset"); - CHECK(plot_map[0].plot.x.size() == 3); - - REQUIRE_NOTHROW(writer_future.get()); -} - -/* -TEST_CASE("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index_sum", "[JFJochReceiver]") { - Logger logger("JFJochIntegrationTest_ZMQ_lysozyme_spot_and_index_sum"); - - RegisterHDF5Filter(); - - size_t nimages = 24; - int64_t ndatastream = 1; - int64_t nmodules = 8; - - JFJochServices services(logger); - JFJochStateMachine state_machine(services, logger); - state_machine.AddDetectorSetup(DetectorGeometry(ndatastream * nmodules, 2, 8, 36)); - - state_machine.NotThreadSafe_Experiment().DataStreams(ndatastream); - state_machine.NotThreadSafe_Experiment().PedestalG0Frames(0).PedestalG1Frames(0).PedestalG2Frames(0) - .SpotFindingPeriod(10ms); - services.Writer("unix:writer_test", "inproc://#1").Receiver(&fpga_receiver); - - logger.Verbose(true); - - std::vector image(RAW_MODULE_SIZE); - - HDF5File data("../../tests/test_data/compression_benchmark.h5", false, false, false); - HDF5DataSet dataset(data, "/entry/data/data"); - HDF5DataSpace file_space(dataset); - - std::vector image_conv(nimages * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - std::vector start = {0, 0, 0}; - std::vector file_size = {nimages, file_space.GetDimensions()[1], file_space.GetDimensions()[2]}; - dataset.ReadVector(image_conv, start, file_size); - - std::vector image_raw_geom( - nimages * state_machine.NotThreadSafe_Experiment().GetModulesNum() * RAW_MODULE_SIZE * sizeof(int16_t)); - for (int i = 0; i < nimages; i++) - ConvertedToRawGeometry(state_machine.NotThreadSafe_Experiment(), - image_raw_geom.data() + - i * RAW_MODULE_SIZE * state_machine.NotThreadSafe_Experiment().GetModulesNum(), - image_conv.data() + i * file_space.GetDimensions()[1] * file_space.GetDimensions()[2]); - - - AcquisitionDeviceGroup aq_devices; - - auto *test = new MockAcquisitionDevice(0, 512); - - std::vector pedestal(RAW_MODULE_SIZE, 3500); - - for (int m = 0; m < state_machine.NotThreadSafe_Experiment().GetModulesNum(0); m++) { - for (int image_num = 1; image_num <= nimages; image_num++) - test->AddModule(image_num, m, (uint16_t *) (image_raw_geom.data() - + (m + (image_num - 1) * - state_machine.NotThreadSafe_Experiment().GetModulesNum( - 0)) - * RAW_MODULE_SIZE)); - } - test->Terminate(); - - aq_devices.Add(std::move(test)); - - ZMQContext zmq_context; - - - ZMQImagePusher pusher(zmq_context, {"inproc://#1"}); - JFJochReceiverService fpga_receiver(tmp_devices, logger, pusher); - JFJochWriterService writer(zmq_context, logger); - - auto writer_server = gRPCServer("unix:writer_test", writer); - - REQUIRE_NOTHROW(state_machine.Initialize()); - logger.Info("Initialized"); - - DatasetSettings setup{}; - setup.ntrigger = 1; - setup.detector_distance_mm = 75; - setup.summation = 3; - setup.file_prefix = "spot_finding_test"; - setup.set_images_per_trigger(nimages / 3); - setup.photon_energy_keV = 12.4; - setup.beam_x_pxl = 1090; - setup.beam_y_pxl = 1136; - setup.mutable_unit_cell()->set_a(36.9); - setup.mutable_unit_cell()->set_b(78.95); - setup.mutable_unit_cell()->set_c(78.95); - setup.mutable_unit_cell()->set_alpha(90.0); - setup.mutable_unit_cell()->set_beta(90.0); - setup.mutable_unit_cell()->set_gamma(90.0); - setup.data_file_count = 1; - - JFJochProtoBuf::DataProcessingSettings settings; - settings.signal_to_noise_threshold = 4; - settings.photon_count_threshold = 5; - settings.min_pix_per_spot = 3; - settings.max_pix_per_spot = 200; - settings.low_resolution_limit = 80.0; - settings.high_resolution_limit = 2.5; - settings.local_bkg_size = 5; - - REQUIRE_NOTHROW(state_machine.SetDataProcessingSettings(settings)); - - REQUIRE_NOTHROW(state_machine.Start(setup)); - logger.Info("Started measurement"); - - REQUIRE_NOTHROW(state_machine.Stop()); - logger.Info("Stopped measurement"); - - auto tmp = state_machine.GetMeasurementStatistics(); - REQUIRE(tmp.has_value()); - auto statistics = tmp.value(); - - REQUIRE(statistics.collection_efficiency == 1.0); - REQUIRE(statistics.images_collected == nimages / 3); - REQUIRE(statistics.has_indexing_rate); - REQUIRE(statistics.indexing_rate == 1.0); - - - writer_server->Shutdown(); -}*/ -//#endif \ No newline at end of file diff --git a/tests/MockAcquisitionDeviceTest.cpp b/tests/MockAcquisitionDeviceTest.cpp deleted file mode 100644 index d96dd4c0..00000000 --- a/tests/MockAcquisitionDeviceTest.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include -#include "../acquisition_device/MockAcquisitionDevice.h" -#include "../receiver/JFJochReceiverTest.h" - -TEST_CASE("MockAcquisitionDevice") { - std::vector module_data(RAW_MODULE_SIZE, 765); - - DiffractionExperiment experiment(DetectorGeometry(8, 2)); - experiment.DataStreams(2); - - experiment.NumTriggers(1).ImagesPerTrigger(5); - MockAcquisitionDevice device(0, 128); - - device.StartAction(experiment); - - - device.AddModule(1,0,module_data.data()); - device.AddModule(1,1,module_data.data()); - device.AddModule(4,0,module_data.data()); - device.AddModule(5,1,module_data.data()); - - device.Terminate(); - device.WaitForActionComplete(); - - REQUIRE(memcmp(device.GetDeviceOutput(0,0)->pixels, module_data.data(), - RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); - REQUIRE(memcmp(device.GetDeviceOutput(0,1)->pixels, module_data.data(), - RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); - REQUIRE(memcmp(device.GetDeviceOutput(3,0)->pixels, module_data.data(), - RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); - REQUIRE(memcmp(device.GetDeviceOutput(4,1)->pixels, module_data.data(), - RAW_MODULE_SIZE * sizeof(uint16_t)) == 0); -} - -TEST_CASE("JFJochReceiverTest_Raw_MockAcquisitionDevice", "[JFJochReceiver]") { - Logger logger("JFJochReceiverTest_Raw_MockAcquisitionDevice"); - logger.Verbose(true); - - std::mt19937 g1(1387); - std::uniform_int_distribution dist(0, 65535); - - DiffractionExperiment x(DetectorGeometry(4)); - const uint16_t nthreads = 4; - - std::vector test_frame(RAW_MODULE_SIZE); - - x.Mode(DetectorMode::Raw); - x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true) - .ImagesPerTrigger(100).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::NO_COMPRESSION); - - AcquisitionDeviceGroup aq_devices; - for (int i = 0; i < x.GetDataStreamsNum(); i++) { - for (auto &j: test_frame) - j = dist(g1); - - auto test = std::make_unique(i, 64); - test->EnableLogging(&logger); - test->SetCustomInternalGeneratorFrame(test_frame); - aq_devices.Add(std::move(test)); - } - - JFJochReceiverOutput output; - bool ret; - REQUIRE_NOTHROW(ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads, false)); - REQUIRE(ret); - REQUIRE(output.efficiency == 1.0); - REQUIRE(output.images_sent == x.GetImageNum()); - REQUIRE(output.compressed_ratio == 1.0); - REQUIRE(output.compressed_size == x.GetImageNum() * x.GetPixelDepth() * x.GetPixelsNum()); - REQUIRE(output.max_image_number_sent == x.GetImageNum() - 1); - REQUIRE(!output.cancelled); -} - - -TEST_CASE("JFJochReceiverTest_Conversion_MockAcquisitionDevice", "[JFJochReceiver]") { - Logger logger("JFJochReceiverTest_Conversion"); - - DiffractionExperiment x(DetectorGeometry(8)); - const uint16_t nthreads = 4; - - x.Mode(DetectorMode::Conversion); - x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).DataStreams(x.GetModulesNum()) - .ImagesPerTrigger(32).DataFileCount(16).PhotonEnergy_keV(12.4).Compression(CompressionAlgorithm::BSHUF_ZSTD); - - AcquisitionDeviceGroup aq_devices; - for (int i = 0; i < x.GetDataStreamsNum(); i++) { - auto test = std::make_unique(i, 64); - test->EnableLogging(&logger); - aq_devices.Add(std::move(test)); - } - - JFJochReceiverOutput output; - bool ret; - REQUIRE_NOTHROW(ret = JFJochReceiverTest(output, logger, aq_devices, x, nthreads, false)); - REQUIRE(ret); - REQUIRE(output.efficiency == 1.0); - REQUIRE(output.images_sent == x.GetImageNum()); - - REQUIRE(!output.cancelled); -} diff --git a/tests/RadialIntegrationTest.cpp b/tests/RadialIntegrationTest.cpp index de7388dc..4d472192 100644 --- a/tests/RadialIntegrationTest.cpp +++ b/tests/RadialIntegrationTest.cpp @@ -66,17 +66,17 @@ TEST_CASE("RadialIntegrationProfile","[RadialIntegration]") { RadialIntegrationProfile profile(mapping, x); - std::vector sum(mapping.GetBinNumber()); - std::vector count(mapping.GetBinNumber()); + std::vector sum(mapping.GetBinNumber()); + std::vector count(mapping.GetBinNumber()); for (int i = 0; i < mapping.GetBinNumber(); i++) { - sum[i] = (i * i * 4) * (1LU<<24); + sum[i] = i * i * 4; count[i] = i; } REQUIRE_NOTHROW(profile.Add(sum, count)); REQUIRE_NOTHROW(profile.Add(sum, count)); - std::vector sum_wr(mapping.GetBinNumber() - 1); + std::vector sum_wr(mapping.GetBinNumber() - 1); REQUIRE_THROWS(profile.Add(sum_wr, count)); Plot plot = profile.GetPlot(); @@ -98,11 +98,11 @@ TEST_CASE("RadialIntegrationProfile_operatorAdd","[RadialIntegration]") { RadialIntegrationProfile profile0(mapping, x), profile1(mapping, x); - std::vector sum(mapping.GetBinNumber()); - std::vector count(mapping.GetBinNumber()); + std::vector sum(mapping.GetBinNumber()); + std::vector count(mapping.GetBinNumber()); for (int i = 0; i < mapping.GetBinNumber(); i++) { - sum[i] = (i * i * 4) * (1LU<<24); + sum[i] = i * i * 4; count[i] = i; } REQUIRE_NOTHROW(profile0.Add(sum, count)); @@ -127,15 +127,15 @@ TEST_CASE("RadialIntegrationProfile_GetMeanValueOfBins","[RadialIntegration]") { RadialIntegrationProfile profile(mapping, x); - std::vector sum(mapping.GetBinNumber()); - std::vector count(mapping.GetBinNumber()); + std::vector sum(mapping.GetBinNumber()); + std::vector count(mapping.GetBinNumber()); for (int i = 0; i < mapping.GetBinNumber(); i++) { - sum[i] = (i * i * 4) * (1LU<<24); + sum[i] = i * i * 4; count[i] = i; } REQUIRE_NOTHROW(profile.Add(sum, count)); - REQUIRE(profile.GetMeanValueOfBins(0,2) == Approx ((sum[0] + sum[1] + sum[2]) /(1LU<<24) / double(count[0] + count[1] + count[2]))); - REQUIRE(profile.GetMeanValueOfBins(5,7) == Approx ((sum[5] + sum[6] + sum[7]) /(1LU<<24) / double (count[5] + count[6] + count[7]))); + REQUIRE(profile.GetMeanValueOfBins(0,2) == Approx ((sum[0] + sum[1] + sum[2]) / double(count[0] + count[1] + count[2]))); + REQUIRE(profile.GetMeanValueOfBins(5,7) == Approx ((sum[5] + sum[6] + sum[7]) / double (count[5] + count[6] + count[7]))); } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 49819fbf..fd058227 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -8,11 +8,7 @@ target_link_libraries(CompressionBenchmark JFJochWriter CommonFunctions) add_executable(HDF5DatasetWriteTest HDF5DatasetWriteTest.cpp) target_link_libraries(HDF5DatasetWriteTest JFJochWriter CommonFunctions) -ADD_EXECUTABLE(JFCalibrationPerfTest JFCalibrationPerfTest.cpp) -TARGET_LINK_LIBRARIES(JFCalibrationPerfTest CommonFunctions) - ADD_EXECUTABLE(jfjoch_writer_test jfjoch_writer_test.cpp) TARGET_LINK_LIBRARIES(jfjoch_writer_test JFJochWriter CommonFunctions) -INSTALL(TARGETS jfjoch_udp_simulator CompressionBenchmark HDF5DatasetWriteTest - jfjoch_writer_test JFCalibrationPerfTest RUNTIME) +INSTALL(TARGETS jfjoch_udp_simulator CompressionBenchmark HDF5DatasetWriteTest jfjoch_writer_test RUNTIME) diff --git a/tools/JFCalibrationPerfTest.cpp b/tools/JFCalibrationPerfTest.cpp deleted file mode 100644 index dfbbedd1..00000000 --- a/tools/JFCalibrationPerfTest.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#include -#include -#include -#include - -#include "../jungfrau/JFPedestalCalc.h" -#include "../common/Logger.h" -#include "../jungfrau/JFCalibration.h" -#include "../jungfrau/JFConversionFloatingPoint.h" -#include "../tests/FPGAUnitTest.h" - -void test_pedestal(Logger &logger) { - size_t nframes = 5000; - DiffractionExperiment x(DetectorGeometry(1)); - x.Mode(DetectorMode::Conversion); - - std::vector data(nframes * RAW_MODULE_SIZE); - - for (size_t i = 0; i < nframes * RAW_MODULE_SIZE; i++) - data[i] = 3000 + (i % 881) + (i % 557); - - JFPedestalCalc calc_cpu(x); - auto start_time = std::chrono::system_clock::now(); - for (int i = 0; i < nframes; i++) - calc_cpu.AnalyzeImage(data.data() + i * RAW_MODULE_SIZE); - - auto end_time = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end_time - start_time); - - logger.Info("CPU pedestal performance: {:5d} us/module {:5.2f} GB/s", std::lround(elapsed.count() / ((double) nframes)), - nframes * RAW_MODULE_SIZE * sizeof(uint16_t) * 1000 * 1000/ ((double) elapsed.count() * 1024 * 1024 * 1024)); -} - -template void test_conversion(Logger &logger) { - size_t nframes = 128; - int64_t nmodules = 8; - int64_t ntries = 8; - - DiffractionExperiment x((DetectorGeometry(nmodules))); - - std::vector input(nframes * nmodules * RAW_MODULE_SIZE); - std::vector output(nframes * nmodules * RAW_MODULE_SIZE); - - for (int i = 0; i < nmodules * nframes; i++) { - std::string image_path = "../../tests/test_data/mod5_raw" + std::to_string(i % 20) + ".bin"; - LoadBinaryFile(image_path, input.data() + i * RAW_MODULE_SIZE, RAW_MODULE_SIZE); - } - - std::vector v(nmodules); - - JFModuleGainCalibration gain_calib = GainCalibrationFromTestFile(); - - for (int m = 0; m < nmodules; m++) { - JFModulePedestal pedestal_g0; - JFModulePedestal pedestal_g1; - JFModulePedestal pedestal_g2; - - for (int i = 0; i < RAW_MODULE_SIZE; i++) { - pedestal_g0.GetPedestal()[i] = 3000 + i % 50 + m * 135; - pedestal_g1.GetPedestal()[i] = 15000 + i % 50 - m * 135; - pedestal_g2.GetPedestal()[i] = 14000 + i % 50 - m * 135; - } - v[m].Setup(gain_calib, pedestal_g0, pedestal_g1, pedestal_g2, 12.4); - } - - x.Mode(DetectorMode::Conversion); - - auto start_time = std::chrono::system_clock::now(); - for (int z = 0; z < ntries; z++) { - for (int i = 0; i < nframes; i++) { - for (int m = 0; m < nmodules; m++) { - v[m].ConvertModule(output.data() + (i * nmodules + m) * RAW_MODULE_SIZE, - input.data() + (i * nmodules + m) * RAW_MODULE_SIZE); - } - } - } - - auto end_time = std::chrono::system_clock::now(); - auto elapsed = std::chrono::duration_cast(end_time - start_time); - - logger.Info("Conversion performance: {:5d} us/module {:5.2f} GB/s", std::lround(elapsed.count() / ((double) (ntries * nframes * nmodules))), - ntries * nframes * nmodules * RAW_MODULE_SIZE * sizeof(uint16_t) * 1000 * 1000/ ((double) elapsed.count() * 1024 * 1024 * 1024)); -} - -int main () { - Logger logger("JFCalibrationPerfTest"); - test_pedestal(logger); - - logger.Info("Floating point conversion"); - test_conversion(logger); -}