From 4efcdaab74cc807c3659a5c0833e41044cc08777 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 10 Nov 2023 11:45:16 +0100 Subject: [PATCH] AcqusitionDevice: Moved to dedicated directory --- CMakeLists.txt | 1 + .../AcquisitionCounters.cpp | 0 .../AcquisitionCounters.h | 0 .../AcquisitionDevice.cpp | 0 .../AcquisitionDevice.h | 0 acquisition_device/CMakeLists.txt | 10 + .../Completion.cpp | 0 {receiver => acquisition_device}/Completion.h | 0 .../FPGAAcquisitionDevice.cpp | 0 .../FPGAAcquisitionDevice.h | 0 .../HLSSimulatedDevice.cpp | 0 .../HLSSimulatedDevice.h | 0 .../MockAcquisitionDevice.cpp | 0 .../MockAcquisitionDevice.h | 0 .../PCIExpressDevice.cpp | 0 .../PCIExpressDevice.h | 0 common/ADUHistogram.h | 2 +- image_analysis/RadialIntegrationProfile.h | 2 +- image_analysis/StrongPixelSet.h | 2 +- receiver/CMakeLists.txt | 14 +- receiver/IBWrappers.cpp | 345 ------------------ receiver/IBWrappers.h | 82 ----- receiver/JFJochReceiver.h | 2 +- tests/AcquisitionCountersTest.cpp | 2 +- tests/FPGAIntegrationTest.cpp | 2 +- tests/FPGANetworkTest.cpp | 2 +- tests/JFJochFullIntegrationTest.cpp | 4 +- tests/JFJochReceiverIntegrationTest.cpp | 2 +- tests/MockAcquisitionDeviceTest.cpp | 2 +- tests/StreamWriterTest.cpp | 2 +- tests/gRPCServerTest.cpp | 2 +- 31 files changed, 25 insertions(+), 453 deletions(-) rename {receiver => acquisition_device}/AcquisitionCounters.cpp (100%) rename {receiver => acquisition_device}/AcquisitionCounters.h (100%) rename {receiver => acquisition_device}/AcquisitionDevice.cpp (100%) rename {receiver => acquisition_device}/AcquisitionDevice.h (100%) create mode 100644 acquisition_device/CMakeLists.txt rename {receiver => acquisition_device}/Completion.cpp (100%) rename {receiver => acquisition_device}/Completion.h (100%) rename {receiver => acquisition_device}/FPGAAcquisitionDevice.cpp (100%) rename {receiver => acquisition_device}/FPGAAcquisitionDevice.h (100%) rename {receiver => acquisition_device}/HLSSimulatedDevice.cpp (100%) rename {receiver => acquisition_device}/HLSSimulatedDevice.h (100%) rename {receiver => acquisition_device}/MockAcquisitionDevice.cpp (100%) rename {receiver => acquisition_device}/MockAcquisitionDevice.h (100%) rename {receiver => acquisition_device}/PCIExpressDevice.cpp (100%) rename {receiver => acquisition_device}/PCIExpressDevice.h (100%) delete mode 100644 receiver/IBWrappers.cpp delete mode 100644 receiver/IBWrappers.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a3cef11..50857795 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,7 @@ SET(jfjoch_executables jfjoch_broker) IF (JFJOCH_COMPILE_TESTS OR JFJOCH_COMPILE_RECEIVER) ADD_SUBDIRECTORY(fpga) + ADD_SUBDIRECTORY(acquisition_device) ADD_SUBDIRECTORY(receiver) ADD_SUBDIRECTORY(image_analysis) LIST(APPEND jfjoch_executables jfjoch_receiver) diff --git a/receiver/AcquisitionCounters.cpp b/acquisition_device/AcquisitionCounters.cpp similarity index 100% rename from receiver/AcquisitionCounters.cpp rename to acquisition_device/AcquisitionCounters.cpp diff --git a/receiver/AcquisitionCounters.h b/acquisition_device/AcquisitionCounters.h similarity index 100% rename from receiver/AcquisitionCounters.h rename to acquisition_device/AcquisitionCounters.h diff --git a/receiver/AcquisitionDevice.cpp b/acquisition_device/AcquisitionDevice.cpp similarity index 100% rename from receiver/AcquisitionDevice.cpp rename to acquisition_device/AcquisitionDevice.cpp diff --git a/receiver/AcquisitionDevice.h b/acquisition_device/AcquisitionDevice.h similarity index 100% rename from receiver/AcquisitionDevice.h rename to acquisition_device/AcquisitionDevice.h diff --git a/acquisition_device/CMakeLists.txt b/acquisition_device/CMakeLists.txt new file mode 100644 index 00000000..5849a0b1 --- /dev/null +++ b/acquisition_device/CMakeLists.txt @@ -0,0 +1,10 @@ +ADD_LIBRARY(JungfraujochAcqusitionDevice STATIC + AcquisitionDevice.cpp AcquisitionDevice.h + AcquisitionCounters.cpp AcquisitionCounters.h + MockAcquisitionDevice.cpp MockAcquisitionDevice.h + HLSSimulatedDevice.cpp HLSSimulatedDevice.h + Completion.cpp Completion.h ../fpga/pcie_driver/ActionConfig.h + PCIExpressDevice.cpp PCIExpressDevice.h + FPGAAcquisitionDevice.cpp FPGAAcquisitionDevice.h) + +TARGET_LINK_LIBRARIES(JungfraujochAcqusitionDevice JungfraujochDevice CommonFunctions HLSSimulation JFCalibration) \ No newline at end of file diff --git a/receiver/Completion.cpp b/acquisition_device/Completion.cpp similarity index 100% rename from receiver/Completion.cpp rename to acquisition_device/Completion.cpp diff --git a/receiver/Completion.h b/acquisition_device/Completion.h similarity index 100% rename from receiver/Completion.h rename to acquisition_device/Completion.h diff --git a/receiver/FPGAAcquisitionDevice.cpp b/acquisition_device/FPGAAcquisitionDevice.cpp similarity index 100% rename from receiver/FPGAAcquisitionDevice.cpp rename to acquisition_device/FPGAAcquisitionDevice.cpp diff --git a/receiver/FPGAAcquisitionDevice.h b/acquisition_device/FPGAAcquisitionDevice.h similarity index 100% rename from receiver/FPGAAcquisitionDevice.h rename to acquisition_device/FPGAAcquisitionDevice.h diff --git a/receiver/HLSSimulatedDevice.cpp b/acquisition_device/HLSSimulatedDevice.cpp similarity index 100% rename from receiver/HLSSimulatedDevice.cpp rename to acquisition_device/HLSSimulatedDevice.cpp diff --git a/receiver/HLSSimulatedDevice.h b/acquisition_device/HLSSimulatedDevice.h similarity index 100% rename from receiver/HLSSimulatedDevice.h rename to acquisition_device/HLSSimulatedDevice.h diff --git a/receiver/MockAcquisitionDevice.cpp b/acquisition_device/MockAcquisitionDevice.cpp similarity index 100% rename from receiver/MockAcquisitionDevice.cpp rename to acquisition_device/MockAcquisitionDevice.cpp diff --git a/receiver/MockAcquisitionDevice.h b/acquisition_device/MockAcquisitionDevice.h similarity index 100% rename from receiver/MockAcquisitionDevice.h rename to acquisition_device/MockAcquisitionDevice.h diff --git a/receiver/PCIExpressDevice.cpp b/acquisition_device/PCIExpressDevice.cpp similarity index 100% rename from receiver/PCIExpressDevice.cpp rename to acquisition_device/PCIExpressDevice.cpp diff --git a/receiver/PCIExpressDevice.h b/acquisition_device/PCIExpressDevice.h similarity index 100% rename from receiver/PCIExpressDevice.h rename to acquisition_device/PCIExpressDevice.h diff --git a/common/ADUHistogram.h b/common/ADUHistogram.h index 837ab698..3d17efc4 100644 --- a/common/ADUHistogram.h +++ b/common/ADUHistogram.h @@ -5,7 +5,7 @@ #include #include -#include "../receiver/AcquisitionDevice.h" +#include "../acquisition_device/AcquisitionDevice.h" class ADUHistogram { mutable std::mutex m; diff --git a/image_analysis/RadialIntegrationProfile.h b/image_analysis/RadialIntegrationProfile.h index 5959322e..168ccdfc 100644 --- a/image_analysis/RadialIntegrationProfile.h +++ b/image_analysis/RadialIntegrationProfile.h @@ -8,7 +8,7 @@ #include #include "RadialIntegrationMapping.h" -#include "../receiver/AcquisitionDevice.h" +#include "../acquisition_device/AcquisitionDevice.h" class RadialIntegrationProfile { mutable std::mutex m; diff --git a/image_analysis/StrongPixelSet.h b/image_analysis/StrongPixelSet.h index c1c24a28..36e79579 100644 --- a/image_analysis/StrongPixelSet.h +++ b/image_analysis/StrongPixelSet.h @@ -8,7 +8,7 @@ #include "../common/DiffractionExperiment.h" #include "../common/Coord.h" #include "../common/DiffractionSpot.h" -#include "../receiver/AcquisitionDevice.h" +#include "../acquisition_device/AcquisitionDevice.h" inline uint32_t strong_pixel_coord(uint16_t col, uint16_t line) { return col + (static_cast(line) << 16u); diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index cf8d0ff8..7191c836 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -1,21 +1,9 @@ -ADD_LIBRARY(JungfraujochHost STATIC - AcquisitionDevice.cpp AcquisitionDevice.h - AcquisitionCounters.cpp AcquisitionCounters.h - MockAcquisitionDevice.cpp MockAcquisitionDevice.h - HLSSimulatedDevice.cpp HLSSimulatedDevice.h - Completion.cpp Completion.h ../fpga/pcie_driver/ActionConfig.h - PCIExpressDevice.cpp PCIExpressDevice.h - IBWrappers.cpp IBWrappers.h - FPGAAcquisitionDevice.cpp FPGAAcquisitionDevice.h) - -TARGET_LINK_LIBRARIES(JungfraujochHost JungfraujochDevice CommonFunctions HLSSimulation JFCalibration) - ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverService.cpp JFJochReceiverService.h JFJochReceiverTest.cpp JFJochReceiverTest.h JFJochReceiver.cpp JFJochReceiver.h) -TARGET_LINK_LIBRARIES(JFJochReceiver ImageAnalysis JungfraujochHost CommonFunctions HLSSimulation) +TARGET_LINK_LIBRARIES(JFJochReceiver ImageAnalysis JungfraujochAcqusitionDevice CommonFunctions HLSSimulation) ADD_EXECUTABLE(jfjoch_receiver jfjoch_receiver.cpp) TARGET_LINK_LIBRARIES(jfjoch_receiver JFJochReceiver) diff --git a/receiver/IBWrappers.cpp b/receiver/IBWrappers.cpp deleted file mode 100644 index 89843163..00000000 --- a/receiver/IBWrappers.cpp +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#ifdef JFJOCH_USE_IBVERBS - -#include -#include -#include - -#ifdef JFJOCH_USE_NUMA -#include -#include -#endif - -#include "IBWrappers.h" -#include "../common/JFJochException.h" - -IBContext::IBContext(const std::string &dev_name) { - struct ibv_device **dev_list; - int num_devices; - - dev_list = ibv_get_device_list(&num_devices); - - if (dev_list == nullptr) throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to get IB devices list."); - - int selected = -1; - for (int i = 0; i < num_devices; i++) { - std::string tmp = std::string(ibv_get_device_name(dev_list[i])); - if (dev_name == tmp) selected = i; - } - - if (selected == -1) throw JFJochException(JFJochExceptionCategory::IBVerbs, "IB device not found."); - - context = ibv_open_device(dev_list[selected]); - if (context == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to open context for IB device."); - - ibv_free_device_list(dev_list); - - if (GetState() != IBV_PORT_ACTIVE) { - ibv_close_device(context); - throw JFJochException(JFJochExceptionCategory::IBVerbs, "IB port is not open"); - } -} - -IBContext::~IBContext() { - ibv_close_device(context); -} - -ibv_port_state IBContext::GetState() { - ibv_port_attr port_attr{}; - if(ibv_query_port(context, 1, &port_attr) != 0) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Cannot query port"); - return port_attr.state; -} - -uint16_t IBContext::GetLid() { - ibv_port_attr port_attr{}; - if(ibv_query_port(context, 1, &port_attr) != 0) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Cannot query port"); - return port_attr.lid; -} - -ibv_context *IBContext::GetPointer() { - return context; -} - -IBCompletionQueue::IBCompletionQueue(IBContext &context, int minimum_size) { - cq = ibv_create_cq(context.GetPointer(), minimum_size, nullptr, nullptr, 0); - if (cq == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to create completion queue."); -} - -int IBCompletionQueue::Poll(int64_t &id, size_t &compl_size) { - ibv_wc wc{}; - int num_comp = ibv_poll_cq(cq, 1, &wc); - - // Error in CQ polling - if (num_comp < 0) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed polling IB Verbs completion queue"); - else if (num_comp > 0) { - // Check for error in work completion - if (wc.status == IBV_WC_RETRY_EXC_ERR) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Remote party can be disconnected or failed (transport retry counter exceeded status in CQ)"); - else if (wc.status != IBV_WC_SUCCESS) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed status of IB Verbs send request " + std::to_string(wc.status), wc.status); - else if (!(wc.wc_flags & IBV_WC_IP_CSUM_OK)) - compl_size = 0; - else - // Frame length in bytes - compl_size = wc.byte_len; - - // Location in buffer is based on work request ID - id = wc.wr_id; - } - return num_comp; -} - -ibv_cq *IBCompletionQueue::GetPointer() { - return cq; -} - -IBCompletionQueue::~IBCompletionQueue() { - ibv_destroy_cq(cq); -} - -IBQueuePair::IBQueuePair(IBProtectionDomain &pd, IBCompletionQueue &cq, size_t send_queue_size, - size_t receive_queue_size) { - ibv_qp_init_attr qp_init_attr{}; - memset(&qp_init_attr, 0, sizeof (qp_init_attr)); - qp_init_attr.send_cq = cq.GetPointer(); - qp_init_attr.recv_cq = cq.GetPointer(); - qp_init_attr.qp_type = IBV_QPT_RAW_PACKET; - qp_init_attr.cap.max_send_wr = send_queue_size; - qp_init_attr.cap.max_recv_wr = receive_queue_size; - qp_init_attr.cap.max_send_sge = 1; // No scatter/gather operations - - qp = ibv_create_qp(pd.GetPointer(), &qp_init_attr); - if (qp == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to create IB queue pair."); -} - -uint32_t IBQueuePair::GetNumber() { - return qp->qp_num; -} - -void IBQueuePair::Init() { - ibv_qp_attr qp_attr{}; - memset(&qp_attr, 0, sizeof(qp_attr)); - - int qp_flags = IBV_QP_STATE | IBV_QP_PORT; - qp_attr.qp_state = IBV_QPS_INIT; - qp_attr.port_num = 1; - - if (ibv_modify_qp(qp, &qp_attr, qp_flags)) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed modify IB queue pair to init."); -} - -void IBQueuePair::ReadyToReceive() { - int qp_flags = IBV_QP_STATE ; - - ibv_qp_attr qp_attr{}; - memset(&qp_attr, 0, sizeof(qp_attr)); - - qp_attr.qp_state = IBV_QPS_RTR; - - if (ibv_modify_qp(qp, &qp_attr, qp_flags)) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to set IB queue pair to ready to receive "); -} - -void IBQueuePair::ReadyToSend() { - int qp_flags = IBV_QP_STATE ; - - ibv_qp_attr qp_attr{}; - memset(&qp_attr, 0, sizeof(qp_attr)); - - qp_attr.qp_state = IBV_QPS_RTS; - - if (ibv_modify_qp(qp, &qp_attr, qp_flags)) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to set IB queue pair to ready to send"); -} - -void IBQueuePair::Reset() { - int qp_flags = IBV_QP_STATE; - - ibv_qp_attr qp_attr{}; - memset(&qp_attr, 0, sizeof(qp_attr)); - - qp_attr.qp_state = IBV_QPS_RESET; - - if (ibv_modify_qp(qp, &qp_attr, qp_flags)) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed modify IB queue pair to reset."); -} - -void IBQueuePair::PostReceiveWR(IBMemoryRegion &mr, uint32_t location, void *pointer, size_t size) { - - ibv_sge ib_sg_entry{}; - ibv_recv_wr ib_wr{}; - ibv_recv_wr *ib_bad_recv_wr = nullptr; - - // pointer to packet buffer size and memory key of each packet buffer - ib_sg_entry.length = size; - ib_sg_entry.lkey = mr.GetLocalKey(); - ib_sg_entry.addr = (uintptr_t)(pointer); - - ib_wr.num_sge = 1; - ib_wr.sg_list = &ib_sg_entry; - ib_wr.next = nullptr; - ib_wr.wr_id = location; - - int ret; - while ((ret = ibv_post_recv(qp, &ib_wr, &ib_bad_recv_wr))) { - if (ret != ENOMEM) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Receiving with IB Verbs failed ", ret); - // ENONEM is not enough location in receive queue; just wait - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } -} - -void IBQueuePair::PostSendWR(IBMemoryRegion &mr, void *pointer, size_t size) { - // Send the frame via RDMA - ibv_sge ib_sg{}; - ibv_send_wr ib_wr{}; - ibv_send_wr *ib_bad_wr = nullptr; - - memset(&ib_sg, 0, sizeof(ib_sg)); - memset(&ib_wr, 0, sizeof(ib_wr)); - - ib_sg.addr = (uintptr_t)(pointer); - ib_sg.length = size; - ib_sg.lkey = mr.GetLocalKey(); - - ib_wr.wr_id = UINT64_MAX; - ib_wr.sg_list = &ib_sg; - ib_wr.num_sge = 1; - - ib_wr.opcode = IBV_WR_SEND; - ib_wr.send_flags = IBV_SEND_SIGNALED; - - int ret; - while ((ret = ibv_post_send(qp, &ib_wr, &ib_bad_wr))) { - if (ret != ENOMEM) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Sending with IB Verbs failed", ret); - // ENONEM is not enough location in send queue; just wait - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } -} - -void IBQueuePair::FlowSteeringIPv4(uint32_t ipv4) { - struct raw_eth_flow_attr { - struct ibv_flow_attr attr; - struct ibv_flow_spec_ipv4 spec_ipv4; - } __attribute__((packed)) flow_attr = { - .attr = { - .comp_mask = 0, - .type = IBV_FLOW_ATTR_NORMAL, - .size = sizeof(flow_attr), - .priority = 0, - .num_of_specs = 1, - .port = 1, - .flags = 0, - }, - .spec_ipv4 = { - .type = IBV_FLOW_SPEC_IPV4, - .size = sizeof(struct ibv_flow_spec_ipv4), - .val = { - .src_ip = 0, - .dst_ip = ipv4 - }, - .mask = { - .src_ip = 0, - .dst_ip = 0xFFFFFFFF - } - } - }; - - - flow = ibv_create_flow(qp, &flow_attr.attr); - if (flow == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to set IB flow steering"); -} - -IBQueuePair::~IBQueuePair() { - if (flow) - ibv_destroy_flow(flow); - ibv_destroy_qp(qp); -} - -IBProtectionDomain::IBProtectionDomain(IBContext &context) { - pd = ibv_alloc_pd(context.GetPointer()); - if (pd == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to allocate IB protection domain."); -} - -IBProtectionDomain::~IBProtectionDomain() { - ibv_dealloc_pd(pd); -} - -ibv_pd *IBProtectionDomain::GetPointer() { - return pd; -} - -IBMemoryRegion::IBMemoryRegion(IBProtectionDomain &pd, void *pointer, size_t size) { - buffer_mr = ibv_reg_mr(pd.GetPointer(), pointer, size, IBV_ACCESS_LOCAL_WRITE); - - if (buffer_mr == nullptr) - throw JFJochException(JFJochExceptionCategory::IBVerbs, "Failed to register IB memory region."); -} - -uint32_t IBMemoryRegion::GetLocalKey() { - return buffer_mr->lkey; -} - -uint32_t IBMemoryRegion::GetRemoteKey() { - return buffer_mr->rkey; -} - -IBMemoryRegion::~IBMemoryRegion() { - ibv_dereg_mr(buffer_mr); -} - -IBRegBuffer::IBRegBuffer(IBProtectionDomain &pd, size_t in_loc_count, size_t in_loc_size, int16_t numa_node) : - loc_count(in_loc_count), loc_size(in_loc_size) { - buffer = (uint8_t *) mmap(nullptr, loc_size * loc_count, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (buffer == nullptr) - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "frame_buffer"); - -#ifdef JFJOCH_USE_NUMA - if (numa_node >= 0) { - unsigned long nodemask = 1L << numa_node;; - if (numa_node > sizeof(nodemask)*8) { - Unmap(); - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Mask too small for NUMA node"); - } - if (mbind(buffer, loc_size * loc_count, MPOL_BIND, &nodemask, sizeof(nodemask)*8, MPOL_MF_STRICT) == -1) { - Unmap(); - throw JFJochException(JFJochExceptionCategory::MemAllocFailed, "Cannot apply NUMA policy"); - } - } -#endif - memset(buffer, 0, loc_size * loc_count); - - mr = std::make_unique(pd, buffer, loc_size * loc_count); -} - -void IBRegBuffer::Unmap() { - munmap(buffer, loc_size * loc_count); -} - -IBRegBuffer::~IBRegBuffer() { - mr.reset(); - Unmap(); -} - -IBMemoryRegion *IBRegBuffer::GetMemoryRegion() { - return mr.get(); -} - -uint8_t *IBRegBuffer::GetLocation(uint64_t location) { - if (location < loc_count) - return buffer + loc_size * location; - else - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Location out of bounds"); -} - -#endif //JFJOCH_USE_IBVERBS diff --git a/receiver/IBWrappers.h b/receiver/IBWrappers.h deleted file mode 100644 index 99b9e692..00000000 --- a/receiver/IBWrappers.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (2019-2023) Paul Scherrer Institute - -#ifdef JFJOCH_USE_IBVERBS -#ifndef JUNGFRAUJOCH_IBWRAPPERS_H -#define JUNGFRAUJOCH_IBWRAPPERS_H - -#include -#include - -class IBContext { - ibv_context *context = nullptr; -public: - explicit IBContext(const std::string &dev_name); - IBContext(const IBContext &context) = delete; - ~IBContext(); - ibv_port_state GetState(); - uint16_t GetLid(); - ibv_context *GetPointer(); -}; - -class IBProtectionDomain { - ibv_pd *pd = nullptr; -public: - explicit IBProtectionDomain(IBContext &context); - IBProtectionDomain(const IBProtectionDomain &context) = delete; - ~IBProtectionDomain(); - ibv_pd *GetPointer(); -}; - -class IBCompletionQueue { - ibv_cq *cq = nullptr; -public: - IBCompletionQueue(IBContext &context, int minimum_size); - int Poll(int64_t &id, size_t &compl_size); - ~IBCompletionQueue(); - ibv_cq *GetPointer(); -}; - - -class IBMemoryRegion { - ibv_mr *buffer_mr = nullptr; -public: - IBMemoryRegion(IBProtectionDomain &pd, void *pointer, size_t size); - IBMemoryRegion(const IBMemoryRegion &mr) = delete; - uint32_t GetLocalKey(); - uint32_t GetRemoteKey(); - ~IBMemoryRegion(); -}; - -class IBQueuePair { - ibv_qp *qp = nullptr; - ibv_flow *flow = nullptr; -public: - IBQueuePair(IBProtectionDomain &pd, IBCompletionQueue &cq, size_t send_queue_size, size_t receive_queue_size); - uint32_t GetNumber(); - void Init(); - void Reset(); - void ReadyToReceive(); - void ReadyToSend(); - - void PostReceiveWR(IBMemoryRegion &mr, uint32_t location, void *pointer, size_t size); - void PostSendWR(IBMemoryRegion &mr, void *pointer, size_t size); - void FlowSteeringIPv4(uint32_t ipv4); - - ~IBQueuePair(); -}; - -class IBRegBuffer { - std::unique_ptr mr; - uint8_t *buffer; - const size_t loc_count; - const size_t loc_size; - void Unmap(); -public: - explicit IBRegBuffer(IBProtectionDomain &pd, size_t loc_count, size_t loc_size, int16_t numa_node = -1); - ~IBRegBuffer(); - uint8_t *GetLocation(uint64_t location); - IBMemoryRegion *GetMemoryRegion(); -}; - -#endif //JUNGFRAUJOCH_IBWRAPPERS_H -#endif // JFJOCH_USE_IBVERBS \ No newline at end of file diff --git a/receiver/JFJochReceiver.h b/receiver/JFJochReceiver.h index 9c3e2781..dd475498 100644 --- a/receiver/JFJochReceiver.h +++ b/receiver/JFJochReceiver.h @@ -10,7 +10,7 @@ #include #include -#include "AcquisitionDevice.h" +#include "../acquisition_device/AcquisitionDevice.h" #include "../common/DiffractionExperiment.h" #include "../common/JFJochException.h" diff --git a/tests/AcquisitionCountersTest.cpp b/tests/AcquisitionCountersTest.cpp index 59e89102..8b43608b 100644 --- a/tests/AcquisitionCountersTest.cpp +++ b/tests/AcquisitionCountersTest.cpp @@ -1,7 +1,7 @@ // Copyright (2019-2023) Paul Scherrer Institute #include -#include "../receiver/AcquisitionCounters.h" +#include "../acquisition_device/AcquisitionCounters.h" TEST_CASE("AcquisitionCountersTest","[AcquisitionDeviceCounters]") { DiffractionExperiment x(DetectorGeometry(2)); diff --git a/tests/FPGAIntegrationTest.cpp b/tests/FPGAIntegrationTest.cpp index 57969e1f..01dbfe58 100644 --- a/tests/FPGAIntegrationTest.cpp +++ b/tests/FPGAIntegrationTest.cpp @@ -5,7 +5,7 @@ #include #include "../jungfrau/JFPedestalCalc.h" -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" #include "FPGAUnitTest.h" using namespace std::literals::chrono_literals; diff --git a/tests/FPGANetworkTest.cpp b/tests/FPGANetworkTest.cpp index 843fd67b..6ea129f8 100644 --- a/tests/FPGANetworkTest.cpp +++ b/tests/FPGANetworkTest.cpp @@ -5,7 +5,7 @@ #include #include -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" // ARP packet - from if_arp.h #pragma pack(push) diff --git a/tests/JFJochFullIntegrationTest.cpp b/tests/JFJochFullIntegrationTest.cpp index 8a48459b..b4585773 100644 --- a/tests/JFJochFullIntegrationTest.cpp +++ b/tests/JFJochFullIntegrationTest.cpp @@ -8,8 +8,8 @@ #include "../writer/HDF5Objects.h" #include "../receiver/JFJochReceiverService.h" #include "FPGAUnitTest.h" -#include "../receiver/MockAcquisitionDevice.h" -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/MockAcquisitionDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" #include "../common/ZMQImagePusher.h" #include "../common/jsonToGrpc.h" diff --git a/tests/JFJochReceiverIntegrationTest.cpp b/tests/JFJochReceiverIntegrationTest.cpp index b2ac65c9..b88ab79e 100644 --- a/tests/JFJochReceiverIntegrationTest.cpp +++ b/tests/JFJochReceiverIntegrationTest.cpp @@ -3,7 +3,7 @@ #include #include "../receiver/JFJochReceiverTest.h" -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" #include "../jungfrau/JFPedestalCalc.h" #include "../common/TestImagePusher.h" diff --git a/tests/MockAcquisitionDeviceTest.cpp b/tests/MockAcquisitionDeviceTest.cpp index 24de598b..2425afc2 100644 --- a/tests/MockAcquisitionDeviceTest.cpp +++ b/tests/MockAcquisitionDeviceTest.cpp @@ -1,7 +1,7 @@ // Copyright (2019-2023) Paul Scherrer Institute #include -#include "../receiver/MockAcquisitionDevice.h" +#include "../acquisition_device/MockAcquisitionDevice.h" #include "../receiver/JFJochReceiverTest.h" TEST_CASE("MockAcquisitionDevice") { diff --git a/tests/StreamWriterTest.cpp b/tests/StreamWriterTest.cpp index dcb0f9d8..eaeb5525 100644 --- a/tests/StreamWriterTest.cpp +++ b/tests/StreamWriterTest.cpp @@ -8,7 +8,7 @@ #include "../writer/HDF5Objects.h" #include "../common/ZMQImagePusher.h" #include "../receiver/JFJochReceiverService.h" -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" TEST_CASE("StreamWriterTest_ZMQ","[JFJochWriter]") { RegisterHDF5Filter(); diff --git a/tests/gRPCServerTest.cpp b/tests/gRPCServerTest.cpp index 6b22038b..5f710fb1 100644 --- a/tests/gRPCServerTest.cpp +++ b/tests/gRPCServerTest.cpp @@ -7,7 +7,7 @@ #include "../common/Logger.h" #include "../receiver/JFJochReceiverService.h" #include "../grpc/JFJochReceiverClient.h" -#include "../receiver/HLSSimulatedDevice.h" +#include "../acquisition_device/HLSSimulatedDevice.h" #include #include "../../common/ZMQImagePusher.h"