restructure zmq socker interface

zmq socket can now return vector of frames. it knows end of transmission with header.data == 0
it can also send vector of frames

network_io/src/ZmqHeader.cpp

@@ -1,5 +1,5 @@
 
-#include "aare/ZmqHeader.hpp"
+#include "aare/network_io/ZmqHeader.hpp"
 
 #include "simdjson.h"
 

network_io/src/ZmqSocket.cpp

@@ -1,5 +1,4 @@
-#include "aare/ZmqSocket.hpp"
-#include <fmt/core.h>
+#include "aare/network_io/ZmqSocket.hpp"
 #include <zmq.h>
 
 namespace aare {

network_io/src/ZmqSocketReceiver.cpp

@@ -1,4 +1,4 @@
-#include "aare/ZmqSocketReceiver.hpp"
+#include "aare/network_io/ZmqSocketReceiver.hpp"
 #include "aare/utils/logger.hpp"
 
 #include <fmt/core.h>
@@ -6,69 +6,109 @@
 
 namespace aare {
 
+/**
+ * @brief Construct a new ZmqSocketReceiver object
+ */
 ZmqSocketReceiver::ZmqSocketReceiver(const std::string &endpoint) {
     m_endpoint = endpoint;
     memset(m_header_buffer, 0, m_max_header_size);
 }
 
+/**
+ * @brief Connect to the given endpoint
+ * subscribe to a Zmq published
+ */
 void ZmqSocketReceiver::connect() {
     m_context = zmq_ctx_new();
     m_socket = zmq_socket(m_context, ZMQ_SUB);
     fmt::print("Setting ZMQ_RCVHWM to {}\n", m_zmq_hwm);
     int rc = zmq_setsockopt(m_socket, ZMQ_RCVHWM, &m_zmq_hwm, sizeof(m_zmq_hwm)); // should be set before connect
     if (rc)
-        throw std::runtime_error(fmt::format("Could not set ZMQ_RCVHWM: {}", strerror(errno)));
+        throw network_io::NetworkError(fmt::format("Could not set ZMQ_RCVHWM: {}", strerror(errno)));
 
     int bufsize = m_potential_frame_size * m_zmq_hwm;
     fmt::print("Setting ZMQ_RCVBUF to: {} MB\n", bufsize / (1024 * 1024));
     rc = zmq_setsockopt(m_socket, ZMQ_RCVBUF, &bufsize, sizeof(bufsize));
     if (rc)
-        throw std::runtime_error(fmt::format("Could not set ZMQ_RCVBUF: {}", strerror(errno)));
+        throw network_io::NetworkError(fmt::format("Could not set ZMQ_RCVBUF: {}", strerror(errno)));
 
     zmq_connect(m_socket, m_endpoint.c_str());
     zmq_setsockopt(m_socket, ZMQ_SUBSCRIBE, "", 0);
 }
 
-size_t ZmqSocketReceiver::receive(ZmqHeader &header, std::byte *data, bool serialized_header) {
-
-    size_t data_bytes_received{};
-
-    if (serialized_header)
-        throw std::runtime_error("Not implemented");
+/**
+ * @brief receive a ZmqHeader
+ * @return ZmqHeader
+ */
+ZmqHeader ZmqSocketReceiver::receive_header() {
 
+    // receive string ZmqHeader
     size_t header_bytes_received = zmq_recv(m_socket, m_header_buffer, m_max_header_size, 0);
 
-    // receive header
     m_header_buffer[header_bytes_received] = '\0'; // make sure we zero terminate
     if (header_bytes_received < 0) {
-        fmt::print("Error receiving header: {}\n", strerror(errno));
-        return -1;
+        throw network_io::NetworkError(LOCATION + "Error receiving header");
     }
     aare::logger::debug("Bytes: ", header_bytes_received, ", Header: ", m_header_buffer);
 
     // parse header
+    ZmqHeader header;
     try {
         std::string header_str(m_header_buffer);
         header.from_string(header_str);
     } catch (const simdjson::simdjson_error &e) {
-        aare::logger::error(LOCATION + "Error parsing header: ", e.what());
-        return -1;
+        throw network_io::NetworkError(LOCATION + "Error parsing header: " + e.what());
+    }
+    return header;
+}
+
+/**
+ * @brief receive data following a ZmqHeader
+ * @param data pointer to data
+ * @param size size of data
+ * @return ZmqHeader
+ */
+int ZmqSocketReceiver::receive_data(std::byte *data, size_t size) {
+    int data_bytes_received = zmq_recv(m_socket, data, size, 0);
+    if (data_bytes_received == -1)
+        network_io::NetworkError("Got half of a multipart msg!!!");
+    aare::logger::debug("Bytes: ", data_bytes_received);
+
+    return data_bytes_received;
+}
+
+ZmqFrame ZmqSocketReceiver::receive_zmqframe() {
+    // receive header from zmq and parse it
+    ZmqHeader header = receive_header();
+    if (!header.data) {
+        // no data following header
+        return {header, Frame(0, 0, 0)};
     }
 
-    // do we have a multipart message (data following header)?
-    int more;
-    size_t more_size = sizeof(more);
-    zmq_getsockopt(m_socket, ZMQ_RCVMORE, &more, &more_size);
-    if (!more) {
-        return 0; // no data following header
-    } else {
-
-        data_bytes_received = zmq_recv(m_socket, data, header.imageSize, 0); // TODO! configurable size!!!!
-        if (data_bytes_received == -1)
-            throw std::runtime_error("Got half of a multipart msg!!!");
-        aare::logger::debug("Bytes: ", data_bytes_received);
+    // receive frame data
+    Frame frame(header.npixelsx, header.npixelsy, header.dynamicRange);
+    int bytes_received = receive_data(frame.data(), frame.size());
+    if (bytes_received == -1) {
+        throw network_io::NetworkError(LOCATION + "Error receiving frame");
     }
-    return data_bytes_received + header_bytes_received;
+    if ((uint32_t)bytes_received != header.imageSize) {
+        throw network_io::NetworkError(
+            fmt::format("{} Expected {} bytes but received {}", LOCATION, header.imageSize, bytes_received));
+    }
+    return {header, std::move(frame)};
+}
+
+std::vector<ZmqFrame> ZmqSocketReceiver::receive_n() {
+    std::vector<ZmqFrame> frames;
+    while (true) {
+        // receive header and frame
+        ZmqFrame zmq_frame = receive_zmqframe();
+        if (!zmq_frame.header.data) {
+            break;
+        }
+        frames.push_back(zmq_frame);
+    }
+    return frames;
 }
 
 } // namespace aare

network_io/src/ZmqSocketSender.cpp

@@ -1,11 +1,18 @@
-#include "aare/ZmqSocketSender.hpp"
-
+#include "aare/network_io/ZmqSocketSender.hpp"
 #include <cassert>
 #include <zmq.h>
 
 namespace aare {
+
+/**
+ * Constructor
+ * @param endpoint ZMQ endpoint
+ */
 ZmqSocketSender::ZmqSocketSender(const std::string &endpoint) { m_endpoint = endpoint; }
 
+/**
+ * bind to the given port
+ */
 void ZmqSocketSender::bind() {
     m_context = zmq_ctx_new();
     m_socket = zmq_socket(m_context, ZMQ_PUB);
@@ -13,15 +20,23 @@ void ZmqSocketSender::bind() {
     assert(rc == 0);
 }
 
-size_t ZmqSocketSender::send(ZmqHeader &header, const std::byte *data, size_t size, bool serialize_header) {
+/**
+ * send a header and data
+ * @param header
+ * @param data pointer to data
+ * @param size size of data
+ * @return number of bytes sent
+ */
+size_t ZmqSocketSender::send(const ZmqHeader &header, const std::byte *data, size_t size) {
     size_t rc;
-    if (serialize_header) {
-        rc = zmq_send(m_socket, &header, sizeof(ZmqHeader), ZMQ_SNDMORE);
-        assert(rc == sizeof(ZmqHeader));
-    } else {
-        std::string header_str = header.to_string();
-        rc = zmq_send(m_socket, header_str.c_str(), header_str.size(), ZMQ_SNDMORE);
-        assert(rc == header_str.size());
+    // if (serialize_header) {
+    //     rc = zmq_send(m_socket, &header, sizeof(ZmqHeader), ZMQ_SNDMORE);
+    //     assert(rc == sizeof(ZmqHeader));
+    std::string header_str = header.to_string();
+    rc = zmq_send(m_socket, header_str.c_str(), header_str.size(), ZMQ_SNDMORE);
+    assert(rc == header_str.size());
+    if (data == nullptr) {
+        return rc;
     }
 
     size_t rc2 = zmq_send(m_socket, data, size, 0);
@@ -29,4 +44,42 @@ size_t ZmqSocketSender::send(ZmqHeader &header, const std::byte *data, size_t si
     return rc + rc2;
 }
 
+/**
+ * Send a frame with a header
+ * @param ZmqFrame that contains a header and a frame
+ * @return number of bytes sent
+ */
+size_t ZmqSocketSender::send(const ZmqFrame &zmq_frame) {
+    const Frame &frame = zmq_frame.frame;
+    // send frame
+    size_t rc = send(zmq_frame.header, frame.data(), frame.size());
+    // send end of message header
+    ZmqHeader end_header = zmq_frame.header;
+    end_header.data = false;
+    size_t rc2 = send(end_header, nullptr, 0);
+
+    return rc + rc2;
+}
+
+/**
+ * Send a vector of headers and frames
+ * @param zmq_frames vector of ZmqFrame
+ * @return number of bytes sent
+ */
+size_t ZmqSocketSender::send(const std::vector<ZmqFrame> &zmq_frames) {
+    size_t rc = 0;
+    for (size_t i = 0; i < zmq_frames.size(); i++) {
+        const ZmqHeader &header = zmq_frames[i].header;
+        const Frame &frame = zmq_frames[i].frame;
+        // send header and frame
+        if (i < zmq_frames.size() - 1) {
+            // send header and frame
+            rc += send(header, frame.data(), frame.size());
+        } else {
+            // send header, frame and end of message header
+            rc += send({header, frame});
+        }
+    }
+    return rc;
+}
 } // namespace aare