receive multimodule (#65)

* add config files for multimodule receiving

* read subfiles with unordered and missing frames

* save work debugging

* Revert "save work debugging"

This reverts commit e791992a05.

* Revert "read subfiles with unordered and missing frames"

This reverts commit 1177fd129d.

* throw when two frames have different frame numbers

* write single part RawFile (working beta)

* correct total number of frames in master file

* add new mythen file with syncd frames

* save work

* save work for receive multimodule
multimodule config results in too much packet loss. needs more debugging.

* setup Task Distributiosn/ parallelization programming model

* read frames with same frame number

* clang-tidy fixes, formatting, add tests

* added second receiver

* Synchronize between zmq streams and merge frames

* improve readability in loop

* fix failing tests

* add simple test for merge frames

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
Co-authored-by: Erik Frojdh <erik.frojdh@gmail.com>

examples/CMakeLists.txt

@@ -1,11 +1,11 @@
 
 set(EXAMPLE_LIST "json_example;logger_example;numpy_read_example;multiport_example;raw_example;zmq_restream_example")
 set(EXAMPLE_LIST "${EXAMPLE_LIST};mythen_example;numpy_write_example;zmq_receiver_example;zmq_sender_example;")    
-set(EXAMPLE_LIST "${EXAMPLE_LIST};cluster_example")
+set(EXAMPLE_LIST "${EXAMPLE_LIST};cluster_example;zmq_multi_receiver;zmq_task_ventilator;zmq_worker;zmq_sink")
 foreach(example ${EXAMPLE_LIST})
     add_executable(${example} ${example}.cpp)
     target_include_directories(${example} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-    target_link_libraries(${example} PUBLIC aare PRIVATE aare_compiler_flags)
+    target_link_libraries(${example} PUBLIC aare PRIVATE aare_compiler_flags libzmq)
 endforeach()
 
 

examples/zmq_multi_receiver.cpp

@@ -0,0 +1,29 @@
+#include "aare/examples/defs.hpp"
+#include "aare/network_io/ZmqMultiReceiver.hpp"
+#include "aare/network_io/ZmqSocketReceiver.hpp"
+#include "aare/network_io/defs.hpp"
+
+#include <boost/program_options.hpp>
+#include <cassert>
+#include <fmt/core.h>
+#include <string>
+using namespace aare;
+namespace po = boost::program_options;
+using namespace std;
+
+int main() {
+    logger::set_verbosity(logger::DEBUG);
+
+    std::string const endpoint1 = "tcp://127.0.0.1:" + std::to_string(5555);
+    std::string const endpoint2 = "tcp://127.0.0.1:" + std::to_string(5556);
+
+    ZmqMultiReceiver socket({endpoint1, endpoint2}, {1, 2});
+
+    socket.connect();
+    while (true) {
+        auto frames = socket.receive_n();
+        logger::info("Received", frames.size(), "frames");
+    }
+
+    return 0;
+}

examples/zmq_receiver_example.cpp

@@ -36,7 +36,7 @@ int main(int argc, char **argv) {
 
     auto port = vm["port"].as<uint16_t>();
 
-    std::string const endpoint = "udp://127.0.0.1:" + std::to_string(port);
+    std::string const endpoint = "tcp://127.0.0.1:" + std::to_string(port);
     aare::ZmqSocketReceiver socket(endpoint);
     socket.connect();
     while (true) {

examples/zmq_restream_example.cpp

@@ -44,7 +44,7 @@ int main(int argc, char **argv) {
         return 1;
     }
     if (vm.count("port") != 1) {
-        aare::logger::error("file is required");
+        aare::logger::error("port is required");
         cout << desc << "\n";
         return 1;
     }
@@ -77,9 +77,9 @@ int main(int argc, char **argv) {
         ZmqHeader header;
         header.frameNumber = frameidx;
         header.data = true;
-        header.npixelsx = frame.rows();
-        header.npixelsy = frame.cols();
-        header.dynamicRange = frame.bitdepth();
+        header.shape.row = frame.rows();
+        header.shape.col = frame.cols();
+        header.bitmode = frame.bitdepth();
         header.size = frame.size();
 
         sender.send({header, frame});

examples/zmq_sender_example.cpp

@@ -23,10 +23,9 @@ int main() {
         }
     }
     aare::ZmqHeader header;
-    header.npixelsx = 1024;
-    header.npixelsy = 1024;
+    header.shape = {1024, 1024};
     header.size = sizeof(uint32_t) * 1024 * 1024;
-    header.dynamicRange = 32;
+    header.bitmode = 32;
 
     std::vector<ZmqFrame> zmq_frames;
     // send two exact frames

examples/zmq_sink.cpp

@@ -0,0 +1,30 @@
+
+#include "aare/network_io/ZmqSink.hpp"
+#include "aare/network_io/ZmqSocketReceiver.hpp"
+#include "aare/network_io/ZmqVentilator.hpp"
+#include "aare/network_io/ZmqWorker.hpp"
+
+#include "zmq.h"
+#include <boost/program_options.hpp>
+#include <cassert>
+#include <fmt/core.h>
+#include <string>
+using namespace aare;
+namespace po = boost::program_options;
+using namespace std;
+
+int main() {
+    logger::set_verbosity(logger::DEBUG);
+    // 1. bind sink to endpoint
+    ZmqSink sink("tcp://*:4322");
+
+    int i = 0;
+    while (true) {
+        // 2. receive Task from ventilator
+        Task *task = sink.pull();
+        logger::info("Received", i++, "tasks");
+
+        Task::destroy(task);
+    }
+    // read the command line arguments
+}

examples/zmq_task_ventilator.cpp

@@ -0,0 +1,72 @@
+#include "aare/network_io/ZmqSocketReceiver.hpp"
+#include "aare/network_io/ZmqVentilator.hpp"
+
+#include "zmq.h"
+#include <boost/program_options.hpp>
+#include <cassert>
+#include <fmt/core.h>
+#include <string>
+using namespace aare;
+namespace po = boost::program_options;
+using namespace std;
+
+string setup(int argc, char **argv) {
+    logger::set_verbosity(logger::DEBUG);
+    po::options_description desc("options");
+    desc.add_options()("help", "produce help message")("port,p", po::value<uint16_t>()->default_value(5555),
+                                                       "port number");
+    po::positional_options_description pd;
+    pd.add("port", 1);
+    po::variables_map vm;
+    try {
+        auto parsed = po::command_line_parser(argc, argv).options(desc).positional(pd).run();
+        po::store(parsed, vm);
+        po::notify(vm);
+
+    } catch (const boost::program_options::error &e) {
+        cout << e.what() << "\n";
+        cout << desc << "\n";
+        exit(1);
+    }
+    if (vm.count("help")) {
+        cout << desc << "\n";
+        exit(1);
+    }
+
+    auto port = vm["port"].as<uint16_t>();
+
+    return "tcp://127.0.0.1:" + to_string(port);
+}
+
+int process(const std::string &endpoint) {
+    // 0. connect to slsReceiver
+    ZmqSocketReceiver receiver(endpoint, ZMQ_SUB);
+    receiver.connect();
+
+    // 1. create ventilator
+    ZmqVentilator ventilator("tcp://*:4321");
+
+    while (true) {
+        // 2. receive frame from slsReceiver
+        ZmqFrame zframe = receiver.receive_zmqframe();
+        if (zframe.header.data == 0)
+            continue;
+        logger::info("Received frame, frame_number=", zframe.header.frameNumber);
+        logger::info(zframe.header.to_string());
+
+        // 3. create task
+        Task *task = Task::init(zframe.frame.data(), zframe.frame.size());
+        task->opcode = (size_t)Task::Operation::PEDESTAL;
+        task->id = zframe.header.frameNumber;
+
+        // 4. push task to ventilator
+        ventilator.push(task);
+        Task::destroy(task);
+    }
+}
+int main(int argc, char **argv) {
+    // read the command line arguments
+    string endpoint = setup(argc, argv);
+    int ret = process(endpoint);
+    return ret;
+}

examples/zmq_worker.cpp

@@ -0,0 +1,35 @@
+
+#include "aare/network_io/ZmqSink.hpp"
+#include "aare/network_io/ZmqSocketReceiver.hpp"
+#include "aare/network_io/ZmqVentilator.hpp"
+#include "aare/network_io/ZmqWorker.hpp"
+
+#include "zmq.h"
+#include <boost/program_options.hpp>
+#include <cassert>
+#include <fmt/core.h>
+#include <string>
+using namespace aare;
+namespace po = boost::program_options;
+using namespace std;
+
+int main() {
+    logger::set_verbosity(logger::DEBUG);
+    // 1. connect to ventilator and sink
+    ZmqWorker worker("tcp://127.0.0.1:4321", "tcp://127.0.0.1:4322");
+
+    while (true) {
+        // 2. receive Task from ventilator
+        Task *ventilator_task = worker.pull();
+        logger::info("Received Task, id=", ventilator_task->id, " data_size=", ventilator_task->data_size);
+
+        Task *sink_task = Task::init(nullptr, 0);
+        sink_task->id = ventilator_task->id;
+        sink_task->opcode = (size_t)Task::Operation::COUNT;
+        worker.push(sink_task);
+
+        Task::destroy(sink_task);
+        Task::destroy(ventilator_task);
+    }
+    // read the command line arguments
+}