receive multimodule (#65)

* add config files for multimodule receiving

* read subfiles with unordered and missing frames

* save work debugging

* Revert "save work debugging"

This reverts commit e791992a05.

* Revert "read subfiles with unordered and missing frames"

This reverts commit 1177fd129d.

* throw when two frames have different frame numbers

* write single part RawFile (working beta)

* correct total number of frames in master file

* add new mythen file with syncd frames

* save work

* save work for receive multimodule
multimodule config results in too much packet loss. needs more debugging.

* setup Task Distributiosn/ parallelization programming model

* read frames with same frame number

* clang-tidy fixes, formatting, add tests

* added second receiver

* Synchronize between zmq streams and merge frames

* improve readability in loop

* fix failing tests

* add simple test for merge frames

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
Co-authored-by: Erik Frojdh <erik.frojdh@gmail.com>

utils/CMakeLists.txt

@@ -1,5 +1,14 @@
 
 add_library(utils STATIC src/logger.cpp)
 target_include_directories(utils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(utils PUBLIC core)
 
 
+
+if(AARE_TESTS)
+    set(TestSources
+            ${CMAKE_CURRENT_SOURCE_DIR}/test/merge_frames.test.cpp
+        )
+    target_sources(tests PRIVATE ${TestSources} )
+    target_link_libraries(tests PRIVATE utils)
+endif()

utils/include/aare/utils/json.hpp

@@ -50,17 +50,18 @@ inline void write_map(std::string &s, const std::string &key, const std::map<std
     }
     s += "}, ";
 }
-inline void write_array(std::string &s, const std::string &key, const std::array<int, 4> &value) {
+
+template <typename T, int N> void write_array(std::string &s, const std::string &key, const std::array<T, N> &value) {
     s += "\"";
     s += key;
     s += "\": [";
-    s += std::to_string(value[0]);
-    s += ", ";
-    s += std::to_string(value[1]);
-    s += ", ";
-    s += std::to_string(value[2]);
-    s += ", ";
-    s += std::to_string(value[3]);
+
+    for (size_t i = 0; i < N - 1; i++) {
+        s += std::to_string(value[i]);
+        s += ", ";
+    }
+    s += std::to_string(value[N - 1]);
+
     s += "], ";
 }
 

utils/include/aare/utils/merge_frames.hpp

@@ -0,0 +1,41 @@
+#pragma once
+#include "aare/core/Frame.hpp"
+#include "aare/core/defs.hpp"
+#include <cassert>
+#include <cstring>
+#include <vector>
+
+namespace aare {
+void merge_frames(std::vector<std::byte *> &part_buffers, size_t part_size, std::byte *merged_frame, const xy &geometry,
+                  size_t rows = 0, size_t cols = 0, size_t bitdepth = 0) {
+
+    assert(part_buffers.size() == geometry.row * geometry.col);
+
+    if (geometry.col == 1) {
+        // get the part from each subfile and copy it to the frame
+        size_t part_idx = 0;
+        for (auto part_buffer : part_buffers) {
+            memcpy(merged_frame + part_idx * part_size, part_buffer, part_size);
+            part_idx++;
+        }
+
+    } else {
+        std::cout << "cols: " << cols << " rows: " << rows << " bitdepth: " << bitdepth << std::endl;
+        assert(cols != 0 && rows != 0 && bitdepth != 0);
+        size_t part_rows = rows / geometry.row;
+        size_t part_cols = cols / geometry.col;
+        // create a buffer that will hold a the frame part
+        size_t part_idx = 0;
+        for (auto part_buffer : part_buffers) {
+            for (size_t cur_row = 0; cur_row < (part_rows); cur_row++) {
+                auto irow = cur_row + (part_idx / geometry.col) * part_rows;
+                auto icol = (part_idx % geometry.col) * part_cols;
+                auto dest = (irow * cols + icol);
+                dest = dest * bitdepth / 8;
+                memcpy(merged_frame + dest, part_buffer + cur_row * part_cols * bitdepth / 8, part_cols * bitdepth / 8);
+            }
+            part_idx++;
+        }
+    }
+}
+} // namespace aare

utils/test/merge_frames.test.cpp

@@ -0,0 +1,38 @@
+#include "aare/utils/merge_frames.hpp"
+#include <catch2/catch_test_macros.hpp>
+
+using namespace aare;
+// void merge_frames(std::vector<std::byte *> &part_buffers, size_t part_size, std::byte *merged_frame, const xy
+// &geometry,
+//                   size_t cols = 0, size_t rows = 0, size_t bitdepth = 0) {
+TEST_CASE("merge frames {2,1}") {
+    xy geo = {2, 1};
+    std::vector<uint32_t> p1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    std::vector<uint32_t> p2 = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19};
+    size_t part_size = p1.size() * sizeof(uint32_t);
+    Frame f(10, 2, 32);
+    std::vector<std::byte *> part_buffers = {reinterpret_cast<std::byte *>(p1.data()),
+                                             reinterpret_cast<std::byte *>(p2.data())};
+    merge_frames(part_buffers, part_size, f.data(), geo);
+
+    auto v = f.view<uint32_t>();
+    for (ssize_t i = 0; i < v.size(); i++) {
+        REQUIRE(v[i] == i);
+    }
+}
+TEST_CASE("merge frames {1,2}") {
+    xy geo = {1, 2};
+    std::vector<uint32_t> p1 = {0, 1, 2, 3, 4, 10, 11, 12, 13, 14};
+    std::vector<uint32_t> p2 = {5, 6, 7, 8, 9, 15, 16, 17, 18, 19};
+    size_t part_size = p1.size() * sizeof(uint32_t);
+    Frame f(2, 10, 32);
+    std::vector<std::byte *> part_buffers = {reinterpret_cast<std::byte *>(p1.data()),
+                                             reinterpret_cast<std::byte *>(p2.data())};
+    merge_frames(part_buffers, part_size, f.data(), geo, 2, 10, 32);
+
+    auto v = f.view<uint32_t>();
+
+    for (ssize_t i = 0; i < v.size(); i++) {
+        REQUIRE(v[i] == i);
+    }
+}