Merge pull request #1342 from slsdetectorgroup/dev/dbitorder

Preserve order in receiver dbit list

slsDetectorSoftware/src/Module.cpp

@@ -2568,17 +2568,15 @@ std::vector<int> Module::getReceiverDbitList() const {
 
 void Module::setReceiverDbitList(std::vector<int> list) {
     LOG(logDEBUG1) << "Setting Receiver Dbit List";
-    if (list.size() > 64) {
-        throw RuntimeError("Dbit list size cannot be greater than 64\n");
-    }
+
     for (auto &it : list) {
         if (it < 0 || it > 63) {
             throw RuntimeError("Dbit list value must be between 0 and 63\n");
         }
     }
-    std::sort(begin(list), end(list));
-    auto last = std::unique(begin(list), end(list));
-    list.erase(last, list.end());
+    auto r = stableRemoveDuplicates(list);
+    if(r)
+        LOG(logWARNING) << "Removed duplicated from receiver dbit list";
 
     StaticVector<int, MAX_RX_DBIT> arg = list;
     sendToReceiver(F_SET_RECEIVER_DBIT_LIST, arg, nullptr);

slsReceiverSoftware/tests/test-ArrangeDataBasedOnBitList.cpp

@@ -64,7 +64,8 @@ class DataProcessorTest : public DataProcessor {
  * num_transceiver_bytes = 2 both bytes have a value of 125
  * num_digital_bytes is variable and is defined by number of samples
  * default num sample is 5
- * all bytes in digital data take a value of 255
+ * all bytes in digital data take a value of 0xFF (alternating bits between 0,
+ * 1)
  */
 class DataProcessorTestFixture {
   public:
@@ -106,7 +107,7 @@ class DataProcessorTestFixture {
                                                  num_random_offset_bytes);
     }
 
-    void set_data() {
+    void set_data(const std::bitset<8> pattern = 0xFF) {
         delete[] data;
         uint64_t max_bytes_per_bit =
             num_samples % 8 == 0 ? num_samples / 8 : num_samples / 8 + 1;
@@ -118,7 +119,8 @@ class DataProcessorTestFixture {
         memset(data, dummy_value, num_analog_bytes); // set to dummy value
         memset(data + num_analog_bytes, 0,
                num_random_offset_bytes); // set to zero
-        memset(data + num_analog_bytes + num_random_offset_bytes, 0xFF,
+        memset(data + num_analog_bytes + num_random_offset_bytes,
+               static_cast<uint8_t>(pattern.to_ulong()),
                num_digital_bytes); // all digital bits are one
         memset(data + num_digital_bytes + num_analog_bytes +
                    num_random_offset_bytes,
@@ -170,7 +172,7 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Remove Trailing Bits",
 TEST_CASE_METHOD(DataProcessorTestFixture, "Reorder all",
                  "[.dataprocessor][.reorder]") {
     // parameters: num_samples, expected_num_digital_bytes,
-    // expected_digital_part
+    // expected_digital_part_for_each_bit
     auto parameters = GENERATE(
         std::make_tuple(5, 64, std::vector<uint8_t>{0b00011111}),
         std::make_tuple(10, 2 * 64, std::vector<uint8_t>{0xFF, 0b00000011}),
@@ -264,11 +266,13 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Arrange bitlist with reorder false",
     // expected_digital_part
     auto parameters = GENERATE(
         std::make_tuple(5, std::vector<int>{1, 4, 5}, 5,
-                        std::vector<uint8_t>{0b00000111}),
+                        std::vector<uint8_t>{0b00000010}),
+        std::make_tuple(5, std::vector<int>{1, 5, 4}, 5,
+                        std::vector<uint8_t>{0b00000100}),
         std::make_tuple(5, std::vector<int>{1, 5, 3, 7, 8, 50, 42, 60, 39}, 10,
-                        std::vector<uint8_t>{0xFF, 0b00000001}),
+                        std::vector<uint8_t>{0b11110000, 0b00000000}),
         std::make_tuple(5, std::vector<int>{1, 5, 3, 7, 8, 50, 42, 60}, 5,
-                        std::vector<uint8_t>{0xFF}));
+                        std::vector<uint8_t>{0b11110000}));
 
     size_t num_samples, expected_num_digital_bytes;
     std::vector<uint8_t> expected_digital_part;
@@ -281,7 +285,7 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Arrange bitlist with reorder false",
     generaldata->SetCtbDbitReorder(false);
 
     set_num_samples(num_samples);
-    set_data();
+    set_data(0b01010101); // set digital data to 0x55 to have alternating bits
 
     size_t expected_size =
         num_analog_bytes + num_transceiver_bytes + expected_num_digital_bytes;
@@ -316,11 +320,15 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Arrange bitlist with reorder true",
     // expected_digital_part
     auto parameters = GENERATE(
         std::make_tuple(5, std::vector<int>{1, 4, 5}, 3,
-                        std::vector<uint8_t>{0b00011111}),
+                        std::vector<uint8_t>{0x00, 0b00011111, 0x00}),
+        std::make_tuple(5, std::vector<int>{1, 5, 4}, 3,
+                        std::vector<uint8_t>{0x00, 0x00, 0b00011111}),
         std::make_tuple(10, std::vector<int>{1, 4, 5}, 6,
-                        std::vector<uint8_t>{0xFF, 0b00000011}),
+                        std::vector<uint8_t>{0x00, 0x00, 0b11111111, 0b00000011,
+                                             0x00, 0x00}),
         std::make_tuple(8, std::vector<int>{1, 5, 3, 7, 8, 50, 42, 60, 39}, 9,
-                        std::vector<uint8_t>{0xFF}));
+                        std::vector<uint8_t>{0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF,
+                                             0xFF, 0xFF, 0x00}));
 
     size_t num_samples, expected_num_digital_bytes;
     std::vector<uint8_t> expected_digital_part;
@@ -333,7 +341,7 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Arrange bitlist with reorder true",
     generaldata->SetCtbDbitReorder(true);
 
     set_num_samples(num_samples);
-    set_data();
+    set_data(0b01010101);
 
     size_t expected_size =
         num_analog_bytes + num_transceiver_bytes + expected_num_digital_bytes;
@@ -343,11 +351,8 @@ TEST_CASE_METHOD(DataProcessorTestFixture, "Arrange bitlist with reorder true",
 
     memset(expected_data, dummy_value, num_analog_bytes);
 
-    for (size_t sample = 0; sample < bitlist.size(); ++sample) {
-        memcpy(expected_data + num_analog_bytes +
-                   expected_digital_part.size() * sample,
-               expected_digital_part.data(), expected_digital_part.size());
-    }
+    memcpy(expected_data + num_analog_bytes, expected_digital_part.data(),
+           expected_digital_part.size());
 
     memset(expected_data + expected_num_digital_bytes + num_analog_bytes,
            dummy_value, num_transceiver_bytes);

slsSupportLib/include/sls/container_utils.h

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <memory>
 #include <numeric>
+#include <set>
 #include <sstream>
 #include <string>
 #include <type_traits>
@@ -155,6 +156,10 @@ template <typename Container> bool hasDuplicates(Container c) {
     return pos != c.end(); // if we found something there are duplicates
 }
 
+/**
+ * @brief Sorts the container and removes duplicated elements
+ * returns true if elements were removed otherwiese false
+ */
 template <typename T>
 typename std::enable_if<is_container<T>::value, bool>::type
 removeDuplicates(T &c) {
@@ -167,6 +172,29 @@ removeDuplicates(T &c) {
     return false;
 }
 
+/**
+ * @brief Removed duplicated entries while preserving the oder
+ * returns true if elements were removed otherwiese false
+ */
+template <typename T>
+typename std::enable_if<is_container<T>::value, bool>::type
+stableRemoveDuplicates(T &c) {
+    auto containerSize = c.size();
+    std::set<typename T::value_type> seen;
+    c.erase(
+        std::remove_if(c.begin(), c.end(),
+            [&](const typename T::value_type& val) {
+                return !seen.insert(val).second;  // erase if already seen
+            }),
+        c.end()
+    );
+    if (c.size() != containerSize) {
+        return true;
+    }
+    return false;
+}
+
+
 } // namespace sls
 
 #endif // CONTAINER_UTILS_H

slsSupportLib/tests/test-container_utils.cpp

@@ -153,13 +153,35 @@ TEST_CASE("check for duplicates in vector of pairs") {
     REQUIRE(hasDuplicates(vec) == true);
 }
 
-TEST_CASE("remove duplicates from vector") {
+TEST_CASE("sorts the vector and remove duplicates") {
     std::vector<int> v{5, 6, 5, 3};
     auto r = removeDuplicates(v);
     CHECK(r == true); // did indeed remove elements
     CHECK(v == std::vector<int>{3, 5, 6});
 }
 
+TEST_CASE("remove duplicates but keep order") {
+    std::vector<int> v{5, 6, 5, 3};
+    auto r = stableRemoveDuplicates(v);
+    CHECK(r == true); // did indeed remove elements
+    CHECK(v == std::vector<int>{5, 6, 3});
+}
+
+TEST_CASE("remove duplicates but keep order, all elements the same ") {
+    std::vector<char> v{'c', 'c', 'c', 'c', 'c', 'c'};
+    auto r = stableRemoveDuplicates(v);
+    CHECK(r == true); // did indeed remove elements
+    CHECK(v == std::vector<char>{'c'});
+}
+
+TEST_CASE("remove duplicates but keep order, pattern ") {
+    std::vector<int> v{8,1,2,8,8,3,2};
+    auto r = stableRemoveDuplicates(v);
+    CHECK(r == true); // did indeed remove elements
+    CHECK(v == std::vector<int>{8,1,2,3});
+}
+
+
 TEST_CASE("remove duplicated empty vector") {
     std::vector<int> v;
     auto r = removeDuplicates(v);
@@ -167,4 +189,11 @@ TEST_CASE("remove duplicated empty vector") {
     CHECK(v == std::vector<int>{});
 }
 
+TEST_CASE("remove duplicated empty vector using stable version") {
+    std::vector<int> v;
+    auto r = stableRemoveDuplicates(v);
+    CHECK(r == false); // no elements to remove
+    CHECK(v == std::vector<int>{});
+}
+
 } // namespace sls