Merge branch 'dev/issue_dont_reorder_digital_data' into dev/cmd_for_ctb_reorder

slsReceiverSoftware/src/DataProcessor.cpp

@@ -358,9 +358,13 @@ void DataProcessor::ProcessAnImage(sls_receiver_header &header, size_t &size,
     if (framePadding && nump < generalData->packetsPerFrame)
         PadMissingPackets(header, data);
 
-    // rearrange ctb digital bits (if ctbDbitlist is not empty)
+    // rearrange ctb digital bits
     if (!ctbDbitList.empty()) {
         ArrangeDbitData(size, data);
+    } else if (reorder) {
+        Reorder(size, data);
+    } else if (ctbDbitOffset > 0) {
+        RemoveTrailingBits(size, data);
     }
 
     // 'stream Image' check has to be done here before crop image
@@ -532,6 +536,111 @@ void DataProcessor::PadMissingPackets(sls_receiver_header header, char *data) {
     }
 }
 
+void DataProcessor::RemoveTrailingBits(size_t &size, char *data) {
+    const size_t nAnalogDataBytes = generalData->GetNumberOfAnalogDatabytes();
+    const size_t nDigitalDataBytes = generalData->GetNumberOfDigitalDatabytes();
+    const size_t nTransceiverDataBytes =
+        generalData->GetNumberOfTransceiverDatabytes();
+
+    const size_t ctbDigitalDataBytes = nDigitalDataBytes - ctbDbitOffset;
+
+    // no digital data
+    if (ctbDigitalDataBytes == 0) {
+        LOG(logWARNING)
+            << "No digital data for call back, yet ctbDbitOffset is non zero.";
+        return;
+    }
+
+    // update size and copy data
+    memmove(data + nAnalogDataBytes,
+            data + nAnalogDataBytes + ctbDigitalDataBytes,
+            ctbDigitalDataBytes + nTransceiverDataBytes);
+
+    size = nAnalogDataBytes + ctbDigitalDataBytes + nTransceiverDataBytes;
+}
+
+void DataProcessor::Reorder(size_t &size, char *data) {
+    const size_t nAnalogDataBytes = generalData->GetNumberOfAnalogDatabytes();
+    const size_t nDigitalDataBytes = generalData->GetNumberOfDigitalDatabytes();
+    const size_t nTransceiverDataBytes =
+        generalData->GetNumberOfTransceiverDatabytes();
+
+    const size_t ctbDigitalDataBytes = nDigitalDataBytes - ctbDbitOffset;
+
+    // no digital data
+    if (ctbDigitalDataBytes == 0) {
+        LOG(logWARNING)
+            << "No digital data for call back, yet reorder is set to 1.";
+        return;
+    }
+
+    auto *source = (uint64_t *)(data + nAnalogDataBytes + ctbDbitOffset);
+
+    const size_t numDigitalSamples = (ctbDigitalDataBytes / sizeof(uint64_t));
+
+    size_t numBytesPerBit =
+        0; // number of bytes per bit in digital data after reordering
+
+    if ((numDigitalSamples % 8) == 0)
+        numBytesPerBit = numDigitalSamples / 8;
+    else
+        numBytesPerBit = numDigitalSamples / 8 + 1;
+
+    size_t totalNumBytes =
+        numBytesPerBit *
+        64; // number of bytes for digital data after reordering
+
+    LOG(logDEBUG1) << "totalNumBytes: " << totalNumBytes
+                   << " nAnalogDataBytes:" << nAnalogDataBytes
+                   << " nDigitalDataBytes: " << nDigitalDataBytes
+                   << " ctbDbitOffset:" << ctbDbitOffset
+                   << " nTransceiverDataBytes:" << nTransceiverDataBytes
+                   << " size:" << size << " numsamples:" << numDigitalSamples;
+
+    std::vector<uint8_t> result(totalNumBytes, 0);
+    uint8_t *dest = &result[0];
+
+    int bitoffset = 0;
+    // reorder
+    for (size_t bi = 0; bi < 64; ++bi) {
+
+        if (bitoffset != 0) {
+            bitoffset = 0;
+            ++dest;
+        }
+
+        for (auto *ptr = source; ptr < (source + numDigitalSamples); ++ptr) {
+            uint8_t bit = (*ptr >> bi) & 1;
+            *dest |= bit << bitoffset; // most significant bits will be padded
+            ++bitoffset;
+
+            if (bitoffset == 8) {
+                bitoffset = 0;
+                ++dest;
+            }
+        }
+    }
+
+    // move transceiver data to not overwrite and avoid gap in memory
+    if (totalNumBytes != nDigitalDataBytes)
+        memmove(data + nAnalogDataBytes + totalNumBytes * sizeof(uint8_t),
+                data + nAnalogDataBytes + nDigitalDataBytes,
+                nTransceiverDataBytes);
+
+    // copy back to memory and update size
+    size = totalNumBytes * sizeof(uint8_t) + nAnalogDataBytes +
+           nTransceiverDataBytes;
+
+    memcpy(data + nAnalogDataBytes, result.data(),
+           totalNumBytes * sizeof(uint8_t));
+
+    LOG(logDEBUG1) << "totalNumBytes: " << totalNumBytes
+                   << " nAnalogDataBytes:" << nAnalogDataBytes
+                   << " ctbDbitOffset:" << ctbDbitOffset
+                   << " nTransceiverDataBytes:" << nTransceiverDataBytes
+                   << " size:" << size;
+}
+
 /** ctb specific */
 void DataProcessor::ArrangeDbitData(size_t &size, char *data) {
     size_t nAnalogDataBytes = generalData->GetNumberOfAnalogDatabytes();
@@ -589,7 +698,7 @@ void DataProcessor::ArrangeDbitData(size_t &size, char *data) {
             for (auto *ptr = source; ptr < (source + numDigitalSamples);) {
                 // get selected bit from each 8 bit
                 uint8_t bit = (*ptr++ >> bi) & 1;
-                *dest |= bit << bitoffset;
+                *dest |= bit << bitoffset; // stored as least significant
                 ++bitoffset;
                 // extract destination in 8 bit batches
                 if (bitoffset == 8) {

slsReceiverSoftware/src/DataProcessor.h

@@ -92,6 +92,26 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
                                                    size_t &, void *),
                                       void *arg);
 
+  protected:
+    /**
+     * Align corresponding digital bits together (CTB only if ctbDbitlist is not
+     * empty)
+     * set variable reorder to true if, data should be rearranged such that
+     * individual digital bits from all samples are consecutive in memory
+     */
+    void ArrangeDbitData(size_t &size, char *data);
+
+    /**
+     * reorder datastream such that individual digital bits from all samples are
+     * stored consecutively in memory
+     */
+    void Reorder(size_t &size, char *data);
+
+    /**
+     * remove trailing bits in digital data stream
+     */
+    void RemoveTrailingBits(size_t &size, char *data);
+
   private:
     void RecordFirstIndex(uint64_t fnum);
 
@@ -138,14 +158,6 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
 
     void PadMissingPackets(sls_receiver_header header, char *data);
 
-    /**
-     * Align corresponding digital bits together (CTB only if ctbDbitlist is not
-     * empty)
-     * set variable reorder to true if, data should be rearranged such that
-     * individual digital bits from all samples are consecutive in memory
-     */
-    void ArrangeDbitData(size_t &size, char *data);
-
     void CropImage(size_t &size, char *data);
 
     static const std::string typeName;
@@ -165,7 +177,7 @@ class DataProcessor : private virtual slsDetectorDefs, public ThreadObject {
     uint32_t streamingTimerInMs;
     uint32_t streamingStartFnum;
     uint32_t currentFreqCount{0};
-    struct timespec timerbegin {};
+    struct timespec timerbegin{};
     bool framePadding;
     std::vector<int> ctbDbitList;
     int ctbDbitOffset{0};

slsReceiverSoftware/tests/CMakeLists.txt

@@ -3,6 +3,9 @@
 target_sources(tests PRIVATE 
     ${CMAKE_CURRENT_SOURCE_DIR}/test-GeneralData.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/test-CircularFifo.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/test-ArrangeDataBasedOnBitList.cpp
 )
 
 target_include_directories(tests PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../src>")
+
+message(STATUS "Resolved path: ${CMAKE_CURRENT_SOURCE_DIR}/../src")

slsReceiverSoftware/tests/test-ArrangeDataBasedOnBitList.cpp

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: LGPL-3.0-or-other
+// Copyright (C) 2025 Contributors to the SLS Detector Package
+/************************************************
+ * @file test-ArrangeDataBasedOnBitList.cpp
+ * @short test case for DataProcessor member function ArrangeDbitData,
+ ***********************************************/
+
+#include "DataProcessor.h"
+#include "GeneralData.h"
+#include "catch.hpp"
+#include <vector>
+
+namespace sls {
+
+// dummy GeneralData class for testing
+class GeneralDataTest : public GeneralData {
+
+  public:
+    int GetNumberOfAnalogDatabytes() { return nAnalogBytes; };
+
+    int GetNumberOfDigitalDatabytes() { return nDigitalBytes; };
+
+    int GetNumberOfTransceiverDatabytes() { return nTransceiverBytes; };
+
+    void SetNumberOfAnalogDatabytes(int value) { nAnalogBytes = value; }
+
+    void SetNumberOfDigitalDatabytes(int value) { nDigitalBytes = value; }
+
+    void SetNumberOfTransceiverDatabytes(int value) {
+        nTransceiverBytes = value;
+    }
+
+  private:
+    int nAnalogBytes;
+    int nDigitalBytes;
+    int nTransceiverBytes;
+};
+
+// oke maybe just make it static
+class DataProcessorTest : public DataProcessor {
+  public:
+    DataProcessorTest() : DataProcessor(0){};
+    ~DataProcessorTest(){};
+    void ArrangeDbitData(size_t &size, char *data) {
+        DataProcessor::ArrangeDbitData(size, data);
+    }
+};
+
+TEST_CASE("Arrange with reorder false") {
+    DataProcessorTest dataprocessor;
+
+    std::vector<int> bitlist{1, 4, 5};
+    dataprocessor.SetCtbDbitList(bitlist);
+
+    size_t num_digital_samples = 5; // size_t or uint8_t ?
+    size_t num_digital_bytes = num_digital_samples * 8;
+    size_t num_analog_bytes = 1;
+    size_t num_transceiver_bytes = 2;
+    size_t random_offset_bytes = 0;
+
+    size_t size = num_analog_bytes + num_digital_bytes + num_transceiver_bytes +
+                  random_offset_bytes;
+
+    char *data = new char[size];
+
+    char dummy_value = static_cast<char>(125);
+    memset(data, dummy_value, num_analog_bytes);
+    memset(data + num_analog_bytes, 0xFF,
+           num_digital_bytes); // all digital bits are one
+    memset(data + num_digital_bytes + num_analog_bytes, dummy_value,
+           num_transceiver_bytes);
+
+    GeneralDataTest *generaldata = new GeneralDataTest;
+    generaldata->SetNumberOfAnalogDatabytes(num_analog_bytes);
+    generaldata->SetNumberOfDigitalDatabytes(num_digital_bytes);
+    generaldata->SetNumberOfTransceiverDatabytes(num_transceiver_bytes);
+
+    dataprocessor.SetGeneralData(generaldata);
+
+    size_t new_num_digital_bytes =
+        (bitlist.size() / 8 + static_cast<size_t>(bitlist.size() % 8 != 0)) *
+        num_digital_samples;
+
+    size_t new_size =
+        num_analog_bytes + num_transceiver_bytes + new_num_digital_bytes;
+
+    char *new_data = new char[new_size];
+
+    memset(new_data, dummy_value, num_analog_bytes);
+
+    // TODO: Make test more explicit and less generic
+    size_t num_bytes_for_bitlist =
+        bitlist.size() / 8 + static_cast<size_t>(bitlist.size() % 8 != 0);
+
+    // 125 7 7 7 7 7 125 125
+    for (size_t sample = 0; sample < num_digital_samples; ++sample) {
+        if (bitlist.size() / 8 != 0) {
+            memset(new_data + sample * num_bytes_for_bitlist + num_analog_bytes,
+                   0xFF,
+                   bitlist.size() / 8); // set to 1
+        } else if (bitlist.size() % 8 != 0) {
+            memset(new_data + sample * num_bytes_for_bitlist +
+                       bitlist.size() / 8 + num_analog_bytes,
+                   static_cast<char>(pow(2, (bitlist.size() % 8)) - 1),
+                   1); // convert binary number to decimal
+        }
+    }
+
+    memset(new_data + new_num_digital_bytes + num_analog_bytes, dummy_value,
+           num_transceiver_bytes);
+
+    dataprocessor.ArrangeDbitData(size, data);
+
+    CHECK(size == new_size);
+
+    CHECK(memcmp(data, new_data, size) == 0);
+
+    // Free allocated memory
+    delete[] data;
+    delete[] new_data;
+    delete generaldata;
+}
+
+// TEST_CASE("Arrange with reorder on") {
+
+//}
+
+// test case reorder on and bitoffset set
+
+// test with different samples
+
+// test with sample number not divisable and ctbitlist not divisable
+
+} // namespace sls