Dev/decode my302 (#254)

This PR adds support for decoding digital data from the my320 test chip.
- Added BitOffset (strong type)
- Expand 24 to 32 bit 
- Python bindings for decoding my302
- Improved docs

RELEASE.md

@@ -1,5 +1,11 @@
 # Release notes
 
+## head
+
+### New Features:
+
+- Expanding 24 to 32 bit data
+- Decoding digital data from Mythen 302
 
 
 ### 2025.11.21

docs/CMakeLists.txt

@@ -12,15 +12,19 @@ set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src)
 set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR})
 
 
-file(GLOB SPHINX_SOURCE_FILES CONFIGURE_DEPENDS "src/*.rst")
+file(GLOB_RECURSE SPHINX_SOURCE_FILES
+     CONFIGURE_DEPENDS
+     RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/src"
+     "${CMAKE_CURRENT_SOURCE_DIR}/src/*.rst"
+)
 
+foreach(relpath IN LISTS SPHINX_SOURCE_FILES)
+    set(src  "${CMAKE_CURRENT_SOURCE_DIR}/src/${relpath}")
+    set(dst  "${SPHINX_BUILD}/src/${relpath}")
 
-
-foreach(filename ${SPHINX_SOURCE_FILES})
-    get_filename_component(fname ${filename} NAME)
-    message(STATUS "Copying ${filename} to ${SPHINX_BUILD}/src/${fname}")
-    configure_file(${filename} "${SPHINX_BUILD}/src/${fname}")  
-endforeach(filename ${SPHINX_SOURCE_FILES})
+    message(STATUS "Copying ${src} to ${dst}")
+    configure_file("${src}" "${dst}" COPYONLY)
+endforeach()
 
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in"

docs/src/index.rst

@@ -22,21 +22,14 @@ AARE
 
 .. toctree::
     :caption: Python API
-    :maxdepth: 1
-    
-    pyFile
-    pycalibration
-    pyCtbRawFile
-    pyClusterFile
-    pyClusterVector
-    pyCluster
-    pyInterpolation
-    pyJungfrauDataFile
-    pyRawFile
-    pyRawMasterFile
-    pyVarClusterFinder
+    :maxdepth: 3
+    :hidden:
 
+    pycalibration
+    python/cluster/index
+    python/file/index
     pyFit
+    
 
 
 .. toctree::

docs/src/pyCtbRawFile.rst

@@ -1,11 +0,0 @@
-
-CtbRawFile
-============
-
-.. py:currentmodule:: aare
-
-.. autoclass:: CtbRawFile
-    :members:
-    :undoc-members:
-    :show-inheritance:
-    :inherited-members:

docs/src/python/cluster/index.rst

@@ -0,0 +1,11 @@
+Cluster & Interpolation
+==========================
+
+.. toctree::
+   :caption: Cluster & Interpolation
+   :maxdepth: 1
+
+   pyCluster
+   pyClusterVector
+   pyInterpolation
+   pyVarClusterFinder

docs/src/python/cluster/pyInterpolation.rst

@@ -21,8 +21,8 @@ Supported are the following :math:`\eta`-functions:
 
 .. py:currentmodule:: aare
 
-.. image:: ../figures/Eta2x2.png
-    :target: ../figures/Eta2x2.png
+.. image:: ../../../figures/Eta2x2.png
+    :target: ../../../figures/Eta2x2.png
     :width: 650px
     :align: center
     :alt: Eta2x2
@@ -35,8 +35,8 @@ Supported are the following :math:`\eta`-functions:
 
 .. autofunction:: calculate_eta2
 
-.. image:: ../figures/Eta2x2Full.png
-    :target: ../figures/Eta2x2Full.png
+.. image:: ../../../figures/Eta2x2Full.png
+    :target: ../../../figures/Eta2x2Full.png
     :width: 650px
     :align: center
     :alt: Eta2x2 Full 
@@ -49,8 +49,8 @@ Supported are the following :math:`\eta`-functions:
 
 .. autofunction:: calculate_full_eta2
 
-.. image:: ../figures/Eta3x3.png
-    :target: ../figures/Eta3x3.png
+.. image:: ../../../figures/Eta3x3.png
+    :target: ../../../figures/Eta3x3.png
     :width: 650px
     :align: center
     :alt: Eta3x3
@@ -63,8 +63,8 @@ Supported are the following :math:`\eta`-functions:
 
 .. autofunction:: calculate_eta3
 
-.. image:: ../figures/Eta3x3Cross.png
-    :target: ../figures/Eta3x3Cross.png
+.. image:: ../../../figures/Eta3x3Cross.png
+    :target: ../../../figures/Eta3x3Cross.png
     :width: 650px
     :align: center
     :alt: Cross Eta3x3

docs/src/python/file/index.rst

@@ -0,0 +1,14 @@
+File I/O
+===================
+
+.. toctree::
+   :caption: File I/O
+   :maxdepth: 1
+
+   pyClusterFile
+   pyCtbRawFile
+   pyFile
+   pyJungfrauDataFile
+   pyRawFile
+   pyRawMasterFile
+   pyTransform

docs/src/python/file/pyCtbRawFile.rst

@@ -0,0 +1,25 @@
+
+CtbRawFile
+============
+
+Read analog, digital and transceiver samples from a raw file containing
+data from the Chip Test Board. Uses :mod:`aare.transform` to decode the 
+data into a format that the user can work with. 
+
+.. code:: python
+
+    import aare
+    from aare.transform import Mythen302Transform
+    my302 = Mythen302Transform(offset = 4)
+
+    with aare.CtbRawFile(fname, transform = my302) as f:
+    for header, data in f:
+       #do something with the data
+
+.. py:currentmodule:: aare
+
+.. autoclass:: CtbRawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/python/file/pyTransform.rst

@@ -0,0 +1,27 @@
+Transform
+===================
+
+The transform module takes data read by :class:`aare.CtbRawFile` and decodes it
+to a useful image format. Depending on detector it supports both analog
+and digital samples. 
+
+For convenience the following transform objects are defined with a short name
+
+.. code:: python
+
+    moench05 = Moench05Transform()
+    moench05_1g = Moench05Transform1g()
+    moench05_old = Moench05TransformOld()
+    matterhorn02 = Matterhorn02Transform()
+    adc_sar_04_64to16 = AdcSar04Transform64to16()
+    adc_sar_05_64to16 = AdcSar05Transform64to16()
+
+.. py:currentmodule:: aare
+
+.. automodule:: aare.transform
+    :members:
+    :undoc-members:
+    :private-members:
+    :special-members: __call__
+    :show-inheritance:
+    :inherited-members:

include/aare/decode.hpp

@@ -1,11 +1,12 @@
 // SPDX-License-Identifier: MPL-2.0
 #pragma once
-
+#include "aare/defs.hpp"
 #include <aare/NDView.hpp>
 #include <cstdint>
 #include <vector>
 namespace aare {
 
+
 uint16_t adc_sar_05_decode64to16(uint64_t input);
 uint16_t adc_sar_04_decode64to16(uint64_t input);
 void adc_sar_05_decode64to16(NDView<uint64_t, 2> input,
@@ -13,6 +14,25 @@ void adc_sar_05_decode64to16(NDView<uint64_t, 2> input,
 void adc_sar_04_decode64to16(NDView<uint64_t, 2> input,
                              NDView<uint16_t, 2> output);
 
+/**
+ * @brief Called with a 32 bit unsigned integer, shift by offset
+ * and then return the lower 24 bits as an 32 bit integer
+ * @param input 32-ibt input value
+ * @param offset (should be in range 0-7 to allow for full 24 bits)
+ * @return uint32_t 
+ */
+uint32_t mask32to24bits(uint32_t input, BitOffset offset={});
+
+/**
+ * @brief Expand 24 bit values in a 8bit buffer to 32bit unsigned integers
+ * Used for detectors with 24bit counters in combination with CTB
+ * 
+ * @param input View of the 24 bit data as uint8_t (no 24bit native data type exists)
+ * @param output Destination of the expanded data (32bit, unsigned) 
+ * @param offset Offset within the first byte to where the data starts (0-7 bits)
+ */
+void expand24to32bit(NDView<uint8_t,1> input, NDView<uint32_t,1> output, BitOffset offset={});
+
 /**
  * @brief Apply custom weights to a 16-bit input value. Will sum up
  * weights[i]**i for each bit i that is set in the input value.

include/aare/defs.hpp

@@ -372,4 +372,15 @@ constexpr uint16_t ADC_MASK =
  */
 template <> DACIndex StringTo(const std::string &arg);
 
+class BitOffset{
+    uint8_t m_offset{};
+    public:
+    BitOffset() = default;
+    explicit BitOffset(uint32_t offset);
+    uint8_t value() const {return m_offset;}
+    bool operator==(const BitOffset& other) const;
+    bool operator<(const BitOffset& other) const;
+
+};
+
 } // namespace aare

python/aare/transform.py

@@ -49,6 +49,43 @@ class Matterhorn02Transform:
         else:
             return np.take(data.view(np.uint16), self.pixel_map[0:counters])
 
+class Mythen302Transform:
+    """
+    Transform Mythen 302 test chip data from a buffer of bytes (uint8_t)
+    to a uint32 numpy array of [64,3] representing channels and counters.
+    Assumes data taken with rx_dbitlist 17 6, rx_dbitreorder 1 and Digital
+    Samples = 2310 [(64x3x24)/2 + some extra]
+
+    .. note::
+
+        The offset is in number of bits 0-7
+
+    """
+    _n_channels = 64
+    _n_counters = 3
+
+    def __init__(self, offset=4):
+        self.offset = offset
+
+    def __call__(self, data : np.ndarray):
+        """
+        Transform buffer of data to a [64,3] np.ndarray of uint32. 
+
+        Parameters
+        ----------
+        data : np.ndarray
+            Expected dtype: uint8
+
+        Returns
+        ----------
+        image : np.ndarray
+            uint32 array of size  64, 3
+        """
+        res = _aare.decode_my302(data, self.offset)
+        res = res.reshape(
+            Mythen302Transform._n_channels, Mythen302Transform._n_counters
+        )
+        return res
 
 #on import generate the pixel maps to avoid doing it every time
 moench05 = Moench05Transform()

python/src/ctb_raw_file.hpp

@@ -96,6 +96,69 @@ void define_ctb_raw_file_io_bindings(py::module &m) {
               return output;
           });
 
+    m.def("expand24to32bit",
+    [](py::array_t<uint8_t, py::array::c_style | py::array::forcecast>
+                 &input, uint32_t offset){
+
+            aare::BitOffset bitoff(offset);
+            py::buffer_info buf = input.request();
+
+            constexpr uint32_t bytes_per_channel = 3; //24 bit
+            py::array_t<uint32_t> output(buf.size/bytes_per_channel);
+
+            NDView<uint8_t, 1> input_view(input.mutable_data(),
+                                             {input.size()});
+            NDView<uint32_t, 1> output_view(output.mutable_data(),
+                                        {output.size()});
+
+            aare::expand24to32bit(input_view, output_view, bitoff);
+            return output;
+
+    });
+
+    m.def("decode_my302",
+    [](py::array_t<uint8_t, py::array::c_style | py::array::forcecast>
+                 &input, uint32_t offset){
+
+            // Physical layout of the chip
+            constexpr size_t channels = 64;
+            constexpr size_t counters = 3;
+            constexpr size_t bytes_per_channel = 3; //24 bit
+            constexpr int n_outputs = 2;
+
+            ssize_t expected_size = channels*counters*bytes_per_channel;
+
+            //If whe have an offset we need one extra byte per output
+            aare::BitOffset bitoff(offset);
+            if(bitoff.value())
+                expected_size += n_outputs;
+
+            if (input.size() != expected_size) {
+                throw std::runtime_error(
+                    fmt::format("{} Expected an input size of {} bytes. Called "
+                                "with input size of {}",
+                                LOCATION, expected_size, input.size()));
+            }
+
+            py::buffer_info buf = input.request();
+            py::array_t<uint32_t> output(channels * counters);
+
+            for (int i = 0; i!=n_outputs; ++i){
+                auto step = input.size()/n_outputs;
+                auto out_step = output.size()/n_outputs;
+                NDView<uint8_t, 1> input_view(input.mutable_data()+step*i,
+                                             {input.size()/n_outputs});
+                NDView<uint32_t, 1> output_view(output.mutable_data()+out_step*i,
+                                            {output.size()/n_outputs});
+
+                aare::expand24to32bit(input_view, output_view, bitoff);
+
+            }
+            
+            return output;
+
+    });
+
     py::class_<CtbRawFile>(m, "CtbRawFile")
         .def(py::init<const std::filesystem::path &>())
         .def("read_frame",

src/CtbRawFile.cpp

@@ -61,7 +61,6 @@ void CtbRawFile::find_subfiles() {
     while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles)))
         m_num_subfiles++;
 
-    fmt::print("Found {} subfiles\n", m_num_subfiles);
 }
 
 void CtbRawFile::open_data_file(size_t subfile_index) {

src/NDView.test.cpp

@@ -5,6 +5,7 @@
 #include <iostream>
 #include <numeric>
 #include <vector>
+#include <cstddef>
 
 using aare::NDView;
 using aare::Shape;
@@ -259,4 +260,12 @@ TEST_CASE("Create a view over a vector") {
     REQUIRE(v.shape()[0] == 12);
     REQUIRE(v[0] == 0);
     REQUIRE(v[11] == 11);
+}
+
+TEST_CASE("NDView over byte"){
+    std::vector<std::byte> buf(5);
+    auto v = aare::make_view(buf);
+    REQUIRE(v.shape()[0] == 5);
+    REQUIRE(v[0] == std::byte{0});
+
 }

src/decode.cpp

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MPL-2.0
 #include "aare/decode.hpp"
+#include <fmt/format.h>
 #include <cmath>
 namespace aare {
 
@@ -105,4 +106,49 @@ void apply_custom_weights(NDView<uint16_t, 1> input, NDView<double, 1> output,
     }
 }
 
+uint32_t mask32to24bits(uint32_t input, BitOffset offset){
+    constexpr uint32_t mask24bits{0xFFFFFF};
+    return (input >> offset.value()) & mask24bits;
+}
+
+void expand24to32bit(NDView<uint8_t,1> input, NDView<uint32_t,1> output, BitOffset bit_offset){
+
+    ssize_t bytes_per_channel = 3; //24bit 
+    ssize_t min_input_size = output.size()*bytes_per_channel;
+
+    //if we have an offset we need one more byte in the input data
+    if (bit_offset.value())
+        min_input_size += 1; 
+
+    if (input.size() < min_input_size)
+        throw std::runtime_error(fmt::format(
+            "{} Mismatch between input and output size. Output "
+            "size of {} with bit offset {} requires an input of at least {} "
+            "bytes. Called with input size: {} output size: {}",
+            LOCATION, output.size(), bit_offset.value(), min_input_size, input.size(), output.size()));
+
+    auto* in = input.data();
+
+    if(bit_offset.value()){
+        //If there is a bit_offset we copy 4 bytes and then
+        //mask out the correct ones. 
+        for (auto& v : output){
+            uint32_t val{};
+            std::memcpy(&val, in, sizeof(val));
+            v = mask32to24bits(val, bit_offset);
+            in += bytes_per_channel;
+         }
+    }else{
+        //If there is no offset we can directly copy the bits
+        //without masking
+        for (auto& v : output){
+            uint32_t val{};
+            std::memcpy(&val, in, 3);
+            v = val;
+            in += bytes_per_channel;
+         }
+    }
+
+}
+
 } // namespace aare

src/decode.test.cpp

@@ -7,6 +7,8 @@
 using Catch::Matchers::WithinAbs;
 #include <vector>
 
+using aare::BitOffset;
+
 TEST_CASE("test_adc_sar_05_decode64to16") {
     uint64_t input = 0;
     uint16_t output = aare::adc_sar_05_decode64to16(input);
@@ -71,4 +73,94 @@ TEST_CASE("test_apply_custom_weights") {
     input = 0b111;
     output = aare::apply_custom_weights(input, weights);
     CHECK_THAT(output, WithinAbs(6.34, 0.001));
+}
+
+TEST_CASE("Mask 32 bit unsigned integer to 24 bit"){
+    //any number less than 2**24 (16777216) should be the same
+    CHECK(aare::mask32to24bits(0)==0);
+    CHECK(aare::mask32to24bits(19)==19);
+    CHECK(aare::mask32to24bits(29875)==29875);
+    CHECK(aare::mask32to24bits(1092177)==1092177);
+    CHECK(aare::mask32to24bits(0xFFFF)==0xFFFF);
+    CHECK(aare::mask32to24bits(0xFFFFFFFF)==0xFFFFFF);
+
+    // Offset specifies that the should ignore 0-7 bits
+    // at the start
+    CHECK(aare::mask32to24bits(0xFFFF, BitOffset(4))==0xFFF);
+    CHECK(aare::mask32to24bits(0xFF0000d9)==0xd9);
+    CHECK(aare::mask32to24bits(0xFF000d9F, BitOffset(4))==0xF000d9);
+    CHECK(aare::mask32to24bits(16777217)==1);
+    CHECK(aare::mask32to24bits(15,BitOffset(7))==0);
+    
+    //Highest bit set to 1 should just be excluded
+    //lowest 4 bits set to 1 
+    CHECK(aare::mask32to24bits(0x8000000f,BitOffset(7))==0);
+    
+}
+
+TEST_CASE("Expand container with 24 bit data to 32"){
+    {
+        uint8_t buffer[] = {
+            0x00, 0x00, 0x00, 
+            0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00,
+        };
+
+        aare::NDView<uint8_t, 1> input(&buffer[0], {9});
+        aare::NDArray<uint32_t, 1> out({3});
+        aare::expand24to32bit(input, out.view());
+
+        CHECK(out(0) == 0);
+        CHECK(out(1) == 0);
+        CHECK(out(2) == 0);
+    }
+    {
+        uint8_t buffer[] = {
+            0x0F, 0x00, 0x00, 
+            0xFF, 0x00, 0x00,
+            0xFF, 0xFF, 0xFF,
+        };
+
+        aare::NDView<uint8_t, 1> input(&buffer[0], {9});
+        aare::NDArray<uint32_t, 1> out({3});
+        aare::expand24to32bit(input, out.view());
+
+        CHECK(out(0) == 0xF);
+        CHECK(out(1) == 0xFF);
+        CHECK(out(2) == 0xFFFFFF);
+    }
+    {
+        uint8_t buffer[] = {
+            0x00, 0x00, 0xFF, 
+            0xFF, 0xFF, 0x00,
+            0x00, 0xFF, 0x00,
+        };
+
+        aare::NDView<uint8_t, 1> input(&buffer[0], {9});
+        aare::NDArray<uint32_t, 1> out({3});
+        aare::expand24to32bit(input, out.view());
+
+        CHECK(out(0) == 0xFF0000);
+        CHECK(out(1) == 0xFFFF);
+        CHECK(out(2) == 0xFF00);
+
+        REQUIRE_THROWS(aare::expand24to32bit(input, out.view(), BitOffset(4)));
+    }
+    {
+        //For use with offset we need an extra byte
+        uint8_t buffer[] = {
+            0x00, 0x00, 0xFF, 
+            0xFF, 0xFF, 0x00,
+            0x00, 0xFF, 0x00, 0x00
+        };
+
+        aare::NDView<uint8_t, 1> input(&buffer[0], {10});
+        aare::NDArray<uint32_t, 1> out({3}); //still output.size == 3
+        aare::expand24to32bit(input, out.view(), BitOffset(4));
+
+        CHECK(out(0) == 0xFFF000);
+        CHECK(out(1) == 0xFFF);
+        CHECK(out(2) == 0xFF0);
+    }
+
 }

src/defs.cpp

@@ -298,4 +298,22 @@ template <> DACIndex StringTo(const std::string &arg) {
                                     "\"");
 }
 
+BitOffset::BitOffset(uint32_t offset){
+    if (offset>7)
+        throw std::runtime_error(fmt::format("{} BitOffset needs to be <8: Called with {}", LOCATION, offset));
+
+    m_offset = static_cast<uint8_t>(offset);
+
+}
+
+bool BitOffset::operator==(const BitOffset& other) const {
+        return m_offset == other.m_offset;
+    }
+
+bool BitOffset::operator<(const BitOffset& other) const {
+        return m_offset < other.m_offset;
+    }
+
+
+
 } // namespace aare

src/defs.test.cpp

@@ -83,6 +83,28 @@ TEST_CASE("DynamicCluster creation") {
     REQUIRE(c2.data() != nullptr);
 }
 
+TEST_CASE("Basic ops on BitOffset"){
+    REQUIRE_THROWS(aare::BitOffset(10));
+
+    aare::BitOffset offset(5);
+    REQUIRE(offset.value()==5);
+
+    aare::BitOffset offset2;
+    REQUIRE(offset2.value()==0);
+
+    aare::BitOffset offset3(offset);
+    REQUIRE(offset3.value()==5);
+
+    REQUIRE(offset==offset3);
+
+    //Now assign offset to offset2 which should get the value 5
+    offset2 = offset;
+    REQUIRE(offset2.value()==5);
+    REQUIRE(offset2==offset);
+}
+
+
+
 // TEST_CASE("cluster set and get data") {
 
 //     aare::DynamicCluster c2(33, 44, aare::Dtype(typeid(double)));