function signature for push back

- Moved definitions of Cluster_2x2 and Cluster_3x3 to it's own file
- Added optional members for ROI, noise_map and gain_map in ClusterFile

**API:**

After creating the ClusterFile the user can set one or all of: roi,
noise_map, gain_map

```python
f = ClusterFile(fname)
f.set_roi(roi) #aare.ROI
f.set_noise_map(noise_map) #numpy array
f.set_gain_map(gain_map) #numpy array
```

**When reading clusters they are evaluated in the order:**

1. If ROI is enabled check that the cluster is within the ROI
1. If noise_map is enabled check that the cluster meets one of the
conditions
    - Center pixel above noise
    - Highest 2x2 sum above 2x noise
    - 3x3 sum above 3x noise
1. If gain_map is set apply the gain map before returning the clusters
(not used for noise cut)

**Open questions:**
1. Check for out of bounds access in noise and gain map?

closes #139 
closes #135 
closes #90

.clang-tidy

@@ -0,0 +1,42 @@
+
+---
+Checks:          '*, 
+                    -altera-*,
+                    -android-cloexec-fopen,
+                    -cppcoreguidelines-pro-bounds-array-to-pointer-decay, 
+                    -cppcoreguidelines-pro-bounds-pointer-arithmetic, 
+                    -fuchsia*,
+                    -readability-else-after-return,
+                    -readability-avoid-const-params-in-decls,
+                    -readability-identifier-length,
+                    -cppcoreguidelines-pro-bounds-constant-array-index,
+                    -cppcoreguidelines-pro-type-reinterpret-cast,
+                    -llvm-header-guard,
+                    -modernize-use-nodiscard,
+                    -misc-non-private-member-variables-in-classes,
+                    -readability-static-accessed-through-instance,
+                    -readability-braces-around-statements,
+                    -readability-isolate-declaration,
+                    -readability-implicit-bool-conversion,
+                    -readability-identifier-length,
+                    -readability-identifier-naming,
+                    -hicpp-signed-bitwise,
+                    -hicpp-no-array-decay, 
+                    -hicpp-braces-around-statements,
+                    -google-runtime-references, 
+                    -google-readability-todo, 
+                    -google-readability-braces-around-statements,
+                    -modernize-use-trailing-return-type,
+                    -llvmlibc-*'
+
+HeaderFilterRegex: \.hpp
+FormatStyle:     none
+CheckOptions:
+  - { key: readability-identifier-naming.NamespaceCase,       value: lower_case }
+ # - { key: readability-identifier-naming.FunctionCase,        value: lower_case }
+  - { key: readability-identifier-naming.ClassCase,           value: CamelCase  }
+ # - { key: readability-identifier-naming.MethodCase,          value: CamelCase  }
+ # - { key: readability-identifier-naming.StructCase,          value: CamelCase  }
+ # - { key: readability-identifier-naming.VariableCase,        value: lower_case }
+  - { key: readability-identifier-naming.GlobalConstantCase,  value: UPPER_CASE }
+...

.gitea/workflows/cmake_build.yml

@@ -0,0 +1,58 @@
+name: Build the package using cmake then documentation
+
+on:
+  workflow_dispatch:
+  push:
+    
+
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        python-version: ["3.12",]
+
+    runs-on: ${{ matrix.platform }}
+
+    # The setup-miniconda action needs this to activate miniconda
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup dev env
+        run: |
+            sudo apt-get update
+            sudo apt-get -y install cmake gcc g++
+
+      - name: Get conda
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+
+      - name: Prepare
+        run: conda install doxygen sphinx=7.1.2 breathe pybind11 sphinx_rtd_theme furo nlohmann_json zeromq fmt numpy
+
+      - name: Build library 
+        run: |
+            mkdir build 
+            cd build 
+            cmake .. -DAARE_SYSTEM_LIBRARIES=ON -DAARE_DOCS=ON 
+            make -j 2 
+            make docs
+
+
+
+
+
+

.github/workflows/build_and_deploy_conda.yml

@@ -1,7 +1,9 @@
-name: Deploy to slsdetectorgroup conda channel
+name: Build pkgs and deploy if on main
 
 on:
-  workflow_dispatch
+  push:
+    branches:
+     - main
 
 jobs:
   build:
@@ -28,7 +30,7 @@ jobs:
           channels: conda-forge
 
       - name: Prepare
-        run: conda install conda-build conda-verify pytest anaconda-client
+        run: conda install conda-build=24.9 conda-verify pytest anaconda-client
 
       - name: Enable upload
         run: conda config --set anaconda_upload yes

.github/workflows/build_conda.yml

@@ -1,14 +1,10 @@
-name: Build packages but don't deploy
+name: Build pkgs and deploy if on main
 
 on:
-  workflow_dispatch:
   push:
     branches:
-     - main
      - developer
 
-    #run on PRs as well?
-
 jobs:
   build:
     strategy:
@@ -34,10 +30,11 @@ jobs:
           channels: conda-forge
 
       - name: Prepare
-        run: conda install conda-build conda-verify pytest anaconda-client
+        run: conda install conda-build=24.9 conda-verify pytest anaconda-client
+
+      - name: Disable upload
+        run: conda config --set anaconda_upload no
 
       - name: Build
-        env:
-          CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
-        run: conda build conda-recipe
+        run: conda build conda-recipe 
 

.github/workflows/build_docs.yml

@@ -3,8 +3,7 @@ name: Build the package using cmake then documentation
 on:
   workflow_dispatch:
   push:
-    branches:
-     - main
+    
 
 
 permissions:
@@ -58,6 +57,7 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     needs: build
+    if: github.ref == 'refs/heads/main'
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

CMakeLists.txt

@@ -31,7 +31,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
 
 # General options
-option(AARE_PYTHON_BINDINGS "Build python bindings" ON)
+option(AARE_PYTHON_BINDINGS "Build python bindings" OFF)
 option(AARE_TESTS "Build tests" OFF)
 option(AARE_BENCHMARKS "Build benchmarks" OFF)
 option(AARE_EXAMPLES "Build examples" OFF)
@@ -39,7 +39,8 @@ option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
 option(AARE_DOCS "Build documentation" OFF)
 option(AARE_VERBOSE "Verbose output" OFF)
 option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
-
+option(AARE_INSTALL_PYTHONEXT "Install the python extension in the install tree under CMAKE_INSTALL_PREFIX/aare/" OFF)
+option(AARE_ASAN "Enable AddressSanitizer" OFF)
 
 # Configure which of the dependencies to use FetchContent for
 option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
@@ -47,6 +48,7 @@ option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON)
 option(AARE_FETCH_CATCH "Use FetchContent to download catch2" ON)
 option(AARE_FETCH_JSON "Use FetchContent to download nlohmann::json" ON)
 option(AARE_FETCH_ZMQ "Use FetchContent to download libzmq" ON)
+option(AARE_FETCH_LMFIT "Use FetchContent to download lmfit" ON)
 
 
 #Convenience option to use system libraries only (no FetchContent)
@@ -58,6 +60,8 @@ if(AARE_SYSTEM_LIBRARIES)
     set(AARE_FETCH_CATCH OFF CACHE BOOL "Disabled FetchContent for catch2" FORCE)
     set(AARE_FETCH_JSON OFF CACHE BOOL "Disabled FetchContent for nlohmann::json" FORCE)
     set(AARE_FETCH_ZMQ OFF CACHE BOOL "Disabled FetchContent for libzmq" FORCE)
+    # Still fetch lmfit when setting AARE_SYSTEM_LIBRARIES since this is not available
+    # on conda-forge
 endif()
 
 if(AARE_VERBOSE)
@@ -75,16 +79,66 @@ endif()
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+if(AARE_FETCH_LMFIT)
+    #TODO! Should we fetch lmfit from the web or inlcude a tar.gz in the repo?
+    set(LMFIT_PATCH_COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/patches/lmfit.patch)
+
+    # For cmake < 3.28 we can't supply EXCLUDE_FROM_ALL to FetchContent_Declare
+    # so we need this workaround
+    if (${CMAKE_VERSION} VERSION_LESS "3.28")
+        FetchContent_Declare(
+            lmfit
+            GIT_REPOSITORY https://jugit.fz-juelich.de/mlz/lmfit.git
+            GIT_TAG        main
+            PATCH_COMMAND ${LMFIT_PATCH_COMMAND}
+            UPDATE_DISCONNECTED 1
+        )
+    else()
+        FetchContent_Declare(
+            lmfit
+            GIT_REPOSITORY https://jugit.fz-juelich.de/mlz/lmfit.git
+            GIT_TAG        main
+            PATCH_COMMAND ${LMFIT_PATCH_COMMAND}
+            UPDATE_DISCONNECTED 1
+            EXCLUDE_FROM_ALL 1
+        )
+    endif()
+    
+    #Disable what we don't need from lmfit
+    set(BUILD_TESTING OFF CACHE BOOL "")
+    set(LMFIT_CPPTEST OFF CACHE BOOL "")
+    set(LIB_MAN OFF CACHE BOOL "")
+    set(LMFIT_CPPTEST OFF CACHE BOOL "")
+    set(BUILD_SHARED_LIBS OFF CACHE BOOL "")
+
+    if (${CMAKE_VERSION} VERSION_LESS "3.28")
+        if(NOT lmfit_POPULATED)
+            FetchContent_Populate(lmfit)
+            add_subdirectory(${lmfit_SOURCE_DIR} ${lmfit_BINARY_DIR} EXCLUDE_FROM_ALL)
+        endif()
+    else()
+        FetchContent_MakeAvailable(lmfit)
+    endif()
+    
+    set_property(TARGET lmfit PROPERTY POSITION_INDEPENDENT_CODE ON)
+else()
+    find_package(lmfit REQUIRED)
+endif()
+
+
 if(AARE_FETCH_ZMQ)
     # Fetchcontent_Declare is deprecated need to find a way to update this
     # for now setting the policy to old is enough
     if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30")
         cmake_policy(SET CMP0169 OLD)
     endif()
+    set(ZMQ_PATCH_COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/patches/libzmq_cmake_version.patch)
     FetchContent_Declare(
         libzmq
         GIT_REPOSITORY https://github.com/zeromq/libzmq.git
         GIT_TAG        v4.3.4
+        PATCH_COMMAND ${ZMQ_PATCH_COMMAND}
+        UPDATE_DISCONNECTED 1
     )
     # Disable unwanted options from libzmq
     set(BUILD_TESTS OFF CACHE BOOL "Switch off libzmq test build")
@@ -126,8 +180,8 @@ if (AARE_FETCH_FMT)
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
         ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
-)
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}   
+    )  
 else()
       find_package(fmt 6 REQUIRED)
 endif()
@@ -145,7 +199,6 @@ if (AARE_FETCH_JSON)
     install(
         TARGETS nlohmann_json
         EXPORT "${TARGETS_EXPORT_NAME}"
-        
     )
     message(STATUS "target: ${NLOHMANN_JSON_TARGET_NAME}")
 else()
@@ -224,13 +277,6 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
     target_compile_options(aare_compiler_flags INTERFACE -O3)
 else()
     message(STATUS "Debug build")
-    target_compile_options(
-        aare_compiler_flags 
-        INTERFACE 
-            -Og 
-            -ggdb3 
-    )
-
 endif()
 
 # Common flags for GCC and Clang
@@ -241,6 +287,7 @@ target_compile_options(
         -Wextra 
         -pedantic 
         -Wshadow 
+        -Wold-style-cast
         -Wnon-virtual-dtor
         -Woverloaded-virtual
         -Wdouble-promotion
@@ -254,7 +301,21 @@ target_compile_options(
 endif() #GCC/Clang specific
 
 
-
+if(AARE_ASAN)
+    message(STATUS "AddressSanitizer enabled")
+    target_compile_options(
+        aare_compiler_flags 
+        INTERFACE 
+            -fsanitize=address,undefined,pointer-compare 
+            -fno-omit-frame-pointer
+    )
+    target_link_libraries(
+        aare_compiler_flags 
+        INTERFACE 
+            -fsanitize=address,undefined,pointer-compare 
+            -fno-omit-frame-pointer
+    )
+endif()
 
 
 
@@ -270,14 +331,21 @@ endif()
 
 set(PUBLICHEADERS
     include/aare/ArrayExpr.hpp
+    include/aare/CalculateEta.hpp
+    include/aare/Cluster.hpp
     include/aare/ClusterFinder.hpp
     include/aare/ClusterFile.hpp
     include/aare/CtbRawFile.hpp
+    include/aare/ClusterVector.hpp
+    include/aare/decode.hpp
     include/aare/defs.hpp
     include/aare/Dtype.hpp
     include/aare/File.hpp
+    include/aare/Fit.hpp
     include/aare/FileInterface.hpp
     include/aare/Frame.hpp
+    include/aare/GainMap.hpp
+    include/aare/geo_helpers.hpp
     include/aare/NDArray.hpp
     include/aare/NDView.hpp
     include/aare/NumpyFile.hpp
@@ -288,30 +356,34 @@ set(PUBLICHEADERS
     include/aare/RawMasterFile.hpp
     include/aare/RawSubFile.hpp
     include/aare/VarClusterFinder.hpp
-
+    include/aare/utils/task.hpp
 )
 
 
 set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Fit.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolator.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.cpp
 )
 
 
 add_library(aare_core STATIC ${SourceFiles})
 target_include_directories(aare_core PUBLIC 
     "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>" 
-    "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+    "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>" 
 )
 
 target_link_libraries(
@@ -322,6 +394,8 @@ target_link_libraries(
     ${STD_FS_LIB} # from helpers.cmake
     PRIVATE 
     aare_compiler_flags 
+    $<BUILD_INTERFACE:lmfit>
+
 )
 
 set_target_properties(aare_core PROPERTIES
@@ -335,17 +409,24 @@ endif()
 
 if(AARE_TESTS)
     set(TestSources
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterVector.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Cluster.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/CalculateEta.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Pedestal.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.test.cpp
 
         )
     target_sources(tests PRIVATE ${TestSources} )
@@ -413,7 +494,7 @@ endif()
 
 add_custom_target(
     clang-tidy
-    COMMAND find \( -path "./src/*" -a -not -path "./src/python/*" -a \( -name "*.cpp"   -not -name "*.test.cpp"\)  \)    -not -name "CircularFifo.hpp" -not -name "ProducerConsumerQueue.hpp" -not -name "VariableSizeClusterFinder.hpp" | xargs -I {} -n 1 -P 10 bash -c "${CLANG_TIDY_COMMAND} --config-file=.clang-tidy -p build {}"
+    COMMAND find \( -path "./src/*" -a -not -path "./src/python/*" -a \( -name "*.cpp"   -not -name "*.test.cpp" \)  \)    -not -name "CircularFifo.hpp" -not -name "ProducerConsumerQueue.hpp" -not -name "VariableSizeClusterFinder.hpp" | xargs -I {} -n 1 -P 10 bash -c "${CLANG_TIDY_COMMAND} --config-file=.clang-tidy -p build {}"
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     COMMENT "linting with clang-tidy"
     VERBATIM
@@ -423,4 +504,4 @@ if(AARE_MASTER_PROJECT)
     set(CMAKE_INSTALL_DIR "share/cmake/${PROJECT_NAME}")
     set(PROJECT_LIBRARIES aare-core aare-compiler-flags )
     include(cmake/package_config.cmake)
-endif()
+endif()

benchmarks/CMakeLists.txt

@@ -1,11 +1,27 @@
-find_package(benchmark REQUIRED)
 
-add_executable(ndarray_benchmark ndarray_benchmark.cpp)
+include(FetchContent)
 
-target_link_libraries(ndarray_benchmark benchmark::benchmark aare_core aare_compiler_flags)
-# target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
 
-set_target_properties(ndarray_benchmark PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
-        # OUTPUT_NAME run_tests
+FetchContent_Declare(
+    benchmark
+    GIT_REPOSITORY https://github.com/google/benchmark.git
+    GIT_TAG        v1.8.3 # Change to the latest version if needed
+)
+
+# Ensure Google Benchmark is built correctly
+set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
+
+FetchContent_MakeAvailable(benchmark)
+
+add_executable(benchmarks)
+
+target_sources(benchmarks PRIVATE ndarray_benchmark.cpp calculateeta_benchmark.cpp)
+
+# Link Google Benchmark and other necessary libraries
+target_link_libraries(benchmarks PRIVATE benchmark::benchmark aare_core aare_compiler_flags)
+
+# Set output properties
+set_target_properties(benchmarks PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+    OUTPUT_NAME run_benchmarks
 )

benchmarks/calculateeta_benchmark.cpp

@@ -0,0 +1,70 @@
+#include "aare/CalculateEta.hpp"
+#include "aare/ClusterFile.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace aare;
+
+class ClusterFixture : public benchmark::Fixture {
+  public:
+    Cluster<int, 2, 2> cluster_2x2{};
+    Cluster<int, 3, 3> cluster_3x3{};
+
+  private:
+    using benchmark::Fixture::SetUp;
+
+    void SetUp([[maybe_unused]] const benchmark::State &state) override {
+        int temp_data[4] = {1, 2, 3, 1};
+        std::copy(std::begin(temp_data), std::end(temp_data),
+                  std::begin(cluster_2x2.data));
+
+        cluster_2x2.x = 0;
+        cluster_2x2.y = 0;
+
+        int temp_data2[9] = {1, 2, 3, 1, 3, 4, 5, 1, 20};
+        std::copy(std::begin(temp_data2), std::end(temp_data2),
+                  std::begin(cluster_3x3.data));
+
+        cluster_3x3.x = 0;
+        cluster_3x3.y = 0;
+    }
+
+    // void TearDown(::benchmark::State& state) {
+    // }
+};
+
+BENCHMARK_F(ClusterFixture, Calculate2x2Eta)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        Eta2 eta = calculate_eta2(cluster_2x2);
+        benchmark::DoNotOptimize(eta);
+    }
+}
+
+// almost takes double the time
+BENCHMARK_F(ClusterFixture,
+            CalculateGeneralEtaFor2x2Cluster)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        Eta2 eta = calculate_eta2<int, 2, 2>(cluster_2x2);
+        benchmark::DoNotOptimize(eta);
+    }
+}
+
+BENCHMARK_F(ClusterFixture, Calculate3x3Eta)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        Eta2 eta = calculate_eta2(cluster_3x3);
+        benchmark::DoNotOptimize(eta);
+    }
+}
+
+// almost takes double the time
+BENCHMARK_F(ClusterFixture,
+            CalculateGeneralEtaFor3x3Cluster)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        Eta2 eta = calculate_eta2<int, 3, 3>(cluster_3x3);
+        benchmark::DoNotOptimize(eta);
+    }
+}
+// BENCHMARK_MAIN();

conda-recipe/meta.yaml

@@ -1,6 +1,8 @@
 package:
   name: aare
-  version: 2024.11.15.dev0 #TODO! how to not duplicate this?
+  version: 2025.4.1 #TODO! how to not duplicate this?
+
+
 
 
 source:

docs/CMakeLists.txt

@@ -12,28 +12,7 @@ set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR})
 
 
 file(GLOB SPHINX_SOURCE_FILES CONFIGURE_DEPENDS "src/*.rst")
-# set(SPHINX_SOURCE_FILES
-#   src/index.rst
-#   src/Installation.rst
-#   src/Requirements.rst
-#   src/NDArray.rst
-#   src/NDView.rst
-#   src/File.rst
-#   src/Frame.rst
-#   src/Dtype.rst
-#   src/ClusterFinder.rst
-#   src/ClusterFile.rst
-#   src/Pedestal.rst
-#   src/RawFile.rst
-#   src/RawSubFile.rst
-#   src/RawMasterFile.rst
-#   src/VarClusterFinder.rst
-#   src/pyVarClusterFinder.rst
-#   src/pyFile.rst
-#   src/pyCtbRawFile.rst
-#   src/pyRawFile.rst
-#   src/pyRawMasterFile.rst
-# )
+
 
 
 foreach(filename ${SPHINX_SOURCE_FILES})

docs/conf.py.in

@@ -29,7 +29,6 @@ version = '@PROJECT_VERSION@'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['breathe',
-              'sphinx_rtd_theme',
               'sphinx.ext.autodoc',
               'sphinx.ext.napoleon',
              ]

docs/src/ClusterFinderMT.rst

@@ -0,0 +1,7 @@
+ClusterFinderMT
+==================
+
+
+.. doxygenclass:: aare::ClusterFinderMT
+   :members:
+   :undoc-members:

docs/src/ClusterVector.rst

@@ -0,0 +1,6 @@
+ClusterVector
+=============
+
+.. doxygenclass:: aare::ClusterVector
+   :members:
+   :undoc-members:

docs/src/index.rst

@@ -30,10 +30,13 @@ AARE
     pyFile
     pyCtbRawFile
     pyClusterFile
+    pyClusterVector
     pyRawFile
     pyRawMasterFile
     pyVarClusterFinder
 
+    pyFit
+
 
 .. toctree::
     :caption: C++ API
@@ -45,7 +48,9 @@ AARE
     File
     Dtype
     ClusterFinder
+    ClusterFinderMT
     ClusterFile
+    ClusterVector
     Pedestal
     RawFile
     RawSubFile

docs/src/pyClusterVector.rst

@@ -0,0 +1,33 @@
+ClusterVector
+================
+
+The ClusterVector, holds clusters from the ClusterFinder. Since it is templated
+in C++  we use a suffix indicating the data type in python. The suffix is
+``_i`` for integer, ``_f`` for float, and ``_d`` for double.
+
+At the moment the functionality from python is limited and it is not supported
+to push_back clusters to the vector. The intended use case is to pass it to 
+C++ functions that support the ClusterVector or to view it as a numpy array.
+
+**View ClusterVector as numpy array**
+
+.. code:: python
+
+    from aare import ClusterFile
+    with ClusterFile("path/to/file") as f:
+        cluster_vector = f.read_frame()
+
+    # Create a copy of the cluster data in a numpy array
+    clusters = np.array(cluster_vector)
+
+    # Avoid copying the data by passing copy=False
+    clusters = np.array(cluster_vector, copy = False)
+
+
+.. py:currentmodule:: aare
+
+.. autoclass:: ClusterVector_i
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyFit.rst

@@ -0,0 +1,19 @@
+
+Fit
+========
+
+.. py:currentmodule:: aare
+
+
+**Functions**
+
+.. autofunction:: gaus
+
+.. autofunction:: pol1
+
+
+**Fitting**
+
+.. autofunction:: fit_gaus
+
+.. autofunction:: fit_pol1

include/aare/CalculateEta.hpp

@@ -0,0 +1,122 @@
+#pragma once
+
+#include "aare/Cluster.hpp"
+#include "aare/ClusterVector.hpp"
+#include "aare/NDArray.hpp"
+
+namespace aare {
+
+typedef enum {
+    cBottomLeft = 0,
+    cBottomRight = 1,
+    cTopLeft = 2,
+    cTopRight = 3
+} corner;
+
+typedef enum {
+    pBottomLeft = 0,
+    pBottom = 1,
+    pBottomRight = 2,
+    pLeft = 3,
+    pCenter = 4,
+    pRight = 5,
+    pTopLeft = 6,
+    pTop = 7,
+    pTopRight = 8
+} pixel;
+
+template <typename T> struct Eta2 {
+    double x;
+    double y;
+    int c;
+    T sum;
+};
+
+/**
+ * @brief Calculate the eta2 values for all clusters in a Clsutervector
+ */
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+NDArray<double, 2> calculate_eta2(const ClusterVector<ClusterType> &clusters) {
+    NDArray<double, 2> eta2({static_cast<int64_t>(clusters.size()), 2});
+
+    for (size_t i = 0; i < clusters.size(); i++) {
+        auto e = calculate_eta2(clusters.at(i));
+        eta2(i, 0) = e.x;
+        eta2(i, 1) = e.y;
+    }
+
+    return eta2;
+}
+
+/**
+ * @brief Calculate the eta2 values for a generic sized cluster and return them
+ * in a Eta2 struct containing etay, etax and the index of the respective 2x2
+ * subcluster.
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+Eta2<T>
+calculate_eta2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &cl) {
+    Eta2<T> eta{};
+
+    auto max_sum = cl.max_sum_2x2();
+    eta.sum = max_sum.first;
+    auto c = max_sum.second;
+
+    size_t index_bottom_left_max_2x2_subcluster =
+        (int(c / (ClusterSizeX - 1))) * ClusterSizeX + c % (ClusterSizeX - 1);
+
+    if ((cl.data[index_bottom_left_max_2x2_subcluster] +
+         cl.data[index_bottom_left_max_2x2_subcluster + 1]) != 0)
+        eta.x = static_cast<double>(
+                    cl.data[index_bottom_left_max_2x2_subcluster + 1]) /
+                static_cast<double>(
+                    (cl.data[index_bottom_left_max_2x2_subcluster] +
+                     cl.data[index_bottom_left_max_2x2_subcluster + 1]));
+
+    if ((cl.data[index_bottom_left_max_2x2_subcluster] +
+         cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]) != 0)
+        eta.y =
+            static_cast<double>(
+                cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]) /
+            static_cast<double>(
+                (cl.data[index_bottom_left_max_2x2_subcluster] +
+                 cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]));
+
+    eta.c = c; // TODO only supported for 2x2 and 3x3 clusters -> at least no
+               // underyling enum class
+    return eta;
+}
+
+// calculates Eta3 for 3x3 cluster based on code from analyze_cluster
+// TODO only supported for 3x3 Clusters
+template <typename T> Eta2<T> calculate_eta3(const Cluster<T, 3, 3> &cl) {
+
+    Eta2<T> eta{};
+
+    T sum = 0;
+
+    std::for_each(std::begin(cl.data), std::end(cl.data),
+                  [&sum](T x) { sum += x; });
+
+    eta.sum = sum;
+
+    eta.c = corner::cBottomLeft;
+
+    if ((cl.data[3] + cl.data[4] + cl.data[5]) != 0)
+
+        eta.x = static_cast<double>(-cl.data[3] + cl.data[3 + 2]) /
+
+                (cl.data[3] + cl.data[4] + cl.data[5]);
+
+    if ((cl.data[1] + cl.data[4] + cl.data[7]) != 0)
+
+        eta.y = static_cast<double>(-cl.data[1] + cl.data[2 * 3 + 1]) /
+
+                (cl.data[1] + cl.data[4] + cl.data[7]);
+
+    return eta;
+}
+
+} // namespace aare

include/aare/CircularFifo.hpp

@@ -0,0 +1,97 @@
+#pragma once
+
+#include <chrono>
+#include <fmt/color.h>
+#include <fmt/format.h>
+#include <memory>
+#include <thread>
+
+#include "aare/ProducerConsumerQueue.hpp"
+
+namespace aare {
+
+template <class ItemType> class CircularFifo {
+    uint32_t fifo_size;
+    aare::ProducerConsumerQueue<ItemType> free_slots;
+    aare::ProducerConsumerQueue<ItemType> filled_slots;
+
+  public:
+    CircularFifo() : CircularFifo(100){};
+    CircularFifo(uint32_t size) : fifo_size(size), free_slots(size + 1), filled_slots(size + 1) {
+
+        // TODO! how do we deal with alignment for writing? alignas???
+        // Do we give the user a chance to provide memory locations?
+        // Templated allocator?
+        for (size_t i = 0; i < fifo_size; ++i) {
+            free_slots.write(ItemType{});
+        }
+    }
+
+    bool next() {
+        // TODO! avoid default constructing ItemType
+        ItemType it;
+        if (!filled_slots.read(it))
+            return false;
+        if (!free_slots.write(std::move(it)))
+            return false;
+        return true;
+    }
+
+    ~CircularFifo() {}
+
+    using value_type = ItemType;
+
+    auto numFilledSlots() const noexcept { return filled_slots.sizeGuess(); }
+    auto numFreeSlots() const noexcept { return free_slots.sizeGuess(); }
+    auto isFull() const noexcept { return filled_slots.isFull(); }
+
+    ItemType pop_free() {
+        ItemType v;
+        while (!free_slots.read(v))
+            ;
+        return std::move(v);
+        // return v;
+    }
+
+    bool try_pop_free(ItemType &v) { return free_slots.read(v); }
+
+    ItemType pop_value(std::chrono::nanoseconds wait, std::atomic<bool> &stopped) {
+        ItemType v;
+        while (!filled_slots.read(v) && !stopped) {
+            std::this_thread::sleep_for(wait);
+        }
+        return std::move(v);
+    }
+
+    ItemType pop_value() {
+        ItemType v;
+        while (!filled_slots.read(v))
+            ;
+        return std::move(v);
+    }
+
+    ItemType *frontPtr() { return filled_slots.frontPtr(); }
+
+    // TODO! Add function to move item from filled to free to be used
+    // with the frontPtr function
+
+    template <class... Args> void push_value(Args &&...recordArgs) {
+        while (!filled_slots.write(std::forward<Args>(recordArgs)...))
+            ;
+    }
+
+    template <class... Args> bool try_push_value(Args &&...recordArgs) {
+        return filled_slots.write(std::forward<Args>(recordArgs)...);
+    }
+
+    template <class... Args> void push_free(Args &&...recordArgs) {
+        while (!free_slots.write(std::forward<Args>(recordArgs)...))
+            ;
+    }
+
+    template <class... Args> bool try_push_free(Args &&...recordArgs) {
+        return free_slots.write(std::forward<Args>(recordArgs)...);
+    }
+};
+
+} // namespace aare

include/aare/Cluster.hpp

@@ -0,0 +1,121 @@
+
+/************************************************
+ * @file Cluster.hpp
+ * @short definition of cluster, where CoordType (x,y) give
+ * the cluster center coordinates and data the actual cluster data
+ * cluster size is given as template parameters
+ ***********************************************/
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <numeric>
+#include <type_traits>
+
+namespace aare {
+
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = int16_t>
+constexpr bool is_valid_cluster =
+    std::is_arithmetic_v<T> && std::is_integral_v<CoordType> &&
+    (ClusterSizeX > 0) && (ClusterSizeY > 0);
+
+// requires clause c++20 maybe update
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = int16_t,
+          typename Enable = std::enable_if_t<
+              is_valid_cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>>
+struct Cluster {
+    CoordType x;
+    CoordType y;
+    T data[ClusterSizeX * ClusterSizeY];
+
+    T sum() const {
+        return std::accumulate(data, data + ClusterSizeX * ClusterSizeY, 0);
+    }
+
+    std::pair<T, int> max_sum_2x2() const {
+
+        constexpr size_t num_2x2_subclusters =
+            (ClusterSizeX - 1) * (ClusterSizeY - 1);
+
+        std::array<T, num_2x2_subclusters> sum_2x2_subcluster;
+        for (size_t i = 0; i < ClusterSizeY - 1; ++i) {
+            for (size_t j = 0; j < ClusterSizeX - 1; ++j)
+                sum_2x2_subcluster[i * (ClusterSizeX - 1) + j] =
+                    data[i * ClusterSizeX + j] +
+                    data[i * ClusterSizeX + j + 1] +
+                    data[(i + 1) * ClusterSizeX + j] +
+                    data[(i + 1) * ClusterSizeX + j + 1];
+        }
+
+        int index = std::max_element(sum_2x2_subcluster.begin(),
+                                     sum_2x2_subcluster.end()) -
+                    sum_2x2_subcluster.begin();
+        return std::make_pair(sum_2x2_subcluster[index], index);
+    }
+};
+
+// Specialization for 2x2 clusters (only one sum exists)
+template <typename T> struct Cluster<T, 2, 2, int16_t> {
+    int16_t x;
+    int16_t y;
+    T data[4];
+
+    T sum() const { return std::accumulate(data, data + 4, 0); }
+
+    std::pair<T, int> max_sum_2x2() const {
+        return std::make_pair(data[0] + data[1] + data[2] + data[3],
+                              0); // Only one possible 2x2 sum
+    }
+};
+
+// Specialization for 3x3 clusters
+template <typename T> struct Cluster<T, 3, 3, int16_t> {
+    int16_t x;
+    int16_t y;
+    T data[9];
+
+    T sum() const { return std::accumulate(data, data + 9, 0); }
+
+    std::pair<T, int> max_sum_2x2() const {
+        std::array<T, 4> sum_2x2_subclusters;
+        sum_2x2_subclusters[0] = data[0] + data[1] + data[3] + data[4];
+        sum_2x2_subclusters[1] = data[1] + data[2] + data[4] + data[5];
+        sum_2x2_subclusters[2] = data[3] + data[4] + data[6] + data[7];
+        sum_2x2_subclusters[3] = data[4] + data[5] + data[7] + data[8];
+        int index = std::max_element(sum_2x2_subclusters.begin(),
+                                     sum_2x2_subclusters.end()) -
+                    sum_2x2_subclusters.begin();
+        return std::make_pair(sum_2x2_subclusters[index], index);
+    }
+};
+
+// Type Traits for is_cluster_type
+template <typename T>
+struct is_cluster : std::false_type {}; // Default case: Not a Cluster
+
+template <typename T, uint8_t X, uint8_t Y, typename CoordType>
+struct is_cluster<Cluster<T, X, Y, CoordType>> : std::true_type {}; // Cluster
+
+template <typename T> constexpr bool is_cluster_v = is_cluster<T>::value;
+
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+struct extract_template_arguments; // Forward declaration
+
+// helper struct to extract template argument
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+struct extract_template_arguments<
+    Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
+
+    using value_type = T;
+    static constexpr int cluster_size_x = ClusterSizeX;
+    static constexpr int cluster_size_y = ClusterSizeY;
+    using coordtype = CoordType;
+};
+
+} // namespace aare

include/aare/ClusterCollector.hpp

@@ -0,0 +1,54 @@
+#pragma once
+#include <atomic>
+#include <thread>
+
+#include "aare/ClusterFinderMT.hpp"
+#include "aare/ClusterVector.hpp"
+#include "aare/ProducerConsumerQueue.hpp"
+
+namespace aare {
+
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+class ClusterCollector {
+    ProducerConsumerQueue<ClusterVector<ClusterType>> *m_source;
+    std::atomic<bool> m_stop_requested{false};
+    std::atomic<bool> m_stopped{true};
+    std::chrono::milliseconds m_default_wait{1};
+    std::thread m_thread;
+    std::vector<ClusterVector<ClusterType>> m_clusters;
+
+    void process() {
+        m_stopped = false;
+        fmt::print("ClusterCollector started\n");
+        while (!m_stop_requested || !m_source->isEmpty()) {
+            if (ClusterVector<ClusterType> *clusters = m_source->frontPtr();
+                clusters != nullptr) {
+                m_clusters.push_back(std::move(*clusters));
+                m_source->popFront();
+            } else {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+        fmt::print("ClusterCollector stopped\n");
+        m_stopped = true;
+    }
+
+  public:
+    ClusterCollector(ClusterFinderMT<ClusterType, uint16_t, double> *source) {
+        m_source = source->sink();
+        m_thread = std::thread(&ClusterCollector::process, this);
+    }
+    void stop() {
+        m_stop_requested = true;
+        m_thread.join();
+    }
+    std::vector<ClusterVector<ClusterType>> steal_clusters() {
+        if (!m_stopped) {
+            throw std::runtime_error("ClusterCollector is still running");
+        }
+        return std::move(m_clusters);
+    }
+};
+
+} // namespace aare

include/aare/ClusterFile.hpp

@@ -1,65 +1,455 @@
+#pragma once
 
-
+#include "aare/Cluster.hpp"
+#include "aare/ClusterVector.hpp"
+#include "aare/GainMap.hpp"
+#include "aare/NDArray.hpp"
 #include "aare/defs.hpp"
 #include <filesystem>
 #include <fstream>
+#include <optional>
 
 namespace aare {
 
-struct Cluster {
-    int16_t x;
-    int16_t y;
-    int32_t data[9];
-};
-
-typedef enum {
-    cBottomLeft = 0,
-    cBottomRight = 1,
-    cTopLeft = 2,
-    cTopRight = 3
-} corner;
-
-typedef enum {
-    pBottomLeft = 0,
-    pBottom = 1,
-    pBottomRight = 2,
-    pLeft = 3,
-    pCenter = 4,
-    pRight = 5,
-    pTopLeft = 6,
-    pTop = 7,
-    pTopRight = 8
-} pixel;
-
-struct ClusterAnalysis {
-    uint32_t c;
-    int32_t tot;
-    double etax;
-    double etay;
-};
-
-
+/*
+Binary cluster file. Expects data to be layed out as:
+int32_t frame_number
+uint32_t number_of_clusters
+int16_t x, int16_t y, int32_t data[9] x number_of_clusters
+int32_t frame_number
+uint32_t number_of_clusters
+....
+*/
 
+// TODO: change to support any type of clusters, e.g. header line with
+// clsuter_size_x, cluster_size_y,
+/**
+ * @brief Class to read and write cluster files
+ * Expects data to be laid out as:
+ *
+ *
+ *       int32_t frame_number
+ *       uint32_t number_of_clusters
+ *       int16_t x, int16_t y, int32_t data[9] * number_of_clusters
+ *       int32_t frame_number
+ *       uint32_t number_of_clusters
+ *       etc.
+ */
+template <typename ClusterType,
+          typename Enable = std::enable_if_t<is_cluster_v<ClusterType>>>
 class ClusterFile {
     FILE *fp{};
-    uint32_t m_num_left{};
-    size_t m_chunk_size{};
+    uint32_t m_num_left{};    /*Number of photons left in frame*/
+    size_t m_chunk_size{};    /*Number of clusters to read at a time*/
+    const std::string m_mode; /*Mode to open the file in*/
+    std::optional<ROI> m_roi; /*Region of interest, will be applied if set*/
+    std::optional<NDArray<int32_t, 2>>
+        m_noise_map; /*Noise map to cut photons, will be applied if set*/
+    std::optional<GainMap> m_gain_map; /*Gain map to apply to the clusters, will
+                                          be applied if set*/
 
   public:
-    ClusterFile(const std::filesystem::path &fname, size_t chunk_size = 1000);
-    std::vector<Cluster> read_clusters(size_t n_clusters);
-    std::vector<Cluster> read_frame(int32_t &out_fnum);
-    std::vector<Cluster>
-    read_cluster_with_cut(size_t n_clusters, double *noise_map, int nx, int ny);
+    /**
+     * @brief Construct a new Cluster File object
+     * @param fname path to the file
+     * @param chunk_size number of clusters to read at a time when iterating
+     * over the file
+     * @param mode mode to open the file in. "r" for reading, "w" for writing,
+     * "a" for appending
+     * @throws std::runtime_error if the file could not be opened
+     */
+    ClusterFile(const std::filesystem::path &fname, size_t chunk_size = 1000,
+                const std::string &mode = "r");
 
-    int analyze_data(int32_t *data, int32_t *t2, int32_t *t3, char *quad,
-                 double *eta2x, double *eta2y, double *eta3x, double *eta3y);
-    int analyze_cluster(Cluster cl, int32_t *t2, int32_t *t3, char *quad,
-                    double *eta2x, double *eta2y, double *eta3x,
-                    double *eta3y);
+    ~ClusterFile();
 
+    /**
+     * @brief Read n_clusters clusters from the file discarding frame numbers.
+     * If EOF is reached the returned vector will have less than n_clusters
+     * clusters
+     */
+    ClusterVector<ClusterType> read_clusters(size_t n_clusters);
+
+    /**
+     * @brief Read a single frame from the file and return the clusters. The
+     * cluster vector will have the frame number set.
+     * @throws std::runtime_error if the file is not opened for reading or the
+     * file pointer not at the beginning of a frame
+     */
+    ClusterVector<ClusterType> read_frame();
+
+    void write_frame(const ClusterVector<ClusterType> &clusters);
+
+    /**
+     * @brief Return the chunk size
+     */
     size_t chunk_size() const { return m_chunk_size; }
 
+    /**
+     * @brief Set the region of interest to use when reading clusters. If set
+     * only clusters within the ROI will be read.
+     */
+    void set_roi(ROI roi);
+
+    /**
+     * @brief Set the noise map to use when reading clusters. If set clusters
+     * below the noise level will be discarded. Selection criteria one of:
+     * Central pixel above noise, highest 2x2 sum above 2 * noise, total sum
+     * above 3 * noise.
+     */
+    void set_noise_map(const NDView<int32_t, 2> noise_map);
+
+    /**
+     * @brief Set the gain map to use when reading clusters. If set the gain map
+     * will be applied to the clusters that pass ROI and noise_map selection.
+     */
+    void set_gain_map(const NDView<double, 2> gain_map);
+
+    void set_gain_map(const GainMap &gain_map);
+
+    void set_gain_map(const GainMap &&gain_map);
+
+    /**
+     * @brief Close the file. If not closed the file will be closed in the
+     * destructor
+     */
+    void close();
+
+  private:
+    ClusterVector<ClusterType> read_clusters_with_cut(size_t n_clusters);
+    ClusterVector<ClusterType> read_clusters_without_cut(size_t n_clusters);
+    ClusterVector<ClusterType> read_frame_with_cut();
+    ClusterVector<ClusterType> read_frame_without_cut();
+    bool is_selected(ClusterType &cl);
+    ClusterType read_one_cluster();
 };
 
+template <typename ClusterType, typename Enable>
+ClusterFile<ClusterType, Enable>::ClusterFile(
+    const std::filesystem::path &fname, size_t chunk_size,
+    const std::string &mode)
+    : m_chunk_size(chunk_size), m_mode(mode) {
+
+    if (mode == "r") {
+        fp = fopen(fname.c_str(), "rb");
+        if (!fp) {
+            throw std::runtime_error("Could not open file for reading: " +
+                                     fname.string());
+        }
+    } else if (mode == "w") {
+        fp = fopen(fname.c_str(), "wb");
+        if (!fp) {
+            throw std::runtime_error("Could not open file for writing: " +
+                                     fname.string());
+        }
+    } else if (mode == "a") {
+        fp = fopen(fname.c_str(), "ab");
+        if (!fp) {
+            throw std::runtime_error("Could not open file for appending: " +
+                                     fname.string());
+        }
+    } else {
+        throw std::runtime_error("Unsupported mode: " + mode);
+    }
+}
+
+template <typename ClusterType, typename Enable>
+ClusterFile<ClusterType, Enable>::~ClusterFile() {
+    close();
+}
+
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::close() {
+    if (fp) {
+        fclose(fp);
+        fp = nullptr;
+    }
+}
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::set_roi(ROI roi) {
+    m_roi = roi;
+}
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::set_noise_map(
+    const NDView<int32_t, 2> noise_map) {
+    m_noise_map = NDArray<int32_t, 2>(noise_map);
+}
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::set_gain_map(
+    const NDView<double, 2> gain_map) {
+    m_gain_map = GainMap(gain_map);
+}
+
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::set_gain_map(const GainMap &gain_map) {
+    m_gain_map = gain_map;
+}
+
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::set_gain_map(const GainMap &&gain_map) {
+    m_gain_map = gain_map;
+}
+
+// TODO generally supported for all clsuter types
+template <typename ClusterType, typename Enable>
+void ClusterFile<ClusterType, Enable>::write_frame(
+    const ClusterVector<ClusterType> &clusters) {
+    if (m_mode != "w" && m_mode != "a") {
+        throw std::runtime_error("File not opened for writing");
+    }
+    if (!(clusters.cluster_size_x() == 3) &&
+        !(clusters.cluster_size_y() == 3)) {
+        throw std::runtime_error("Only 3x3 clusters are supported");
+    }
+    int32_t frame_number = clusters.frame_number();
+    fwrite(&frame_number, sizeof(frame_number), 1, fp);
+    uint32_t n_clusters = clusters.size();
+    fwrite(&n_clusters, sizeof(n_clusters), 1, fp);
+    fwrite(clusters.data(), clusters.item_size(), clusters.size(), fp);
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType>
+ClusterFile<ClusterType, Enable>::read_clusters(size_t n_clusters) {
+    if (m_mode != "r") {
+        throw std::runtime_error("File not opened for reading");
+    }
+    if (m_noise_map || m_roi) {
+        return read_clusters_with_cut(n_clusters);
+    } else {
+        return read_clusters_without_cut(n_clusters);
+    }
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType>
+ClusterFile<ClusterType, Enable>::read_clusters_without_cut(size_t n_clusters) {
+    if (m_mode != "r") {
+        throw std::runtime_error("File not opened for reading");
+    }
+
+    ClusterVector<ClusterType> clusters(n_clusters);
+
+    int32_t iframe = 0; // frame number needs to be 4 bytes!
+    size_t nph_read = 0;
+    uint32_t nn = m_num_left;
+    uint32_t nph = m_num_left; // number of clusters in frame needs to be 4
+
+    // auto buf = reinterpret_cast<Cluster3x3 *>(clusters.data());
+    auto buf = clusters.data();
+    // if there are photons left from previous frame read them first
+    if (nph) {
+        if (nph > n_clusters) {
+            // if we have more photons left in the frame then photons to read we
+            // read directly the requested number
+            nn = n_clusters;
+        } else {
+            nn = nph;
+        }
+        nph_read += fread((buf + nph_read * clusters.item_size()),
+                          clusters.item_size(), nn, fp);
+        m_num_left = nph - nn; // write back the number of photons left
+    }
+
+    if (nph_read < n_clusters) {
+        // keep on reading frames and photons until reaching n_clusters
+        while (fread(&iframe, sizeof(iframe), 1, fp)) {
+            clusters.set_frame_number(iframe);
+            // read number of clusters in frame
+            if (fread(&nph, sizeof(nph), 1, fp)) {
+                if (nph > (n_clusters - nph_read))
+                    nn = n_clusters - nph_read;
+                else
+                    nn = nph;
+
+                nph_read += fread((buf + nph_read * clusters.item_size()),
+                                  clusters.item_size(), nn, fp);
+                m_num_left = nph - nn;
+            }
+            if (nph_read >= n_clusters)
+                break;
+        }
+    }
+
+    // Resize the vector to the number of clusters.
+    // No new allocation, only change bounds.
+    clusters.resize(nph_read);
+    if (m_gain_map)
+        m_gain_map->apply_gain_map(clusters);
+    return clusters;
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType>
+ClusterFile<ClusterType, Enable>::read_clusters_with_cut(size_t n_clusters) {
+    ClusterVector<ClusterType> clusters;
+    clusters.reserve(n_clusters);
+
+    // if there are photons left from previous frame read them first
+    if (m_num_left) {
+        while (m_num_left && clusters.size() < n_clusters) {
+            ClusterType c = read_one_cluster();
+            if (is_selected(c)) {
+                clusters.push_back(c);
+            }
+        }
+    }
+
+    // we did not have enough clusters left in the previous frame
+    // keep on reading frames until reaching n_clusters
+    if (clusters.size() < n_clusters) {
+        // sanity check
+        if (m_num_left) {
+            throw std::runtime_error(
+                LOCATION + "Entered second loop with clusters left\n");
+        }
+
+        int32_t frame_number = 0; // frame number needs to be 4 bytes!
+        while (fread(&frame_number, sizeof(frame_number), 1, fp)) {
+            if (fread(&m_num_left, sizeof(m_num_left), 1, fp)) {
+                clusters.set_frame_number(
+                    frame_number); // cluster vector will hold the last frame
+                                   // number
+                while (m_num_left && clusters.size() < n_clusters) {
+                    ClusterType c = read_one_cluster();
+                    if (is_selected(c)) {
+                        clusters.push_back(c);
+                    }
+                }
+            }
+
+            // we have enough clusters, break out of the outer while loop
+            if (clusters.size() >= n_clusters)
+                break;
+        }
+    }
+    if (m_gain_map)
+        m_gain_map->apply_gain_map(clusters);
+
+    return clusters;
+}
+
+template <typename ClusterType, typename Enable>
+ClusterType ClusterFile<ClusterType, Enable>::read_one_cluster() {
+    ClusterType c;
+    auto rc = fread(&c, sizeof(c), 1, fp);
+    if (rc != 1) {
+        throw std::runtime_error(LOCATION + "Could not read cluster");
+    }
+    --m_num_left;
+    return c;
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType> ClusterFile<ClusterType, Enable>::read_frame() {
+    if (m_mode != "r") {
+        throw std::runtime_error(LOCATION + "File not opened for reading");
+    }
+    if (m_noise_map || m_roi) {
+        return read_frame_with_cut();
+    } else {
+        return read_frame_without_cut();
+    }
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType>
+ClusterFile<ClusterType, Enable>::read_frame_without_cut() {
+    if (m_mode != "r") {
+        throw std::runtime_error("File not opened for reading");
+    }
+    if (m_num_left) {
+        throw std::runtime_error(
+            "There are still photons left in the last frame");
+    }
+    int32_t frame_number;
+    if (fread(&frame_number, sizeof(frame_number), 1, fp) != 1) {
+        throw std::runtime_error(LOCATION + "Could not read frame number");
+    }
+
+    int32_t n_clusters; // Saved as 32bit integer in the cluster file
+    if (fread(&n_clusters, sizeof(n_clusters), 1, fp) != 1) {
+        throw std::runtime_error(LOCATION +
+                                 "Could not read number of clusters");
+    }
+
+    ClusterVector<ClusterType> clusters(n_clusters);
+    clusters.set_frame_number(frame_number);
+
+    if (fread(clusters.data(), clusters.item_size(), n_clusters, fp) !=
+        static_cast<size_t>(n_clusters)) {
+        throw std::runtime_error(LOCATION + "Could not read clusters");
+    }
+    clusters.resize(n_clusters);
+    if (m_gain_map)
+        m_gain_map->apply_gain_map(clusters);
+    return clusters;
+}
+
+template <typename ClusterType, typename Enable>
+ClusterVector<ClusterType>
+ClusterFile<ClusterType, Enable>::read_frame_with_cut() {
+    if (m_mode != "r") {
+        throw std::runtime_error("File not opened for reading");
+    }
+    if (m_num_left) {
+        throw std::runtime_error(
+            "There are still photons left in the last frame");
+    }
+    int32_t frame_number;
+    if (fread(&frame_number, sizeof(frame_number), 1, fp) != 1) {
+        throw std::runtime_error("Could not read frame number");
+    }
+
+    if (fread(&m_num_left, sizeof(m_num_left), 1, fp) != 1) {
+        throw std::runtime_error("Could not read number of clusters");
+    }
+
+    ClusterVector<ClusterType> clusters;
+    clusters.reserve(m_num_left);
+    clusters.set_frame_number(frame_number);
+    while (m_num_left) {
+        ClusterType c = read_one_cluster();
+        if (is_selected(c)) {
+            clusters.push_back(c);
+        }
+    }
+    if (m_gain_map)
+        m_gain_map->apply_gain_map(clusters);
+    return clusters;
+}
+
+template <typename ClusterType, typename Enable>
+bool ClusterFile<ClusterType, Enable>::is_selected(ClusterType &cl) {
+    // Should fail fast
+    if (m_roi) {
+        if (!(m_roi->contains(cl.x, cl.y))) {
+            return false;
+        }
+    }
+
+    auto cluster_size_x = extract_template_arguments<
+        std::remove_reference_t<decltype(cl)>>::cluster_size_x;
+    auto cluster_size_y = extract_template_arguments<
+        std::remove_reference_t<decltype(cl)>>::cluster_size_y;
+
+    size_t cluster_center_index =
+        (cluster_size_x / 2) + (cluster_size_y / 2) * cluster_size_x;
+
+    if (m_noise_map) {
+        auto sum_1x1 = cl.data[cluster_center_index]; // central pixel
+        auto sum_2x2 = cl.max_sum_2x2().first; // highest sum of 2x2 subclusters
+        auto total_sum = cl.sum();             // sum of all pixels
+
+        auto noise =
+            (*m_noise_map)(cl.y, cl.x); // TODO! check if this is correct
+        if (sum_1x1 <= noise || sum_2x2 <= 2 * noise ||
+            total_sum <= 3 * noise) {
+            return false;
+        }
+    }
+    // we passed all checks
+    return true;
+}
+
 } // namespace aare

include/aare/ClusterFileSink.hpp

@@ -0,0 +1,62 @@
+#pragma once
+#include <atomic>
+#include <filesystem>
+#include <thread>
+
+#include "aare/ClusterFinderMT.hpp"
+#include "aare/ClusterVector.hpp"
+#include "aare/ProducerConsumerQueue.hpp"
+
+namespace aare {
+
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+class ClusterFileSink {
+    ProducerConsumerQueue<ClusterVector<ClusterType>> *m_source;
+    std::atomic<bool> m_stop_requested{false};
+    std::atomic<bool> m_stopped{true};
+    std::chrono::milliseconds m_default_wait{1};
+    std::thread m_thread;
+    std::ofstream m_file;
+
+    void process() {
+        m_stopped = false;
+        fmt::print("ClusterFileSink started\n");
+        while (!m_stop_requested || !m_source->isEmpty()) {
+            if (ClusterVector<ClusterType> *clusters = m_source->frontPtr();
+                clusters != nullptr) {
+                // Write clusters to file
+                int32_t frame_number =
+                    clusters->frame_number(); // TODO! Should we store frame
+                                              // number already as int?
+                uint32_t num_clusters = clusters->size();
+                m_file.write(reinterpret_cast<const char *>(&frame_number),
+                             sizeof(frame_number));
+                m_file.write(reinterpret_cast<const char *>(&num_clusters),
+                             sizeof(num_clusters));
+                m_file.write(reinterpret_cast<const char *>(clusters->data()),
+                             clusters->size() * clusters->item_size());
+                m_source->popFront();
+            } else {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+        fmt::print("ClusterFileSink stopped\n");
+        m_stopped = true;
+    }
+
+  public:
+    ClusterFileSink(ClusterFinderMT<ClusterType, uint16_t, double> *source,
+                    const std::filesystem::path &fname) {
+        m_source = source->sink();
+        m_thread = std::thread(&ClusterFileSink::process, this);
+        m_file.open(fname, std::ios::binary);
+    }
+    void stop() {
+        m_stop_requested = true;
+        m_thread.join();
+        m_file.close();
+    }
+};
+
+} // namespace aare

include/aare/ClusterFileV2.hpp

@@ -1,15 +1,16 @@
 #pragma once
 #include "aare/core/defs.hpp"
 #include <filesystem>
-#include <string>
 #include <fmt/format.h>
+#include <string>
 
 namespace aare {
 struct ClusterHeader {
     int32_t frame_number;
     int32_t n_clusters;
     std::string to_string() const {
-        return "frame_number: " + std::to_string(frame_number) + ", n_clusters: " + std::to_string(n_clusters);
+        return "frame_number: " + std::to_string(frame_number) +
+               ", n_clusters: " + std::to_string(n_clusters);
     }
 };
 
@@ -24,7 +25,8 @@ struct ClusterV2_ {
                 data_str += std::to_string(d) + ", ";
             }
             data_str += "]";
-            return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", data: " + data_str;
+            return "x: " + std::to_string(x) + ", y: " + std::to_string(y) +
+                   ", data: " + data_str;
         }
         return "x: " + std::to_string(x) + ", y: " + std::to_string(y);
     }
@@ -34,27 +36,31 @@ struct ClusterV2 {
     ClusterV2_ cluster;
     int32_t frame_number;
     std::string to_string() const {
-        return "frame_number: " + std::to_string(frame_number) + ", " + cluster.to_string();
+        return "frame_number: " + std::to_string(frame_number) + ", " +
+               cluster.to_string();
     }
 };
 
 /**
  * @brief
- * important not: fp always points to the clusters header and does not point to individual clusters
+ * important not: fp always points to the clusters header and does not point to
+ * individual clusters
  *
  */
 class ClusterFileV2 {
-    std::filesystem::path m_fpath; 
+    std::filesystem::path m_fpath;
     std::string m_mode;
     FILE *fp{nullptr};
 
-    void check_open(){
+    void check_open() {
         if (!fp)
-            throw std::runtime_error(fmt::format("File: {} not open", m_fpath.string()));
+            throw std::runtime_error(
+                fmt::format("File: {} not open", m_fpath.string()));
     }
 
   public:
-    ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode): m_fpath(fpath), m_mode(mode) {
+    ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode)
+        : m_fpath(fpath), m_mode(mode) {
         if (m_mode != "r" && m_mode != "w")
             throw std::invalid_argument("mode must be 'r' or 'w'");
         if (m_mode == "r" && !std::filesystem::exists(m_fpath))
@@ -77,7 +83,7 @@ class ClusterFileV2 {
         check_open();
 
         ClusterHeader header;
-        fread(&header, sizeof(ClusterHeader), 1, fp); 
+        fread(&header, sizeof(ClusterHeader), 1, fp);
         std::vector<ClusterV2_> clusters_(header.n_clusters);
         fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp);
         std::vector<ClusterV2> clusters;
@@ -117,7 +123,7 @@ class ClusterFileV2 {
 
     size_t write(std::vector<std::vector<ClusterV2>> const &clusters) {
         check_open();
-        if (m_mode != "w") 
+        if (m_mode != "w")
             throw std::runtime_error("File not opened in write mode");
 
         size_t n_clusters = 0;

include/aare/ClusterFinder.hpp

@@ -1,4 +1,6 @@
 #pragma once
+#include "aare/ClusterFile.hpp"
+#include "aare/ClusterVector.hpp"
 #include "aare/Dtype.hpp"
 #include "aare/NDArray.hpp"
 #include "aare/NDView.hpp"
@@ -8,251 +10,154 @@
 
 namespace aare {
 
-/** enum to define the event types */
-enum eventType {
-    PEDESTAL,   /** pedestal */
-    NEIGHBOUR,  /** neighbour i.e. below threshold, but in the cluster of a
-                   photon */
-    PHOTON,     /** photon i.e. above threshold */
-    PHOTON_MAX, /** maximum of a cluster satisfying the photon conditions */
-    NEGATIVE_PEDESTAL, /** negative value, will not be accounted for as pedestal
-                          in order to avoid drift of the pedestal towards
-                          negative values */
-    UNDEFINED_EVENT = -1 /** undefined */
-};
-
-template <typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
+template <typename ClusterType = Cluster<int32_t, 3, 3>,
+          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
 class ClusterFinder {
     Shape<2> m_image_size;
-    const int m_cluster_sizeX;
-    const int m_cluster_sizeY;
-    const double m_threshold;
-    const double m_nSigma;
-    const double c2;
-    const double c3;
+    const PEDESTAL_TYPE m_nSigma;
+    const PEDESTAL_TYPE c2;
+    const PEDESTAL_TYPE c3;
     Pedestal<PEDESTAL_TYPE> m_pedestal;
+    ClusterVector<ClusterType> m_clusters;
+
+    static const uint8_t ClusterSizeX =
+        extract_template_arguments<ClusterType>::cluster_size_x;
+    static const uint8_t ClusterSizeY =
+        extract_template_arguments<ClusterType>::cluster_size_x;
+    using CT = typename extract_template_arguments<ClusterType>::value_type;
 
   public:
-    ClusterFinder(Shape<2> image_size, Shape<2>cluster_size, double nSigma = 5.0,
-                  double threshold = 0.0)
-        : m_image_size(image_size), m_cluster_sizeX(cluster_size[0]), m_cluster_sizeY(cluster_size[1]),
-          m_threshold(threshold), m_nSigma(nSigma),
-          c2(sqrt((m_cluster_sizeY + 1) / 2 * (m_cluster_sizeX + 1) / 2)),
-          c3(sqrt(m_cluster_sizeX * m_cluster_sizeY)),
-          m_pedestal(image_size[0], image_size[1]) {
-
-              // c2 = sqrt((cluster_sizeY + 1) / 2 * (cluster_sizeX + 1) / 2);
-              // c3 = sqrt(cluster_sizeX * cluster_sizeY);
-          };
+    /**
+     * @brief Construct a new ClusterFinder object
+     * @param image_size size of the image
+     * @param cluster_size size of the cluster (x, y)
+     * @param nSigma number of sigma above the pedestal to consider a photon
+     * @param capacity initial capacity of the cluster vector
+     *
+     */
+    ClusterFinder(Shape<2> image_size, PEDESTAL_TYPE nSigma = 5.0,
+                  size_t capacity = 1000000)
+        : m_image_size(image_size), m_nSigma(nSigma),
+          c2(sqrt((ClusterSizeY + 1) / 2 * (ClusterSizeX + 1) / 2)),
+          c3(sqrt(ClusterSizeX * ClusterSizeY)),
+          m_pedestal(image_size[0], image_size[1]), m_clusters(capacity) {};
 
     void push_pedestal_frame(NDView<FRAME_TYPE, 2> frame) {
         m_pedestal.push(frame);
     }
 
-    NDArray<PEDESTAL_TYPE, 2> pedestal() {
-        return m_pedestal.mean();
+    NDArray<PEDESTAL_TYPE, 2> pedestal() { return m_pedestal.mean(); }
+    NDArray<PEDESTAL_TYPE, 2> noise() { return m_pedestal.std(); }
+    void clear_pedestal() { m_pedestal.clear(); }
+
+    /**
+     * @brief Move the clusters from the ClusterVector in the ClusterFinder to a
+     * new ClusterVector and return it.
+     * @param realloc_same_capacity if true the new ClusterVector will have the
+     * same capacity as the old one
+     *
+     */
+    ClusterVector<ClusterType>
+    steal_clusters(bool realloc_same_capacity = false) {
+        ClusterVector<ClusterType> tmp = std::move(m_clusters);
+        if (realloc_same_capacity)
+            m_clusters = ClusterVector<ClusterType>(tmp.capacity());
+        else
+            m_clusters = ClusterVector<ClusterType>{};
+        return tmp;
     }
+    void find_clusters(NDView<FRAME_TYPE, 2> frame, uint64_t frame_number = 0) {
+        // // TODO! deal with even size clusters
+        // // currently 3,3 -> +/- 1
+        // //  4,4 -> +/- 2
+        int dy = ClusterSizeY / 2;
+        int dx = ClusterSizeX / 2;
+        int has_center_pixel_x =
+            ClusterSizeX %
+            2; // for even sized clusters there is no proper cluster center and
+               // even amount of pixels around the center
+        int has_center_pixel_y = ClusterSizeY % 2;
 
-    std::vector<DynamicCluster>
-    find_clusters_without_threshold(NDView<FRAME_TYPE, 2> frame,
-                                    // Pedestal<PEDESTAL_TYPE> &pedestal,
-                                    bool late_update = false) {
-        struct pedestal_update {
-            int x;
-            int y;
-            FRAME_TYPE value;
-        };
-        std::vector<pedestal_update> pedestal_updates;
-
-        std::vector<DynamicCluster> clusters;
-        std::vector<std::vector<eventType>> eventMask;
-        for (int i = 0; i < frame.shape(0); i++) {
-            eventMask.push_back(std::vector<eventType>(frame.shape(1)));
-        }
-        long double val;
-        long double max;
-
+        m_clusters.set_frame_number(frame_number);
+        std::vector<CT> cluster_data(ClusterSizeX * ClusterSizeY);
         for (int iy = 0; iy < frame.shape(0); iy++) {
             for (int ix = 0; ix < frame.shape(1); ix++) {
-                // initialize max and total
-                max = std::numeric_limits<FRAME_TYPE>::min();
-                long double total = 0;
-                eventMask[iy][ix] = PEDESTAL;
 
-                for (short ir = -(m_cluster_sizeY / 2);
-                     ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                    for (short ic = -(m_cluster_sizeX / 2);
-                         ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                PEDESTAL_TYPE max = std::numeric_limits<FRAME_TYPE>::min();
+                PEDESTAL_TYPE total = 0;
+
+                // What can we short circuit here?
+                PEDESTAL_TYPE rms = m_pedestal.std(iy, ix);
+                PEDESTAL_TYPE value = (frame(iy, ix) - m_pedestal.mean(iy, ix));
+
+                if (value < -m_nSigma * rms)
+                    continue; // NEGATIVE_PEDESTAL go to next pixel
+                              // TODO! No pedestal update???
+
+                for (int ir = -dy; ir < dy + has_center_pixel_y; ir++) {
+                    for (int ic = -dx; ic < dx + has_center_pixel_x; ic++) {
                         if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
                             iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                            val = frame(iy + ir, ix + ic) -
-                                  m_pedestal.mean(iy + ir, ix + ic);
+                            PEDESTAL_TYPE val =
+                                frame(iy + ir, ix + ic) -
+                                m_pedestal.mean(iy + ir, ix + ic);
+
                             total += val;
-                            if (val > max) {
-                                max = val;
-                            }
+                            max = std::max(max, val);
                         }
                     }
                 }
-                auto rms = m_pedestal.std(iy, ix);
-
-                if (frame(iy, ix) - m_pedestal.mean(iy, ix) < -m_nSigma * rms) {
-                    eventMask[iy][ix] = NEGATIVE_PEDESTAL;
-                    continue;
-                } else if (max > m_nSigma * rms) {
-                    eventMask[iy][ix] = PHOTON;
 
+                if ((max > m_nSigma * rms)) {
+                    if (value < max)
+                        continue; // Not max go to the next pixel
+                                  // but also no pedestal update
                 } else if (total > c3 * m_nSigma * rms) {
-                    eventMask[iy][ix] = PHOTON;
+                    // pass
                 } else {
-                    if (late_update) {
-                        pedestal_updates.push_back({ix, iy, frame(iy, ix)});
-                    } else {
-                        m_pedestal.push(iy, ix, frame(iy, ix));
-                    }
-                    continue;
+                    // m_pedestal.push(iy, ix, frame(iy, ix));   // Safe option
+                    m_pedestal.push_fast(
+                        iy, ix,
+                        frame(iy,
+                              ix)); // Assume we have reached n_samples in the
+                                    // pedestal, slight performance improvement
+                    continue;       // It was a pedestal value nothing to store
                 }
-                if (eventMask[iy][ix] == PHOTON &&
-                    (frame(iy, ix) - m_pedestal.mean(iy, ix)) >= max) {
-                    eventMask[iy][ix] = PHOTON_MAX;
-                    DynamicCluster cluster(m_cluster_sizeX, m_cluster_sizeY,
-                                    Dtype(typeid(PEDESTAL_TYPE)));
-                    cluster.x = ix;
-                    cluster.y = iy;
-                    short i = 0;
 
-                    for (short ir = -(m_cluster_sizeY / 2);
-                         ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                        for (short ic = -(m_cluster_sizeX / 2);
-                             ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                // Store cluster
+                if (value == max) {
+                    // Zero out the cluster data
+                    std::fill(cluster_data.begin(), cluster_data.end(), 0);
+
+                    // Fill the cluster data since we have a photon to store
+                    // It's worth redoing the look since most of the time we
+                    // don't have a photon
+                    int i = 0;
+                    for (int ir = -dy; ir < dy + has_center_pixel_y; ir++) {
+                        for (int ic = -dx; ic < dx + has_center_pixel_y; ic++) {
                             if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
                                 iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                                PEDESTAL_TYPE tmp =
-                                    static_cast<PEDESTAL_TYPE>(
-                                        frame(iy + ir, ix + ic)) -
-                                    m_pedestal.mean(iy + ir, ix + ic);
-                                cluster.set<PEDESTAL_TYPE>(i, tmp);
+                                CT tmp =
+                                    static_cast<CT>(frame(iy + ir, ix + ic)) -
+                                    static_cast<CT>(
+                                        m_pedestal.mean(iy + ir, ix + ic));
+                                cluster_data[i] =
+                                    tmp; // Watch for out of bounds access
                                 i++;
                             }
                         }
                     }
-                    clusters.push_back(cluster);
+
+                    ClusterType new_cluster{};
+                    new_cluster.x = ix;
+                    new_cluster.y = iy;
+                    std::copy(cluster_data.begin(), cluster_data.end(),
+                              new_cluster.data);
+                    // Add the cluster to the output ClusterVector
+                    m_clusters.push_back(new_cluster);
                 }
             }
         }
-        if (late_update) {
-            for (auto &update : pedestal_updates) {
-                m_pedestal.push(update.y, update.x, update.value);
-            }
-        }
-        return clusters;
-    }
-
-    // template <typename FRAME_TYPE, typename PEDESTAL_TYPE>
-    std::vector<DynamicCluster>
-    find_clusters_with_threshold(NDView<FRAME_TYPE, 2> frame,
-                                 Pedestal<PEDESTAL_TYPE> &pedestal) {
-        assert(m_threshold > 0);
-        std::vector<DynamicCluster> clusters;
-        std::vector<std::vector<eventType>> eventMask;
-        for (int i = 0; i < frame.shape(0); i++) {
-            eventMask.push_back(std::vector<eventType>(frame.shape(1)));
-        }
-        double tthr, tthr1, tthr2;
-
-        NDArray<FRAME_TYPE, 2> rest({frame.shape(0), frame.shape(1)});
-        NDArray<int, 2> nph({frame.shape(0), frame.shape(1)});
-        // convert to n photons
-        // nph = (frame-pedestal.mean()+0.5*m_threshold)/m_threshold; // can be
-        // optimized with expression templates?
-        for (int iy = 0; iy < frame.shape(0); iy++) {
-            for (int ix = 0; ix < frame.shape(1); ix++) {
-                auto val = frame(iy, ix) - pedestal.mean(iy, ix);
-                nph(iy, ix) = (val + 0.5 * m_threshold) / m_threshold;
-                nph(iy, ix) = nph(iy, ix) < 0 ? 0 : nph(iy, ix);
-                rest(iy, ix) = val - nph(iy, ix) * m_threshold;
-            }
-        }
-        // iterate over frame pixels
-        for (int iy = 0; iy < frame.shape(0); iy++) {
-            for (int ix = 0; ix < frame.shape(1); ix++) {
-                eventMask[iy][ix] = PEDESTAL;
-                // initialize max and total
-                FRAME_TYPE max = std::numeric_limits<FRAME_TYPE>::min();
-                long double total = 0;
-                if (rest(iy, ix) <= 0.25 * m_threshold) {
-                    pedestal.push(iy, ix, frame(iy, ix));
-                    continue;
-                }
-                eventMask[iy][ix] = NEIGHBOUR;
-                // iterate over cluster pixels around the current pixel (ix,iy)
-                for (short ir = -(m_cluster_sizeY / 2);
-                     ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                    for (short ic = -(m_cluster_sizeX / 2);
-                         ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                        if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
-                            iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                            auto val = frame(iy + ir, ix + ic) -
-                                       pedestal.mean(iy + ir, ix + ic);
-                            total += val;
-                            if (val > max) {
-                                max = val;
-                            }
-                        }
-                    }
-                }
-
-                auto rms = pedestal.std(iy, ix);
-                if (m_nSigma == 0) {
-                    tthr = m_threshold;
-                    tthr1 = m_threshold;
-                    tthr2 = m_threshold;
-                } else {
-                    tthr = m_nSigma * rms;
-                    tthr1 = m_nSigma * rms * c3;
-                    tthr2 = m_nSigma * rms * c2;
-
-                    if (m_threshold > 2 * tthr)
-                        tthr = m_threshold - tthr;
-                    if (m_threshold > 2 * tthr1)
-                        tthr1 = tthr - tthr1;
-                    if (m_threshold > 2 * tthr2)
-                        tthr2 = tthr - tthr2;
-                }
-                if (total > tthr1 || max > tthr) {
-                    eventMask[iy][ix] = PHOTON;
-                    nph(iy, ix) += 1;
-                    rest(iy, ix) -= m_threshold;
-                } else {
-                    pedestal.push(iy, ix, frame(iy, ix));
-                    continue;
-                }
-                if (eventMask[iy][ix] == PHOTON &&
-                    frame(iy, ix) - pedestal.mean(iy, ix) >= max) {
-                    eventMask[iy][ix] = PHOTON_MAX;
-                    DynamicCluster cluster(m_cluster_sizeX, m_cluster_sizeY,
-                                    Dtype(typeid(FRAME_TYPE)));
-                    cluster.x = ix;
-                    cluster.y = iy;
-                    short i = 0;
-                    for (short ir = -(m_cluster_sizeY / 2);
-                         ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                        for (short ic = -(m_cluster_sizeX / 2);
-                             ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                            if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
-                                iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                                auto tmp = frame(iy + ir, ix + ic) -
-                                           pedestal.mean(iy + ir, ix + ic);
-                                cluster.set<FRAME_TYPE>(i, tmp);
-                                i++;
-                            }
-                        }
-                    }
-                    clusters.push_back(cluster);
-                }
-            }
-        }
-        return clusters;
     }
 };
 

include/aare/ClusterFinderMT.hpp

@@ -0,0 +1,274 @@
+#pragma once
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "aare/ClusterFinder.hpp"
+#include "aare/NDArray.hpp"
+#include "aare/ProducerConsumerQueue.hpp"
+
+namespace aare {
+
+enum class FrameType {
+    DATA,
+    PEDESTAL,
+};
+
+struct FrameWrapper {
+    FrameType type;
+    uint64_t frame_number;
+    NDArray<uint16_t, 2> data;
+};
+
+/**
+ * @brief ClusterFinderMT is a multi-threaded version of ClusterFinder. It uses
+ * a producer-consumer queue to distribute the frames to the threads. The
+ * clusters are collected in a single output queue.
+ * @tparam FRAME_TYPE type of the frame data
+ * @tparam PEDESTAL_TYPE type of the pedestal data
+ * @tparam CT type of the cluster data
+ */
+template <typename ClusterType = Cluster<int32_t, 3, 3>,
+          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
+class ClusterFinderMT {
+
+    using CT = typename extract_template_arguments<ClusterType>::value_type;
+    size_t m_current_thread{0};
+    size_t m_n_threads{0};
+    using Finder = ClusterFinder<ClusterType, FRAME_TYPE, PEDESTAL_TYPE>;
+    using InputQueue = ProducerConsumerQueue<FrameWrapper>;
+    using OutputQueue = ProducerConsumerQueue<ClusterVector<ClusterType>>;
+    std::vector<std::unique_ptr<InputQueue>> m_input_queues;
+    std::vector<std::unique_ptr<OutputQueue>> m_output_queues;
+
+    OutputQueue m_sink{1000}; // All clusters go into this queue
+
+    std::vector<std::unique_ptr<Finder>> m_cluster_finders;
+    std::vector<std::thread> m_threads;
+    std::thread m_collect_thread;
+    std::chrono::milliseconds m_default_wait{1};
+
+    std::atomic<bool> m_stop_requested{false};
+    std::atomic<bool> m_processing_threads_stopped{true};
+
+    /**
+     * @brief Function called by the processing threads. It reads the frames
+     * from the input queue and processes them.
+     */
+    void process(int thread_id) {
+        auto cf = m_cluster_finders[thread_id].get();
+        auto q = m_input_queues[thread_id].get();
+        bool realloc_same_capacity = true;
+
+        while (!m_stop_requested || !q->isEmpty()) {
+            if (FrameWrapper *frame = q->frontPtr(); frame != nullptr) {
+
+                switch (frame->type) {
+                case FrameType::DATA:
+                    cf->find_clusters(frame->data.view(), frame->frame_number);
+                    m_output_queues[thread_id]->write(
+                        cf->steal_clusters(realloc_same_capacity));
+                    break;
+
+                case FrameType::PEDESTAL:
+                    m_cluster_finders[thread_id]->push_pedestal_frame(
+                        frame->data.view());
+                    break;
+                }
+
+                // frame is processed now discard it
+                m_input_queues[thread_id]->popFront();
+            } else {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+    }
+
+    /**
+     * @brief Collect all the clusters from the output queues and write them to
+     * the sink
+     */
+    void collect() {
+        bool empty = true;
+        while (!m_stop_requested || !empty || !m_processing_threads_stopped) {
+            empty = true;
+            for (auto &queue : m_output_queues) {
+                if (!queue->isEmpty()) {
+
+                    while (!m_sink.write(std::move(*queue->frontPtr()))) {
+                        std::this_thread::sleep_for(m_default_wait);
+                    }
+                    queue->popFront();
+                    empty = false;
+                }
+            }
+        }
+    }
+
+  public:
+    /**
+     * @brief Construct a new ClusterFinderMT object
+     * @param image_size size of the image
+     * @param cluster_size size of the cluster
+     * @param nSigma number of sigma above the pedestal to consider a photon
+     * @param capacity initial capacity of the cluster vector. Should match
+     * expected number of clusters in a frame per frame.
+     * @param n_threads number of threads to use
+     */
+    ClusterFinderMT(Shape<2> image_size, PEDESTAL_TYPE nSigma = 5.0,
+                    size_t capacity = 2000, size_t n_threads = 3)
+        : m_n_threads(n_threads) {
+        for (size_t i = 0; i < n_threads; i++) {
+            m_cluster_finders.push_back(
+                std::make_unique<
+                    ClusterFinder<ClusterType, FRAME_TYPE, PEDESTAL_TYPE>>(
+                    image_size, nSigma, capacity));
+        }
+        for (size_t i = 0; i < n_threads; i++) {
+            m_input_queues.emplace_back(std::make_unique<InputQueue>(200));
+            m_output_queues.emplace_back(std::make_unique<OutputQueue>(200));
+        }
+        // TODO! Should we start automatically?
+        start();
+    }
+
+    /**
+     * @brief Return the sink queue where all the clusters are collected
+     * @warning You need to empty this queue otherwise the cluster finder will
+     * wait forever
+     */
+    ProducerConsumerQueue<ClusterVector<ClusterType>> *sink() {
+        return &m_sink;
+    }
+
+    /**
+     * @brief Start all processing threads
+     */
+    void start() {
+        m_processing_threads_stopped = false;
+        m_stop_requested = false;
+
+        for (size_t i = 0; i < m_n_threads; i++) {
+            m_threads.push_back(
+                std::thread(&ClusterFinderMT::process, this, i));
+        }
+
+        m_collect_thread = std::thread(&ClusterFinderMT::collect, this);
+    }
+
+    /**
+     * @brief Stop all processing threads
+     */
+    void stop() {
+        m_stop_requested = true;
+
+        for (auto &thread : m_threads) {
+            thread.join();
+        }
+        m_threads.clear();
+
+        m_processing_threads_stopped = true;
+        m_collect_thread.join();
+    }
+
+    /**
+     * @brief Wait for all the queues to be empty. Mostly used for timing tests.
+     */
+    void sync() {
+        for (auto &q : m_input_queues) {
+            while (!q->isEmpty()) {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+        for (auto &q : m_output_queues) {
+            while (!q->isEmpty()) {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+        while (!m_sink.isEmpty()) {
+            std::this_thread::sleep_for(m_default_wait);
+        }
+    }
+
+    /**
+     * @brief Push a pedestal frame to all the cluster finders. The frames is
+     * expected to be dark. No photon finding is done. Just pedestal update.
+     */
+    void push_pedestal_frame(NDView<FRAME_TYPE, 2> frame) {
+        FrameWrapper fw{FrameType::PEDESTAL, 0,
+                        NDArray(frame)}; // TODO! copies the data!
+
+        for (auto &queue : m_input_queues) {
+            while (!queue->write(fw)) {
+                std::this_thread::sleep_for(m_default_wait);
+            }
+        }
+    }
+
+    /**
+     * @brief Push the frame to the queue of the next available thread. Function
+     * returns once the frame is in a queue.
+     * @note Spin locks with a default wait if the queue is full.
+     */
+    void find_clusters(NDView<FRAME_TYPE, 2> frame, uint64_t frame_number = 0) {
+        FrameWrapper fw{FrameType::DATA, frame_number,
+                        NDArray(frame)}; // TODO! copies the data!
+        while (!m_input_queues[m_current_thread % m_n_threads]->write(fw)) {
+            std::this_thread::sleep_for(m_default_wait);
+        }
+        m_current_thread++;
+    }
+
+    void clear_pedestal() {
+        if (!m_processing_threads_stopped) {
+            throw std::runtime_error("ClusterFinderMT is still running");
+        }
+        for (auto &cf : m_cluster_finders) {
+            cf->clear_pedestal();
+        }
+    }
+
+    /**
+     * @brief Return the pedestal currently used by the cluster finder
+     * @param thread_index index of the thread
+     */
+    auto pedestal(size_t thread_index = 0) {
+        if (m_cluster_finders.empty()) {
+            throw std::runtime_error("No cluster finders available");
+        }
+        if (!m_processing_threads_stopped) {
+            throw std::runtime_error("ClusterFinderMT is still running");
+        }
+        if (thread_index >= m_cluster_finders.size()) {
+            throw std::runtime_error("Thread index out of range");
+        }
+        return m_cluster_finders[thread_index]->pedestal();
+    }
+
+    /**
+     * @brief Return the noise currently used by the cluster finder
+     * @param thread_index index of the thread
+     */
+    auto noise(size_t thread_index = 0) {
+        if (m_cluster_finders.empty()) {
+            throw std::runtime_error("No cluster finders available");
+        }
+        if (!m_processing_threads_stopped) {
+            throw std::runtime_error("ClusterFinderMT is still running");
+        }
+        if (thread_index >= m_cluster_finders.size()) {
+            throw std::runtime_error("Thread index out of range");
+        }
+        return m_cluster_finders[thread_index]->noise();
+    }
+
+    // void push(FrameWrapper&& frame) {
+    //     //TODO! need to loop until we are successful
+    //     auto rc = m_input_queue.write(std::move(frame));
+    //     fmt::print("pushed frame {}\n", rc);
+    // }
+};
+
+} // namespace aare

include/aare/ClusterVector.hpp

@@ -0,0 +1,389 @@
+#pragma once
+#include "aare/Cluster.hpp" //TODO maybe store in seperate file !!!
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <numeric>
+#include <vector>
+
+#include <fmt/core.h>
+
+#include "aare/Cluster.hpp"
+#include "aare/NDView.hpp"
+
+namespace aare {
+
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+class ClusterVector; // Forward declaration
+
+/**
+ * @brief ClusterVector is a container for clusters of various sizes. It uses a
+ * contiguous memory buffer to store the clusters. It is templated on the data
+ * type and the coordinate type of the clusters.
+ * @note push_back can invalidate pointers to elements in the container
+ * @warning ClusterVector is currently move only to catch unintended copies, but
+ * this might change since there are probably use cases where copying is needed.
+ * @tparam T data type of the pixels in the cluster
+ * @tparam CoordType data type of the x and y coordinates of the cluster
+ * (normally int16_t)
+ */
+#if 0
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
+
+    std::byte *m_data{};
+    size_t m_size{0};
+    size_t m_capacity;
+    uint64_t m_frame_number{0}; // TODO! Check frame number size and type
+    /** 
+    Format string used in the python bindings to create a numpy
+    array from the buffer
+    = - native byte order
+    h - short
+    d - double
+    i - int
+    */
+    constexpr static char m_fmt_base[] = "=h:x:\nh:y:\n({},{}){}:data:";
+
+  public:
+    using value_type = T;
+    using ClusterType = Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>;
+
+    /**
+     * @brief Construct a new ClusterVector object
+     * @param capacity initial capacity of the buffer in number of clusters
+     * @param frame_number frame number of the clusters. Default is 0, which is
+     * also used to indicate that the clusters come from many frames
+     */
+    ClusterVector(size_t capacity = 1024, uint64_t frame_number = 0)
+        : m_capacity(capacity), m_frame_number(frame_number) {
+        allocate_buffer(m_capacity);
+    }
+
+    ~ClusterVector() { delete[] m_data; }
+
+    // Move constructor
+    ClusterVector(ClusterVector &&other) noexcept
+        : m_data(other.m_data), m_size(other.m_size),
+          m_capacity(other.m_capacity), m_frame_number(other.m_frame_number) {
+        other.m_data = nullptr;
+        other.m_size = 0;
+        other.m_capacity = 0;
+    }
+
+    // Move assignment operator
+    ClusterVector &operator=(ClusterVector &&other) noexcept {
+        if (this != &other) {
+            delete[] m_data;
+            m_data = other.m_data;
+            m_size = other.m_size;
+            m_capacity = other.m_capacity;
+            m_frame_number = other.m_frame_number;
+            other.m_data = nullptr;
+            other.m_size = 0;
+            other.m_capacity = 0;
+            other.m_frame_number = 0;
+        }
+        return *this;
+    }
+
+    /**
+     * @brief Reserve space for at least capacity clusters
+     * @param capacity number of clusters to reserve space for
+     * @note If capacity is less than the current capacity, the function does
+     * nothing.
+     */
+    void reserve(size_t capacity) {
+        if (capacity > m_capacity) {
+            allocate_buffer(capacity);
+        }
+    }
+
+    /**
+     * @brief Add a cluster to the vector
+     */
+    void push_back(const ClusterType &cluster) {
+        if (m_size == m_capacity) {
+            allocate_buffer(m_capacity * 2);
+        }
+        std::byte *ptr = element_ptr(m_size);
+        *reinterpret_cast<CoordType *>(ptr) = cluster.x;
+        ptr += sizeof(CoordType);
+        *reinterpret_cast<CoordType *>(ptr) = cluster.y;
+        ptr += sizeof(CoordType);
+
+        std::memcpy(ptr, cluster.data, ClusterSizeX * ClusterSizeY * sizeof(T));
+
+        m_size++;
+    }
+
+    ClusterVector &operator+=(const ClusterVector &other) {
+        if (m_size + other.m_size > m_capacity) {
+            allocate_buffer(m_capacity + other.m_size);
+        }
+        std::copy(other.m_data, other.m_data + other.m_size * item_size(),
+                  m_data + m_size * item_size());
+        m_size += other.m_size;
+        return *this;
+    }
+
+    /**
+     * @brief Sum the pixels in each cluster
+     * @return std::vector<T> vector of sums for each cluster
+     */
+    /*
+    std::vector<T> sum() {
+        std::vector<T> sums(m_size);
+        const size_t stride = item_size();
+        const size_t n_pixels = ClusterSizeX * ClusterSizeY;
+        std::byte *ptr = m_data + 2 * sizeof(CoordType); // skip x and y
+
+        for (size_t i = 0; i < m_size; i++) {
+            sums[i] =
+                std::accumulate(reinterpret_cast<T *>(ptr),
+                                reinterpret_cast<T *>(ptr) + n_pixels, T{});
+            ptr += stride;
+        }
+        return sums;
+    }
+    */
+
+    /**
+     * @brief Sum the pixels in the 2x2 subcluster with the biggest pixel sum in
+     * each cluster
+     * @return std::vector<T> vector of sums for each cluster
+     */ //TODO if underlying container is a vector use std::for_each
+    /*
+    std::vector<T> sum_2x2() {
+        std::vector<T> sums_2x2(m_size);
+
+        for (size_t i = 0; i < m_size; i++) {
+            sums_2x2[i] = at(i).max_sum_2x2;
+        }
+        return sums_2x2;
+    }
+    */
+
+    /**
+     * @brief Return the number of clusters in the vector
+     */
+    size_t size() const { return m_size; }
+
+    uint8_t cluster_size_x() const { return ClusterSizeX; }
+
+    uint8_t cluster_size_y() const { return ClusterSizeY; }
+
+    /**
+     * @brief Return the capacity of the buffer in number of clusters. This is
+     * the number of clusters that can be stored in the current buffer without
+     * reallocation.
+     */
+    size_t capacity() const { return m_capacity; }
+
+    /**
+     * @brief Return the size in bytes of a single cluster
+     */
+    size_t item_size() const {
+        return 2 * sizeof(CoordType) + ClusterSizeX * ClusterSizeY * sizeof(T);
+    }
+
+    /**
+     * @brief Return the offset in bytes for the i-th cluster
+     */
+    size_t element_offset(size_t i) const { return item_size() * i; }
+
+    /**
+     * @brief Return a pointer to the i-th cluster
+     */
+    std::byte *element_ptr(size_t i) { return m_data + element_offset(i); }
+
+    /**
+     * @brief Return a pointer to the i-th cluster
+     */
+    const std::byte *element_ptr(size_t i) const {
+        return m_data + element_offset(i);
+    }
+
+    std::byte *data() { return m_data; }
+    std::byte const *data() const { return m_data; }
+
+    /**
+     * @brief Return a reference to the i-th cluster casted to type V
+     * @tparam V type of the cluster
+     */
+    ClusterType &at(size_t i) {
+        return *reinterpret_cast<ClusterType *>(element_ptr(i));
+    }
+
+    const ClusterType &at(size_t i) const {
+        return *reinterpret_cast<const ClusterType *>(element_ptr(i));
+    }
+
+    template <typename V> const V &at(size_t i) const {
+        return *reinterpret_cast<const V *>(element_ptr(i));
+    }
+
+    const std::string_view fmt_base() const {
+        // TODO! how do we match on coord_t?
+        return m_fmt_base;
+    }
+
+    /** 
+     * @brief Return the frame number of the clusters. 0 is used to indicate
+     * that the clusters come from many frames
+     */
+    uint64_t frame_number() const { return m_frame_number; }
+
+    void set_frame_number(uint64_t frame_number) {
+        m_frame_number = frame_number;
+    }
+
+    /** 
+     * @brief Resize the vector to contain new_size clusters. If new_size is
+     * greater than the current capacity, a new buffer is allocated. If the size
+     * is smaller no memory is freed, size is just updated.
+     * @param new_size new size of the vector
+     * @warning The additional clusters are not initialized
+     */
+    void resize(size_t new_size) {
+        // TODO! Should we initialize the new clusters?
+        if (new_size > m_capacity) {
+            allocate_buffer(new_size);
+        }
+        m_size = new_size;
+    }
+
+  private:
+    void allocate_buffer(size_t new_capacity) {
+        size_t num_bytes = item_size() * new_capacity;
+        std::byte *new_data = new std::byte[num_bytes]{};
+        std::copy(m_data, m_data + item_size() * m_size, new_data);
+        delete[] m_data;
+        m_data = new_data;
+        m_capacity = new_capacity;
+    }
+};
+#endif
+
+/**
+ * @brief ClusterVector is a container for clusters of various sizes. It
+ * uses a contiguous memory buffer to store the clusters. It is templated on
+ * the data type and the coordinate type of the clusters.
+ * @note push_back can invalidate pointers to elements in the container
+ * @warning ClusterVector is currently move only to catch unintended copies,
+ * but this might change since there are probably use cases where copying is
+ * needed.
+ * @tparam T data type of the pixels in the cluster
+ * @tparam CoordType data type of the x and y coordinates of the cluster
+ * (normally int16_t)
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
+
+    std::vector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> m_data{};
+    uint64_t m_frame_number{0}; // TODO! Check frame number size and type
+
+  public:
+    using value_type = T;
+    using ClusterType = Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>;
+
+    /**
+     * @brief Construct a new ClusterVector object
+     * @param capacity initial capacity of the buffer in number of clusters
+     * @param frame_number frame number of the clusters. Default is 0, which is
+     * also used to indicate that the clusters come from many frames
+     */
+    ClusterVector(size_t capacity = 1024, uint64_t frame_number = 0)
+        : m_frame_number(frame_number) {
+        m_data.reserve(capacity);
+    }
+
+    // Move constructor
+    ClusterVector(ClusterVector &&other) noexcept
+        : m_data(other.m_data), m_frame_number(other.m_frame_number) {
+        other.m_data.clear();
+    }
+
+    // Move assignment operator
+    ClusterVector &operator=(ClusterVector &&other) noexcept {
+        if (this != &other) {
+            m_data = other.m_data;
+            m_frame_number = other.m_frame_number;
+            other.m_data.clear();
+            other.m_frame_number = 0;
+        }
+        return *this;
+    }
+
+    /**
+     * @brief Reserve space for at least capacity clusters
+     * @param capacity number of clusters to reserve space for
+     * @note If capacity is less than the current capacity, the function does
+     * nothing.
+     */
+    void reserve(size_t capacity) { m_data.reserve(capacity); }
+
+    void resize(size_t size) { m_data.resize(size); }
+
+    void push_back(const ClusterType &cluster) { m_data.push_back(cluster); }
+
+    ClusterVector &operator+=(const ClusterVector &other) {
+        m_data.insert(m_data.end(), other.begin(), other.end());
+
+        return *this;
+    }
+
+    /**
+     * @brief Return the number of clusters in the vector
+     */
+    size_t size() const { return m_data.size(); }
+
+    uint8_t cluster_size_x() const { return ClusterSizeX; }
+
+    uint8_t cluster_size_y() const { return ClusterSizeY; }
+
+    /**
+     * @brief Return the capacity of the buffer in number of clusters. This is
+     * the number of clusters that can be stored in the current buffer without
+     * reallocation.
+     */
+    size_t capacity() const { return m_data.capacity(); }
+
+    const auto begin() const { return m_data.begin(); }
+
+    const auto end() const { return m_data.end(); }
+
+    /**
+     * @brief Return the size in bytes of a single cluster
+     */
+    size_t item_size() const {
+        return 2 * sizeof(CoordType) + ClusterSizeX * ClusterSizeY * sizeof(T);
+    }
+
+    ClusterType *data() { return m_data.data(); }
+    ClusterType const *data() const { return m_data.data(); }
+
+    /**
+     * @brief Return a reference to the i-th cluster casted to type V
+     * @tparam V type of the cluster
+     */
+    ClusterType &at(size_t i) { return m_data[i]; }
+
+    const ClusterType &at(size_t i) const { return m_data[i]; }
+
+    /**
+     * @brief Return the frame number of the clusters. 0 is used to indicate
+     * that the clusters come from many frames
+     */
+    uint64_t frame_number() const { return m_frame_number; }
+
+    void set_frame_number(uint64_t frame_number) {
+        m_frame_number = frame_number;
+    }
+};
+
+} // namespace aare

include/aare/File.hpp

@@ -36,6 +36,8 @@ class File {
     File(File &&other) noexcept;
     File& operator=(File &&other) noexcept;
     ~File() = default;
+
+    // void close();                             //!< close the file
       
     Frame read_frame();                       //!< read one frame from the file at the current position
     Frame read_frame(size_t frame_index);     //!< read one frame at the position given by frame number
@@ -44,6 +46,7 @@ class File {
     void read_into(std::byte *image_buf);
     void read_into(std::byte *image_buf, size_t n_frames);
     
+    size_t frame_number();                    //!< get the frame number at the current position
     size_t frame_number(size_t frame_index);  //!< get the frame number at the given frame index
     size_t bytes_per_frame() const;           
     size_t pixels_per_frame() const;    

include/aare/Fit.hpp

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <cmath>
+#include <fmt/core.h>
+#include <vector>
+
+#include "aare/NDArray.hpp"
+
+namespace aare {
+
+namespace func {
+double gaus(const double x, const double *par);
+NDArray<double, 1> gaus(NDView<double, 1> x, NDView<double, 1> par);
+
+double pol1(const double x, const double *par);
+NDArray<double, 1> pol1(NDView<double, 1> x, NDView<double, 1> par);
+
+} // namespace func
+
+
+/**
+ * @brief Estimate the initial parameters for a Gaussian fit
+ */
+std::array<double, 3> gaus_init_par(const NDView<double, 1> x, const NDView<double, 1> y);
+
+std::array<double, 2> pol1_init_par(const NDView<double, 1> x, const NDView<double, 1> y);
+
+static constexpr int DEFAULT_NUM_THREADS = 4;
+
+/**
+ * @brief Fit a 1D Gaussian to data.
+ * @param data data to fit
+ * @param x x values
+ */
+NDArray<double, 1> fit_gaus(NDView<double, 1> x, NDView<double, 1> y);
+
+
+/**
+ * @brief Fit a 1D Gaussian to each pixel. Data layout [row, col, values]
+ * @param x x values
+ * @param y y vales, layout [row, col, values]
+ * @param n_threads number of threads to use
+ */
+
+NDArray<double, 3> fit_gaus(NDView<double, 1> x, NDView<double, 3> y,
+                            int n_threads = DEFAULT_NUM_THREADS);
+
+
+
+
+/**
+ * @brief Fit a 1D Gaussian with error estimates
+ * @param x x values
+ * @param y y vales, layout [row, col, values]
+ * @param y_err error in y, layout [row, col, values]
+ * @param par_out output parameters
+ * @param par_err_out output error parameters
+ */
+void fit_gaus(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
+              NDView<double, 1> par_out, NDView<double, 1> par_err_out,
+              double& chi2);
+
+/**
+ * @brief Fit a 1D Gaussian to each pixel with error estimates. Data layout
+ * [row, col, values]
+ * @param x x values
+ * @param y y vales, layout [row, col, values]
+ * @param y_err error in y, layout [row, col, values]
+ * @param par_out output parameters, layout [row, col, values]
+ * @param par_err_out output parameter errors, layout [row, col, values]
+ * @param n_threads number of threads to use
+ */
+void fit_gaus(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out, NDView<double, 2> chi2_out,
+              int n_threads = DEFAULT_NUM_THREADS
+              );
+
+NDArray<double, 1> fit_pol1(NDView<double, 1> x, NDView<double, 1> y);
+
+NDArray<double, 3> fit_pol1(NDView<double, 1> x, NDView<double, 3> y,
+                            int n_threads = DEFAULT_NUM_THREADS);
+
+void fit_pol1(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
+              NDView<double, 1> par_out, NDView<double, 1> par_err_out, double& chi2);
+
+// TODO! not sure we need to offer the different version in C++
+void fit_pol1(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out,NDView<double, 2> chi2_out,
+              int n_threads = DEFAULT_NUM_THREADS);
+
+
+} // namespace aare

include/aare/GainMap.hpp

@@ -0,0 +1,58 @@
+/************************************************
+ * @file ApplyGainMap.hpp
+ * @short function to apply gain map of image size to a vector of clusters
+ ***********************************************/
+
+#pragma once
+#include "aare/Cluster.hpp"
+#include "aare/ClusterVector.hpp"
+#include "aare/NDArray.hpp"
+#include "aare/NDView.hpp"
+#include <memory>
+
+namespace aare {
+
+class GainMap {
+
+  public:
+    explicit GainMap(const NDArray<double, 2> &gain_map)
+        : m_gain_map(gain_map) {};
+
+    explicit GainMap(const NDView<double, 2> gain_map) {
+        m_gain_map = NDArray<double, 2>(gain_map);
+    }
+
+    template <typename ClusterType,
+              typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+    void apply_gain_map(ClusterVector<ClusterType> &clustervec) {
+        // in principle we need to know the size of the image for this lookup
+        size_t ClusterSizeX = clustervec.cluster_size_x();
+        size_t ClusterSizeY = clustervec.cluster_size_y();
+
+        using T = typename ClusterVector<ClusterType>::value_type;
+
+        int64_t index_cluster_center_x = ClusterSizeX / 2;
+        int64_t index_cluster_center_y = ClusterSizeY / 2;
+        for (size_t i = 0; i < clustervec.size(); i++) {
+            auto &cl = clustervec.at(i);
+
+            if (cl.x > 0 && cl.y > 0 && cl.x < m_gain_map.shape(1) - 1 &&
+                cl.y < m_gain_map.shape(0) - 1) {
+                for (size_t j = 0; j < ClusterSizeX * ClusterSizeY; j++) {
+                    size_t x = cl.x + j % ClusterSizeX - index_cluster_center_x;
+                    size_t y = cl.y + j / ClusterSizeX - index_cluster_center_y;
+                    cl.data[j] = cl.data[j] * static_cast<T>(m_gain_map(y, x));
+                }
+            } else {
+                memset(cl.data, 0,
+                       ClusterSizeX * ClusterSizeY *
+                           sizeof(T)); // clear edge clusters
+            }
+        }
+    }
+
+  private:
+    NDArray<double, 2> m_gain_map{};
+};
+
+} // end of namespace aare

include/aare/Interpolator.hpp

@@ -0,0 +1,132 @@
+#pragma once
+
+#include "aare/CalculateEta.hpp"
+#include "aare/Cluster.hpp"
+#include "aare/ClusterFile.hpp" //Cluster_3x3
+#include "aare/ClusterVector.hpp"
+#include "aare/NDArray.hpp"
+#include "aare/NDView.hpp"
+#include "aare/algorithm.hpp"
+
+namespace aare {
+
+struct Photon {
+    double x;
+    double y;
+    double energy;
+};
+
+class Interpolator {
+    NDArray<double, 3> m_ietax;
+    NDArray<double, 3> m_ietay;
+
+    NDArray<double, 1> m_etabinsx;
+    NDArray<double, 1> m_etabinsy;
+    NDArray<double, 1> m_energy_bins;
+
+  public:
+    Interpolator(NDView<double, 3> etacube, NDView<double, 1> xbins,
+                 NDView<double, 1> ybins, NDView<double, 1> ebins);
+    NDArray<double, 3> get_ietax() { return m_ietax; }
+    NDArray<double, 3> get_ietay() { return m_ietay; }
+
+    template <typename ClusterType,
+              typename Eanble = std::enable_if_t<is_cluster_v<ClusterType>>>
+    std::vector<Photon> interpolate(const ClusterVector<ClusterType> &clusters);
+};
+
+// TODO: generalize to support any clustertype!!! otherwise add std::enable_if_t
+// to only take Cluster2x2 and Cluster3x3
+template <typename ClusterType, typename Enable>
+std::vector<Photon>
+Interpolator::interpolate(const ClusterVector<ClusterType> &clusters) {
+    std::vector<Photon> photons;
+    photons.reserve(clusters.size());
+
+    if (clusters.cluster_size_x() == 3 || clusters.cluster_size_y() == 3) {
+        for (size_t i = 0; i < clusters.size(); i++) {
+
+            auto cluster = clusters.at(i);
+            auto eta = calculate_eta2(cluster);
+
+            Photon photon;
+            photon.x = cluster.x;
+            photon.y = cluster.y;
+            photon.energy = eta.sum;
+
+            // auto ie = nearest_index(m_energy_bins, photon.energy)-1;
+            // auto ix = nearest_index(m_etabinsx, eta.x)-1;
+            // auto iy = nearest_index(m_etabinsy, eta.y)-1;
+            // Finding the index of the last element that is smaller
+            // should work fine as long as we have many bins
+            auto ie = last_smaller(m_energy_bins, photon.energy);
+            auto ix = last_smaller(m_etabinsx, eta.x);
+            auto iy = last_smaller(m_etabinsy, eta.y);
+
+            // fmt::print("ex: {}, ix: {}, iy: {}\n", ie, ix, iy);
+
+            double dX, dY;
+            // cBottomLeft = 0,
+            // cBottomRight = 1,
+            // cTopLeft = 2,
+            // cTopRight = 3
+            switch (eta.c) {
+            case cTopLeft:
+                dX = -1.;
+                dY = 0;
+                break;
+            case cTopRight:;
+                dX = 0;
+                dY = 0;
+                break;
+            case cBottomLeft:
+                dX = -1.;
+                dY = -1.;
+                break;
+            case cBottomRight:
+                dX = 0.;
+                dY = -1.;
+                break;
+            }
+            photon.x += m_ietax(ix, iy, ie) * 2 + dX;
+            photon.y += m_ietay(ix, iy, ie) * 2 + dY;
+            photons.push_back(photon);
+        }
+    } else if (clusters.cluster_size_x() == 2 ||
+               clusters.cluster_size_y() == 2) {
+        for (size_t i = 0; i < clusters.size(); i++) {
+            auto cluster = clusters.at(i);
+            auto eta = calculate_eta2(cluster);
+
+            Photon photon;
+            photon.x = cluster.x;
+            photon.y = cluster.y;
+            photon.energy = eta.sum;
+
+            // Now do some actual interpolation.
+            // Find which energy bin the cluster is in
+            //  auto ie = nearest_index(m_energy_bins, photon.energy)-1;
+            //  auto ix = nearest_index(m_etabinsx, eta.x)-1;
+            //  auto iy = nearest_index(m_etabinsy, eta.y)-1;
+            // Finding the index of the last element that is smaller
+            // should work fine as long as we have many bins
+            auto ie = last_smaller(m_energy_bins, photon.energy);
+            auto ix = last_smaller(m_etabinsx, eta.x);
+            auto iy = last_smaller(m_etabinsy, eta.y);
+
+            photon.x += m_ietax(ix, iy, ie) *
+                        2; // eta goes between 0 and 1 but we could move the hit
+                           // anywhere in the 2x2
+            photon.y += m_ietay(ix, iy, ie) * 2;
+            photons.push_back(photon);
+        }
+
+    } else {
+        throw std::runtime_error(
+            "Only 3x3 and 2x2 clusters are supported for interpolation");
+    }
+
+    return photons;
+}
+
+} // namespace aare

include/aare/NDArray.hpp

@@ -69,6 +69,11 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
         std::copy(v.begin(), v.end(), begin());
     }
 
+    template<size_t Size>
+    NDArray(const std::array<T, Size>& arr) : NDArray<T,1>({Size}) {
+        std::copy(arr.begin(), arr.end(), begin());
+    }
+
     // Move constructor
     NDArray(NDArray &&other) noexcept
         : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)),
@@ -87,7 +92,7 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     // Conversion operator from array expression to array
     template <typename E>
     NDArray(ArrayExpr<E, Ndim> &&expr) : NDArray(expr.shape()) {
-        for (int i = 0; i < size_; ++i) {
+        for (size_t i = 0; i < size_; ++i) {
             data_[i] = expr[i];
         }
     }
@@ -97,6 +102,9 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     auto begin() { return data_; }
     auto end() { return data_ + size_; }
 
+    auto begin() const { return data_; }
+    auto end() const { return data_ + size_; }
+
     using value_type = T;
 
     NDArray &operator=(NDArray &&other) noexcept; // Move assign
@@ -105,6 +113,20 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     NDArray &operator-=(const NDArray &other);
     NDArray &operator*=(const NDArray &other);
 
+    //Write directly to the data array, or create a new one
+    template<size_t Size>
+    NDArray<T,1>& operator=(const std::array<T,Size> &other){
+        if(Size != size_){
+            delete[] data_;
+            size_ = Size;
+            data_ = new T[size_];
+        }
+        for (size_t i = 0; i < Size; ++i) {
+            data_[i] = other[i];
+        }
+        return *this;
+    }
+
     // NDArray& operator/=(const NDArray& other);
 
     template <typename V> NDArray &operator/=(const NDArray<V, Ndim> &other) {
@@ -135,6 +157,11 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
 
     NDArray &operator&=(const T & /*mask*/);
 
+  
+
+
+    
+
     void sqrt() {
         for (int i = 0; i < size_; ++i) {
             data_[i] = std::sqrt(data_[i]);
@@ -159,11 +186,11 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     }
 
     // TODO! is int the right type for index?
-    T &operator()(int i) { return data_[i]; }
-    const T &operator()(int i) const { return data_[i]; }
+    T &operator()(int64_t i) { return data_[i]; }
+    const T &operator()(int64_t i) const { return data_[i]; }
 
-    T &operator[](int i) { return data_[i]; }
-    const T &operator[](int i) const { return data_[i]; }
+    T &operator[](int64_t i) { return data_[i]; }
+    const T &operator[](int64_t i) const { return data_[i]; }
 
     T *data() { return data_; }
     std::byte *buffer() { return reinterpret_cast<std::byte *>(data_); }
@@ -318,6 +345,9 @@ NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const T &value) {
     return *this;
 }
 
+
+
+
 template <typename T, int64_t Ndim>
 NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const T &value) {
     NDArray result = *this;
@@ -361,12 +391,12 @@ NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const T &value) {
     result *= value;
     return result;
 }
-template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
-    if (shape_[0] < 20 && shape_[1] < 20)
-        Print_all();
-    else
-        Print_some();
-}
+// template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
+//     if (shape_[0] < 20 && shape_[1] < 20)
+//         Print_all();
+//     else
+//         Print_some();
+// }
 
 template <typename T, int64_t Ndim>
 std::ostream &operator<<(std::ostream &os, const NDArray<T, Ndim> &arr) {
@@ -418,4 +448,6 @@ NDArray<T, Ndim> load(const std::string &pathname,
     return img;
 }
 
+
+
 } // namespace aare

include/aare/NDView.hpp

@@ -1,5 +1,5 @@
 #pragma once
-
+#include "aare/defs.hpp"
 #include "aare/ArrayExpr.hpp"
 
 #include <algorithm>
@@ -99,6 +99,15 @@ template <typename T, int64_t Ndim = 2> class NDView : public ArrayExpr<NDView<T
 
     NDView &operator/=(const NDView &other) { return elemenwise(other, std::divides<T>()); }
 
+
+    template<size_t Size>
+    NDView& operator=(const std::array<T, Size> &arr) {
+        if(size() != arr.size())
+            throw std::runtime_error(LOCATION + "Array and NDView size mismatch");
+        std::copy(arr.begin(), arr.end(), begin());
+        return *this;
+    }
+
     NDView &operator=(const T val) {
         for (auto it = begin(); it != end(); ++it)
             *it = val;

include/aare/Pedestal.hpp

@@ -18,34 +18,48 @@ template <typename SUM_TYPE = double> class Pedestal {
 
     uint32_t m_samples;
     NDArray<uint32_t, 2> m_cur_samples;
+    
+    //TODO! in case of int needs to be changed to uint64_t
     NDArray<SUM_TYPE, 2> m_sum;
     NDArray<SUM_TYPE, 2> m_sum2;
 
+    //Cache mean since it is used over and over in the ClusterFinder
+    //This optimization is related to the access pattern of the ClusterFinder
+    //Relies on having more reads than pushes to the pedestal
+    NDArray<SUM_TYPE, 2> m_mean; 
+
   public:
     Pedestal(uint32_t rows, uint32_t cols, uint32_t n_samples = 1000)
         : m_rows(rows), m_cols(cols), m_samples(n_samples),
           m_cur_samples(NDArray<uint32_t, 2>({rows, cols}, 0)),
           m_sum(NDArray<SUM_TYPE, 2>({rows, cols})),
-          m_sum2(NDArray<SUM_TYPE, 2>({rows, cols})) {
+          m_sum2(NDArray<SUM_TYPE, 2>({rows, cols})),
+          m_mean(NDArray<SUM_TYPE, 2>({rows, cols})) {
         assert(rows > 0 && cols > 0 && n_samples > 0);
         m_sum = 0;
         m_sum2 = 0;
+        m_mean = 0;
     }
     ~Pedestal() = default;
 
     NDArray<SUM_TYPE, 2> mean() {
-        NDArray<SUM_TYPE, 2> mean_array({m_rows, m_cols});
-        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
-            mean_array(i / m_cols, i % m_cols) = mean(i / m_cols, i % m_cols);
-        }
-        return mean_array;
+        return m_mean;
     }
 
     SUM_TYPE mean(const uint32_t row, const uint32_t col) const {
+        return m_mean(row, col);
+    }
+
+    SUM_TYPE std(const uint32_t row, const uint32_t col) const {
+        return std::sqrt(variance(row, col));
+    }
+
+    SUM_TYPE variance(const uint32_t row, const uint32_t col) const {
         if (m_cur_samples(row, col) == 0) {
             return 0.0;
         }
-        return m_sum(row, col) / m_cur_samples(row, col);
+        return m_sum2(row, col) / m_cur_samples(row, col) -
+               mean(row, col) * mean(row, col);
     }
 
     NDArray<SUM_TYPE, 2> variance() {
@@ -57,13 +71,7 @@ template <typename SUM_TYPE = double> class Pedestal {
         return variance_array;
     }
 
-    SUM_TYPE variance(const uint32_t row, const uint32_t col) const {
-        if (m_cur_samples(row, col) == 0) {
-            return 0.0;
-        }
-        return m_sum2(row, col) / m_cur_samples(row, col) -
-               mean(row, col) * mean(row, col);
-    }
+    
 
     NDArray<SUM_TYPE, 2> std() {
         NDArray<SUM_TYPE, 2> standard_deviation_array({m_rows, m_cols});
@@ -75,14 +83,13 @@ template <typename SUM_TYPE = double> class Pedestal {
         return standard_deviation_array;
     }
 
-    SUM_TYPE std(const uint32_t row, const uint32_t col) const {
-        return std::sqrt(variance(row, col));
-    }
+    
 
     void clear() {
-        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
-            clear(i / m_cols, i % m_cols);
-        }
+        m_sum = 0;
+        m_sum2 = 0;
+        m_cur_samples = 0;
+        m_mean = 0;
     }
 
     
@@ -91,8 +98,11 @@ template <typename SUM_TYPE = double> class Pedestal {
         m_sum(row, col) = 0;
         m_sum2(row, col) = 0;
         m_cur_samples(row, col) = 0;
+        m_mean(row, col) = 0;
     }
-    // frame level operations
+    
+
+
     template <typename T> void push(NDView<T, 2> frame) {
         assert(frame.size() == m_rows * m_cols);
 
@@ -102,17 +112,37 @@ template <typename SUM_TYPE = double> class Pedestal {
                 "Frame shape does not match pedestal shape");
         }
 
-        for (uint32_t row = 0; row < m_rows; row++) {
-            for (uint32_t col = 0; col < m_cols; col++) {
+        for (size_t row = 0; row < m_rows; row++) {
+            for (size_t col = 0; col < m_cols; col++) {
                 push<T>(row, col, frame(row, col));
             }
         }
-
-        // // TODO: test the effect of #pragma omp parallel for
-        // for (uint32_t index = 0; index < m_rows * m_cols; index++) {
-        //     push<T>(index / m_cols, index % m_cols, frame(index));
-        // }
     }
+
+    /**
+     * Push but don't update the cached mean. Speeds up the process
+     * when initializing the pedestal.
+     * 
+     */
+    template <typename T> void push_no_update(NDView<T, 2> frame) {
+        assert(frame.size() == m_rows * m_cols);
+
+        // TODO! move away from m_rows, m_cols
+        if (frame.shape() != std::array<int64_t, 2>{m_rows, m_cols}) {
+            throw std::runtime_error(
+                "Frame shape does not match pedestal shape");
+        }
+
+        for (size_t row = 0; row < m_rows; row++) {
+            for (size_t col = 0; col < m_cols; col++) {
+                push_no_update<T>(row, col, frame(row, col));
+            }
+        }
+    }
+
+
+
+
     template <typename T> void push(Frame &frame) {
         assert(frame.rows() == static_cast<size_t>(m_rows) &&
                frame.cols() == static_cast<size_t>(m_cols));
@@ -132,18 +162,48 @@ template <typename SUM_TYPE = double> class Pedestal {
     template <typename T>
     void push(const uint32_t row, const uint32_t col, const T val_) {
         SUM_TYPE val = static_cast<SUM_TYPE>(val_);
-        const uint32_t idx = index(row, col);
-        if (m_cur_samples(idx) < m_samples) {
-            m_sum(idx) += val;
-            m_sum2(idx) += val * val;
-            m_cur_samples(idx)++;
+        if (m_cur_samples(row, col) < m_samples) {
+            m_sum(row, col) += val;
+            m_sum2(row, col) += val * val;
+            m_cur_samples(row, col)++;
         } else {
-            m_sum(idx) += val - m_sum(idx) / m_cur_samples(idx);
-            m_sum2(idx) += val * val - m_sum2(idx) / m_cur_samples(idx);
+            m_sum(row, col) += val - m_sum(row, col) / m_samples;
+            m_sum2(row, col) += val * val - m_sum2(row, col) / m_samples;
+        }
+        //Since we just did a push we know that m_cur_samples(row, col) is at least 1
+        m_mean(row, col) = m_sum(row, col) / m_cur_samples(row, col);
+    }
+
+    template <typename T>
+    void push_no_update(const uint32_t row, const uint32_t col, const T val_) {
+        SUM_TYPE val = static_cast<SUM_TYPE>(val_);
+        if (m_cur_samples(row, col) < m_samples) {
+            m_sum(row, col) += val;
+            m_sum2(row, col) += val * val;
+            m_cur_samples(row, col)++;
+        } else {
+            m_sum(row, col) += val - m_sum(row, col) / m_cur_samples(row, col);
+            m_sum2(row, col) += val * val - m_sum2(row, col) / m_cur_samples(row, col);
         }
     }
-    uint32_t index(const uint32_t row, const uint32_t col) const {
-        return row * m_cols + col;
-    };
+
+    /**
+     * @brief Update the mean of the pedestal. This is used after having done
+     * push_no_update. It is not necessary to call this function after push.
+     */
+    void update_mean(){
+        m_mean = m_sum / m_cur_samples;
+    }
+
+    template<typename T>
+    void push_fast(const uint32_t row, const uint32_t col, const T val_){
+        //Assume we reached the steady state where all pixels have
+        //m_samples samples
+        SUM_TYPE val = static_cast<SUM_TYPE>(val_);
+        m_sum(row, col) += val - m_sum(row, col) / m_samples;
+        m_sum2(row, col) += val * val - m_sum2(row, col) / m_samples;
+        m_mean(row, col) = m_sum(row, col) / m_samples;
+    }
+
 };
 } // namespace aare

include/aare/PixelMap.hpp

@@ -7,6 +7,8 @@ namespace aare {
 
 NDArray<ssize_t, 2> GenerateMoench03PixelMap();
 NDArray<ssize_t, 2> GenerateMoench05PixelMap();
+NDArray<ssize_t, 2> GenerateMoench05PixelMap1g();
+NDArray<ssize_t, 2> GenerateMoench05PixelMapOld();
 
 //Matterhorn02
 NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap();

include/aare/ProducerConsumerQueue.hpp

@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @author Bo Hu (bhu@fb.com)
+// @author Jordan DeLong (delong.j@fb.com)
+
+// Changes made by PSD Detector Group:
+// Copied: Line 34 constexpr std::size_t hardware_destructive_interference_size = 128; from folly/lang/Align.h
+// Changed extension to .hpp
+// Changed namespace to aare
+
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <memory>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+constexpr std::size_t hardware_destructive_interference_size = 128;
+namespace aare {
+
+/*
+ * ProducerConsumerQueue is a one producer and one consumer queue
+ * without locks.
+ */
+template <class T> struct ProducerConsumerQueue {
+    typedef T value_type;
+
+    ProducerConsumerQueue(const ProducerConsumerQueue &) = delete;
+    ProducerConsumerQueue &operator=(const ProducerConsumerQueue &) = delete;
+
+    
+    ProducerConsumerQueue(ProducerConsumerQueue &&other){
+        size_ = other.size_;
+        records_ = other.records_;
+        other.records_ = nullptr;
+        readIndex_ = other.readIndex_.load(std::memory_order_acquire);
+        writeIndex_ = other.writeIndex_.load(std::memory_order_acquire);
+    }
+    ProducerConsumerQueue &operator=(ProducerConsumerQueue &&other){
+        size_ = other.size_;
+        records_ = other.records_;
+        other.records_ = nullptr;
+        readIndex_ = other.readIndex_.load(std::memory_order_acquire);
+        writeIndex_ = other.writeIndex_.load(std::memory_order_acquire);
+        return *this;
+    }
+    
+    
+    ProducerConsumerQueue():ProducerConsumerQueue(2){};
+    // size must be >= 2.
+    //
+    // Also, note that the number of usable slots in the queue at any
+    // given time is actually (size-1), so if you start with an empty queue,
+    // isFull() will return true after size-1 insertions.
+    explicit ProducerConsumerQueue(uint32_t size)
+        : size_(size), records_(static_cast<T *>(std::malloc(sizeof(T) * size))), readIndex_(0), writeIndex_(0) {
+        assert(size >= 2);
+        if (!records_) {
+            throw std::bad_alloc();
+        }
+    }
+
+    ~ProducerConsumerQueue() {
+        // We need to destruct anything that may still exist in our queue.
+        // (No real synchronization needed at destructor time: only one
+        // thread can be doing this.)
+        if (!std::is_trivially_destructible<T>::value) {
+            size_t readIndex = readIndex_;
+            size_t endIndex = writeIndex_;
+            while (readIndex != endIndex) {
+                records_[readIndex].~T();
+                if (++readIndex == size_) {
+                    readIndex = 0;
+                }
+            }
+        }
+
+        std::free(records_);
+    }
+
+    template <class... Args> bool write(Args &&...recordArgs) {
+        auto const currentWrite = writeIndex_.load(std::memory_order_relaxed);
+        auto nextRecord = currentWrite + 1;
+        if (nextRecord == size_) {
+            nextRecord = 0;
+        }
+        if (nextRecord != readIndex_.load(std::memory_order_acquire)) {
+            new (&records_[currentWrite]) T(std::forward<Args>(recordArgs)...);
+            writeIndex_.store(nextRecord, std::memory_order_release);
+            return true;
+        }
+
+        // queue is full
+        return false;
+    }
+
+    // move (or copy) the value at the front of the queue to given variable
+    bool read(T &record) {
+        auto const currentRead = readIndex_.load(std::memory_order_relaxed);
+        if (currentRead == writeIndex_.load(std::memory_order_acquire)) {
+            // queue is empty
+            return false;
+        }
+
+        auto nextRecord = currentRead + 1;
+        if (nextRecord == size_) {
+            nextRecord = 0;
+        }
+        record = std::move(records_[currentRead]);
+        records_[currentRead].~T();
+        readIndex_.store(nextRecord, std::memory_order_release);
+        return true;
+    }
+
+    // pointer to the value at the front of the queue (for use in-place) or
+    // nullptr if empty.
+    T *frontPtr() {
+        auto const currentRead = readIndex_.load(std::memory_order_relaxed);
+        if (currentRead == writeIndex_.load(std::memory_order_acquire)) {
+            // queue is empty
+            return nullptr;
+        }
+        return &records_[currentRead];
+    }
+
+    // queue must not be empty
+    void popFront() {
+        auto const currentRead = readIndex_.load(std::memory_order_relaxed);
+        assert(currentRead != writeIndex_.load(std::memory_order_acquire));
+
+        auto nextRecord = currentRead + 1;
+        if (nextRecord == size_) {
+            nextRecord = 0;
+        }
+        records_[currentRead].~T();
+        readIndex_.store(nextRecord, std::memory_order_release);
+    }
+
+    bool isEmpty() const {
+        return readIndex_.load(std::memory_order_acquire) == writeIndex_.load(std::memory_order_acquire);
+    }
+
+    bool isFull() const {
+        auto nextRecord = writeIndex_.load(std::memory_order_acquire) + 1;
+        if (nextRecord == size_) {
+            nextRecord = 0;
+        }
+        if (nextRecord != readIndex_.load(std::memory_order_acquire)) {
+            return false;
+        }
+        // queue is full
+        return true;
+    }
+
+    // * If called by consumer, then true size may be more (because producer may
+    //   be adding items concurrently).
+    // * If called by producer, then true size may be less (because consumer may
+    //   be removing items concurrently).
+    // * It is undefined to call this from any other thread.
+    size_t sizeGuess() const {
+        int ret = writeIndex_.load(std::memory_order_acquire) - readIndex_.load(std::memory_order_acquire);
+        if (ret < 0) {
+            ret += size_;
+        }
+        return ret;
+    }
+
+    // maximum number of items in the queue.
+    size_t capacity() const { return size_ - 1; }
+
+  private:
+    using AtomicIndex = std::atomic<unsigned int>;
+
+    char pad0_[hardware_destructive_interference_size];
+    // const uint32_t size_;
+    uint32_t size_;
+    // T *const records_;
+    T* records_;
+
+    alignas(hardware_destructive_interference_size) AtomicIndex readIndex_;
+    alignas(hardware_destructive_interference_size) AtomicIndex writeIndex_;
+
+    char pad1_[hardware_destructive_interference_size - sizeof(AtomicIndex)];
+};
+
+} // namespace aare

include/aare/RawFile.hpp

@@ -34,15 +34,19 @@ class RawFile : public FileInterface {
     size_t n_subfile_parts{}; // d0,d1...dn
     //TODO! move to vector of SubFile instead of pointers
     std::vector<std::vector<RawSubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
-    std::vector<xy> positions;
-    std::vector<ModuleGeometry> m_module_pixel_0;
+    // std::vector<xy> positions;
+    
     ModuleConfig cfg{0, 0};
 
     RawMasterFile m_master;
 
     size_t m_current_frame{};
-    size_t m_rows{};
-    size_t m_cols{};
+
+    // std::vector<ModuleGeometry> m_module_pixel_0;
+    // size_t m_rows{};
+    // size_t m_cols{};
+
+    DetectorGeometry m_geometry;
 
   public:
     /**
@@ -111,11 +115,12 @@ class RawFile : public FileInterface {
      */
     static DetectorHeader read_header(const std::filesystem::path &fname);
 
-    void update_geometry_with_roi();
+    // void update_geometry_with_roi();
     int find_number_of_subfiles();
 
     void open_subfiles();
     void find_geometry();
 };
 
+
 } // namespace aare

include/aare/RawMasterFile.hpp

@@ -14,6 +14,7 @@ namespace aare {
  * @brief Implementation used in RawMasterFile to parse the file name
  */
 class RawFileNameComponents {
+    bool m_old_scheme{false};
     std::filesystem::path m_base_path{};
     std::string m_base_name{};
     std::string m_ext{};
@@ -35,6 +36,7 @@ class RawFileNameComponents {
     const std::string &base_name() const;
     const std::string &ext() const;
     int file_index() const;
+    void set_old_scheme(bool old_scheme);
 };
 
 class ScanParameters {
@@ -60,17 +62,6 @@ class ScanParameters {
 };
 
 
-struct ROI{
-  size_t xmin{};
-  size_t xmax{};
-  size_t ymin{};
-  size_t ymax{};
-
-  size_t height() const { return ymax - ymin; }
-  size_t width() const { return xmax - xmin; }
-}__attribute__((packed));
-
-
 /**
  * @brief Class for parsing a master file either in our .json format or the old
  * .raw format
@@ -88,10 +79,10 @@ class RawMasterFile {
     size_t m_pixels_x{};
     size_t m_bitdepth{};
 
-    xy m_geometry;
+    xy m_geometry{};
 
     size_t m_max_frames_per_file{};
-    uint32_t m_adc_mask{};
+    // uint32_t m_adc_mask{}; // TODO! implement reading
     FrameDiscardPolicy m_frame_discard_policy{};
     size_t m_frame_padding{};
 

include/aare/RawSubFile.hpp

@@ -16,6 +16,7 @@ namespace aare {
 class RawSubFile {
   protected:
     std::ifstream m_file;
+    DetectorType m_detector_type;
     size_t m_bitdepth;
     std::filesystem::path m_fname;
     size_t m_rows{};
@@ -25,7 +26,7 @@ class RawSubFile {
     uint32_t m_pos_row{};
     uint32_t m_pos_col{};
  
-    DetectorType m_detector_type;
+    
     std::optional<NDArray<ssize_t, 2>> m_pixel_map;
 
   public:
@@ -63,8 +64,11 @@ class RawSubFile {
 
     size_t bytes_per_frame() const { return m_bytes_per_frame; }
     size_t pixels_per_frame() const { return m_rows * m_cols; }
-    size_t bytes_per_pixel() const { return m_bitdepth / 8; }
+    size_t bytes_per_pixel() const { return m_bitdepth / bits_per_byte; }
 
+private:
+  template <typename T>
+  void read_with_map(std::byte *image_buf);
 
 };
 

include/aare/VarClusterFinder.hpp

@@ -7,7 +7,7 @@
 
 #include "aare/NDArray.hpp"
 
-const int MAX_CLUSTER_SIZE = 200;
+const int MAX_CLUSTER_SIZE = 50;
 namespace aare {
 
 template <typename T> class VarClusterFinder {
@@ -226,7 +226,7 @@ template <typename T> void VarClusterFinder<T>::single_pass(NDView<T, 2> img) {
 
 template <typename T> void VarClusterFinder<T>::first_pass() {
 
-    for (int i = 0; i < original_.size(); ++i) {
+    for (size_t i = 0; i < original_.size(); ++i) {
         if (use_noise_map)
             threshold_ = 5 * noiseMap(i);
         binary_(i) = (original_(i) > threshold_);
@@ -250,17 +250,17 @@ template <typename T> void VarClusterFinder<T>::first_pass() {
 
 template <typename T> void VarClusterFinder<T>::second_pass() {
 
-    for (int64_t i = 0; i != labeled_.size(); ++i) {
-        auto current_label = labeled_(i);
-        if (current_label != 0) {
-            auto it = child.find(current_label);
+    for (size_t i = 0; i != labeled_.size(); ++i) {
+        auto cl = labeled_(i);
+        if (cl != 0) {
+            auto it = child.find(cl);
             while (it != child.end()) {
-                current_label = it->second;
-                it = child.find(current_label);
+                cl = it->second;
+                it = child.find(cl);
                 // do this once before doing the second pass?
                 // all values point to the final one...
             }
-            labeled_(i) = current_label;
+            labeled_(i) = cl;
         }
     }
 }
@@ -271,7 +271,7 @@ template <typename T> void VarClusterFinder<T>::store_clusters() {
     // Do we always have monotonic increasing
     // labels? Then vector?
     // here the translation is label -> Hit
-    std::unordered_map<int, Hit> h_size;
+    std::unordered_map<int, Hit> h_map;
     for (int i = 0; i < shape_[0]; ++i) {
         for (int j = 0; j < shape_[1]; ++j) {
             if (labeled_(i, j) != 0 || false
@@ -280,7 +280,7 @@ template <typename T> void VarClusterFinder<T>::store_clusters() {
                 // (i+1 < shape_[0] and labeled_(i+1, j) != 0) or
                 // (j+1 < shape_[1] and labeled_(i, j+1) != 0)
             ) {
-                Hit &record = h_size[labeled_(i, j)];
+                Hit &record = h_map[labeled_(i, j)];
                 if (record.size < MAX_CLUSTER_SIZE) {
                     record.rows[record.size] = i;
                     record.cols[record.size] = j;
@@ -300,7 +300,7 @@ template <typename T> void VarClusterFinder<T>::store_clusters() {
         }
     }
 
-    for (const auto &h : h_size)
+    for (const auto &h : h_map)
         hits.push_back(h.second);
 }
 

include/aare/algorithm.hpp

@@ -0,0 +1,55 @@
+
+#pragma once
+#include <algorithm>
+#include <array>
+#include <vector>
+#include <aare/NDArray.hpp>
+
+namespace aare {
+/**
+ * @brief Find the index of the last element smaller than val
+ * assume a sorted array
+ */
+template <typename T>
+size_t last_smaller(const T* first, const T* last, T val) {
+    for (auto iter = first+1; iter != last; ++iter) {
+        if (*iter > val) {
+            return std::distance(first, iter-1);
+        }
+    }
+    return std::distance(first, last-1);
+}
+
+template <typename T>
+size_t last_smaller(const NDArray<T, 1>& arr, T val) {
+    return last_smaller(arr.begin(), arr.end(), val);
+}
+
+
+template <typename T>
+size_t nearest_index(const T* first, const T* last, T val) {
+    auto iter = std::min_element(first, last,
+    [val](T a, T b) {
+        return std::abs(a - val) < std::abs(b - val);
+    });
+    return std::distance(first, iter);
+}
+
+template <typename T>
+size_t nearest_index(const NDArray<T, 1>& arr, T val) {
+    return nearest_index(arr.begin(), arr.end(), val);
+}
+
+template <typename T>
+size_t nearest_index(const std::vector<T>& vec, T val) {
+    return nearest_index(vec.data(), vec.data()+vec.size(), val);
+}
+
+template <typename T, size_t N>
+size_t nearest_index(const std::array<T,N>& arr, T val) {
+    return nearest_index(arr.data(), arr.data()+arr.size(), val);
+}
+
+
+
+} // namespace aare

include/aare/decode.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <cstdint>
+#include <aare/NDView.hpp>
+namespace aare {
+
+
+uint16_t adc_sar_05_decode64to16(uint64_t input);
+uint16_t adc_sar_04_decode64to16(uint64_t input);
+void adc_sar_05_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output);
+void adc_sar_04_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output);
+
+} // namespace aare

include/aare/defs.hpp

@@ -1,11 +1,9 @@
 #pragma once
 
 #include "aare/Dtype.hpp"
-// #include "aare/utils/logger.hpp"
 
 #include <array>
 #include <stdexcept>
-
 #include <cassert>
 #include <cstdint>
 #include <cstring>
@@ -38,16 +36,19 @@
 
 namespace aare {
 
+inline constexpr size_t bits_per_byte = 8;
+
 void assert_failed(const std::string &msg);
 
 
+
 class DynamicCluster {
   public:
     int cluster_sizeX;
     int cluster_sizeY;
     int16_t x;
     int16_t y;
-    Dtype dt;
+    Dtype dt; // 4 bytes
 
   private:
     std::byte *m_data;
@@ -93,7 +94,7 @@ class DynamicCluster {
         (sizeof(T) == dt.bytes())
             ? 0
             : throw std::invalid_argument("[ERROR] Type size mismatch");
-        return memcpy(m_data + idx * dt.bytes(), &val, (size_t)dt.bytes());
+        return memcpy(m_data + idx * dt.bytes(), &val, dt.bytes());
     }
 
     template <typename T> std::string to_string() const {
@@ -179,25 +180,70 @@ template <typename T> struct t_xy {
 using xy = t_xy<uint32_t>;
 
 
+/**
+ * @brief Class to hold the geometry of a module. Where pixel 0 is located and the size of the module
+ */
 struct ModuleGeometry{
-    int x{};
-    int y{};
+    int origin_x{};
+    int origin_y{};
     int height{};
     int width{};
+    int row_index{};
+    int col_index{}; 
 };
 
+/**
+ * @brief Class to hold the geometry of a detector. Number of modules, their size and where pixel 0 
+ * for each module is located
+ */
+struct DetectorGeometry{
+    int modules_x{};
+    int modules_y{};
+    int pixels_x{};
+    int pixels_y{};
+    int module_gap_row{};
+    int module_gap_col{};
+    std::vector<ModuleGeometry> module_pixel_0;
+};
+
+struct ROI{
+    int64_t xmin{};
+    int64_t xmax{};
+    int64_t ymin{};
+    int64_t ymax{};
+  
+    int64_t height() const { return ymax - ymin; }
+    int64_t width() const { return xmax - xmin; }
+    bool contains(int64_t x, int64_t y) const {
+        return x >= xmin && x < xmax && y >= ymin && y < ymax;
+    }
+  };
+
 
 using dynamic_shape = std::vector<int64_t>;
 
 //TODO! Can we uniform enums between the libraries?
+
+/**
+ * @brief Enum class to identify different detectors. 
+ * The values are the same as in slsDetectorPackage
+ * Different spelling to avoid confusion with the slsDetectorPackage
+ */
 enum class DetectorType {
-    Jungfrau,
+    //Standard detectors match the enum values from slsDetectorPackage
+    Generic,
     Eiger,
-    Mythen3,
-    Moench,
-    Moench03,
-    Moench03_old,
+    Gotthard,
+    Jungfrau,
     ChipTestBoard,
+    Moench,
+    Mythen3,
+    Gotthard2,
+    Xilinx_ChipTestBoard,
+
+    //Additional detectors used for defining processing. Variants of the standard ones.
+    Moench03=100,
+    Moench03_old,
     Unknown
 };
 

include/aare/geo_helpers.hpp

@@ -0,0 +1,16 @@
+#pragma once
+#include "aare/defs.hpp"
+#include "aare/RawMasterFile.hpp" //ROI refactor away
+namespace aare{
+
+/**
+ * @brief Update the detector geometry given a region of interest
+ * 
+ * @param geo 
+ * @param roi 
+ * @return DetectorGeometry 
+ */
+DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, ROI roi);
+
+
+} // namespace aare

include/aare/utils/par.hpp

@@ -0,0 +1,18 @@
+#include <thread>
+#include <vector>
+#include <utility>
+
+namespace aare {
+
+    template<typename F>
+    void RunInParallel(F func, const std::vector<std::pair<int, int>>& tasks) {
+        // auto tasks = split_task(0, y.shape(0), n_threads);
+        std::vector<std::thread> threads;
+        for (auto &task : tasks) {
+            threads.push_back(std::thread(func, task.first, task.second));
+        }
+        for (auto &thread : threads) {
+            thread.join();
+        }
+    }
+} // namespace aare

include/aare/utils/task.hpp

@@ -0,0 +1,8 @@
+
+#include <utility>
+#include <vector>
+
+namespace aare {
+std::vector<std::pair<int, int>> split_task(int first, int last, int n_threads);
+
+} // namespace aare

patches/libzmq_cmake_version.patch

@@ -0,0 +1,18 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index dd3d8eb9..c0187747 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -1,11 +1,8 @@
+ # CMake build script for ZeroMQ
+ project(ZeroMQ)
+ 
+-if(${CMAKE_SYSTEM_NAME} STREQUAL Darwin)
+-  cmake_minimum_required(VERSION 3.0.2)
+-else()
+-  cmake_minimum_required(VERSION 2.8.12)
+-endif()
++cmake_minimum_required(VERSION 3.15)
++message(STATUS "Patched cmake version")
+ 
+ include(CheckIncludeFiles)
+ include(CheckCCompilerFlag)

patches/lmfit.patch

@@ -0,0 +1,13 @@
+diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
+index 4efb7ed..6533660 100644
+--- a/lib/CMakeLists.txt
++++ b/lib/CMakeLists.txt
+@@ -11,7 +11,7 @@ target_compile_definitions(${lib} PRIVATE "LMFIT_EXPORT") # for Windows DLL expo
+ 
+ target_include_directories(${lib}
+     PUBLIC
+-    $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/>
++    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/>
+     $<INSTALL_INTERFACE:include/>
+     )
+ 

pyproject.toml

@@ -4,7 +4,8 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "aare"
-version = "2024.11.15.dev0"
+version = "2025.4.1"
+
 
 
 [tool.scikit-build]
@@ -12,4 +13,5 @@ cmake.verbose = true
 
 [tool.scikit-build.cmake.define]
 AARE_PYTHON_BINDINGS = "ON"
-AARE_SYSTEM_LIBRARIES = "ON"
+AARE_SYSTEM_LIBRARIES = "ON"
+AARE_INSTALL_PYTHONEXT = "ON"

python/CMakeLists.txt

@@ -1,5 +1,5 @@
 
-find_package (Python 3.10 COMPONENTS Interpreter Development)
+find_package (Python 3.10 COMPONENTS Interpreter Development REQUIRED)
 
 # Download or find pybind11 depending on configuration
 if(AARE_FETCH_PYBIND11)
@@ -28,8 +28,11 @@ target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
 set( PYTHON_FILES
     aare/__init__.py
     aare/CtbRawFile.py
+    aare/func.py
+    aare/RawFile.py
     aare/transform.py
     aare/ScanParameters.py
+    aare/utils.py
 )
 
 # Copy the python files to the build directory
@@ -41,10 +44,24 @@ set_target_properties(_aare PROPERTIES
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/aare
 )
 
+set(PYTHON_EXAMPLES
+    examples/play.py
+    examples/fits.py
+)
 
 
-# Copy the examples/scripts to the build directory
-configure_file(examples/play.py ${CMAKE_BINARY_DIR}/play.py)
+# Copy the python examples to the build directory
+foreach(FILE ${PYTHON_EXAMPLES})
+    configure_file(${FILE} ${CMAKE_BINARY_DIR}/${FILE}  )
+    message(STATUS "Copying ${FILE} to ${CMAKE_BINARY_DIR}/${FILE}")
+endforeach(FILE ${PYTHON_EXAMPLES})
 
 
-install(TARGETS _aare DESTINATION aare)
+if(AARE_INSTALL_PYTHONEXT)
+    install(TARGETS _aare
+        EXPORT "${TARGETS_EXPORT_NAME}"
+        LIBRARY DESTINATION aare
+    )
+
+    install(FILES ${PYTHON_FILES} DESTINATION aare)
+endif()

python/aare/CtbRawFile.py

@@ -2,18 +2,21 @@
 from . import _aare
 import numpy as np
 from .ScanParameters import ScanParameters
+
 class CtbRawFile(_aare.CtbRawFile):
     """File reader for the CTB raw file format.
     
     Args:  
             fname (pathlib.Path | str): Path to the file to be read.
+            chunk_size (int): Number of frames to read at a time. Default is 1.
             transform (function): Function to apply to the data after reading it. 
                 The function should take a numpy array of type uint8 and return one
                 or several numpy arrays.
     """
-    def __init__(self, fname, transform = None):
+    def __init__(self, fname, chunk_size = 1, transform = None):
         super().__init__(fname)
-        self.transform = transform
+        self._chunk_size = chunk_size
+        self._transform = transform
 
 
     def read_frame(self, frame_index: int | None = None ) -> tuple:
@@ -42,8 +45,8 @@ class CtbRawFile(_aare.CtbRawFile):
             header = header[0]
 
 
-        if self.transform:
-            res = self.transform(data)
+        if self._transform:
+            res = self._transform(data)
             if isinstance(res, tuple):
                 return header, *res
             else:
@@ -59,6 +62,10 @@ class CtbRawFile(_aare.CtbRawFile):
             Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine
             where to start reading from.
 
+        If the number of frames requested is larger than the number of frames left in the file,
+        the function will read the remaining frames. If no frames are left in the file
+        a RuntimeError is raised.
+
         Args:
             n_frames (int): Number of frames to read.
 
@@ -68,6 +75,12 @@ class CtbRawFile(_aare.CtbRawFile):
         Raises:
             RuntimeError: If EOF is reached.
         """
+        # Calculate the number of frames to actually read
+        n_frames = min(n_frames, self.frames_in_file - self.tell())
+        if n_frames == 0:
+            raise RuntimeError("No frames left in file.")
+
+
         # Do the first read to figure out what we have
         tmp_header, tmp_data = self.read_frame()
         
@@ -87,10 +100,12 @@ class CtbRawFile(_aare.CtbRawFile):
 
     def read(self) -> tuple:
         """Read the entire file.
+        Seeks to the beginning of the file before reading.
 
         Returns:
             tuple: header, data
         """
+        self.seek(0)
         return self.read_n(self.frames_in_file)
 
     def seek(self, frame_index:int) -> None:
@@ -101,7 +116,7 @@ class CtbRawFile(_aare.CtbRawFile):
         """
         super().seek(frame_index)
 
-    def tell() -> int:
+    def tell(self) -> int:
         """Return the current frame position in the file.
 
         Returns:
@@ -164,7 +179,12 @@ class CtbRawFile(_aare.CtbRawFile):
     
     def __next__(self):
         try:
-            return self.read_frame()
+            if self._chunk_size == 1:
+                return self.read_frame()
+            else:
+                return self.read_n(self._chunk_size)
+            
+                
         except RuntimeError:
             # TODO! find a good way to check that we actually have the right exception
             raise StopIteration

python/aare/RawFile.py

@@ -0,0 +1,66 @@
+from . import _aare
+import numpy as np
+from .ScanParameters import ScanParameters
+
+class RawFile(_aare.RawFile):
+    def __init__(self, fname, chunk_size = 1):
+        super().__init__(fname)
+        self._chunk_size = chunk_size
+
+
+    def read(self) -> tuple:
+        """Read the entire file.
+        Seeks to the beginning of the file before reading.
+
+        Returns:
+            tuple: header, data
+        """
+        self.seek(0)
+        return self.read_n(self.total_frames)
+
+    @property
+    def scan_parameters(self):
+        """Return the scan parameters.
+
+        Returns:
+            ScanParameters: Scan parameters.
+        """
+        return ScanParameters(self.master.scan_parameters)
+    
+    @property
+    def master(self):
+        """Return the master file.
+
+        Returns:
+            RawMasterFile: Master file.
+        """
+        return super().master
+    
+    def __len__(self) -> int:
+        """Return the number of frames in the file.
+
+        Returns:
+            int: Number of frames in file.
+        """
+        return super().frames_in_file
+
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __iter__(self):
+        return self
+    
+    def __next__(self):
+        try:
+            if self._chunk_size == 1:
+                return self.read_frame()
+            else:
+                return self.read_n(self._chunk_size)
+            
+                
+        except RuntimeError:
+            # TODO! find a good way to check that we actually have the right exception
+            raise StopIteration

python/aare/__init__.py

@@ -2,10 +2,23 @@
 from . import _aare
 
 
-from ._aare import File, RawFile, RawMasterFile, RawSubFile
-from ._aare import Pedestal, ClusterFinder, VarClusterFinder
+# from ._aare import File, RawMasterFile, RawSubFile
+# from ._aare import Pedestal_d, Pedestal_f, ClusterFinder, VarClusterFinder
 from ._aare import DetectorType
-from ._aare import ClusterFile
+from ._aare import ClusterFile_Cluster3x3i as ClusterFile
+from ._aare import hitmap
+from ._aare import ROI
 
+# from ._aare import ClusterFinderMT, ClusterCollector, ClusterFileSink, ClusterVector_i
+
+from ._aare import fit_gaus, fit_pol1
+from ._aare import Interpolator
 from .CtbRawFile import CtbRawFile
-from .ScanParameters import ScanParameters
+from .RawFile import RawFile
+from .ScanParameters import ScanParameters
+
+from .utils import random_pixels, random_pixel, flat_list
+
+
+#make functions available in the top level API
+from .func import *

python/aare/func.py

@@ -0,0 +1 @@
+from ._aare import gaus, pol1

python/aare/transform.py

@@ -2,6 +2,14 @@ import numpy as np
 from . import _aare
 
 
+class AdcSar04Transform64to16:
+    def __call__(self, data):
+        return _aare.adc_sar_04_decode64to16(data)
+
+class AdcSar05Transform64to16:
+    def __call__(self, data):
+        return _aare.adc_sar_05_decode64to16(data)
+
 class Moench05Transform:
     #Could be moved to C++ without changing the interface
     def __init__(self):
@@ -9,6 +17,24 @@ class Moench05Transform:
 
     def __call__(self, data):
         return np.take(data.view(np.uint16), self.pixel_map)
+    
+
+class Moench05Transform1g:
+    #Could be moved to C++ without changing the interface
+    def __init__(self):
+        self.pixel_map = _aare.GenerateMoench05PixelMap1g()
+
+    def __call__(self, data):
+        return np.take(data.view(np.uint16), self.pixel_map)
+    
+
+class Moench05TransformOld:
+    #Could be moved to C++ without changing the interface
+    def __init__(self):
+        self.pixel_map = _aare.GenerateMoench05PixelMapOld()
+
+    def __call__(self, data):
+        return np.take(data.view(np.uint16), self.pixel_map)
 
 
 class Matterhorn02Transform:
@@ -25,4 +51,8 @@ class Matterhorn02Transform:
 
 #on import generate the pixel maps to avoid doing it every time
 moench05 = Moench05Transform()
-matterhorn02 = Matterhorn02Transform()
+moench05_1g = Moench05Transform1g()
+moench05_old = Moench05TransformOld()
+matterhorn02 = Matterhorn02Transform()
+adc_sar_04_64to16 = AdcSar04Transform64to16()
+adc_sar_05_64to16 = AdcSar05Transform64to16()

python/aare/utils.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+def random_pixels(n_pixels, xmin=0, xmax=512, ymin=0, ymax=1024):
+    """Return a list of random pixels.
+    
+    Args:
+        n_pixels (int): Number of pixels to return.
+        rows (int): Number of rows in the image.
+        cols (int): Number of columns in the image.
+        
+    Returns:
+        list: List of (row, col) tuples.
+    """
+    return [(np.random.randint(ymin, ymax), np.random.randint(xmin, xmax)) for _ in range(n_pixels)]
+
+
+def random_pixel(xmin=0, xmax=512, ymin=0, ymax=1024):
+    """Return a random pixel.
+    
+    Returns:
+        tuple: (row, col)
+    """
+    return random_pixels(1, xmin, xmax, ymin, ymax)[0]
+
+def flat_list(xss):
+    """Flatten a list of lists."""
+    return [x for xs in xss for x in xs]

python/examples/fits.py

@@ -0,0 +1,79 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from aare import fit_gaus, fit_pol1
+from aare import gaus, pol1
+
+textpm = f"±"  #
+textmu = f"μ"  #
+textsigma = f"σ"  #
+
+
+
+# ================================= Gauss fit =================================
+# Parameters
+mu = np.random.uniform(1, 100)  # Mean of Gaussian
+sigma = np.random.uniform(4, 20)  # Standard deviation
+num_points = 10000  # Number of points for smooth distribution
+noise_sigma = 100
+
+# Generate Gaussian distribution
+data = np.random.normal(mu, sigma, num_points)
+
+# Generate errors for each point
+errors = np.abs(np.random.normal(0, sigma, num_points))  # Errors with mean 0, std 0.5
+
+# Create subplot
+fig0, ax0 = plt.subplots(1, 1, num=0, figsize=(12, 8))
+
+x = np.histogram(data, bins=30)[1][:-1] + 0.05
+y = np.histogram(data, bins=30)[0]
+yerr = errors[:30]
+
+
+# Add the errors as error bars in the step plot
+ax0.errorbar(x, y, yerr=yerr, fmt=". ", capsize=5)
+ax0.grid()
+
+par, err = fit_gaus(x, y, yerr)
+print(par, err)
+
+x = np.linspace(x[0], x[-1], 1000)
+ax0.plot(x, gaus(x, par), marker="")
+ax0.set(xlabel="x", ylabel="Counts", title=f"A0 = {par[0]:0.2f}{textpm}{err[0]:0.2f}\n"
+                                           f"{textmu} = {par[1]:0.2f}{textpm}{err[1]:0.2f}\n"
+                                           f"{textsigma} = {par[2]:0.2f}{textpm}{err[2]:0.2f}\n"
+                                           f"(init: {textmu}: {mu:0.2f}, {textsigma}: {sigma:0.2f})")
+fig0.tight_layout()
+
+
+
+# ================================= pol1 fit =================================
+# Parameters
+n_points = 40
+
+# Generate random slope and intercept (origin)
+slope = np.random.uniform(-10, 10)  # Random slope between 0.5 and 2.0
+intercept = np.random.uniform(-10, 10)  # Random intercept between -10 and 10
+
+# Generate random x values
+x_values = np.random.uniform(-10, 10, n_points)
+
+# Calculate y values based on the linear function y = mx + b + error
+errors = np.abs(np.random.normal(0, np.random.uniform(1, 5), n_points))
+var_points = np.random.normal(0, np.random.uniform(0.1, 2), n_points)
+y_values = slope * x_values + intercept + var_points
+
+fig1, ax1 = plt.subplots(1, 1, num=1, figsize=(12, 8))
+ax1.errorbar(x_values, y_values, yerr=errors, fmt=". ", capsize=5)
+par, err = fit_pol1(x_values, y_values, errors)
+
+
+x = np.linspace(np.min(x_values), np.max(x_values), 1000)
+ax1.plot(x, pol1(x, par), marker="")
+ax1.set(xlabel="x", ylabel="y", title=f"a = {par[0]:0.2f}{textpm}{err[0]:0.2f}\n"
+                                      f"b = {par[1]:0.2f}{textpm}{err[1]:0.2f}\n"
+                                      f"(init: {slope:0.2f}, {intercept:0.2f})")
+fig1.tight_layout()
+
+plt.show()
+

python/examples/play.py

@@ -1,15 +1,79 @@
-import matplotlib.pyplot as plt
+import sys
+sys.path.append('/home/l_msdetect/erik/aare/build')
+
+from aare._aare import ClusterVector_i, Interpolator
+
+import pickle 
 import numpy as np
-plt.ion()
-from pathlib import Path
-from aare import ClusterFile
-
-base = Path('~/data/aare_test_data/clusters').expanduser()
-
-f = ClusterFile(base / 'beam_En700eV_-40deg_300V_10us_d0_f0_100.clust')
-# f = ClusterFile(base / 'single_frame_97_clustrers.clust')
+import matplotlib.pyplot as plt
+import boost_histogram as bh
+import torch
+import math
+import time
 
 
-for i in range(10):
-    fn, cl = f.read_frame()
-    print(fn, cl.size)
+
+def gaussian_2d(mx, my, sigma = 1, res=100, grid_size = 2):
+    """
+    Generate a 2D gaussian as position mx, my, with sigma=sigma. 
+    The gaussian is placed on a 2x2 pixel matrix with resolution 
+    res in one dimesion.
+    """
+    x = torch.linspace(0, pixel_size*grid_size, res)
+    x,y = torch.meshgrid(x,x, indexing="ij")
+    return 1 / (2*math.pi*sigma**2) * \
+      torch.exp(-((x - my)**2 / (2*sigma**2) + (y - mx)**2 / (2*sigma**2)))
+
+scale = 1000 #Scale factor when converting to integer
+pixel_size = 25 #um
+grid = 2
+resolution = 100
+sigma_um = 10
+xa = np.linspace(0,grid*pixel_size,resolution)
+ticks = [0, 25, 50]
+
+hit = np.array((20,20))
+etahist_fname = "/home/l_msdetect/erik/tmp/test_hist.pkl"
+
+local_resolution = 99
+grid_size = 3
+xaxis = np.linspace(0,grid_size*pixel_size, local_resolution)
+t = gaussian_2d(hit[0],hit[1], grid_size = grid_size, sigma = 10, res = local_resolution)
+pixels = t.reshape(grid_size, t.shape[0] // grid_size, grid_size, t.shape[1] // grid_size).sum(axis = 3).sum(axis = 1)
+pixels = pixels.numpy()
+pixels = (pixels*scale).astype(np.int32)
+v = ClusterVector_i(3,3)
+v.push_back(1,1, pixels)
+
+with open(etahist_fname, "rb") as f:
+        hist = pickle.load(f)
+eta = hist.view().copy()
+etabinsx = np.array(hist.axes.edges.T[0].flat)
+etabinsy = np.array(hist.axes.edges.T[1].flat)
+ebins = np.array(hist.axes.edges.T[2].flat)
+p = Interpolator(eta, etabinsx[0:-1], etabinsy[0:-1], ebins[0:-1])
+
+
+
+
+#Generate the hit
+
+
+
+
+tmp = p.interpolate(v)
+print(f'tmp:{tmp}')
+pos = np.array((tmp['x'], tmp['y']))*25
+
+
+print(pixels)
+fig, ax = plt.subplots(figsize = (7,7))
+ax.pcolormesh(xaxis, xaxis, t)
+ax.plot(*pos, 'o')
+ax.set_xticks([0,25,50,75])
+ax.set_yticks([0,25,50,75])
+ax.set_xlim(0,75)
+ax.set_ylim(0,75)
+ax.grid()
+print(f'{hit=}')
+print(f'{pos=}')

python/src/cluster.hpp

@@ -1,4 +1,8 @@
+#include "aare/ClusterCollector.hpp"
+#include "aare/ClusterFileSink.hpp"
 #include "aare/ClusterFinder.hpp"
+#include "aare/ClusterFinderMT.hpp"
+#include "aare/ClusterVector.hpp"
 #include "aare/NDView.hpp"
 #include "aare/Pedestal.hpp"
 #include "np_helper.hpp"
@@ -7,46 +11,276 @@
 #include <filesystem>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
+#include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
+using pd_type = double;
 
-void define_cluster_finder_bindings(py::module &m) {
-    py::class_<ClusterFinder<uint16_t, double>>(m, "ClusterFinder")
-        .def(py::init<Shape<2>, Shape<2>>())
+using namespace aare;
+
+template <typename Type, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+void define_cluster(py::module &m, const std::string &typestr) {
+    auto class_name = fmt::format("Cluster{}", typestr);
+
+    using ClusterType =
+        Cluster<Type, ClusterSizeX, ClusterSizeY, CoordType, void>;
+    py::class_<Cluster<Type, ClusterSizeX, ClusterSizeY, CoordType, void>>(
+        m, class_name.c_str())
+
+        .def(py::init([](uint8_t x, uint8_t y, py::array_t<Type> data) {
+            py::buffer_info buf_info = data.request();
+            Type *ptr = static_cast<Type *>(buf_info.ptr);
+            Cluster<Type, ClusterSizeX, ClusterSizeY, CoordType, void> cluster;
+            cluster.x = x;
+            cluster.y = y;
+            std::copy(ptr, ptr + ClusterSizeX * ClusterSizeY,
+                      cluster.data); // Copy array contents
+            return cluster;
+        }))
+
+        //.def(py::init<>())
+        .def_readwrite("x", &ClusterType::x)
+        .def_readwrite("y", &ClusterType::y)
+        .def_property(
+            "data",
+            [](ClusterType &c) -> py::array {
+                return py::array(py::buffer_info(
+                    c.data, sizeof(Type),
+                    py::format_descriptor<Type>::format(), // Type
+                                                           // format
+                    1, // Number of dimensions
+                    {static_cast<ssize_t>(ClusterSizeX *
+                                          ClusterSizeY)}, // Shape (flattened)
+                    {sizeof(Type)} // Stride (step size between elements)
+                    ));
+            },
+            [](ClusterType &c, py::array_t<Type> arr) {
+                py::buffer_info buf_info = arr.request();
+                Type *ptr = static_cast<Type *>(buf_info.ptr);
+                std::copy(ptr, ptr + ClusterSizeX * ClusterSizeY,
+                          c.data); // TODO dont iterate over centers!!!
+            });
+}
+
+template <typename Type, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = uint16_t>
+void define_cluster_vector(py::module &m, const std::string &typestr) {
+    using ClusterType =
+        Cluster<Type, ClusterSizeX, ClusterSizeY, uint16_t, void>;
+    auto class_name = fmt::format("ClusterVector_{}", typestr);
+
+    py::class_<ClusterVector<ClusterType>>(m, class_name.c_str(),
+                                           py::buffer_protocol())
+
+        .def(py::init()) // TODO change!!!
+        /*
+        .def("push_back",
+             [](ClusterVector<ClusterType> &self, ClusterType &cl) {
+                 //  auto view = make_view_2d(data);
+                 self.push_back(cl);
+             })
+        */
+        /*
+        .def(
+            "push_back",
+            [](ClusterVector<ClusterType> &self, py::object obj) {
+                ClusterType &cl = py::cast<ClusterType &>(obj);
+                self.push_back(cl);
+            },
+            py::arg("cluster"))
+        */
+
+        .def("push_back",
+             [](ClusterVector<ClusterType> &self, const ClusterType &cluster) {
+                 self.push_back(cluster);
+             })
+
+        //.def("push_back", &ClusterVector<ClusterType>::push_back) //TODO
+        // implement push_back
+        .def_property_readonly("size", &ClusterVector<ClusterType>::size)
+        .def("item_size", &ClusterVector<ClusterType>::item_size)
+        .def_property_readonly("fmt",
+                               [typestr]() { return fmt_format<ClusterType>; })
+        /*
+        .def("sum",
+             [](ClusterVector<ClusterType> &self) {
+                 auto *vec = new std::vector<T>(self.sum());
+                 return return_vector(vec);
+             })
+        .def("sum_2x2",
+             [](ClusterVector<ClusterType> &self) {
+                 auto *vec = new std::vector<T>(self.sum_2x2());
+                 return return_vector(vec);
+             })
+        */
+        .def_property_readonly("cluster_size_x",
+                               &ClusterVector<ClusterType>::cluster_size_x)
+        .def_property_readonly("cluster_size_y",
+                               &ClusterVector<ClusterType>::cluster_size_y)
+        .def_property_readonly("capacity",
+                               &ClusterVector<ClusterType>::capacity)
+        .def_property("frame_number", &ClusterVector<ClusterType>::frame_number,
+                      &ClusterVector<ClusterType>::set_frame_number)
+        .def_buffer(
+            [typestr](ClusterVector<ClusterType> &self) -> py::buffer_info {
+                return py::buffer_info(
+                    self.data(),             /* Pointer to buffer */
+                    self.item_size(),        /* Size of one scalar */
+                    fmt_format<ClusterType>, /* Format descriptor */
+                    1,                       /* Number of dimensions */
+                    {self.size()},           /* Buffer dimensions */
+                    {self.item_size()} /* Strides (in bytes) for each index */
+                );
+            });
+}
+
+template <typename ClusterType>
+void define_cluster_finder_mt_bindings(py::module &m,
+                                       const std::string &typestr) {
+    auto class_name = fmt::format("ClusterFinderMT_{}", typestr);
+
+    py::class_<ClusterFinderMT<ClusterType, uint16_t, pd_type>>(
+        m, class_name.c_str())
+        .def(py::init<Shape<2>, pd_type, size_t, size_t>(),
+             py::arg("image_size"), py::arg("n_sigma") = 5.0,
+             py::arg("capacity") = 2048, py::arg("n_threads") = 3)
         .def("push_pedestal_frame",
-             [](ClusterFinder<uint16_t, double> &self,
+             [](ClusterFinderMT<ClusterType, uint16_t, pd_type> &self,
                 py::array_t<uint16_t> frame) {
                  auto view = make_view_2d(frame);
                  self.push_pedestal_frame(view);
              })
-        .def("pedestal",
-             [](ClusterFinder<uint16_t, double> &self) {
-                 auto m = new NDArray<double, 2>{};
-                 *m = self.pedestal();
-                 return return_image_data(m);
-             })
-        .def("find_clusters_without_threshold",
-             [](ClusterFinder<uint16_t, double> &self,
+        .def(
+            "find_clusters",
+            [](ClusterFinderMT<ClusterType, uint16_t, pd_type> &self,
+               py::array_t<uint16_t> frame, uint64_t frame_number) {
+                auto view = make_view_2d(frame);
+                self.find_clusters(view, frame_number);
+                return;
+            },
+            py::arg(), py::arg("frame_number") = 0)
+        .def("clear_pedestal",
+             &ClusterFinderMT<ClusterType, uint16_t, pd_type>::clear_pedestal)
+        .def("sync", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::sync)
+        .def("stop", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::stop)
+        .def("start", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::start)
+        .def(
+            "pedestal",
+            [](ClusterFinderMT<ClusterType, uint16_t, pd_type> &self,
+               size_t thread_index) {
+                auto pd = new NDArray<pd_type, 2>{};
+                *pd = self.pedestal(thread_index);
+                return return_image_data(pd);
+            },
+            py::arg("thread_index") = 0)
+        .def(
+            "noise",
+            [](ClusterFinderMT<ClusterType, uint16_t, pd_type> &self,
+               size_t thread_index) {
+                auto arr = new NDArray<pd_type, 2>{};
+                *arr = self.noise(thread_index);
+                return return_image_data(arr);
+            },
+            py::arg("thread_index") = 0);
+}
+
+template <typename ClusterType>
+void define_cluster_collector_bindings(py::module &m,
+                                       const std::string &typestr) {
+    auto class_name = fmt::format("ClusterCollector_{}", typestr);
+
+    py::class_<ClusterCollector<ClusterType>>(m, class_name.c_str())
+        .def(py::init<ClusterFinderMT<ClusterType, uint16_t, double> *>())
+        .def("stop", &ClusterCollector<ClusterType>::stop)
+        .def(
+            "steal_clusters",
+            [](ClusterCollector<ClusterType> &self) {
+                auto v = new std::vector<ClusterVector<ClusterType>>(
+                    self.steal_clusters());
+                return v;
+            },
+            py::return_value_policy::take_ownership);
+}
+
+template <typename ClusterType>
+void define_cluster_file_sink_bindings(py::module &m,
+                                       const std::string &typestr) {
+    auto class_name = fmt::format("ClusterFileSink_{}", typestr);
+
+    py::class_<ClusterFileSink<ClusterType>>(m, class_name.c_str())
+        .def(py::init<ClusterFinderMT<ClusterType, uint16_t, double> *,
+                      const std::filesystem::path &>())
+        .def("stop", &ClusterFileSink<ClusterType>::stop);
+}
+
+template <typename ClusterType>
+void define_cluster_finder_bindings(py::module &m, const std::string &typestr) {
+    auto class_name = fmt::format("ClusterFinder_{}", typestr);
+
+    py::class_<ClusterFinder<ClusterType, uint16_t, pd_type>>(
+        m, class_name.c_str())
+        .def(py::init<Shape<2>, pd_type, size_t>(), py::arg("image_size"),
+             py::arg("n_sigma") = 5.0, py::arg("capacity") = 1'000'000)
+        .def("push_pedestal_frame",
+             [](ClusterFinder<ClusterType, uint16_t, pd_type> &self,
                 py::array_t<uint16_t> frame) {
                  auto view = make_view_2d(frame);
-                 auto clusters = self.find_clusters_without_threshold(view);
-                 return clusters;
-             });
+                 self.push_pedestal_frame(view);
+             })
+        .def("clear_pedestal",
+             &ClusterFinder<ClusterType, uint16_t, pd_type>::clear_pedestal)
+        .def_property_readonly(
+            "pedestal",
+            [](ClusterFinder<ClusterType, uint16_t, pd_type> &self) {
+                auto pd = new NDArray<pd_type, 2>{};
+                *pd = self.pedestal();
+                return return_image_data(pd);
+            })
+        .def_property_readonly(
+            "noise",
+            [](ClusterFinder<ClusterType, uint16_t, pd_type> &self) {
+                auto arr = new NDArray<pd_type, 2>{};
+                *arr = self.noise();
+                return return_image_data(arr);
+            })
+        .def(
+            "steal_clusters",
+            [](ClusterFinder<ClusterType, uint16_t, pd_type> &self,
+               bool realloc_same_capacity) {
+                auto v = new ClusterVector<ClusterType>(
+                    self.steal_clusters(realloc_same_capacity));
+                return v;
+            },
+            py::arg("realloc_same_capacity") = false)
+        .def(
+            "find_clusters",
+            [](ClusterFinder<ClusterType, uint16_t, pd_type> &self,
+               py::array_t<uint16_t> frame, uint64_t frame_number) {
+                auto view = make_view_2d(frame);
+                self.find_clusters(view, frame_number);
+                return;
+            },
+            py::arg(), py::arg("frame_number") = 0);
 
-    py::class_<DynamicCluster>(m, "DynamicCluster", py::buffer_protocol())
-        .def(py::init<int, int, Dtype>())
-        .def("size", &DynamicCluster::size)
-        .def("begin", &DynamicCluster::begin)
-        .def("end", &DynamicCluster::end)
-        .def_readwrite("x", &DynamicCluster::x)
-        .def_readwrite("y", &DynamicCluster::y)
-        .def_buffer([](DynamicCluster &c) -> py::buffer_info {
-            return py::buffer_info(c.data(), c.dt.bytes(), c.dt.format_descr(),
-                                   1, {c.size()}, {c.dt.bytes()});
-        })
+    m.def("hitmap",
+          [](std::array<size_t, 2> image_size, ClusterVector<ClusterType> &cv) {
+              py::array_t<int32_t> hitmap(image_size);
+              auto r = hitmap.mutable_unchecked<2>();
 
-        .def("__repr__", [](const DynamicCluster &a) {
-            return "<DynamicCluster: x: " + std::to_string(a.x) +
-                   ", y: " + std::to_string(a.y) + ">";
-        });
-}
+              // Initialize hitmap to 0
+              for (py::ssize_t i = 0; i < r.shape(0); i++)
+                  for (py::ssize_t j = 0; j < r.shape(1); j++)
+                      r(i, j) = 0;
+
+              size_t stride = cv.item_size();
+              auto ptr = cv.data();
+              for (size_t i = 0; i < cv.size(); i++) {
+                  auto x = *reinterpret_cast<int16_t *>(ptr);
+                  auto y = *reinterpret_cast<int16_t *>(ptr + sizeof(int16_t));
+                  r(y, x) += 1;
+                  ptr += stride;
+              }
+              return hitmap;
+          });
+}

python/src/cluster_file.hpp

@@ -1,7 +1,7 @@
+#include "aare/CalculateEta.hpp"
 #include "aare/ClusterFile.hpp"
 #include "aare/defs.hpp"
 
-
 #include <cstdint>
 #include <filesystem>
 #include <pybind11/iostream.h>
@@ -11,40 +11,81 @@
 #include <pybind11/stl/filesystem.h>
 #include <string>
 
+// Disable warnings for unused parameters, as we ignore some
+// in the __exit__ method
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+
 namespace py = pybind11;
 using namespace ::aare;
 
-void define_cluster_file_io_bindings(py::module &m) {
-    PYBIND11_NUMPY_DTYPE(Cluster, x, y, data);
+template <typename ClusterType>
+void define_cluster_file_io_bindings(py::module &m,
+                                     const std::string &typestr) {
+    // PYBIND11_NUMPY_DTYPE(Cluster<int, 3, 3>, x, y,
+    // data); // is this used - maybe use as cluster type
 
-    py::class_<ClusterFile>(m, "ClusterFile")
-        .def(py::init<const std::filesystem::path &, size_t>(), py::arg(), py::arg("chunk_size") = 1000)
-        .def("read_clusters",
-             [](ClusterFile &self, size_t n_clusters) {
-                 auto* vec = new std::vector<Cluster>(self.read_clusters(n_clusters));
-                 return return_vector(vec);
-             })
+    auto class_name = fmt::format("ClusterFile_{}", typestr);
+
+    py::class_<ClusterFile<ClusterType>>(m, class_name.c_str())
+        .def(py::init<const std::filesystem::path &, size_t,
+                      const std::string &>(),
+             py::arg(), py::arg("chunk_size") = 1000, py::arg("mode") = "r")
+        .def(
+            "read_clusters",
+            [](ClusterFile<ClusterType> &self, size_t n_clusters) {
+                auto v = new ClusterVector<ClusterType>(
+                    self.read_clusters(n_clusters));
+                return v;
+            },
+            py::return_value_policy::take_ownership)
         .def("read_frame",
-             [](ClusterFile &self) {
-                 int32_t frame_number;
-                 auto* vec = new std::vector<Cluster>(self.read_frame(frame_number));
-                 return py::make_tuple(frame_number, return_vector(vec));
+             [](ClusterFile<ClusterType> &self) {
+                 auto v = new ClusterVector<ClusterType>(self.read_frame());
+                 return v;
              })
-        .def("read_cluster_with_cut",
-             [](ClusterFile &self, size_t n_clusters, py::array_t<double> noise_map, int nx, int ny) {
-                 auto view = make_view_2d(noise_map);
-                 auto* vec = new std::vector<Cluster>(self.read_cluster_with_cut(n_clusters, view.data(), nx, ny));
-                 return return_vector(vec);
+        .def("set_roi", &ClusterFile<ClusterType>::set_roi)
+        .def(
+            "set_noise_map",
+            [](ClusterFile<ClusterType> &self, py::array_t<int32_t> noise_map) {
+                auto view = make_view_2d(noise_map);
+                self.set_noise_map(view);
+            })
+
+        .def("set_gain_map",
+             [](ClusterFile<ClusterType> &self, py::array_t<double> gain_map) {
+                 auto view = make_view_2d(gain_map);
+                 self.set_gain_map(view);
              })
-        .def("__enter__", [](ClusterFile &self) { return &self; })
-        .def("__exit__", [](ClusterFile &self, py::args args) { return; })
-        .def("__iter__", [](ClusterFile &self) { return &self; })
-        .def("__next__", [](ClusterFile &self) {
-            auto vec =  new std::vector<Cluster>(self.read_clusters(self.chunk_size()));
-            if(vec->size() == 0) {
+
+        // void set_gain_map(const GainMap &gain_map); //TODO do i need a
+        // gainmap constructor?
+
+        .def("close", &ClusterFile<ClusterType>::close)
+        .def("write_frame", &ClusterFile<ClusterType>::write_frame)
+        .def("__enter__", [](ClusterFile<ClusterType> &self) { return &self; })
+        .def("__exit__",
+             [](ClusterFile<ClusterType> &self,
+                const std::optional<pybind11::type> &exc_type,
+                const std::optional<pybind11::object> &exc_value,
+                const std::optional<pybind11::object> &traceback) {
+                 self.close();
+             })
+        .def("__iter__", [](ClusterFile<ClusterType> &self) { return &self; })
+        .def("__next__", [](ClusterFile<ClusterType> &self) {
+            auto v = new ClusterVector<ClusterType>(
+                self.read_clusters(self.chunk_size()));
+            if (v->size() == 0) {
                 throw py::stop_iteration();
             }
-            return return_vector(vec);
+            return v;
         });
 
-}
+    m.def("calculate_eta2",
+          [](const aare::ClusterVector<ClusterType> &clusters) {
+              auto eta2 = new NDArray<double, 2>(calculate_eta2(clusters));
+              return return_image_data(eta2);
+          });
+}
+
+#pragma GCC diagnostic pop

python/src/ctb_raw_file.hpp

@@ -0,0 +1,99 @@
+
+#include "aare/CtbRawFile.hpp"
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/RawFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/RawSubFile.hpp"
+
+#include "aare/defs.hpp"
+#include "aare/decode.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_ctb_raw_file_io_bindings(py::module &m) {
+
+m.def("adc_sar_05_decode64to16", [](py::array_t<uint8_t> input) {
+
+    
+    if(input.ndim() != 2){
+        throw std::runtime_error("Only 2D arrays are supported at this moment"); 
+    }
+
+    //Create a 2D output array with the same shape as the input
+    std::vector<ssize_t> shape{input.shape(0), input.shape(1)/static_cast<int64_t>(bits_per_byte)};
+    py::array_t<uint16_t> output(shape);
+
+    //Create a view of the input and output arrays
+    NDView<uint64_t, 2> input_view(reinterpret_cast<uint64_t*>(input.mutable_data()), {output.shape(0), output.shape(1)});
+    NDView<uint16_t, 2> output_view(output.mutable_data(), {output.shape(0), output.shape(1)});
+
+    adc_sar_05_decode64to16(input_view, output_view);
+
+    return output;
+});
+
+
+m.def("adc_sar_04_decode64to16", [](py::array_t<uint8_t> input) {
+
+    
+    if(input.ndim() != 2){
+        throw std::runtime_error("Only 2D arrays are supported at this moment"); 
+    }
+
+    //Create a 2D output array with the same shape as the input
+    std::vector<ssize_t> shape{input.shape(0), input.shape(1)/static_cast<int64_t>(bits_per_byte)};
+    py::array_t<uint16_t> output(shape);
+
+    //Create a view of the input and output arrays
+    NDView<uint64_t, 2> input_view(reinterpret_cast<uint64_t*>(input.mutable_data()), {output.shape(0), output.shape(1)});
+    NDView<uint16_t, 2> output_view(output.mutable_data(), {output.shape(0), output.shape(1)});
+
+    adc_sar_04_decode64to16(input_view, output_view);
+
+    return output;
+});
+
+ py::class_<CtbRawFile>(m, "CtbRawFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("read_frame",
+             [](CtbRawFile &self) {
+                 size_t image_size = self.image_size_in_bytes();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(1);
+                 shape.push_back(image_size);
+
+                 py::array_t<DetectorHeader> header(1);
+
+                 // always read bytes
+                 image = py::array_t<uint8_t>(shape);
+
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def("seek", &CtbRawFile::seek)
+        .def("tell", &CtbRawFile::tell)
+        .def("master", &CtbRawFile::master)
+
+        .def_property_readonly("image_size_in_bytes",
+                               &CtbRawFile::image_size_in_bytes)
+
+        .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
+
+}

python/src/file.hpp

@@ -20,6 +20,11 @@
 namespace py = pybind11;
 using namespace ::aare;
 
+//Disable warnings for unused parameters, as we ignore some
+//in the __exit__ method
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+
 void define_file_io_bindings(py::module &m) {
 
 
@@ -38,36 +43,7 @@ void define_file_io_bindings(py::module &m) {
                          bunchId, timestamp, modId, row, column, reserved,
                          debug, roundRNumber, detType, version, packetMask);
 
-    py::class_<CtbRawFile>(m, "CtbRawFile")
-        .def(py::init<const std::filesystem::path &>())
-        .def("read_frame",
-             [](CtbRawFile &self) {
-                 size_t image_size = self.image_size_in_bytes();
-                 py::array image;
-                 std::vector<ssize_t> shape;
-                 shape.reserve(2);
-                 shape.push_back(1);
-                 shape.push_back(image_size);
-
-                 py::array_t<DetectorHeader> header(1);
-
-                 // always read bytes
-                 image = py::array_t<uint8_t>(shape);
-
-                 self.read_into(
-                     reinterpret_cast<std::byte *>(image.mutable_data()),
-                     header.mutable_data());
-
-                 return py::make_tuple(header, image);
-             })
-        .def("seek", &CtbRawFile::seek)
-        .def("tell", &CtbRawFile::tell)
-        .def("master", &CtbRawFile::master)
-
-        .def_property_readonly("image_size_in_bytes",
-                               &CtbRawFile::image_size_in_bytes)
-
-        .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
+   
 
     py::class_<File>(m, "File")
         .def(py::init([](const std::filesystem::path &fname) {
@@ -80,7 +56,8 @@ void define_file_io_bindings(py::module &m) {
         .def(py::init<const std::filesystem::path &, const std::string &,
                       const FileConfig &>())
 
-        .def("frame_number", &File::frame_number)
+        .def("frame_number", py::overload_cast<>(&File::frame_number))
+        .def("frame_number", py::overload_cast<size_t>(&File::frame_number))
         .def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
         .def_property_readonly("pixels_per_frame", &File::pixels_per_frame)
         .def("seek", &File::seek)
@@ -133,13 +110,15 @@ void define_file_io_bindings(py::module &m) {
                  return image;
              })
         .def("read_n", [](File &self, size_t n_frames) {
-            const uint8_t item_size = self.bytes_per_pixel();
+            //adjust for actual frames left in the file
+            n_frames = std::min(n_frames, self.total_frames()-self.tell());
+            if(n_frames == 0){
+                throw std::runtime_error("No frames left in file");
+            }
+            std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
+            
             py::array image;
-            std::vector<ssize_t> shape;
-            shape.reserve(3);
-            shape.push_back(n_frames);
-            shape.push_back(self.rows());
-            shape.push_back(self.cols());
+            const uint8_t item_size = self.bytes_per_pixel();
             if (item_size == 1) {
                 image = py::array_t<uint8_t>(shape);
             } else if (item_size == 2) {
@@ -150,8 +129,41 @@ void define_file_io_bindings(py::module &m) {
             self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
                            n_frames);
             return image;
+        })
+        .def("__enter__", [](File &self) { return &self; })
+        .def("__exit__",
+             [](File &self,
+                const std::optional<pybind11::type> &exc_type,
+                const std::optional<pybind11::object> &exc_value,
+                const std::optional<pybind11::object> &traceback) {
+                //  self.close();
+             })
+        .def("__iter__", [](File &self) { return &self; })
+        .def("__next__", [](File &self) {
+
+            try{
+                const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+            }catch(std::runtime_error &e){
+                throw py::stop_iteration();
+            }
         });
 
+
     py::class_<FileConfig>(m, "FileConfig")
         .def(py::init<>())
         .def_readwrite("rows", &FileConfig::rows)
@@ -183,6 +195,8 @@ void define_file_io_bindings(py::module &m) {
 
     py::class_<ROI>(m, "ROI")
         .def(py::init<>())
+        .def(py::init<int64_t, int64_t, int64_t, int64_t>(), py::arg("xmin"),
+             py::arg("xmax"), py::arg("ymin"), py::arg("ymax"))
         .def_readwrite("xmin", &ROI::xmin)
         .def_readwrite("xmax", &ROI::xmax)
         .def_readwrite("ymin", &ROI::ymin)
@@ -194,7 +208,7 @@ void define_file_io_bindings(py::module &m) {
             return fmt::format("<ROI: xmin: {} xmax: {} ymin: {} ymax: {}>", self.xmin, self.xmax, self.ymin, self.ymax);
         })
         .def("__iter__", [](const ROI &self) {
-            return py::make_iterator(&self.xmin, &self.ymax+1);
+            return py::make_iterator(&self.xmin, &self.ymax+1); //NOLINT
         });
 
     
@@ -231,7 +245,7 @@ void define_file_io_bindings(py::module &m) {
                  return image;
              });
 
-
+#pragma GCC diagnostic pop
     // py::class_<ClusterHeader>(m, "ClusterHeader")
     //     .def(py::init<>())
     //     .def_readwrite("frame_number", &ClusterHeader::frame_number)

python/src/fit.hpp

@@ -0,0 +1,250 @@
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl_bind.h>
+
+#include "aare/Fit.hpp"
+
+namespace py = pybind11;
+using namespace pybind11::literals;
+
+
+void define_fit_bindings(py::module &m) {
+
+    // TODO! Evaluate without converting to double
+    m.def(
+        "gaus",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
+            auto x_view = make_view_1d(x);
+            auto par_view = make_view_1d(par);
+            auto y = new NDArray<double, 1>{aare::func::gaus(x_view, par_view)};
+            return return_image_data(y);
+        },
+        R"(
+        Evaluate a 1D Gaussian function for all points in x using parameters par.
+
+        Parameters
+        ----------
+        x : array_like
+            The points at which to evaluate the Gaussian function.
+        par : array_like
+            The parameters of the Gaussian function. The first element is the amplitude, the second element is the mean, and the third element is the standard deviation.
+        )",
+        py::arg("x"), py::arg("par"));
+
+    m.def(
+        "pol1",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
+            auto x_view = make_view_1d(x);
+            auto par_view = make_view_1d(par);
+            auto y = new NDArray<double, 1>{aare::func::pol1(x_view, par_view)};
+            return return_image_data(y);
+        },
+        R"(
+        Evaluate a 1D polynomial function for all points in x using parameters par. (p0+p1*x)
+        
+        Parameters
+        ----------
+        x : array_like
+            The points at which to evaluate the polynomial function.
+        par : array_like
+            The parameters of the polynomial function. The first element is the intercept, and the second element is the slope.    
+        )",
+        py::arg("x"), py::arg("par"));
+
+
+    m.def(
+        "fit_gaus",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y,
+           int n_threads) {
+            if (y.ndim() == 3) {
+                auto par = new NDArray<double, 3>{};
+                auto y_view = make_view_3d(y);
+                auto x_view = make_view_1d(x);
+                *par = aare::fit_gaus(x_view, y_view, n_threads);
+                return return_image_data(par);
+            } else if (y.ndim() == 1) {
+                auto par = new NDArray<double, 1>{};
+                auto y_view = make_view_1d(y);
+                auto x_view = make_view_1d(x);
+                *par = aare::fit_gaus(x_view, y_view);
+                return return_image_data(par);
+            } else {
+                throw std::runtime_error("Data must be 1D or 3D");
+            }
+        },
+        R"(
+
+Fit a 1D Gaussian to data.
+
+Parameters
+----------
+x : array_like
+    The x values.
+y : array_like
+    The y values.
+n_threads : int, optional
+    The number of threads to use. Default is 4.
+)",
+        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
+
+    m.def(
+        "fit_gaus",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
+           int n_threads) {
+
+            if (y.ndim() == 3) {
+                // Allocate memory for the output
+                // Need to have pointers to allow python to manage
+                // the memory
+                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 3});
+                auto par_err =
+                    new NDArray<double, 3>({y.shape(0), y.shape(1), 3});
+                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
+
+                // Make views of the numpy arrays
+                auto y_view = make_view_3d(y);
+                auto y_view_err = make_view_3d(y_err);
+                auto x_view = make_view_1d(x);
+
+                aare::fit_gaus(x_view, y_view, y_view_err, par->view(),
+                               par_err->view(), chi2->view(), n_threads);
+
+                return py::dict("par"_a = return_image_data(par),
+                                "par_err"_a = return_image_data(par_err),
+                                "chi2"_a = return_image_data(chi2),
+                                "Ndf"_a = y.shape(2) - 3);
+            } else if (y.ndim() == 1) {
+                // Allocate memory for the output
+                // Need to have pointers to allow python to manage
+                // the memory
+                auto par = new NDArray<double, 1>({3});
+                auto par_err = new NDArray<double, 1>({3});
+
+                // Decode the numpy arrays
+                auto y_view = make_view_1d(y);
+                auto y_view_err = make_view_1d(y_err);
+                auto x_view = make_view_1d(x);
+
+
+                double chi2 = 0;
+                aare::fit_gaus(x_view, y_view, y_view_err, par->view(),
+                               par_err->view(), chi2);
+
+                return py::dict("par"_a = return_image_data(par),
+                                "par_err"_a = return_image_data(par_err),
+                                "chi2"_a = chi2, "Ndf"_a = y.size() - 3);
+
+            } else {
+                throw std::runtime_error("Data must be 1D or 3D");
+            }
+        },
+        R"(
+
+Fit a 1D Gaussian to data with error estimates.
+
+Parameters
+----------
+x : array_like
+    The x values.
+y : array_like
+    The y values.
+y_err : array_like
+    The error in the y values.
+n_threads : int, optional
+    The number of threads to use. Default is 4.
+)",
+        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
+
+    m.def(
+        "fit_pol1",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y,
+           int n_threads) {
+            if (y.ndim() == 3) {
+                auto par = new NDArray<double, 3>{};
+
+                auto x_view = make_view_1d(x);
+                auto y_view = make_view_3d(y);
+                *par = aare::fit_pol1(x_view, y_view, n_threads);
+                return return_image_data(par);
+            } else if (y.ndim() == 1) {
+                auto par = new NDArray<double, 1>{};
+                auto x_view = make_view_1d(x);
+                auto y_view = make_view_1d(y);
+                *par = aare::fit_pol1(x_view, y_view);
+                return return_image_data(par);
+            } else {
+                throw std::runtime_error("Data must be 1D or 3D");
+            }
+        },
+        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
+
+    m.def(
+        "fit_pol1",
+        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y,
+           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
+           int n_threads) {
+            if (y.ndim() == 3) {
+                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 2});
+
+                auto par_err =
+                    new NDArray<double, 3>({y.shape(0), y.shape(1), 2});
+
+                auto y_view = make_view_3d(y);
+                auto y_view_err = make_view_3d(y_err);
+                auto x_view = make_view_1d(x);
+
+                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
+
+                aare::fit_pol1(x_view, y_view, y_view_err, par->view(),
+                               par_err->view(), chi2->view(), n_threads);
+                return py::dict("par"_a = return_image_data(par),
+                                "par_err"_a = return_image_data(par_err), 
+                                "chi2"_a = return_image_data(chi2),
+                                "Ndf"_a = y.shape(2) - 2);
+
+
+            } else if (y.ndim() == 1) {
+                auto par = new NDArray<double, 1>({2});
+                auto par_err = new NDArray<double, 1>({2});
+
+                auto y_view = make_view_1d(y);
+                auto y_view_err = make_view_1d(y_err);
+                auto x_view = make_view_1d(x);
+
+                double chi2 = 0;
+
+                aare::fit_pol1(x_view, y_view, y_view_err, par->view(),
+                               par_err->view(), chi2);
+                return py::dict("par"_a = return_image_data(par),
+                                "par_err"_a = return_image_data(par_err),
+                                "chi2"_a = chi2, "Ndf"_a = y.size() - 2);
+
+            } else {
+                throw std::runtime_error("Data must be 1D or 3D");
+            }
+        },
+        R"(
+Fit a 1D polynomial to data with error estimates.
+
+Parameters
+----------
+x : array_like
+    The x values.
+y : array_like
+    The y values.
+y_err : array_like
+    The error in the y values.
+n_threads : int, optional
+    The number of threads to use. Default is 4.
+)",
+        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
+}

python/src/interpolation.hpp

@@ -0,0 +1,81 @@
+#include "aare/Interpolator.hpp"
+#include "aare/NDArray.hpp"
+#include "aare/NDView.hpp"
+#include "np_helper.hpp"
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+template <typename ClusterType>
+void register_interpolate(py::class_<aare::Interpolator> &interpolator,
+                          const std::string &typestr) {
+    auto name = fmt::format("interpolate_{}", typestr);
+
+    interpolator.def(name.c_str(),
+                     [](aare::Interpolator &self,
+                        const ClusterVector<ClusterType> &clusters) {
+                         auto photons = self.interpolate<ClusterType>(clusters);
+                         auto *ptr = new std::vector<Photon>{photons};
+                         return return_vector(ptr);
+                     });
+}
+
+void define_interpolation_bindings(py::module &m) {
+
+    PYBIND11_NUMPY_DTYPE(aare::Photon, x, y, energy);
+
+    auto interpolator =
+        py::class_<aare::Interpolator>(m, "Interpolator")
+            .def(py::init([](py::array_t<double, py::array::c_style |
+                                                     py::array::forcecast>
+                                 etacube,
+                             py::array_t<double> xbins,
+                             py::array_t<double> ybins,
+                             py::array_t<double> ebins) {
+                return Interpolator(make_view_3d(etacube), make_view_1d(xbins),
+                                    make_view_1d(ybins), make_view_1d(ebins));
+            }))
+            .def("get_ietax",
+                 [](Interpolator &self) {
+                     auto *ptr = new NDArray<double, 3>{};
+                     *ptr = self.get_ietax();
+                     return return_image_data(ptr);
+                 })
+            .def("get_ietay", [](Interpolator &self) {
+                auto *ptr = new NDArray<double, 3>{};
+                *ptr = self.get_ietay();
+                return return_image_data(ptr);
+            });
+
+    register_interpolate<Cluster<int, 3, 3>>(interpolator, "Cluster3x3i");
+    register_interpolate<Cluster<float, 3, 3>>(interpolator, "Cluster3x3f");
+    register_interpolate<Cluster<double, 3, 3>>(interpolator, "Cluster3x3d");
+    register_interpolate<Cluster<int, 2, 2>>(interpolator, "Cluster2x2i");
+    register_interpolate<Cluster<float, 2, 2>>(interpolator, "Cluster2x2f");
+    register_interpolate<Cluster<double, 2, 2>>(interpolator, "Cluster2x2d");
+
+    // TODO! Evaluate without converting to double
+    m.def(
+        "hej",
+        []() {
+            // auto boost_histogram = py::module_::import("boost_histogram");
+            // py::object axis =
+            // boost_histogram.attr("axis").attr("Regular")(10, 0.0, 10.0);
+            // py::object histogram = boost_histogram.attr("Histogram")(axis);
+            // return histogram;
+            // return h;
+        },
+        R"(
+        Evaluate a 1D Gaussian function for all points in x using parameters par.
+
+        Parameters
+        ----------
+        x : array_like
+            The points at which to evaluate the Gaussian function.
+        par : array_like
+            The parameters of the Gaussian function. The first element is the amplitude, the second element is the mean, and the third element is the standard deviation.
+        )");
+}

python/src/module.cpp

@@ -1,14 +1,17 @@
-//Files with bindings to the different classes
+// Files with bindings to the different classes
+#include "cluster.hpp"
+#include "cluster_file.hpp"
+#include "ctb_raw_file.hpp"
 #include "file.hpp"
+#include "fit.hpp"
+#include "interpolation.hpp"
+#include "pedestal.hpp"
+#include "pixel_map.hpp"
 #include "raw_file.hpp"
 #include "raw_master_file.hpp"
 #include "var_cluster.hpp"
-#include "pixel_map.hpp"
-#include "pedestal.hpp"
-#include "cluster.hpp"
-#include "cluster_file.hpp"
 
-//Pybind stuff
+// Pybind stuff
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
@@ -17,11 +20,61 @@ namespace py = pybind11;
 PYBIND11_MODULE(_aare, m) {
     define_file_io_bindings(m);
     define_raw_file_io_bindings(m);
+    define_ctb_raw_file_io_bindings(m);
     define_raw_master_file_bindings(m);
     define_var_cluster_finder_bindings(m);
     define_pixel_map_bindings(m);
-    define_pedestal_bindings<double>(m, "Pedestal");
-    define_pedestal_bindings<float>(m, "Pedestal_float32");
-    define_cluster_finder_bindings(m);
-    define_cluster_file_io_bindings(m);
-}
+    define_pedestal_bindings<double>(m, "Pedestal_d");
+    define_pedestal_bindings<float>(m, "Pedestal_f");
+    define_fit_bindings(m);
+    define_interpolation_bindings(m);
+
+    define_cluster_file_io_bindings<Cluster<int, 3, 3>>(m, "Cluster3x3i");
+    define_cluster_file_io_bindings<Cluster<double, 3, 3>>(m, "Cluster3x3d");
+    define_cluster_file_io_bindings<Cluster<float, 3, 3>>(m, "Cluster3x3f");
+    define_cluster_file_io_bindings<Cluster<int, 2, 2>>(m, "Cluster2x2i");
+    define_cluster_file_io_bindings<Cluster<float, 2, 2>>(m, "Cluster2x2f");
+    define_cluster_file_io_bindings<Cluster<double, 2, 2>>(m, "Cluster2x2d");
+
+    define_cluster_vector<int, 3, 3, uint16_t>(m, "Cluster3x3i");
+    define_cluster_vector<double, 3, 3, uint16_t>(m, "Cluster3x3d");
+    define_cluster_vector<float, 3, 3, uint16_t>(m, "Cluster3x3f");
+    define_cluster_vector<int, 2, 2, uint16_t>(m, "Cluster2x2i");
+    define_cluster_vector<double, 2, 2, uint16_t>(m, "Cluster2x2d");
+    define_cluster_vector<float, 2, 2, uint16_t>(m, "Cluster2x2f");
+
+    define_cluster_finder_bindings<Cluster<int, 3, 3>>(m, "Cluster3x3i");
+    define_cluster_finder_bindings<Cluster<double, 3, 3>>(m, "Cluster3x3d");
+    define_cluster_finder_bindings<Cluster<float, 3, 3>>(m, "Cluster3x3f");
+    define_cluster_finder_bindings<Cluster<int, 2, 2>>(m, "Cluster2x2i");
+    define_cluster_finder_bindings<Cluster<double, 2, 2>>(m, "Cluster2x2d");
+    define_cluster_finder_bindings<Cluster<float, 2, 2>>(m, "Cluster2x2f");
+
+    define_cluster_finder_mt_bindings<Cluster<int, 3, 3>>(m, "Cluster3x3i");
+    define_cluster_finder_mt_bindings<Cluster<double, 3, 3>>(m, "Cluster3x3d");
+    define_cluster_finder_mt_bindings<Cluster<float, 3, 3>>(m, "Cluster3x3f");
+    define_cluster_finder_mt_bindings<Cluster<int, 2, 2>>(m, "Cluster2x2i");
+    define_cluster_finder_mt_bindings<Cluster<double, 2, 2>>(m, "Cluster2x2d");
+    define_cluster_finder_mt_bindings<Cluster<float, 2, 2>>(m, "Cluster2x2f");
+
+    define_cluster_file_sink_bindings<Cluster<int, 3, 3>>(m, "Cluster3x3i");
+    define_cluster_file_sink_bindings<Cluster<double, 3, 3>>(m, "Cluster3x3d");
+    define_cluster_file_sink_bindings<Cluster<float, 3, 3>>(m, "Cluster3x3f");
+    define_cluster_file_sink_bindings<Cluster<int, 2, 2>>(m, "Cluster2x2i");
+    define_cluster_file_sink_bindings<Cluster<double, 2, 2>>(m, "Cluster2x2d");
+    define_cluster_file_sink_bindings<Cluster<float, 2, 2>>(m, "Cluster2x2f");
+
+    define_cluster_collector_bindings<Cluster<int, 3, 3>>(m, "Cluster3x3i");
+    define_cluster_collector_bindings<Cluster<double, 3, 3>>(m, "Cluster3x3f");
+    define_cluster_collector_bindings<Cluster<float, 3, 3>>(m, "Cluster3x3d");
+    define_cluster_collector_bindings<Cluster<int, 2, 2>>(m, "Cluster2x2i");
+    define_cluster_collector_bindings<Cluster<double, 2, 2>>(m, "Cluster2x2f");
+    define_cluster_collector_bindings<Cluster<float, 2, 2>>(m, "Cluster2x2d");
+
+    define_cluster<int, 3, 3, uint16_t>(m, "3x3i");
+    define_cluster<float, 3, 3, uint16_t>(m, "3x3f");
+    define_cluster<double, 3, 3, uint16_t>(m, "3x3d");
+    define_cluster<int, 2, 2, uint16_t>(m, "2x2i");
+    define_cluster<float, 2, 2, uint16_t>(m, "2x2f");
+    define_cluster<double, 2, 2, uint16_t>(m, "2x2d");
+}

python/src/np_helper.hpp

@@ -10,6 +10,7 @@
 #include "aare/NDView.hpp"
 
 namespace py = pybind11;
+using namespace aare;
 
 // Pass image data back to python as a numpy array
 template <typename T, int64_t Ndim>
@@ -39,78 +40,47 @@ template <typename T> py::array return_vector(std::vector<T> *vec) {
                           free_when_done); // numpy array references this parent
 }
 
-// template <typename Reader> py::array do_read(Reader &r, size_t n_frames) {
-//     py::array image;
-//     if (n_frames == 0)
-//         n_frames = r.total_frames();
-
-//     std::array<ssize_t, 3> shape{static_cast<ssize_t>(n_frames), r.rows(),
-//                                  r.cols()};
-//     const uint8_t item_size = r.bytes_per_pixel();
-//     if (item_size == 1) {
-//         image = py::array_t<uint8_t, py::array::c_style | py::array::forcecast>(
-//             shape);
-//     } else if (item_size == 2) {
-//         image =
-//             py::array_t<uint16_t, py::array::c_style | py::array::forcecast>(
-//                 shape);
-//     } else if (item_size == 4) {
-//         image =
-//             py::array_t<uint32_t, py::array::c_style | py::array::forcecast>(
-//                 shape);
-//     }
-//     r.read_into(reinterpret_cast<std::byte *>(image.mutable_data()), n_frames);
-//     return image;
-// }
-
-// py::array return_frame(pl::Frame *ptr) {
-//     py::capsule free_when_done(ptr, [](void *f) {
-//         pl::Frame *foo = reinterpret_cast<pl::Frame *>(f);
-//         delete foo;
-//     });
-
-//     const uint8_t item_size = ptr->bytes_per_pixel();
-//     std::vector<ssize_t> shape;
-//     for (auto val : ptr->shape())
-//         if (val > 1)
-//             shape.push_back(val);
-
-//     std::vector<ssize_t> strides;
-//     if (shape.size() == 1)
-//         strides.push_back(item_size);
-//     else if (shape.size() == 2) {
-//         strides.push_back(item_size * shape[1]);
-//         strides.push_back(item_size);
-//     }
-
-//     if (item_size == 1)
-//         return py::array_t<uint8_t>(
-//             shape, strides,
-//             reinterpret_cast<uint8_t *>(ptr->data()), free_when_done);
-//     else if (item_size == 2)
-//         return py::array_t<uint16_t>(shape, strides,
-//                                      reinterpret_cast<uint16_t *>(ptr->data()),
-//                                      free_when_done);
-//     else if (item_size == 4)
-//         return py::array_t<uint32_t>(shape, strides,
-//                                      reinterpret_cast<uint32_t *>(ptr->data()),
-//                                      free_when_done);
-//     return {};
-// }
-
 // todo rewrite generic
-template <class T, int Flags> auto get_shape_3d(py::array_t<T, Flags> arr) {
+template <class T, int Flags>
+auto get_shape_3d(const py::array_t<T, Flags> &arr) {
     return aare::Shape<3>{arr.shape(0), arr.shape(1), arr.shape(2)};
 }
 
-template <class T, int Flags> auto make_view_3d(py::array_t<T, Flags> arr) {
+template <class T, int Flags> auto make_view_3d(py::array_t<T, Flags> &arr) {
     return aare::NDView<T, 3>(arr.mutable_data(), get_shape_3d<T, Flags>(arr));
 }
 
-template <class T, int Flags> auto get_shape_2d(py::array_t<T, Flags> arr) {
+template <class T, int Flags>
+auto get_shape_2d(const py::array_t<T, Flags> &arr) {
     return aare::Shape<2>{arr.shape(0), arr.shape(1)};
 }
 
-template <class T, int Flags> auto make_view_2d(py::array_t<T, Flags> arr) {
+template <class T, int Flags>
+auto get_shape_1d(const py::array_t<T, Flags> &arr) {
+    return aare::Shape<1>{arr.shape(0)};
+}
+
+template <class T, int Flags> auto make_view_2d(py::array_t<T, Flags> &arr) {
     return aare::NDView<T, 2>(arr.mutable_data(), get_shape_2d<T, Flags>(arr));
-}
+}
+template <class T, int Flags> auto make_view_1d(py::array_t<T, Flags> &arr) {
+    return aare::NDView<T, 1>(arr.mutable_data(), get_shape_1d<T, Flags>(arr));
+}
+
+template <typename ClusterType> struct fmt_format_trait; // forward declaration
+
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+struct fmt_format_trait<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
+
+    static std::string value() {
+        return fmt::format("T{{{}:x;{}:y;{}:data;}}",
+                           py::format_descriptor<CoordType>::format(),
+                           py::format_descriptor<CoordType>::format(),
+                           fmt::format("{}{}", ClusterSizeX * ClusterSizeY,
+                                       py::format_descriptor<T>::format()));
+    }
+};
+
+template <typename ClusterType>
+auto fmt_format = fmt_format_trait<ClusterType>::value();

python/src/pedestal.hpp

@@ -15,19 +15,19 @@ template <typename SUM_TYPE> void define_pedestal_bindings(py::module &m, const
         .def(py::init<int, int>())
         .def("mean",
              [](Pedestal<SUM_TYPE> &self) {
-                 auto m = new NDArray<SUM_TYPE, 2>{};
-                 *m = self.mean();
-                 return return_image_data(m);
+                 auto mea = new NDArray<SUM_TYPE, 2>{};
+                 *mea = self.mean();
+                 return return_image_data(mea);
              })
         .def("variance", [](Pedestal<SUM_TYPE> &self) {
-            auto m = new NDArray<SUM_TYPE, 2>{};
-            *m = self.variance();
-            return return_image_data(m);
+            auto var = new NDArray<SUM_TYPE, 2>{};
+            *var = self.variance();
+            return return_image_data(var);
         })
         .def("std", [](Pedestal<SUM_TYPE> &self) {
-            auto m = new NDArray<SUM_TYPE, 2>{};
-            *m = self.std();
-            return return_image_data(m);
+            auto std = new NDArray<SUM_TYPE, 2>{};
+            *std = self.std();
+            return return_image_data(std);
         })
         .def("clear", py::overload_cast<>(&Pedestal<SUM_TYPE>::clear))
         .def_property_readonly("rows", &Pedestal<SUM_TYPE>::rows)
@@ -43,5 +43,10 @@ template <typename SUM_TYPE> void define_pedestal_bindings(py::module &m, const
         .def("push", [](Pedestal<SUM_TYPE> &pedestal, py::array_t<uint16_t> &f) {
             auto v = make_view_2d(f);
             pedestal.push(v);
-        });
+        })
+        .def("push_no_update", [](Pedestal<SUM_TYPE> &pedestal, py::array_t<uint16_t, py::array::c_style> &f) {
+            auto v = make_view_2d(f);
+            pedestal.push_no_update(v);
+        }, py::arg().noconvert())
+        .def("update_mean", &Pedestal<SUM_TYPE>::update_mean);
 }

python/src/pixel_map.hpp

@@ -20,6 +20,14 @@ void define_pixel_map_bindings(py::module &m) {
     .def("GenerateMoench05PixelMap", []() {
         auto ptr = new NDArray<ssize_t,2>(GenerateMoench05PixelMap());
         return return_image_data(ptr);
+    })
+        .def("GenerateMoench05PixelMap1g", []() {
+        auto ptr = new NDArray<ssize_t,2>(GenerateMoench05PixelMap1g());
+        return return_image_data(ptr);
+    })
+    .def("GenerateMoench05PixelMapOld", []() {
+        auto ptr = new NDArray<ssize_t,2>(GenerateMoench05PixelMapOld());
+        return return_image_data(ptr);
     })
     .def("GenerateMH02SingleCounterPixelMap", []() {
         auto ptr = new NDArray<ssize_t,2>(GenerateMH02SingleCounterPixelMap());

python/src/raw_file.hpp

@@ -25,7 +25,6 @@ void define_raw_file_io_bindings(py::module &m) {
         .def(py::init<const std::filesystem::path &>())
         .def("read_frame",
              [](RawFile &self) {
-                 size_t image_size = self.bytes_per_frame();
                  py::array image;
                  std::vector<ssize_t> shape;
                  shape.reserve(2);
@@ -49,32 +48,44 @@ void define_raw_file_io_bindings(py::module &m) {
 
                  return py::make_tuple(header, image);
              })
-        .def("read_n",
-             [](RawFile &self, size_t n_frames) {
-                 py::array image;
-                 std::vector<ssize_t> shape;
-                 shape.reserve(3);
-                 shape.push_back(n_frames);
-                 shape.push_back(self.rows());
-                 shape.push_back(self.cols());
+        .def(
+            "read_n",
+            [](RawFile &self, size_t n_frames) {
+                // adjust for actual frames left in the file
+                n_frames =
+                    std::min(n_frames, self.total_frames() - self.tell());
+                if (n_frames == 0) {
+                    throw std::runtime_error("No frames left in file");
+                }
+                std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
+            
+                // return headers from all subfiles
+                py::array_t<DetectorHeader> header;
+                if (self.n_mod() == 1) {
+                    header = py::array_t<DetectorHeader>(n_frames);
+                } else {
+                    header = py::array_t<DetectorHeader>({self.n_mod(), n_frames});
+                }
+                // py::array_t<DetectorHeader> header({self.n_mod(), n_frames});
 
-                 // return headers from all subfiles
-                 py::array_t<DetectorHeader> header({self.n_mod(), n_frames});
+                py::array image;
+                const uint8_t item_size = self.bytes_per_pixel();
+                if (item_size == 1) {
+                    image = py::array_t<uint8_t>(shape);
+                } else if (item_size == 2) {
+                    image = py::array_t<uint16_t>(shape);
+                } else if (item_size == 4) {
+                    image = py::array_t<uint32_t>(shape);
+                }
+                self.read_into(
+                    reinterpret_cast<std::byte *>(image.mutable_data()),
+                    n_frames, header.mutable_data());
 
-                 const uint8_t item_size = self.bytes_per_pixel();
-                 if (item_size == 1) {
-                     image = py::array_t<uint8_t>(shape);
-                 } else if (item_size == 2) {
-                     image = py::array_t<uint16_t>(shape);
-                 } else if (item_size == 4) {
-                     image = py::array_t<uint32_t>(shape);
-                 }
-                 self.read_into(
-                     reinterpret_cast<std::byte *>(image.mutable_data()),
-                     n_frames, header.mutable_data());
-
-                 return py::make_tuple(header, image);
-             })
+                return py::make_tuple(header, image);
+            },
+            R"(
+             Read n frames from the file.
+             )")
         .def("frame_number", &RawFile::frame_number)
         .def_property_readonly("bytes_per_frame", &RawFile::bytes_per_frame)
         .def_property_readonly("pixels_per_frame", &RawFile::pixels_per_frame)

python/src/var_cluster.hpp

@@ -19,15 +19,24 @@ using namespace::aare;
 
 void define_var_cluster_finder_bindings(py::module &m) {
     PYBIND11_NUMPY_DTYPE(VarClusterFinder<double>::Hit, size, row, col,
-                         reserved, energy, max);
+                         reserved, energy, max, rows, cols, enes);
 
     py::class_<VarClusterFinder<double>>(m, "VarClusterFinder")
         .def(py::init<Shape<2>, double>())
         .def("labeled",
              [](VarClusterFinder<double> &self) {
-                 auto ptr = new NDArray<int, 2>(self.labeled());
+                 auto *ptr = new NDArray<int, 2>(self.labeled());
                  return return_image_data(ptr);
              })
+        .def("set_noiseMap",
+            [](VarClusterFinder<double> &self,
+                py::array_t<double, py::array::c_style | py::array::forcecast>
+                    noise_map) {
+                auto noise_map_span = make_view_2d(noise_map);
+                self.set_noiseMap(noise_map_span);
+            })
+        .def("set_peripheralThresholdFactor",
+             &VarClusterFinder<double>::set_peripheralThresholdFactor)
         .def("find_clusters",
              [](VarClusterFinder<double> &self,
                 py::array_t<double, py::array::c_style | py::array::forcecast>
@@ -35,6 +44,30 @@ void define_var_cluster_finder_bindings(py::module &m) {
                  auto view = make_view_2d(img);
                  self.find_clusters(view);
              })
+        .def("find_clusters_X",
+             [](VarClusterFinder<double> &self,
+                py::array_t<double, py::array::c_style | py::array::forcecast>
+                    img) {
+                 auto img_span = make_view_2d(img);
+                 self.find_clusters_X(img_span);
+             })
+        .def("single_pass",
+             [](VarClusterFinder<double> &self,
+                py::array_t<double, py::array::c_style | py::array::forcecast>
+                    img) {
+                 auto img_span = make_view_2d(img);
+                 self.single_pass(img_span);
+             })
+        .def("hits",
+             [](VarClusterFinder<double> &self) {
+                 auto ptr = new std::vector<VarClusterFinder<double>::Hit>(
+                     self.steal_hits());
+                 return return_vector(ptr);
+             })
+        .def("clear_hits",
+             [](VarClusterFinder<double> &self) {
+                 self.clear_hits();
+             })
         .def("steal_hits",
              [](VarClusterFinder<double> &self) {
                  auto ptr = new std::vector<VarClusterFinder<double>::Hit>(

python/tests/test_Cluster.py

@@ -0,0 +1,64 @@
+import pytest 
+import numpy as np
+
+from _aare import ClusterVector_Cluster3x3i, Interpolator, Cluster3x3i, ClusterFinder_Cluster3x3i
+
+def test_ClusterVector(): 
+    """Test ClusterVector""" 
+
+    clustervector = ClusterVector_Cluster3x3i()
+    assert clustervector.cluster_size_x == 3
+    assert clustervector.cluster_size_y == 3
+    assert clustervector.item_size() == 4+9*4
+    assert clustervector.frame_number == 0
+    assert clustervector.capacity == 1024
+    assert clustervector.size == 0
+
+    cluster = Cluster3x3i(0,0,np.ones(9, dtype=np.int32))
+    
+    clustervector.push_back(cluster) 
+    assert clustervector.size == 1
+
+    #push_back - check size
+
+
+
+def test_Interpolator(): 
+    """Test Interpolator"""
+
+    ebins = np.linspace(0,10, 20, dtype=np.float64)
+    xbins = np.linspace(0, 5, 30, dtype=np.float64)
+    ybins = np.linspace(0, 5, 30, dtype=np.float64)
+
+    etacube = np.zeros(shape=[30, 30, 20], dtype=np.float64)
+    interpolator = Interpolator(etacube, xbins, ybins, ebins)
+
+    assert interpolator.get_ietax().shape == (30,30,20)
+    assert interpolator.get_ietay().shape == (30,30,20)
+    clustervector = ClusterVector_Cluster3x3i()
+    
+    cluster = Cluster3x3i(0,0, np.ones(9, dtype=np.int32))
+    #clustervector.push_back(cluster) 
+    #num_clusters = 1; 
+
+    #assert interpolator.interpolate_Cluster3x3i(clustervector).shape == (num_clusters, 3)
+
+
+#def test_cluster_file(): 
+
+#def test_cluster_finder(): 
+    #"""Test ClusterFinder""" 
+
+    #clusterfinder = ClusterFinder_Cluster3x3i([100,100])
+
+    #clusterfinder.find_clusters()
+
+    #clusters = clusterfinder.steal_clusters()
+
+    #print("cluster size: ", clusters.size()) 
+
+
+
+
+
+

src/CalculateEta.test.cpp

@@ -0,0 +1,62 @@
+/************************************************
+ * @file CalculateEta.test.cpp
+ * @short test case to calculate_eta2
+ ***********************************************/
+
+#include "aare/CalculateEta.hpp"
+#include "aare/Cluster.hpp"
+#include "aare/ClusterFile.hpp"
+
+// #include "catch.hpp"
+#include <array>
+#include <catch2/catch_all.hpp>
+#include <catch2/catch_test_macros.hpp>
+
+using namespace aare;
+
+using ClusterTypes =
+    std::variant<Cluster<int, 2, 2>, Cluster<int, 3, 3>, Cluster<int, 5, 5>,
+                 Cluster<int, 4, 2>, Cluster<int, 2, 3>>;
+
+auto get_test_parameters() {
+    return GENERATE(
+        std::make_tuple(ClusterTypes{Cluster<int, 2, 2>{0, 0, {1, 2, 3, 1}}},
+                        Eta2<int>{2. / 3, 3. / 4, corner::cBottomLeft, 7}),
+        std::make_tuple(
+            ClusterTypes{Cluster<int, 3, 3>{0, 0, {1, 2, 3, 4, 5, 6, 1, 2, 7}}},
+            Eta2<int>{6. / 11, 2. / 7, corner::cTopRight, 20}),
+        std::make_tuple(ClusterTypes{Cluster<int, 5, 5>{
+                            0, 0, {1, 6, 7, 6, 5, 4, 3, 2, 1, 8, 8, 9, 2,
+                                   1, 4, 5, 6, 7, 8, 4, 1, 1, 1, 1, 1}}},
+                        Eta2<int>{9. / 17, 5. / 13, 8, 28}),
+        std::make_tuple(
+            ClusterTypes{Cluster<int, 4, 2>{0, 0, {1, 4, 7, 2, 5, 6, 4, 3}}},
+            Eta2<int>{7. / 11, 6. / 10, 1, 21}),
+        std::make_tuple(
+            ClusterTypes{Cluster<int, 2, 3>{0, 0, {1, 3, 2, 3, 4, 2}}},
+            Eta2<int>{3. / 5, 4. / 6, 1, 11}));
+}
+
+TEST_CASE("compute_largest_2x2_subcluster", "[.eta_calculation]") {
+    auto [cluster, expected_eta] = get_test_parameters();
+
+    auto [sum, index] = std::visit(
+        [](const auto &clustertype) { return clustertype.max_sum_2x2(); },
+        cluster);
+    CHECK(expected_eta.c == index);
+    CHECK(expected_eta.sum == sum);
+}
+
+TEST_CASE("calculate_eta2", "[.eta_calculation]") {
+
+    auto [cluster, expected_eta] = get_test_parameters();
+
+    auto eta = std::visit(
+        [](const auto &clustertype) { return calculate_eta2(clustertype); },
+        cluster);
+
+    CHECK(eta.x == expected_eta.x);
+    CHECK(eta.y == expected_eta.y);
+    CHECK(eta.c == expected_eta.c);
+    CHECK(eta.sum == expected_eta.sum);
+}

src/Cluster.test.cpp

@@ -0,0 +1,28 @@
+/************************************************
+ * @file test-Cluster.cpp
+ * @short test case for generic Cluster, ClusterVector, and calculate_eta2
+ ***********************************************/
+
+#include "aare/Cluster.hpp"
+#include "aare/CalculateEta.hpp"
+#include "aare/ClusterFile.hpp"
+
+// #include "catch.hpp"
+#include <array>
+#include <catch2/catch_all.hpp>
+#include <catch2/catch_test_macros.hpp>
+
+using namespace aare;
+
+TEST_CASE("Correct Instantiation of Cluster and ClusterVector",
+          "[.cluster][.instantiation]") {
+
+    CHECK(is_valid_cluster<double, 3, 3>);
+    CHECK(is_valid_cluster<double, 3, 2>);
+    CHECK(not is_valid_cluster<int, 0, 0>);
+    CHECK(not is_valid_cluster<std::string, 2, 2>);
+    CHECK(not is_valid_cluster<int, 2, 2, double>);
+
+    CHECK(not is_cluster_v<int>);
+    CHECK(is_cluster_v<Cluster<int, 3, 3>>);
+}

src/ClusterFile.cpp

@@ -1,312 +0,0 @@
-#include "aare/ClusterFile.hpp"
-
-namespace aare {
-
-ClusterFile::ClusterFile(const std::filesystem::path &fname, size_t chunk_size): m_chunk_size(chunk_size) {
-    fp = fopen(fname.c_str(), "rb");
-    if (!fp) {
-        throw std::runtime_error("Could not open file: " + fname.string());
-    }
-}
-
-std::vector<Cluster> ClusterFile::read_clusters(size_t n_clusters) {
-    std::vector<Cluster> clusters(n_clusters);
-
-    int32_t iframe = 0; // frame number needs to be 4 bytes!
-    size_t nph_read = 0;
-    uint32_t nn = m_num_left;
-    uint32_t nph = m_num_left; // number of clusters in frame needs to be 4
-
-    auto buf = reinterpret_cast<Cluster *>(clusters.data());
-    // if there are photons left from previous frame read them first
-    if (nph) {
-        if (nph > n_clusters) {
-            // if we have more photons left in the frame then photons to read we
-            // read directly the requested number
-            nn = n_clusters;
-        } else {
-            nn = nph;
-        }
-        nph_read += fread((void *)(buf + nph_read), sizeof(Cluster), nn, fp);
-        m_num_left = nph - nn; // write back the number of photons left
-    }
-
-    if (nph_read < n_clusters) {
-        // keep on reading frames and photons until reaching n_clusters
-        while (fread(&iframe, sizeof(iframe), 1, fp)) {
-            // read number of clusters in frame
-            if (fread(&nph, sizeof(nph), 1, fp)) {
-                if (nph > (n_clusters - nph_read))
-                    nn = n_clusters - nph_read;
-                else
-                    nn = nph;
-
-                nph_read +=
-                    fread((void *)(buf + nph_read), sizeof(Cluster), nn, fp);
-                m_num_left = nph - nn;
-            }
-            if (nph_read >= n_clusters)
-                break;
-        }
-    }
-
-    // Resize the vector to the number of clusters.
-    // No new allocation, only change bounds.
-    clusters.resize(nph_read);
-    return clusters;
-}
-
-std::vector<Cluster> ClusterFile::read_frame(int32_t &out_fnum) {
-    if (m_num_left) {
-        throw std::runtime_error("There are still photons left in the last frame");
-    }
-
-    if (fread(&out_fnum, sizeof(out_fnum), 1, fp) != 1) {
-        throw std::runtime_error("Could not read frame number");
-    }
-
-    int n_clusters;
-    if (fread(&n_clusters, sizeof(n_clusters), 1, fp) != 1) {
-        throw std::runtime_error("Could not read number of clusters");
-    }
-    std::vector<Cluster> clusters(n_clusters);
-
-    if (fread(clusters.data(), sizeof(Cluster), n_clusters, fp) != n_clusters) {
-        throw std::runtime_error("Could not read clusters");
-    }
-    return clusters;
-
-}
-
-std::vector<Cluster> ClusterFile::read_cluster_with_cut(size_t n_clusters,
-                                                        double *noise_map,
-                                                        int nx, int ny) {
-
-    std::vector<Cluster> clusters(n_clusters);
-    // size_t read_clusters_with_cut(FILE *fp, size_t n_clusters, Cluster *buf,
-    //                               uint32_t *n_left, double *noise_map, int
-    //                               nx, int ny) {
-    int iframe = 0;
-    //     uint32_t nph = *n_left;
-    uint32_t nph = m_num_left;
-    //     uint32_t nn = *n_left;
-    uint32_t nn = m_num_left;
-    size_t nph_read = 0;
-
-    int32_t t2max, tot1;
-    int32_t tot3;
-    // Cluster *ptr = buf;
-    Cluster *ptr = clusters.data();
-    int good = 1;
-    double noise;
-    // read photons left from previous frame
-    if (noise_map)
-        printf("Using noise map\n");
-
-    if (nph) {
-        if (nph > n_clusters) {
-            // if we have more photons left in the frame then photons to
-            // read we read directly the requested number
-            nn = n_clusters;
-        } else {
-            nn = nph;
-        }
-        for (size_t iph = 0; iph < nn; iph++) {
-            // read photons 1 by 1
-            size_t n_read = fread((void *)(ptr), sizeof(Cluster), 1, fp);
-            if (n_read != 1) {
-                clusters.resize(nph_read);
-                return clusters;
-            }
-            // TODO! error handling on read
-            good = 1;
-            if (noise_map) {
-                if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 && ptr->y < ny) {
-                    tot1 = ptr->data[4];
-                    analyze_cluster(*ptr, &t2max, &tot3, NULL, NULL, NULL, NULL,
-                                    NULL);
-                    noise = noise_map[ptr->y * nx + ptr->x];
-                    if (tot1 > noise || t2max > 2 * noise || tot3 > 3 * noise) {
-                        ;
-                    } else {
-                        good = 0;
-                        printf("%d %d %f %d %d %d\n", ptr->x, ptr->y, noise,
-                               tot1, t2max, tot3);
-                    }
-                } else {
-                    printf("Bad pixel number %d %d\n", ptr->x, ptr->y);
-                    good = 0;
-                }
-            }
-            if (good) {
-                ptr++;
-                nph_read++;
-            }
-            (m_num_left)--;
-            if (nph_read >= n_clusters)
-                break;
-        }
-    }
-        if (nph_read < n_clusters) {
-    //         // keep on reading frames and photons until reaching n_clusters
-            while (fread(&iframe, sizeof(iframe), 1, fp)) {
-    //             // printf("%d\n",nph_read);
-
-                if (fread(&nph, sizeof(nph), 1, fp)) {
-    //                 // printf("** %d\n",nph);
-                    m_num_left = nph;
-                    for (size_t iph = 0; iph < nph; iph++) {
-    //                     // read photons 1 by 1
-                        size_t n_read =
-                            fread((void *)(ptr), sizeof(Cluster), 1, fp);
-                        if (n_read != 1) {
-                            clusters.resize(nph_read);
-                            return clusters;
-                            // return nph_read;
-                        }
-                        good = 1;
-                        if (noise_map) {
-                            if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 &&
-                                ptr->y < ny) {
-                                tot1 = ptr->data[4];
-                                analyze_cluster(*ptr, &t2max, &tot3, NULL,
-                                NULL,
-                                                NULL, NULL, NULL);
-                                // noise = noise_map[ptr->y * nx + ptr->x];
-                                noise = noise_map[ptr->y + ny * ptr->x];
-    			    if (tot1 > noise || t2max > 2 * noise ||
-                                    tot3 > 3 * noise) {
-                                    ;
-                                } else
-                                    good = 0;
-                            } else {
-                                printf("Bad pixel number %d %d\n", ptr->x,
-                                ptr->y); good = 0;
-                            }
-                        }
-                        if (good) {
-                            ptr++;
-    			nph_read++;
-                        }
-    		    (m_num_left)--;
-                        if (nph_read >= n_clusters)
-                            break;
-                    }
-                }
-                if (nph_read >= n_clusters)
-                    break;
-            }
-        }
-        // printf("%d\n",nph_read);
-        clusters.resize(nph_read);
-        return clusters;
-
-}
-
-int ClusterFile::analyze_cluster(Cluster cl, int32_t *t2, int32_t *t3, char *quad,
-                    double *eta2x, double *eta2y, double *eta3x,
-                    double *eta3y) {
-
-    return analyze_data(cl.data, t2, t3, quad, eta2x, eta2y, eta3x, eta3y);
-}
-
-int ClusterFile::analyze_data(int32_t *data, int32_t *t2, int32_t *t3, char *quad,
-                 double *eta2x, double *eta2y, double *eta3x, double *eta3y) {
-
-    int ok = 1;
-
-    int32_t tot2[4];
-    int32_t t2max = 0;
-    char c = 0;
-    int32_t val, tot3;
-
-    tot3 = 0;
-    for (int i = 0; i < 4; i++)
-        tot2[i] = 0;
-
-    for (int ix = 0; ix < 3; ix++) {
-        for (int iy = 0; iy < 3; iy++) {
-            val = data[iy * 3 + ix];
-            //	printf ("%d ",data[iy * 3 + ix]);
-            tot3 += val;
-            if (ix <= 1 && iy <= 1)
-                tot2[cBottomLeft] += val;
-            if (ix >= 1 && iy <= 1)
-                tot2[cBottomRight] += val;
-            if (ix <= 1 && iy >= 1)
-                tot2[cTopLeft] += val;
-            if (ix >= 1 && iy >= 1)
-                tot2[cTopRight] += val;
-        }
-        //	printf ("\n");
-    }
-    // printf ("\n");
-
-    if (t2 || quad) {
-
-        t2max = tot2[0];
-        c = cBottomLeft;
-        for (int i = 1; i < 4; i++) {
-            if (tot2[i] > t2max) {
-                t2max = tot2[i];
-                c = i;
-            }
-        }
-	//printf("*** %d %d %d %d -- %d\n",tot2[0],tot2[1],tot2[2],tot2[3],t2max);
-        if (quad)
-            *quad = c;
-        if (t2)
-            *t2 = t2max;
-    }
-    if (t3)
-        *t3 = tot3;
-
-    if (eta2x || eta2y) {
-        if (eta2x)
-            *eta2x = 0;
-        if (eta2y)
-            *eta2y = 0;
-        switch (c) {
-        case cBottomLeft:
-            if (eta2x && (data[3] + data[4]) != 0)
-                *eta2x = (double)(data[4]) / (data[3] + data[4]);
-            if (eta2y && (data[1] + data[4]) != 0)
-                *eta2y = (double)(data[4]) / (data[1] + data[4]);
-            break;
-        case cBottomRight:
-            if (eta2x && (data[2] + data[5]) != 0)
-                *eta2x = (double)(data[5]) / (data[4] + data[5]);
-            if (eta2y && (data[1] + data[4]) != 0)
-                *eta2y = (double)(data[4]) / (data[1] + data[4]);
-            break;
-        case cTopLeft:
-            if (eta2x && (data[7] + data[4]) != 0)
-                *eta2x = (double)(data[4]) / (data[3] + data[4]);
-            if (eta2y && (data[7] + data[4]) != 0)
-                *eta2y = (double)(data[7]) / (data[7] + data[4]);
-            break;
-        case cTopRight:
-            if (eta2x && t2max != 0)
-                *eta2x = (double)(data[5]) / (data[5] + data[4]);
-            if (eta2y && t2max != 0)
-                *eta2y = (double)(data[7]) / (data[7] + data[4]);
-            break;
-        default:;
-        }
-    }
-
-    if (eta3x || eta3y) {
-        if (eta3x && (data[3] + data[4] + data[5]) != 0)
-            *eta3x = (double)(-data[3] + data[3 + 2]) /
-                     (data[3] + data[4] + data[5]);
-        if (eta3y && (data[1] + data[4] + data[7]) != 0)
-            *eta3y = (double)(-data[1] + data[2 * 3 + 1]) /
-                     (data[1] + data[4] + data[7]);
-    }
-
-    return ok;
-}
-
-
-
-} // namespace aare

src/ClusterFile.test.cpp

@@ -0,0 +1,75 @@
+#include "aare/ClusterFile.hpp"
+#include "test_config.hpp"
+
+#include "aare/defs.hpp"
+#include <catch2/catch_test_macros.hpp>
+#include <filesystem>
+
+using aare::Cluster;
+using aare::ClusterFile;
+
+TEST_CASE("Read one frame from a a cluster file", "[.integration]") {
+    // We know that the frame has 97 clusters
+    auto fpath =
+        test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    ClusterFile<Cluster<int32_t, 3, 3>> f(fpath);
+    auto clusters = f.read_frame();
+    REQUIRE(clusters.size() == 97);
+    REQUIRE(clusters.frame_number() == 135);
+}
+
+TEST_CASE("Read one frame using ROI", "[.integration]") {
+    // We know that the frame has 97 clusters
+    auto fpath =
+        test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    ClusterFile<Cluster<int32_t, 3, 3>> f(fpath);
+    aare::ROI roi;
+    roi.xmin = 0;
+    roi.xmax = 50;
+    roi.ymin = 200;
+    roi.ymax = 249;
+    f.set_roi(roi);
+    auto clusters = f.read_frame();
+    REQUIRE(clusters.size() == 49);
+    REQUIRE(clusters.frame_number() == 135);
+
+    // Check that all clusters are within the ROI
+    for (size_t i = 0; i < clusters.size(); i++) {
+        auto c = clusters.at(i);
+        REQUIRE(c.x >= roi.xmin);
+        REQUIRE(c.x <= roi.xmax);
+        REQUIRE(c.y >= roi.ymin);
+        REQUIRE(c.y <= roi.ymax);
+    }
+}
+
+TEST_CASE("Read clusters from single frame file", "[.integration]") {
+
+    auto fpath =
+        test_data_path() / "clusters" / "single_frame_97_clustrers.clust";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    SECTION("Read fewer clusters than available") {
+        ClusterFile<Cluster<int32_t, 3, 3>> f(fpath);
+        auto clusters = f.read_clusters(50);
+        REQUIRE(clusters.size() == 50);
+        REQUIRE(clusters.frame_number() == 135);
+    }
+    SECTION("Read more clusters than available") {
+        ClusterFile<Cluster<int32_t, 3, 3>> f(fpath);
+        // 100 is the maximum number of clusters read
+        auto clusters = f.read_clusters(100);
+        REQUIRE(clusters.size() == 97);
+        REQUIRE(clusters.frame_number() == 135);
+    }
+    SECTION("Read all clusters") {
+        ClusterFile<Cluster<int32_t, 3, 3>> f(fpath);
+        auto clusters = f.read_clusters(97);
+        REQUIRE(clusters.size() == 97);
+        REQUIRE(clusters.frame_number() == 135);
+    }
+}

src/ClusterFinder.test.cpp

@@ -1,19 +1,18 @@
 #include "aare/ClusterFinder.hpp"
 #include "aare/Pedestal.hpp"
-#include <catch2/matchers/catch_matchers_floating_point.hpp>
 #include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_floating_point.hpp>
 #include <chrono>
 #include <random>
 
 using namespace aare;
 
-//TODO! Find a way to test the cluster finder
-
-
+// TODO! Find a way to test the cluster finder
 
 // class ClusterFinderUnitTest : public ClusterFinder {
 //   public:
-//     ClusterFinderUnitTest(int cluster_sizeX, int cluster_sizeY, double nSigma = 5.0, double threshold = 0.0)
+//     ClusterFinderUnitTest(int cluster_sizeX, int cluster_sizeY, double nSigma
+//     = 5.0, double threshold = 0.0)
 //         : ClusterFinder(cluster_sizeX, cluster_sizeY, nSigma, threshold) {}
 //     double get_c2() { return c2; }
 //     double get_c3() { return c3; }
@@ -37,8 +36,8 @@ using namespace aare;
 //     REQUIRE_THAT(cf.get_c3(), Catch::Matchers::WithinRel(c3, 1e-9));
 // }
 
-TEST_CASE("Construct a cluster finder"){
-    ClusterFinder clusterFinder({400,400}, {3,3});
+TEST_CASE("Construct a cluster finder") {
+    ClusterFinder clusterFinder({400, 400});
     // REQUIRE(clusterFinder.get_cluster_sizeX() == 3);
     // REQUIRE(clusterFinder.get_cluster_sizeY() == 3);
     // REQUIRE(clusterFinder.get_threshold() == 1);
@@ -49,16 +48,17 @@ TEST_CASE("Construct a cluster finder"){
 //     aare::Pedestal pedestal(10, 10, 5);
 //     NDArray<double, 2> frame({10, 10});
 //     frame = 0;
-//     ClusterFinder clusterFinder(3, 3, 1, 1); // 3x3 cluster, 1 nSigma, 1 threshold
+//     ClusterFinder clusterFinder(3, 3, 1, 1); // 3x3 cluster, 1 nSigma, 1
+//     threshold
 
-//     auto clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
+//     auto clusters =
+//     clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
 
 //     REQUIRE(clusters.size() == 0);
 
 //     frame(5, 5) = 10;
-//     clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
-//     REQUIRE(clusters.size() == 1);
-//     REQUIRE(clusters[0].x == 5);
+//     clusters = clusterFinder.find_clusters_without_threshold(frame.span(),
+//     pedestal); REQUIRE(clusters.size() == 1); REQUIRE(clusters[0].x == 5);
 //     REQUIRE(clusters[0].y == 5);
 //     for (int i = 0; i < 3; i++) {
 //         for (int j = 0; j < 3; j++) {

src/ClusterVector.test.cpp

@@ -0,0 +1,233 @@
+#include "aare/ClusterVector.hpp"
+#include <cstdint>
+
+#include <catch2/catch_all.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_floating_point.hpp>
+
+using aare::Cluster;
+using aare::ClusterVector;
+
+TEST_CASE("ClusterVector 2x2 int32_t capacity 4, push back then read",
+          "[.ClusterVector]") {
+
+    ClusterVector<Cluster<int32_t, 2, 2>> cv(4);
+    REQUIRE(cv.capacity() == 4);
+    REQUIRE(cv.size() == 0);
+    REQUIRE(cv.cluster_size_x() == 2);
+    REQUIRE(cv.cluster_size_y() == 2);
+    // int16_t, int16_t, 2x2 int32_t = 20 bytes
+    REQUIRE(cv.item_size() == 20);
+
+    // Create a cluster and push back into the vector
+    Cluster<int32_t, 2, 2> c1 = {1, 2, {3, 4, 5, 6}};
+    cv.push_back(c1);
+    REQUIRE(cv.size() == 1);
+    REQUIRE(cv.capacity() == 4);
+
+    auto c2 = cv.at(0);
+
+    // Check that the data is the same
+    REQUIRE(c1.x == c2.x);
+    REQUIRE(c1.y == c2.y);
+    for (size_t i = 0; i < 4; i++) {
+        REQUIRE(c1.data[i] == c2.data[i]);
+    }
+}
+
+TEST_CASE("Summing 3x1 clusters of int64", "[.ClusterVector]") {
+    ClusterVector<Cluster<int32_t, 3, 1>> cv(2);
+    REQUIRE(cv.capacity() == 2);
+    REQUIRE(cv.size() == 0);
+    REQUIRE(cv.cluster_size_x() == 3);
+    REQUIRE(cv.cluster_size_y() == 1);
+
+    // Create a cluster and push back into the vector
+    Cluster<int32_t, 3, 1> c1 = {1, 2, {3, 4, 5}};
+    cv.push_back(c1);
+    REQUIRE(cv.capacity() == 2);
+    REQUIRE(cv.size() == 1);
+
+    Cluster<int32_t, 3, 1> c2 = {6, 7, {8, 9, 10}};
+    cv.push_back(c2);
+    REQUIRE(cv.capacity() == 2);
+    REQUIRE(cv.size() == 2);
+
+    Cluster<int32_t, 3, 1> c3 = {11, 12, {13, 14, 15}};
+    cv.push_back(c3);
+    REQUIRE(cv.capacity() == 4);
+    REQUIRE(cv.size() == 3);
+
+    /*
+    auto sums = cv.sum();
+    REQUIRE(sums.size() == 3);
+    REQUIRE(sums[0] == 12);
+    REQUIRE(sums[1] == 27);
+    REQUIRE(sums[2] == 42);
+    */
+}
+
+TEST_CASE("Storing floats", "[.ClusterVector]") {
+    ClusterVector<Cluster<float, 2, 4>> cv(10);
+    REQUIRE(cv.capacity() == 10);
+    REQUIRE(cv.size() == 0);
+    REQUIRE(cv.cluster_size_x() == 2);
+    REQUIRE(cv.cluster_size_y() == 4);
+
+    // Create a cluster and push back into the vector
+    Cluster<float, 2, 4> c1 = {1, 2, {3.0, 4.0, 5.0, 6.0, 3.0, 4.0, 5.0, 6.0}};
+    cv.push_back(c1);
+    REQUIRE(cv.capacity() == 10);
+    REQUIRE(cv.size() == 1);
+
+    Cluster<float, 2, 4> c2 = {
+        6, 7, {8.0, 9.0, 10.0, 11.0, 8.0, 9.0, 10.0, 11.0}};
+    cv.push_back(c2);
+    REQUIRE(cv.capacity() == 10);
+    REQUIRE(cv.size() == 2);
+
+    /*
+    auto sums = cv.sum();
+    REQUIRE(sums.size() == 2);
+    REQUIRE_THAT(sums[0], Catch::Matchers::WithinAbs(36.0, 1e-6));
+    REQUIRE_THAT(sums[1], Catch::Matchers::WithinAbs(76.0, 1e-6));
+    */
+}
+
+TEST_CASE("Push back more than initial capacity", "[.ClusterVector]") {
+
+    ClusterVector<Cluster<int32_t, 2, 2>> cv(2);
+    auto initial_data = cv.data();
+    Cluster<int32_t, 2, 2> c1 = {1, 2, {3, 4, 5, 6}};
+    cv.push_back(c1);
+    REQUIRE(cv.size() == 1);
+    REQUIRE(cv.capacity() == 2);
+
+    Cluster<int32_t, 2, 2> c2 = {6, 7, {8, 9, 10, 11}};
+    cv.push_back(c2);
+    REQUIRE(cv.size() == 2);
+    REQUIRE(cv.capacity() == 2);
+
+    Cluster<int32_t, 2, 2> c3 = {11, 12, {13, 14, 15, 16}};
+    cv.push_back(c3);
+    REQUIRE(cv.size() == 3);
+    REQUIRE(cv.capacity() == 4);
+
+    Cluster<int32_t, 2, 2> *ptr =
+        reinterpret_cast<Cluster<int32_t, 2, 2> *>(cv.data());
+    REQUIRE(ptr[0].x == 1);
+    REQUIRE(ptr[0].y == 2);
+    REQUIRE(ptr[1].x == 6);
+    REQUIRE(ptr[1].y == 7);
+    REQUIRE(ptr[2].x == 11);
+    REQUIRE(ptr[2].y == 12);
+
+    // We should have allocated a new buffer, since we outgrew the initial
+    // capacity
+    REQUIRE(initial_data != cv.data());
+}
+
+TEST_CASE("Concatenate two cluster vectors where the first has enough capacity",
+          "[.ClusterVector]") {
+    ClusterVector<Cluster<int32_t, 2, 2>> cv1(12);
+    Cluster<int32_t, 2, 2> c1 = {1, 2, {3, 4, 5, 6}};
+    cv1.push_back(c1);
+    Cluster<int32_t, 2, 2> c2 = {6, 7, {8, 9, 10, 11}};
+    cv1.push_back(c2);
+
+    ClusterVector<Cluster<int32_t, 2, 2>> cv2(2);
+    Cluster<int32_t, 2, 2> c3 = {11, 12, {13, 14, 15, 16}};
+    cv2.push_back(c3);
+    Cluster<int32_t, 2, 2> c4 = {16, 17, {18, 19, 20, 21}};
+    cv2.push_back(c4);
+
+    cv1 += cv2;
+    REQUIRE(cv1.size() == 4);
+    REQUIRE(cv1.capacity() == 12);
+
+    Cluster<int32_t, 2, 2> *ptr =
+        reinterpret_cast<Cluster<int32_t, 2, 2> *>(cv1.data());
+    REQUIRE(ptr[0].x == 1);
+    REQUIRE(ptr[0].y == 2);
+    REQUIRE(ptr[1].x == 6);
+    REQUIRE(ptr[1].y == 7);
+    REQUIRE(ptr[2].x == 11);
+    REQUIRE(ptr[2].y == 12);
+    REQUIRE(ptr[3].x == 16);
+    REQUIRE(ptr[3].y == 17);
+}
+
+TEST_CASE("Concatenate two cluster vectors where we need to allocate",
+          "[.ClusterVector]") {
+    ClusterVector<Cluster<int32_t, 2, 2>> cv1(2);
+    Cluster<int32_t, 2, 2> c1 = {1, 2, {3, 4, 5, 6}};
+    cv1.push_back(c1);
+    Cluster<int32_t, 2, 2> c2 = {6, 7, {8, 9, 10, 11}};
+    cv1.push_back(c2);
+
+    ClusterVector<Cluster<int32_t, 2, 2>> cv2(2);
+    Cluster<int32_t, 2, 2> c3 = {11, 12, {13, 14, 15, 16}};
+    cv2.push_back(c3);
+    Cluster<int32_t, 2, 2> c4 = {16, 17, {18, 19, 20, 21}};
+    cv2.push_back(c4);
+
+    cv1 += cv2;
+    REQUIRE(cv1.size() == 4);
+    REQUIRE(cv1.capacity() == 4);
+
+    Cluster<int32_t, 2, 2> *ptr =
+        reinterpret_cast<Cluster<int32_t, 2, 2> *>(cv1.data());
+    REQUIRE(ptr[0].x == 1);
+    REQUIRE(ptr[0].y == 2);
+    REQUIRE(ptr[1].x == 6);
+    REQUIRE(ptr[1].y == 7);
+    REQUIRE(ptr[2].x == 11);
+    REQUIRE(ptr[2].y == 12);
+    REQUIRE(ptr[3].x == 16);
+    REQUIRE(ptr[3].y == 17);
+}
+
+struct ClusterTestData {
+    uint8_t ClusterSizeX;
+    uint8_t ClusterSizeY;
+    std::vector<int64_t> index_map_x;
+    std::vector<int64_t> index_map_y;
+};
+
+TEST_CASE("Gain Map Calculation Index Map", "[.ClusterVector][.gain_map]") {
+
+    auto clustertestdata = GENERATE(
+        ClusterTestData{3,
+                        3,
+                        {-1, 0, 1, -1, 0, 1, -1, 0, 1},
+                        {-1, -1, -1, 0, 0, 0, 1, 1, 1}},
+        ClusterTestData{
+            4,
+            4,
+            {-2, -1, 0, 1, -2, -1, 0, 1, -2, -1, 0, 1, -2, -1, 0, 1},
+            {-2, -2, -2, -2, -1, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1}},
+        ClusterTestData{2, 2, {-1, 0, -1, 0}, {-1, -1, 0, 0}},
+        ClusterTestData{5,
+                        5,
+                        {-2, -1, 0,  1,  2, -2, -1, 0,  1,  2, -2, -1, 0,
+                         1,  2,  -2, -1, 0, 1,  2,  -2, -1, 0, 1,  2},
+                        {-2, -2, -2, -2, -2, -1, -1, -1, -1, -1, 0, 0, 0,
+                         0,  0,  1,  1,  1,  1,  1,  2,  2,  2,  2, 2}});
+
+    uint8_t ClusterSizeX = clustertestdata.ClusterSizeX;
+    uint8_t ClusterSizeY = clustertestdata.ClusterSizeY;
+
+    std::vector<int64_t> index_map_x(ClusterSizeX * ClusterSizeY);
+    std::vector<int64_t> index_map_y(ClusterSizeX * ClusterSizeY);
+
+    int64_t index_cluster_center_x = ClusterSizeX / 2;
+    int64_t index_cluster_center_y = ClusterSizeY / 2;
+
+    for (size_t j = 0; j < ClusterSizeX * ClusterSizeY; j++) {
+        index_map_x[j] = j % ClusterSizeX - index_cluster_center_x;
+        index_map_y[j] = j / ClusterSizeX - index_cluster_center_y;
+    }
+
+    CHECK(index_map_x == clustertestdata.index_map_x);
+    CHECK(index_map_y == clustertestdata.index_map_y);
+}

src/CtbRawFile.cpp

@@ -16,7 +16,7 @@ CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) {
 
 void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) {
     if(m_current_frame >= m_master.frames_in_file()){
-        throw std::runtime_error(LOCATION + "End of file reached");
+        throw std::runtime_error(LOCATION + " End of file reached");
     }
 
     if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){

src/Dtype.cpp

@@ -70,7 +70,7 @@ uint8_t Dtype::bitdepth() const {
 /**
  * @brief Get the number of bytes of the data type
  */
-size_t Dtype::bytes() const { return bitdepth() / 8; }
+size_t Dtype::bytes() const { return bitdepth() / bits_per_byte; }
 
 /**
  * @brief Construct a DType object from a TypeIndex

src/File.cpp

@@ -45,6 +45,8 @@ File& File::operator=(File &&other) noexcept {
     return *this;
 }
 
+// void File::close() { file_impl->close(); }
+
 Frame File::read_frame() { return file_impl->read_frame(); }
 Frame File::read_frame(size_t frame_index) {
     return file_impl->read_frame(frame_index);
@@ -58,6 +60,8 @@ void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
 void File::read_into(std::byte *image_buf, size_t n_frames) {
     file_impl->read_into(image_buf, n_frames);
 }
+
+size_t File::frame_number() { return file_impl->frame_number(tell()); }
 size_t File::frame_number(size_t frame_index) {
     return file_impl->frame_number(frame_index);
 }
@@ -69,7 +73,7 @@ size_t File::tell() const { return file_impl->tell(); }
 size_t File::rows() const { return file_impl->rows(); }
 size_t File::cols() const { return file_impl->cols(); }
 size_t File::bitdepth() const { return file_impl->bitdepth(); }
-size_t File::bytes_per_pixel() const { return file_impl->bitdepth() / 8; }
+size_t File::bytes_per_pixel() const { return file_impl->bitdepth() / bits_per_byte; }
 
 DetectorType File::detector_type() const { return file_impl->detector_type(); }
 

src/Fit.cpp

@@ -0,0 +1,276 @@
+#include "aare/Fit.hpp"
+#include "aare/utils/task.hpp"
+#include "aare/utils/par.hpp"
+#include <lmcurve2.h>
+#include <lmfit.hpp>
+#include <thread>
+
+#include <array>
+
+
+namespace aare {
+
+namespace func {
+
+double gaus(const double x, const double *par) {
+    return par[0] * exp(-pow(x - par[1], 2) / (2 * pow(par[2], 2)));
+}
+
+NDArray<double, 1> gaus(NDView<double, 1> x, NDView<double, 1> par) {
+    NDArray<double, 1> y({x.shape(0)}, 0);
+    for (size_t i = 0; i < x.size(); i++) {
+        y(i) = gaus(x(i), par.data());
+    }
+    return y;
+}
+
+double pol1(const double x, const double *par) { return par[0] * x + par[1]; }
+
+NDArray<double, 1> pol1(NDView<double, 1> x, NDView<double, 1> par) {
+    NDArray<double, 1> y({x.shape()}, 0);
+    for (size_t i = 0; i < x.size(); i++) {
+        y(i) = pol1(x(i), par.data());
+    }
+    return y;
+}
+
+} // namespace func
+
+NDArray<double, 1> fit_gaus(NDView<double, 1> x, NDView<double, 1> y) {
+    NDArray<double, 1> result = gaus_init_par(x, y);
+    lm_status_struct status;
+
+    lmcurve(result.size(), result.data(), x.size(), x.data(), y.data(),
+            aare::func::gaus, &lm_control_double, &status);
+
+    return result;
+}
+
+NDArray<double, 3> fit_gaus(NDView<double, 1> x, NDView<double, 3> y,
+                            int n_threads) {
+    NDArray<double, 3> result({y.shape(0), y.shape(1), 3}, 0);
+
+    auto process = [&x, &y, &result](ssize_t first_row, ssize_t last_row) {
+        for (ssize_t row = first_row; row < last_row; row++) {
+            for (ssize_t col = 0; col < y.shape(1); col++) {
+                NDView<double, 1> values(&y(row, col, 0), {y.shape(2)});
+                auto res = fit_gaus(x, values);
+                result(row, col, 0) = res(0);
+                result(row, col, 1) = res(1);
+                result(row, col, 2) = res(2);
+            }
+        }
+    };
+
+    auto tasks = split_task(0, y.shape(0), n_threads);
+    RunInParallel(process, tasks);
+    return result;
+}
+
+std::array<double, 3> gaus_init_par(const NDView<double, 1> x, const NDView<double, 1> y) {
+    std::array<double, 3> start_par{0, 0, 0};
+    auto e = std::max_element(y.begin(), y.end());
+    auto idx = std::distance(y.begin(), e);
+
+    start_par[0] = *e; // For amplitude we use the maximum value
+    start_par[1] =
+        x[idx]; // For the mean we use the x value of the maximum value
+
+    // For sigma we estimate the fwhm and divide by 2.35
+    // assuming equally spaced x values
+    auto delta = x[1] - x[0];
+    start_par[2] =
+        std::count_if(y.begin(), y.end(),
+                      [e, delta](double val) { return val > *e / 2; }) *
+        delta / 2.35;
+
+    return start_par;
+}
+
+
+std::array<double, 2> pol1_init_par(const NDView<double, 1> x, const NDView<double, 1> y){
+        // Estimate the initial parameters for the fit
+        std::array<double, 2> start_par{0, 0};
+
+    
+        auto y2 = std::max_element(y.begin(), y.end());
+        auto x2 = x[std::distance(y.begin(), y2)];
+        auto y1 = std::min_element(y.begin(), y.end());
+        auto x1 = x[std::distance(y.begin(), y1)];
+    
+        start_par[0] =
+            (*y2 - *y1) / (x2 - x1); // For amplitude we use the maximum value
+        start_par[1] =
+            *y1 - ((*y2 - *y1) / (x2 - x1)) *
+                      x1; // For the mean we use the x value of the maximum value
+        return start_par;
+}
+
+void fit_gaus(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
+              NDView<double, 1> par_out, NDView<double, 1> par_err_out,
+              double &chi2) {
+
+    // Check that we have the correct sizes
+    if (y.size() != x.size() || y.size() != y_err.size() ||
+        par_out.size() != 3 || par_err_out.size() != 3) {
+        throw std::runtime_error("Data, x, data_err must have the same size "
+                                 "and par_out, par_err_out must have size 3");
+    }
+
+
+    // /* Collection of output parameters for status info. */
+    // typedef struct {
+    //     double fnorm;  /* norm of the residue vector fvec. */
+    //     int nfev;      /* actual number of iterations. */
+    //     int outcome;   /* Status indicator. Nonnegative values are used as
+    //     index
+    //                     for the message text lm_infmsg, set in lmmin.c. */
+    //     int userbreak; /* Set when function evaluation requests termination.
+    //     */
+    // } lm_status_struct;
+
+
+    lm_status_struct status;
+    par_out = gaus_init_par(x, y);
+    std::array<double, 9> cov{0, 0, 0, 0, 0, 0, 0 , 0 , 0};
+
+    // void lmcurve2( const int n_par, double *par, double *parerr, double *covar, const int m_dat, const double *t, const double *y, const double *dy, double (*f)( const double ti, const double *par ), const lm_control_struct *control, lm_status_struct *status);
+    // n_par - Number of free variables. Length of parameter vector par.
+    // par - Parameter vector. On input, it must contain a reasonable guess. On output, it contains the solution found to minimize ||r||.
+    // parerr - Parameter uncertainties vector. Array of length n_par or NULL. On output, unless it or covar is NULL, it contains the weighted parameter uncertainties for the found parameters.
+    // covar - Covariance matrix. Array of length n_par * n_par or NULL. On output, unless it is NULL, it contains the covariance matrix.
+    // m_dat - Number of data points. Length of vectors t, y, dy. Must statisfy n_par <= m_dat.
+    // t - Array of length m_dat. Contains the abcissae (time, or "x") for which function f will be evaluated.
+    // y - Array of length m_dat. Contains the ordinate values that shall be fitted.
+    // dy - Array of length m_dat. Contains the standard deviations of the values y.
+    // f - A user-supplied parametric function f(ti;par).
+    // control - Parameter collection for tuning the fit procedure. In most cases, the default &lm_control_double is adequate. If f is only computed with single-precision accuracy, &lm_control_float should be used. Parameters are explained in lmmin2(3).
+    // status - A record used to return information about the minimization process: For details, see lmmin2(3).
+
+    lmcurve2(par_out.size(), par_out.data(), par_err_out.data(), cov.data(),
+             x.size(), x.data(), y.data(), y_err.data(), aare::func::gaus,
+             &lm_control_double, &status);
+
+    // Calculate chi2
+    chi2 = 0;
+    for (size_t i = 0; i < y.size(); i++) {
+        chi2 += std::pow((y(i) - func::gaus(x(i), par_out.data())) / y_err(i), 2);
+    }
+}
+
+void fit_gaus(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out,  NDView<double, 2> chi2_out,
+
+              int n_threads) {
+
+    auto process = [&](ssize_t first_row, ssize_t last_row) {
+        for (ssize_t row = first_row; row < last_row; row++) {
+            for (ssize_t col = 0; col < y.shape(1); col++) {
+                NDView<double, 1> y_view(&y(row, col, 0), {y.shape(2)});
+                NDView<double, 1> y_err_view(&y_err(row, col, 0),
+                                             {y_err.shape(2)});
+                NDView<double, 1> par_out_view(&par_out(row, col, 0),
+                                               {par_out.shape(2)});
+                NDView<double, 1> par_err_out_view(&par_err_out(row, col, 0),
+                                                   {par_err_out.shape(2)});
+ 
+                fit_gaus(x, y_view, y_err_view, par_out_view, par_err_out_view,
+                         chi2_out(row, col));
+
+            }
+        }
+    };
+
+    auto tasks = split_task(0, y.shape(0), n_threads);
+    RunInParallel(process, tasks);
+}
+
+void fit_pol1(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
+              NDView<double, 1> par_out, NDView<double, 1> par_err_out, double& chi2) {
+
+    // Check that we have the correct sizes
+    if (y.size() != x.size() || y.size() != y_err.size() ||
+        par_out.size() != 2 || par_err_out.size() != 2) {
+        throw std::runtime_error("Data, x, data_err must have the same size "
+                                 "and par_out, par_err_out must have size 2");
+    }
+
+    lm_status_struct status;
+    par_out = pol1_init_par(x, y);
+    std::array<double, 4> cov{0, 0, 0, 0};
+
+    lmcurve2(par_out.size(), par_out.data(), par_err_out.data(), cov.data(),
+             x.size(), x.data(), y.data(), y_err.data(), aare::func::pol1,
+             &lm_control_double, &status);
+
+    // Calculate chi2
+    chi2 = 0;
+    for (size_t i = 0; i < y.size(); i++) {
+        chi2 += std::pow((y(i) - func::pol1(x(i), par_out.data())) / y_err(i), 2);
+    }
+}
+
+void fit_pol1(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out, NDView<double, 2> chi2_out,
+              int n_threads) {
+
+    auto process = [&](ssize_t first_row, ssize_t last_row) {
+        for (ssize_t row = first_row; row < last_row; row++) {
+            for (ssize_t col = 0; col < y.shape(1); col++) {
+                NDView<double, 1> y_view(&y(row, col, 0), {y.shape(2)});
+                NDView<double, 1> y_err_view(&y_err(row, col, 0),
+                                             {y_err.shape(2)});
+                NDView<double, 1> par_out_view(&par_out(row, col, 0),
+                                               {par_out.shape(2)});
+                NDView<double, 1> par_err_out_view(&par_err_out(row, col, 0),
+                                                   {par_err_out.shape(2)});
+
+                fit_pol1(x, y_view, y_err_view, par_out_view, par_err_out_view, chi2_out(row, col));
+
+            }
+        }
+    };
+
+    auto tasks = split_task(0, y.shape(0), n_threads);
+    RunInParallel(process, tasks);
+
+}
+
+NDArray<double, 1> fit_pol1(NDView<double, 1> x, NDView<double, 1> y) {
+    // // Check that we have the correct sizes
+    // if (y.size() != x.size() || y.size() != y_err.size() ||
+    // par_out.size() != 2 || par_err_out.size() != 2) {
+    // throw std::runtime_error("Data, x, data_err must have the same size "
+    //                      "and par_out, par_err_out must have size 2");
+    // }
+    NDArray<double, 1> par = pol1_init_par(x, y);
+
+    lm_status_struct status;
+    lmcurve(par.size(), par.data(), x.size(), x.data(), y.data(),
+            aare::func::pol1, &lm_control_double, &status);
+
+    return par;
+}
+
+NDArray<double, 3> fit_pol1(NDView<double, 1> x, NDView<double, 3> y,
+                            int n_threads) {
+    NDArray<double, 3> result({y.shape(0), y.shape(1), 2}, 0);
+
+    auto process = [&](ssize_t first_row, ssize_t last_row) {
+        for (ssize_t row = first_row; row < last_row; row++) {
+            for (ssize_t col = 0; col < y.shape(1); col++) {
+                NDView<double, 1> values(&y(row, col, 0), {y.shape(2)});
+                auto res = fit_pol1(x, values);
+                result(row, col, 0) = res(0);
+                result(row, col, 1) = res(1);
+            }
+        }
+    };
+
+    auto tasks = split_task(0, y.shape(0), n_threads);
+
+    RunInParallel(process, tasks);
+    return result;
+}
+
+} // namespace aare

src/Frame.test.cpp

@@ -19,7 +19,7 @@ TEST_CASE("Construct a frame") {
     // data should be initialized to 0
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
+            uint8_t *data = reinterpret_cast<uint8_t *>(frame.pixel_ptr(i, j));
             REQUIRE(data != nullptr);
             REQUIRE(*data == 0);
         }
@@ -40,7 +40,7 @@ TEST_CASE("Set a value in a 8 bit frame") {
     // only the value we did set should be non-zero
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
+            uint8_t *data = reinterpret_cast<uint8_t *>(frame.pixel_ptr(i, j));
             REQUIRE(data != nullptr);
             if (i == 5 && j == 7) {
                 REQUIRE(*data == value);
@@ -65,7 +65,7 @@ TEST_CASE("Set a value in a 64 bit frame") {
     // only the value we did set should be non-zero
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint64_t *data = (uint64_t *)frame.pixel_ptr(i, j);
+            uint64_t *data = reinterpret_cast<uint64_t *>(frame.pixel_ptr(i, j));
             REQUIRE(data != nullptr);
             if (i == 5 && j == 7) {
                 REQUIRE(*data == value);
@@ -149,4 +149,5 @@ TEST_CASE("test explicit copy constructor") {
     REQUIRE(frame2.bitdepth() == bitdepth);
     REQUIRE(frame2.bytes() == rows * cols * bitdepth / 8);
     REQUIRE(frame2.data() != data);
-}
+}
+

src/Interpolator.cpp

@@ -0,0 +1,56 @@
+#include "aare/Interpolator.hpp"
+
+namespace aare {
+
+Interpolator::Interpolator(NDView<double, 3> etacube, NDView<double, 1> xbins,
+                           NDView<double, 1> ybins, NDView<double, 1> ebins)
+    : m_ietax(etacube), m_ietay(etacube), m_etabinsx(xbins), m_etabinsy(ybins),
+      m_energy_bins(ebins) {
+    if (etacube.shape(0) != xbins.size() || etacube.shape(1) != ybins.size() ||
+        etacube.shape(2) != ebins.size()) {
+        throw std::invalid_argument(
+            "The shape of the etacube does not match the shape of the bins");
+    }
+
+    // Cumulative sum in the x direction
+    for (ssize_t i = 1; i < m_ietax.shape(0); i++) {
+        for (ssize_t j = 0; j < m_ietax.shape(1); j++) {
+            for (ssize_t k = 0; k < m_ietax.shape(2); k++) {
+                m_ietax(i, j, k) += m_ietax(i - 1, j, k);
+            }
+        }
+    }
+
+    // Normalize by the highest row, if norm less than 1 don't do anything
+    for (ssize_t i = 0; i < m_ietax.shape(0); i++) {
+        for (ssize_t j = 0; j < m_ietax.shape(1); j++) {
+            for (ssize_t k = 0; k < m_ietax.shape(2); k++) {
+                auto val = m_ietax(m_ietax.shape(0) - 1, j, k);
+                double norm = val < 1 ? 1 : val;
+                m_ietax(i, j, k) /= norm;
+            }
+        }
+    }
+
+    // Cumulative sum in the y direction
+    for (ssize_t i = 0; i < m_ietay.shape(0); i++) {
+        for (ssize_t j = 1; j < m_ietay.shape(1); j++) {
+            for (ssize_t k = 0; k < m_ietay.shape(2); k++) {
+                m_ietay(i, j, k) += m_ietay(i, j - 1, k);
+            }
+        }
+    }
+
+    // Normalize by the highest column, if norm less than 1 don't do anything
+    for (ssize_t i = 0; i < m_ietay.shape(0); i++) {
+        for (ssize_t j = 0; j < m_ietay.shape(1); j++) {
+            for (ssize_t k = 0; k < m_ietay.shape(2); k++) {
+                auto val = m_ietay(i, m_ietay.shape(1) - 1, k);
+                double norm = val < 1 ? 1 : val;
+                m_ietay(i, j, k) /= norm;
+            }
+        }
+    }
+}
+
+} // namespace aare

src/NDArray.test.cpp

@@ -2,6 +2,7 @@
 #include <array>
 #include <catch2/benchmark/catch_benchmark.hpp>
 #include <catch2/catch_test_macros.hpp>
+#include <numeric>
 
 using aare::NDArray;
 using aare::NDView;
@@ -34,6 +35,24 @@ TEST_CASE("Construct from an NDView") {
     }
 }
 
+TEST_CASE("3D NDArray from NDView"){
+    std::vector<int> data(27);
+    std::iota(data.begin(), data.end(), 0);
+    NDView<int, 3> view(data.data(), Shape<3>{3, 3, 3});
+    NDArray<int, 3> image(view);
+    REQUIRE(image.shape() == view.shape());
+    REQUIRE(image.size() == view.size());
+    REQUIRE(image.data() != view.data());
+
+    for(int64_t i=0; i<image.shape(0); i++){
+        for(int64_t j=0; j<image.shape(1); j++){
+            for(int64_t k=0; k<image.shape(2); k++){
+                REQUIRE(image(i, j, k) == view(i, j, k));
+            }
+        }
+    }
+}
+
 TEST_CASE("1D image") {
     std::array<int64_t, 1> shape{{20}};
     NDArray<short, 1> img(shape, 3);
@@ -379,4 +398,32 @@ TEST_CASE("Elementwise operations on images") {
             REQUIRE(A(i) == a_val);
         }
     }
+}
+
+TEST_CASE("Assign an std::array to a 1D NDArray") {
+    NDArray<int, 1> a{{5}, 0};
+    std::array<int, 5> b{1, 2, 3, 4, 5};
+    a = b;
+    for (uint32_t i = 0; i < a.size(); ++i) {
+        REQUIRE(a(i) == b[i]);
+    }
+}
+
+TEST_CASE("Assign an std::array to a 1D NDArray of a different size") {
+    NDArray<int, 1> a{{3}, 0};
+    std::array<int, 5> b{1, 2, 3, 4, 5};
+    a = b;
+
+    REQUIRE(a.size() == 5);
+    for (uint32_t i = 0; i < a.size(); ++i) {
+        REQUIRE(a(i) == b[i]);
+    }
+}
+
+TEST_CASE("Construct an NDArray from an std::array") {
+    std::array<int, 5> b{1, 2, 3, 4, 5};
+    NDArray<int, 1> a(b);
+    for (uint32_t i = 0; i < a.size(); ++i) {
+        REQUIRE(a(i) == b[i]);
+    }
 }

src/PixelMap.cpp

@@ -31,6 +31,49 @@ NDArray<ssize_t, 2> GenerateMoench03PixelMap() {
 }
 
 NDArray<ssize_t, 2> GenerateMoench05PixelMap() {
+    std::array<int, 3> adc_numbers = {5, 9, 1};  
+    NDArray<ssize_t, 2> order_map({160, 150});
+    int n_pixel = 0;
+    for (int row = 0; row < 160; row++) {
+        for (int i_col = 0; i_col < 50; i_col++) {
+            n_pixel = row * 50 + i_col;
+            for (int i_sc = 0; i_sc < 3; i_sc++) {
+                int col = 50 * i_sc + i_col;
+                int adc_nr = adc_numbers[i_sc];
+                int i_analog = n_pixel * 12 + adc_nr;  
+
+                // analog_frame[row * 150 + col] = analog_data[i_analog] & 0x3FFF;
+                order_map(row, col) = i_analog;
+                
+            }
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 2> GenerateMoench05PixelMap1g() {
+    std::array<int, 3> adc_numbers = {1, 2, 0}; 
+    NDArray<ssize_t, 2> order_map({160, 150});
+    int n_pixel = 0;
+    for (int row = 0; row < 160; row++) {
+        for (int i_col = 0; i_col < 50; i_col++) {
+            n_pixel = row * 50 + i_col;
+            for (int i_sc = 0; i_sc < 3; i_sc++) {
+                int col = 50 * i_sc + i_col;
+                int adc_nr = adc_numbers[i_sc];
+                int i_analog = n_pixel * 3 + adc_nr;  
+
+
+                // analog_frame[row * 150 + col] = analog_data[i_analog] & 0x3FFF;
+                order_map(row, col) = i_analog;
+                
+            }
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 2> GenerateMoench05PixelMapOld() {
     std::array<int, 3> adc_numbers = {9, 13, 1};
     NDArray<ssize_t, 2> order_map({160, 150});
     int n_pixel = 0;

src/RawFile.cpp

@@ -1,6 +1,7 @@
 #include "aare/RawFile.hpp"
 #include "aare/PixelMap.hpp"
 #include "aare/defs.hpp"
+#include "aare/geo_helpers.hpp"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -17,9 +18,15 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
         n_subfiles = find_number_of_subfiles(); // f0,f1...fn
         n_subfile_parts =
             m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
-        find_geometry();
-        update_geometry_with_roi();
 
+        
+
+        find_geometry();
+
+        if (m_master.roi()){
+            m_geometry = update_geometry_with_roi(m_geometry, m_master.roi().value());
+        }
+        
         open_subfiles();
     } else {
         throw std::runtime_error(LOCATION +
@@ -69,9 +76,12 @@ size_t RawFile::n_mod() const { return n_subfile_parts; }
 
 
 size_t RawFile::bytes_per_frame() {
-    return m_rows * m_cols * m_master.bitdepth() / 8;
+    return m_geometry.pixels_x * m_geometry.pixels_y * m_master.bitdepth() / bits_per_byte;
+}
+size_t RawFile::pixels_per_frame() { 
+    // return m_rows * m_cols; 
+    return m_geometry.pixels_x * m_geometry.pixels_y;
 }
-size_t RawFile::pixels_per_frame() { return m_rows * m_cols; }
 
 DetectorType RawFile::detector_type() const { return m_master.detector_type(); }
 
@@ -89,8 +99,8 @@ void RawFile::seek(size_t frame_index) {
 size_t RawFile::tell() { return m_current_frame; };
 
 size_t RawFile::total_frames() const { return m_master.frames_in_file(); }
-size_t RawFile::rows() const { return m_rows; }
-size_t RawFile::cols() const { return m_cols; }
+size_t RawFile::rows() const { return m_geometry.pixels_y; }
+size_t RawFile::cols() const { return m_geometry.pixels_x; }
 size_t RawFile::bitdepth() const { return m_master.bitdepth(); }
 xy RawFile::geometry() { return m_master.geometry(); }
 
@@ -99,14 +109,11 @@ void RawFile::open_subfiles() {
         for (size_t i = 0; i != n_subfiles; ++i) {
             auto v = std::vector<RawSubFile *>(n_subfile_parts);
             for (size_t j = 0; j != n_subfile_parts; ++j) {
-                fmt::print("{} pos: {},{}\n", j,positions[j].row, positions[j].col);
-                
-                auto pos = m_module_pixel_0[j];
-                fmt::print("{} pos: {},{}\n", j,pos.y, pos.x);
+                auto pos = m_geometry.module_pixel_0[j];
                 v[j] = new RawSubFile(m_master.data_fname(j, i),
                                       m_master.detector_type(), pos.height,
                                       pos.width, m_master.bitdepth(),
-                                      positions[j].row, positions[j].col);
+                                      pos.row_index, pos.col_index);
 
             }
             subfiles.push_back(v);
@@ -149,112 +156,49 @@ int RawFile::find_number_of_subfiles() {
 
 RawMasterFile RawFile::master() const { return m_master; }
 
+/**
+ * @brief Find the geometry of the detector by opening all the subfiles and
+ * reading the headers. 
+ */
 void RawFile::find_geometry() {
+    
+    //Hold the maximal row and column number found
+    //Later used for calculating the total number of rows and columns
     uint16_t r{};
     uint16_t c{};
 
 
     for (size_t i = 0; i < n_subfile_parts; i++) {
-        auto h = this->read_header(m_master.data_fname(i, 0));
+        auto h = read_header(m_master.data_fname(i, 0));
         r = std::max(r, h.row);
         c = std::max(c, h.column);
-        positions.push_back({h.row, h.column});
+        // positions.push_back({h.row, h.column});
+       
         ModuleGeometry g;
-        g.x = h.column * m_master.pixels_x();
-        g.y = h.row * m_master.pixels_y();
+        g.origin_x = h.column * m_master.pixels_x();
+        g.origin_y = h.row * m_master.pixels_y();
+        g.row_index = h.row;
+        g.col_index = h.column;
         g.width = m_master.pixels_x();
         g.height = m_master.pixels_y();
-        m_module_pixel_0.push_back(g);
+        m_geometry.module_pixel_0.push_back(g);
 
     }
 
     r++;
     c++;
 
-    m_rows = (r * m_master.pixels_y());
-    m_cols = (c * m_master.pixels_x());
-
-    m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
-
-#ifdef AARE_VERBOSE
-    fmt::print("\nRawFile::find_geometry()\n");
-    for (size_t i = 0; i < m_module_pixel_0.size(); i++) {
-        fmt::print("Module {} at position: (r:{},c:{})\n", i,
-                   m_module_pixel_0[i].y, m_module_pixel_0[i].x);
-    }
-    fmt::print("Image size: {}x{}\n\n", m_rows, m_cols);
-#endif
-}
-
-void RawFile::update_geometry_with_roi() {
-    // TODO! implement this
-    if (m_master.roi()) {
-        auto roi = m_master.roi().value();
-
-        // TODO! can we do this cleaner?
-        int pos_y = 0;
-        int pos_y_increment = 0;
-        for (size_t row = 0; row < m_master.geometry().row; row++) {
-            int pos_x = 0;
-            for (size_t col = 0; col < m_master.geometry().col; col++) {
-                auto &m = m_module_pixel_0[row * m_master.geometry().col + col];
-                auto original_height = m.height;
-                auto original_width = m.width;
-
-                // module is to the left of the roi
-                if (m.x + m.width < roi.xmin) {
-                    m.width = 0;
-
-                    // roi is in module
-                } else {
-                    // here we only arrive when the roi is in or to the left of
-                    // the module
-                    if (roi.xmin > m.x) {
-                        m.width -= roi.xmin - m.x;
-                    }
-                    if (roi.xmax < m.x + m.width) {
-                        m.width -= m.x + original_width - roi.xmax;
-                    }
-                    m.x = pos_x;
-                    pos_x += m.width;
-                }
-
-                if (m.y + m.height < roi.ymin) {
-                    m.height = 0;
-                } else {
-                    if ((roi.ymin > m.y) && (roi.ymin < m.y + m.height)) {
-                        m.height -= roi.ymin - m.y;
-
-                    }
-                    if (roi.ymax < m.y + m.height) {
-                        m.height -= m.y + original_height - roi.ymax;
-                    }
-                    m.y = pos_y;
-                    pos_y_increment = m.height;
-                }
-            }
-            // increment pos_y
-            pos_y += pos_y_increment;
-        }
-
-        m_rows = roi.height();
-        m_cols = roi.width();
-    }
-
-#ifdef AARE_VERBOSE
-    fmt::print("RawFile::update_geometry_with_roi()\n");
-    for (const auto &m : m_module_pixel_0) {
-        fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x,
-                   m.height, m.width);
-    }
-    fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols);
-    fmt::print("\n");
-#endif
+    m_geometry.pixels_y = (r * m_master.pixels_y());
+    m_geometry.pixels_x  = (c * m_master.pixels_x());
+    m_geometry.modules_x = c;
+    m_geometry.modules_y = r;
+    m_geometry.pixels_y += static_cast<size_t>((r - 1) * cfg.module_gap_row);
 
 }
 
+
 Frame RawFile::get_frame(size_t frame_index) {
-    auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
+    auto f = Frame(m_geometry.pixels_y, m_geometry.pixels_x, Dtype::from_bitdepth(m_master.bitdepth()));
     std::byte *frame_buffer = f.data();
     get_frame_into(frame_index, frame_buffer);
     return f;
@@ -266,8 +210,7 @@ size_t RawFile::bytes_per_pixel() const {
 }
 
 void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
-
-    if (frame_index > total_frames()) {
+    if (frame_index >= total_frames()) {
         throw std::runtime_error(LOCATION + "Frame number out of range");
     }
     std::vector<size_t> frame_numbers(n_subfile_parts);
@@ -279,6 +222,10 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
     if (n_subfile_parts != 1) {
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto subfile_id = frame_index / m_master.max_frames_per_file();
+            if (subfile_id >= subfiles.size()) {
+                throw std::runtime_error(LOCATION +
+                                         " Subfile out of range. Possible missing data.");
+            }
             frame_numbers[part_idx] =
                 subfiles[subfile_id][part_idx]->frame_number(
                     frame_index % m_master.max_frames_per_file());
@@ -312,14 +259,19 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto corrected_idx = frame_indices[part_idx];
             auto subfile_id = corrected_idx / m_master.max_frames_per_file();
+            if (subfile_id >= subfiles.size()) {
+                throw std::runtime_error(LOCATION +
+                                         " Subfile out of range. Possible missing data.");
+            }
 
             // This is where we start writing
-            auto offset = (m_module_pixel_0[part_idx].y * m_cols +
-                          m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8;
+            auto offset = (m_geometry.module_pixel_0[part_idx].origin_y * m_geometry.pixels_x +
+                m_geometry.module_pixel_0[part_idx].origin_x)*m_master.bitdepth()/8;
 
-            if (m_module_pixel_0[part_idx].x!=0)
+            if (m_geometry.module_pixel_0[part_idx].origin_x!=0)
                 throw std::runtime_error(LOCATION + "Implementation error. x pos not 0.");
-
+            
+            //TODO! Risk for out of range access
             subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
             subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header);
             if (header)
@@ -340,21 +292,25 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
         // level
 
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
-            auto pos = m_module_pixel_0[part_idx];
+            auto pos = m_geometry.module_pixel_0[part_idx];
             auto corrected_idx = frame_indices[part_idx];
             auto subfile_id = corrected_idx / m_master.max_frames_per_file();
+            if (subfile_id >= subfiles.size()) {
+                throw std::runtime_error(LOCATION +
+                                         " Subfile out of range. Possible missing data.");
+            }
 
             subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
             subfiles[subfile_id][part_idx]->read_into(part_buffer, header);
             if(header)
                 ++header;
 
-            for (size_t cur_row = 0; cur_row < (pos.height);
+            for (size_t cur_row = 0; cur_row < static_cast<size_t>(pos.height);
                  cur_row++) {
 
-                auto irow = (pos.y + cur_row);
-                auto icol = pos.x;
-                auto dest = (irow * this->m_cols + icol);
+                auto irow = (pos.origin_y + cur_row);
+                auto icol = pos.origin_x;
+                auto dest = (irow * this->m_geometry.pixels_x + icol);
                 dest = dest * m_master.bitdepth() / 8;
                 memcpy(frame_buffer + dest,
                        part_buffer + cur_row * pos.width *
@@ -400,4 +356,8 @@ RawFile::~RawFile() {
     }
 }
 
+
+
+
+
 } // namespace aare

src/RawFile.test.cpp

@@ -1,10 +1,13 @@
 #include "aare/File.hpp"
+#include "aare/RawMasterFile.hpp" //needed for ROI
+#include "aare/RawFile.hpp"
 
 #include <catch2/catch_test_macros.hpp>
 #include <filesystem>
 
 #include "test_config.hpp"
 
+
 using aare::File;
 
 TEST_CASE("Read number of frames from a jungfrau raw file", "[.integration]") {
@@ -148,3 +151,5 @@ TEST_CASE("Read file with unordered frames", "[.integration]") {
     File f(fpath);
     REQUIRE_THROWS((f.read_frame()));
 }
+
+

src/RawMasterFile.cpp

@@ -37,11 +37,24 @@ std::filesystem::path RawFileNameComponents::master_fname() const {
 }
 
 std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id,
-                                                        size_t file_id) const {
-    return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name, mod_id,
+                                                        size_t file_id
+                                                        ) const {
+    
+    
+    
+    std::string fmt = "{}_d{}_f{}_{}.raw";
+    //Before version X we used to name the data files f000000000000
+    if (m_old_scheme) {
+        fmt = "{}_d{}_f{:012}_{}.raw";
+    }
+    return m_base_path / fmt::format(fmt, m_base_name, mod_id,
                                      file_id, m_file_index);
 }
 
+void RawFileNameComponents::set_old_scheme(bool old_scheme) {
+    m_old_scheme = old_scheme;
+}
+
 const std::filesystem::path &RawFileNameComponents::base_path() const {
     return m_base_path;
 }
@@ -314,10 +327,22 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
             // do the actual parsing
             if (key == "Version") {
                 m_version = value;
+
+                //TODO!: How old versions can we handle?
+                auto v = std::stod(value);
+
+                //TODO! figure out exactly when we did the change
+                //This enables padding of f to 12 digits
+                if (v<4.0)
+                    m_fnc.set_old_scheme(true);
+
             } else if (key == "TimeStamp") {
 
             } else if (key == "Detector Type") {
                 m_type = StringTo<DetectorType>(value);
+                if(m_type==DetectorType::Moench){
+                    m_type = DetectorType::Moench03_old;
+                }
             } else if (key == "Timing Mode") {
                 m_timing_mode = StringTo<TimingMode>(value);
             } else if (key == "Image Size") {
@@ -352,6 +377,12 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
                 pos = value.find(',');
                 m_pixels_x = std::stoi(value.substr(1, pos));
                 m_pixels_y = std::stoi(value.substr(pos + 1));
+            }else if(key == "row"){
+                pos = value.find('p');
+                m_pixels_y = std::stoi(value.substr(0, pos));
+            }else if(key == "col"){
+                pos = value.find('p');
+                m_pixels_x = std::stoi(value.substr(0, pos));
             } else if (key == "Total Frames") {
                 m_total_frames_expected = std::stoi(value);
             } else if (key == "Dynamic Range") {
@@ -360,6 +391,9 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
                 m_quad = std::stoi(value);
             } else if (key == "Max Frames Per File") {
                 m_max_frames_per_file = std::stoi(value);
+            }else if(key == "Max. Frames Per File"){
+                //Version 3.0 way of writing it
+                m_max_frames_per_file = std::stoi(value);
             } else if (key == "Geometry") {
                 pos = value.find(',');
                 m_geometry = {
@@ -368,5 +402,19 @@ void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
             }
         }
     }
+    if (m_pixels_x == 400 && m_pixels_y == 400) {
+        m_type = DetectorType::Moench03_old;
+    }
+
+
+    //TODO! Look for d0, d1...dn and update geometry
+    if(m_geometry.col == 0 && m_geometry.row == 0){
+        m_geometry = {1,1};
+        fmt::print("Warning: No geometry found in master file. Assuming 1x1\n");
+    }
+
+    //TODO! Read files and find actual frames
+    if(m_frames_in_file==0)
+        m_frames_in_file = m_total_frames_expected;
 }
 } // namespace aare

src/RawMasterFile.test.cpp

@@ -31,6 +31,16 @@ TEST_CASE("Construction of master file name and data files"){
     REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
 }
 
+TEST_CASE("Construction of master file name and data files using old scheme"){
+    RawFileNameComponents m("test_master_1.raw");
+    m.set_old_scheme(true);
+    REQUIRE(m.master_fname() == "test_master_1.raw");
+    REQUIRE(m.data_fname(0, 0) == "test_d0_f000000000000_1.raw");
+    REQUIRE(m.data_fname(1, 0) == "test_d1_f000000000000_1.raw");
+    REQUIRE(m.data_fname(0, 1) == "test_d0_f000000000001_1.raw");
+    REQUIRE(m.data_fname(1, 1) == "test_d1_f000000000001_1.raw");
+}
+
 TEST_CASE("Master file name does not fit pattern"){
     REQUIRE_THROWS(RawFileNameComponents("somefile.json"));
     REQUIRE_THROWS(RawFileNameComponents("another_test_d0_f0_1.raw"));

src/RawSubFile.cpp

@@ -9,14 +9,14 @@ namespace aare {
 RawSubFile::RawSubFile(const std::filesystem::path &fname,
                        DetectorType detector, size_t rows, size_t cols,
                        size_t bitdepth, uint32_t pos_row, uint32_t pos_col)
-    : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
-      m_detector_type(detector),
-      m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row), m_pos_col(pos_col) {
+    : m_detector_type(detector), m_bitdepth(bitdepth), m_fname(fname),
+      m_rows(rows), m_cols(cols),
+      m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row),
+      m_pos_col(pos_col) {
     if (m_detector_type == DetectorType::Moench03_old) {
         m_pixel_map = GenerateMoench03PixelMap();
-    }else if(m_detector_type == DetectorType::Eiger && m_pos_row % 2 == 0){
+    } else if (m_detector_type == DetectorType::Eiger && m_pos_row % 2 == 0) {
         m_pixel_map = GenerateEigerFlipRowsPixelMap();
-        fmt::print("Flipping rows\n");
     }
 
     if (std::filesystem::exists(fname)) {
@@ -44,7 +44,7 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname,
 
 void RawSubFile::seek(size_t frame_index) {
     if (frame_index >= n_frames) {
-        throw std::runtime_error("Frame number out of range");
+        throw std::runtime_error(LOCATION + fmt::format("Frame index {} out of range in a file with {} frames", frame_index, n_frames));
     }
     m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index);
 }
@@ -53,37 +53,48 @@ size_t RawSubFile::tell() {
     return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame());
 }
 
-
 void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
-    if(header){
-         m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
+    if (header) {
+        m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
     } else {
         m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
     }
 
-    //TODO! expand support for different bitdepths
-    if(m_pixel_map){
+    // TODO! expand support for different bitdepths
+    if (m_pixel_map) {
         // read into a temporary buffer and then copy the data to the buffer
         // in the correct order
-        // currently this only supports 16 bit data!
-        auto part_buffer = new std::byte[bytes_per_frame()];
-        m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
-        auto *data = reinterpret_cast<uint16_t *>(image_buf);
-        auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
-        for (size_t i = 0; i < pixels_per_frame(); i++) {
-            data[i] = part_data[(*m_pixel_map)(i)];
+        // TODO! add 4 bit support
+        if(m_bitdepth == 8){
+            read_with_map<uint8_t>(image_buf);
+        }else if (m_bitdepth == 16) {
+            read_with_map<uint16_t>(image_buf);
+        } else if (m_bitdepth == 32) {
+            read_with_map<uint32_t>(image_buf);
+        }else{
+            throw std::runtime_error("Unsupported bitdepth for read with pixel map");
         }
-        delete[] part_buffer;
+
     } else {
         // read directly into the buffer
         m_file.read(reinterpret_cast<char *>(image_buf), bytes_per_frame());
     }
 }
 
+template <typename T>
+void RawSubFile::read_with_map(std::byte *image_buf) {
+    auto part_buffer = new std::byte[bytes_per_frame()];
+    m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
+    auto *data = reinterpret_cast<T *>(image_buf);
+    auto *part_data = reinterpret_cast<T *>(part_buffer);
+    for (size_t i = 0; i < pixels_per_frame(); i++) {
+        data[i] = part_data[(*m_pixel_map)(i)];
+    }
+    delete[] part_buffer;
+}
 size_t RawSubFile::rows() const { return m_rows; }
 size_t RawSubFile::cols() const { return m_cols; }
 
-
 void RawSubFile::get_part(std::byte *buffer, size_t frame_index) {
     seek(frame_index);
     read_into(buffer, nullptr);
@@ -96,5 +107,4 @@ size_t RawSubFile::frame_number(size_t frame_index) {
     return h.frameNumber;
 }
 
-
 } // namespace aare

src/algorithm.test.cpp

@@ -0,0 +1,69 @@
+
+
+#include <aare/algorithm.hpp>
+#include <catch2/catch_test_macros.hpp>
+
+TEST_CASE("Find the closed index in a 1D array", "[algorithm]") {
+    aare::NDArray<double, 1> arr({5});
+    for (size_t i = 0; i < arr.size(); i++) {
+        arr[i] = i;
+    }
+    // arr 0, 1, 2, 3, 4
+    REQUIRE(aare::nearest_index(arr, 2.3) == 2);
+    REQUIRE(aare::nearest_index(arr, 2.6) == 3);
+    REQUIRE(aare::nearest_index(arr, 45.0) == 4);
+    REQUIRE(aare::nearest_index(arr, 0.0) == 0);
+    REQUIRE(aare::nearest_index(arr, -1.0) == 0);
+}
+
+TEST_CASE("Passing integers to nearest_index works", "[algorithm]") {
+    aare::NDArray<int, 1> arr({5});
+    for (size_t i = 0; i < arr.size(); i++) {
+        arr[i] = i;
+    }
+    // arr 0, 1, 2, 3, 4
+    REQUIRE(aare::nearest_index(arr, 2) == 2);
+    REQUIRE(aare::nearest_index(arr, 3) == 3);
+    REQUIRE(aare::nearest_index(arr, 45) == 4);
+    REQUIRE(aare::nearest_index(arr, 0) == 0);
+    REQUIRE(aare::nearest_index(arr, -1) == 0);
+}
+
+TEST_CASE("nearest_index works with std::vector", "[algorithm]") {
+    std::vector<double> vec = {0, 1, 2, 3, 4};
+    REQUIRE(aare::nearest_index(vec, 2.123) == 2);
+    REQUIRE(aare::nearest_index(vec, 2.66) == 3);
+    REQUIRE(aare::nearest_index(vec, 4555555.0) == 4);
+    REQUIRE(aare::nearest_index(vec, 0.0) == 0);
+    REQUIRE(aare::nearest_index(vec, -10.0) == 0);
+}
+
+TEST_CASE("nearest index works with std::array", "[algorithm]") {
+    std::array<double, 5> arr = {0, 1, 2, 3, 4};
+    REQUIRE(aare::nearest_index(arr, 2.123) == 2);
+    REQUIRE(aare::nearest_index(arr, 2.501) == 3);
+    REQUIRE(aare::nearest_index(arr, 4555555.0) == 4);
+    REQUIRE(aare::nearest_index(arr, 0.0) == 0);
+    REQUIRE(aare::nearest_index(arr, -10.0) == 0);
+}
+
+TEST_CASE("last smaller", "[algorithm]") {
+    aare::NDArray<double, 1> arr({5});
+    for (size_t i = 0; i < arr.size(); i++) {
+        arr[i] = i;
+    }
+    // arr 0, 1, 2, 3, 4
+    REQUIRE(aare::last_smaller(arr, -10.0) == 0);
+    REQUIRE(aare::last_smaller(arr, 0.0) == 0);
+    REQUIRE(aare::last_smaller(arr, 2.3) == 2);
+    REQUIRE(aare::last_smaller(arr, 253.) == 4);
+}
+
+TEST_CASE("returns last bin strictly smaller", "[algorithm]") {
+    aare::NDArray<double, 1> arr({5});
+    for (size_t i = 0; i < arr.size(); i++) {
+        arr[i] = i;
+    }
+    // arr 0, 1, 2, 3, 4
+    REQUIRE(aare::last_smaller(arr, 2.0) == 2);
+}

src/decode.cpp

@@ -0,0 +1,61 @@
+#include "aare/decode.hpp"
+
+namespace aare {
+
+uint16_t adc_sar_05_decode64to16(uint64_t input){
+
+    //we want bits 29,19,28,18,31,21,27,20,24,23,25,22 and then pad to 16
+    uint16_t output = 0;
+    output |= ((input >> 22) & 1) << 11;
+    output |= ((input >> 25) & 1) << 10;
+    output |= ((input >> 23) & 1) << 9;
+    output |= ((input >> 24) & 1) << 8;
+    output |= ((input >> 20) & 1) << 7;
+    output |= ((input >> 27) & 1) << 6;
+    output |= ((input >> 21) & 1) << 5;
+    output |= ((input >> 31) & 1) << 4;
+    output |= ((input >> 18) & 1) << 3;
+    output |= ((input >> 28) & 1) << 2;
+    output |= ((input >> 19) & 1) << 1;
+    output |= ((input >> 29) & 1) << 0;
+    return output;
+}
+
+void adc_sar_05_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output){
+    for(int64_t i = 0; i < input.shape(0); i++){
+        for(int64_t j = 0; j < input.shape(1); j++){
+            output(i,j) = adc_sar_05_decode64to16(input(i,j));
+        }
+    }
+}
+
+uint16_t adc_sar_04_decode64to16(uint64_t input){
+
+    // bit_map = array([15,17,19,21,23,4,6,8,10,12,14,16] LSB->MSB
+    uint16_t output = 0;
+    output |= ((input >> 16) & 1) << 11;
+    output |= ((input >> 14) & 1) << 10;
+    output |= ((input >> 12) & 1) << 9;
+    output |= ((input >> 10) & 1) << 8;
+    output |= ((input >> 8) & 1) << 7;
+    output |= ((input >> 6) & 1) << 6;
+    output |= ((input >> 4) & 1) << 5;
+    output |= ((input >> 23) & 1) << 4;
+    output |= ((input >> 21) & 1) << 3;
+    output |= ((input >> 19) & 1) << 2;
+    output |= ((input >> 17) & 1) << 1;
+    output |= ((input >> 15) & 1) << 0;
+    return output;
+}
+
+void adc_sar_04_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output){
+    for(int64_t i = 0; i < input.shape(0); i++){
+        for(int64_t j = 0; j < input.shape(1); j++){
+            output(i,j) = adc_sar_04_decode64to16(input(i,j));
+        }
+    }
+}
+
+
+
+} // namespace aare

src/defs.cpp

@@ -21,23 +21,37 @@ void assert_failed(const std::string &msg)
  */
 template <> std::string ToString(DetectorType arg) {
     switch (arg) {
-    case DetectorType::Jungfrau:
-        return "Jungfrau";
+    case DetectorType::Generic:
+        return "Generic";
     case DetectorType::Eiger:
         return "Eiger";
-    case DetectorType::Mythen3:
-        return "Mythen3";
+    case DetectorType::Gotthard:
+        return "Gotthard";
+    case DetectorType::Jungfrau:
+        return "Jungfrau";
+    case DetectorType::ChipTestBoard:
+        return "ChipTestBoard";
     case DetectorType::Moench:
         return "Moench";
+    case DetectorType::Mythen3:
+        return "Mythen3";
+    case DetectorType::Gotthard2:
+        return "Gotthard2";
+    case DetectorType::Xilinx_ChipTestBoard:
+        return "Xilinx_ChipTestBoard";
+
+    //Custom ones
     case DetectorType::Moench03:
         return "Moench03";
     case DetectorType::Moench03_old:
         return "Moench03_old";
-    case DetectorType::ChipTestBoard:
-        return "ChipTestBoard";
-    default:
+    case DetectorType::Unknown:
         return "Unknown";
+
+    //no default case to trigger compiler warning if not all 
+    //enum values are handled
     }
+    throw std::runtime_error("Could not decode detector to string");
 }
 
 /**
@@ -47,21 +61,34 @@ template <> std::string ToString(DetectorType arg) {
  * @throw runtime_error if the string does not match any DetectorType
  */
 template <> DetectorType StringTo(const std::string &arg) {
-    if (arg == "Jungfrau")
-        return DetectorType::Jungfrau;
+    if (arg == "Generic")
+        return DetectorType::Generic;
     if (arg == "Eiger")
         return DetectorType::Eiger;
-    if (arg == "Mythen3")
-        return DetectorType::Mythen3;
+    if (arg == "Gotthard")
+        return DetectorType::Gotthard;
+    if (arg == "Jungfrau")
+        return DetectorType::Jungfrau;
+    if (arg == "ChipTestBoard")
+        return DetectorType::ChipTestBoard;
     if (arg == "Moench")
         return DetectorType::Moench;
+    if (arg == "Mythen3")
+        return DetectorType::Mythen3;
+    if (arg == "Gotthard2")
+        return DetectorType::Gotthard2;
+    if (arg == "Xilinx_ChipTestBoard")
+        return DetectorType::Xilinx_ChipTestBoard;
+
+    //Custom ones
     if (arg == "Moench03")
         return DetectorType::Moench03;
     if (arg == "Moench03_old")
         return DetectorType::Moench03_old;
-    if (arg == "ChipTestBoard")
-        return DetectorType::ChipTestBoard;
-    throw std::runtime_error("Could not decode dector from: \"" + arg + "\"");
+    if (arg == "Unknown")
+        return DetectorType::Unknown;
+    
+    throw std::runtime_error("Could not decode detector from: \"" + arg + "\"");
 }
 
 /**

src/defs.test.cpp

@@ -1,12 +1,59 @@
 #include "aare/defs.hpp"
-// #include "aare/utils/floats.hpp"
 
 #include <catch2/catch_test_macros.hpp>
 #include <string>
+
+using aare::ToString;
+using aare::StringTo;
+
 TEST_CASE("Enum to string conversion") {
-    // By the way I don't think the enum string conversions should be in the defs.hpp file
+    // TODO! By the way I don't think the enum string conversions should be in the defs.hpp file
     // but let's use this to show a test
+    REQUIRE(ToString(aare::DetectorType::Generic) == "Generic");
+    REQUIRE(ToString(aare::DetectorType::Eiger) == "Eiger");
+    REQUIRE(ToString(aare::DetectorType::Gotthard) == "Gotthard");
     REQUIRE(ToString(aare::DetectorType::Jungfrau) == "Jungfrau");
+    REQUIRE(ToString(aare::DetectorType::ChipTestBoard) == "ChipTestBoard");
+    REQUIRE(ToString(aare::DetectorType::Moench) == "Moench");
+    REQUIRE(ToString(aare::DetectorType::Mythen3) == "Mythen3");
+    REQUIRE(ToString(aare::DetectorType::Gotthard2) == "Gotthard2");
+    REQUIRE(ToString(aare::DetectorType::Xilinx_ChipTestBoard) == "Xilinx_ChipTestBoard");
+    REQUIRE(ToString(aare::DetectorType::Moench03) == "Moench03");
+    REQUIRE(ToString(aare::DetectorType::Moench03_old) == "Moench03_old");
+    REQUIRE(ToString(aare::DetectorType::Unknown) == "Unknown");
+}
+
+TEST_CASE("String to enum"){
+    REQUIRE(StringTo<aare::DetectorType>("Generic") == aare::DetectorType::Generic);
+    REQUIRE(StringTo<aare::DetectorType>("Eiger") == aare::DetectorType::Eiger);
+    REQUIRE(StringTo<aare::DetectorType>("Gotthard") == aare::DetectorType::Gotthard);
+    REQUIRE(StringTo<aare::DetectorType>("Jungfrau") == aare::DetectorType::Jungfrau);
+    REQUIRE(StringTo<aare::DetectorType>("ChipTestBoard") == aare::DetectorType::ChipTestBoard);
+    REQUIRE(StringTo<aare::DetectorType>("Moench") == aare::DetectorType::Moench);
+    REQUIRE(StringTo<aare::DetectorType>("Mythen3") == aare::DetectorType::Mythen3);
+    REQUIRE(StringTo<aare::DetectorType>("Gotthard2") == aare::DetectorType::Gotthard2);
+    REQUIRE(StringTo<aare::DetectorType>("Xilinx_ChipTestBoard") == aare::DetectorType::Xilinx_ChipTestBoard);
+    REQUIRE(StringTo<aare::DetectorType>("Moench03") == aare::DetectorType::Moench03);
+    REQUIRE(StringTo<aare::DetectorType>("Moench03_old") == aare::DetectorType::Moench03_old);
+    REQUIRE(StringTo<aare::DetectorType>("Unknown") == aare::DetectorType::Unknown);
+}
+
+TEST_CASE("Enum values"){
+    //Since some of the enums are written to file we need to make sure 
+    //they match the value in the slsDetectorPackage
+
+    REQUIRE(static_cast<int>(aare::DetectorType::Generic) == 0);
+    REQUIRE(static_cast<int>(aare::DetectorType::Eiger) == 1);
+    REQUIRE(static_cast<int>(aare::DetectorType::Gotthard) == 2);
+    REQUIRE(static_cast<int>(aare::DetectorType::Jungfrau) == 3);
+    REQUIRE(static_cast<int>(aare::DetectorType::ChipTestBoard) == 4);
+    REQUIRE(static_cast<int>(aare::DetectorType::Moench) == 5);
+    REQUIRE(static_cast<int>(aare::DetectorType::Mythen3) == 6);
+    REQUIRE(static_cast<int>(aare::DetectorType::Gotthard2) == 7);
+    REQUIRE(static_cast<int>(aare::DetectorType::Xilinx_ChipTestBoard) == 8);
+
+    //Not included
+    REQUIRE(static_cast<int>(aare::DetectorType::Moench03) == 100);
 }
 
 TEST_CASE("DynamicCluster creation") {

src/geo_helpers.cpp

@@ -0,0 +1,71 @@
+
+#include "aare/geo_helpers.hpp"
+#include "fmt/core.h"
+
+namespace aare{
+
+DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi) {
+    #ifdef AARE_VERBOSE
+    fmt::println("update_geometry_with_roi() called with ROI: {} {} {} {}",
+                 roi.xmin, roi.xmax, roi.ymin, roi.ymax);
+    fmt::println("Geometry: {} {} {} {} {} {}",
+                 geo.modules_x, geo.modules_y, geo.pixels_x, geo.pixels_y, geo.module_gap_row, geo.module_gap_col);
+    #endif
+    int pos_y = 0;
+    int pos_y_increment = 0;
+    for (int row = 0; row < geo.modules_y; row++) {
+        int pos_x = 0;
+        for (int col = 0; col < geo.modules_x; col++) {
+            auto &m = geo.module_pixel_0[row * geo.modules_x + col];
+            auto original_height = m.height;
+            auto original_width = m.width;
+
+            // module is to the left of the roi
+            if (m.origin_x + m.width < roi.xmin) {
+                m.width = 0;
+
+                // roi is in module
+            } else {
+                // here we only arrive when the roi is in or to the left of
+                // the module
+                if (roi.xmin > m.origin_x) {
+                    m.width -= roi.xmin - m.origin_x;
+                }
+                if (roi.xmax < m.origin_x + original_width) {
+                    m.width -= m.origin_x + original_width - roi.xmax;
+                }
+                m.origin_x = pos_x;
+                pos_x += m.width;
+            }
+
+            if (m.origin_y + m.height < roi.ymin) {
+                m.height = 0;
+            } else {
+                if ((roi.ymin > m.origin_y) && (roi.ymin < m.origin_y + m.height)) {
+                    m.height -= roi.ymin - m.origin_y;
+
+                }
+                if (roi.ymax < m.origin_y + original_height) {
+                    m.height -= m.origin_y + original_height - roi.ymax;
+                }
+                m.origin_y = pos_y;
+                pos_y_increment = m.height;
+            }
+            #ifdef AARE_VERBOSE
+    fmt::println("Module {} {} {} {}", m.origin_x, m.origin_y, m.width, m.height);
+    #endif
+        }
+        // increment pos_y
+        pos_y += pos_y_increment;
+    }
+
+    // m_rows = roi.height();
+    // m_cols = roi.width();
+    geo.pixels_x = roi.width();
+    geo.pixels_y = roi.height();
+
+    return geo;
+
+}
+
+} // namespace aare

src/geo_helpers.test.cpp

@@ -0,0 +1,230 @@
+#include "aare/File.hpp"
+#include "aare/RawMasterFile.hpp" //needed for ROI
+#include "aare/RawFile.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+#include <filesystem>
+
+#include "aare/geo_helpers.hpp"
+#include "test_config.hpp"
+
+TEST_CASE("Simple ROIs on one module"){
+    // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi)
+    aare::DetectorGeometry geo;
+    
+
+    aare::ModuleGeometry mod; 
+    mod.origin_x = 0;
+    mod.origin_y = 0;
+    mod.width = 1024;
+    mod.height = 512;
+
+
+    
+    geo.pixels_x = 1024;
+    geo.pixels_y = 512;
+    geo.modules_x = 1;
+    geo.modules_y = 1;
+    geo.module_pixel_0.push_back(mod);
+
+    SECTION("ROI is the whole module"){
+        aare::ROI roi;
+        roi.xmin = 0;
+        roi.xmax = 1024;
+        roi.ymin = 0;
+        roi.ymax = 512;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 1024);
+        REQUIRE(updated_geo.pixels_y == 512);
+        REQUIRE(updated_geo.modules_x == 1);
+        REQUIRE(updated_geo.modules_y == 1);
+        REQUIRE(updated_geo.module_pixel_0[0].height == 512);
+        REQUIRE(updated_geo.module_pixel_0[0].width == 1024);
+    }
+    SECTION("ROI is the top left corner of the module"){
+        aare::ROI roi;
+        roi.xmin = 100;
+        roi.xmax = 200;
+        roi.ymin = 150;
+        roi.ymax = 200;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 100);
+        REQUIRE(updated_geo.pixels_y == 50);
+        REQUIRE(updated_geo.modules_x == 1);
+        REQUIRE(updated_geo.modules_y == 1);
+        REQUIRE(updated_geo.module_pixel_0[0].height == 50);
+        REQUIRE(updated_geo.module_pixel_0[0].width == 100);
+    }
+
+    SECTION("ROI is a small square"){
+        aare::ROI roi;
+        roi.xmin = 1000;
+        roi.xmax = 1010;
+        roi.ymin = 500;
+        roi.ymax = 510;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 10);
+        REQUIRE(updated_geo.pixels_y == 10);
+        REQUIRE(updated_geo.modules_x == 1);
+        REQUIRE(updated_geo.modules_y == 1);
+        REQUIRE(updated_geo.module_pixel_0[0].height == 10);
+        REQUIRE(updated_geo.module_pixel_0[0].width == 10);
+    }
+    SECTION("ROI is a few columns"){
+        aare::ROI roi;
+        roi.xmin = 750;
+        roi.xmax = 800;
+        roi.ymin = 0;
+        roi.ymax = 512;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 50);
+        REQUIRE(updated_geo.pixels_y == 512);
+        REQUIRE(updated_geo.modules_x == 1);
+        REQUIRE(updated_geo.modules_y == 1);
+        REQUIRE(updated_geo.module_pixel_0[0].height == 512);
+        REQUIRE(updated_geo.module_pixel_0[0].width == 50);
+    }
+}
+
+
+
+TEST_CASE("Two modules side by side"){
+    // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi)
+    aare::DetectorGeometry geo;
+    
+
+    aare::ModuleGeometry mod; 
+    mod.origin_x = 0;
+    mod.origin_y = 0;
+    mod.width = 1024;
+    mod.height = 512;
+
+    geo.pixels_x = 2048;
+    geo.pixels_y = 512;
+    geo.modules_x = 2;
+    geo.modules_y = 1;
+
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 1024;
+    geo.module_pixel_0.push_back(mod);
+
+    SECTION("ROI is the whole image"){
+        aare::ROI roi;
+        roi.xmin = 0;
+        roi.xmax = 2048;
+        roi.ymin = 0;
+        roi.ymax = 512;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 2048);
+        REQUIRE(updated_geo.pixels_y == 512);
+        REQUIRE(updated_geo.modules_x == 2);
+        REQUIRE(updated_geo.modules_y == 1);
+    }
+    SECTION("rectangle on both modules"){
+        aare::ROI roi;
+        roi.xmin = 800;
+        roi.xmax = 1300;
+        roi.ymin = 200;
+        roi.ymax = 499;
+        auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+        REQUIRE(updated_geo.pixels_x == 500);
+        REQUIRE(updated_geo.pixels_y == 299);
+        REQUIRE(updated_geo.modules_x == 2);
+        REQUIRE(updated_geo.modules_y == 1);
+        REQUIRE(updated_geo.module_pixel_0[0].height == 299);
+        REQUIRE(updated_geo.module_pixel_0[0].width == 224);
+        REQUIRE(updated_geo.module_pixel_0[1].height == 299);
+        REQUIRE(updated_geo.module_pixel_0[1].width == 276);
+    }   
+}
+
+TEST_CASE("Three modules side by side"){
+    // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi)
+    aare::DetectorGeometry geo;
+    aare::ROI roi;
+    roi.xmin = 700;
+    roi.xmax = 2500;
+    roi.ymin = 0;
+    roi.ymax = 123;
+
+    aare::ModuleGeometry mod; 
+    mod.origin_x = 0;
+    mod.origin_y = 0;
+    mod.width = 1024;
+    mod.height = 512;
+
+    geo.pixels_x = 3072;
+    geo.pixels_y = 512;
+    geo.modules_x = 3;
+    geo.modules_y = 1;
+
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 1024;
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 2048;
+    geo.module_pixel_0.push_back(mod);
+
+    auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+    REQUIRE(updated_geo.pixels_x == 1800);
+    REQUIRE(updated_geo.pixels_y == 123);
+    REQUIRE(updated_geo.modules_x == 3);
+    REQUIRE(updated_geo.modules_y == 1);
+    REQUIRE(updated_geo.module_pixel_0[0].height == 123);
+    REQUIRE(updated_geo.module_pixel_0[0].width == 324);
+    REQUIRE(updated_geo.module_pixel_0[1].height == 123);
+    REQUIRE(updated_geo.module_pixel_0[1].width == 1024);
+    REQUIRE(updated_geo.module_pixel_0[2].height == 123);
+    REQUIRE(updated_geo.module_pixel_0[2].width == 452);
+}
+
+TEST_CASE("Four modules as a square"){
+    // DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, aare::ROI roi)
+    aare::DetectorGeometry geo;
+    aare::ROI roi;
+    roi.xmin = 500;
+    roi.xmax = 2000;
+    roi.ymin = 500;
+    roi.ymax = 600;
+
+    aare::ModuleGeometry mod; 
+    mod.origin_x = 0;
+    mod.origin_y = 0;
+    mod.width = 1024;
+    mod.height = 512;
+
+    geo.pixels_x = 2048;
+    geo.pixels_y = 1024;
+    geo.modules_x = 2;
+    geo.modules_y = 2;
+
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 1024;
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 0;
+    mod.origin_y = 512;
+    geo.module_pixel_0.push_back(mod);
+    mod.origin_x = 1024;
+    geo.module_pixel_0.push_back(mod);
+
+    auto updated_geo = aare::update_geometry_with_roi(geo, roi);
+
+    REQUIRE(updated_geo.pixels_x == 1500);
+    REQUIRE(updated_geo.pixels_y == 100);
+    REQUIRE(updated_geo.modules_x == 2);
+    REQUIRE(updated_geo.modules_y == 2);
+    REQUIRE(updated_geo.module_pixel_0[0].height == 12);
+    REQUIRE(updated_geo.module_pixel_0[0].width == 524);
+    REQUIRE(updated_geo.module_pixel_0[1].height == 12);
+    REQUIRE(updated_geo.module_pixel_0[1].width == 976);
+    REQUIRE(updated_geo.module_pixel_0[2].height == 88);
+    REQUIRE(updated_geo.module_pixel_0[2].width == 524);
+    REQUIRE(updated_geo.module_pixel_0[3].height == 88);
+    REQUIRE(updated_geo.module_pixel_0[3].width == 976);
+}

src/utils/task.cpp

@@ -0,0 +1,30 @@
+#include "aare/utils/task.hpp"
+
+namespace aare {
+
+std::vector<std::pair<int, int>> split_task(int first, int last,
+                                            int n_threads) {
+    std::vector<std::pair<int, int>> vec;
+    vec.reserve(n_threads);
+
+    int n_frames = last - first;
+
+    if (n_threads >= n_frames) {
+        for (int i = 0; i != n_frames; ++i) {
+            vec.push_back({i, i + 1});
+        }
+        return vec;
+    }
+
+    int step = (n_frames) / n_threads;
+    for (int i = 0; i != n_threads; ++i) {
+        int start = step * i;
+        int stop = step * (i + 1);
+        if (i == n_threads - 1)
+            stop = last;
+        vec.push_back({start, stop});
+    }
+    return vec;
+}
+
+} // namespace aare

src/utils/task.test.cpp

@@ -0,0 +1,32 @@
+#include "aare/utils/task.hpp"
+
+#include <catch2/matchers/catch_matchers_floating_point.hpp>
+#include <catch2/catch_test_macros.hpp>
+
+
+TEST_CASE("Split a range into multiple tasks"){    
+
+    auto tasks = aare::split_task(0, 10, 3);
+    REQUIRE(tasks.size() == 3);
+    REQUIRE(tasks[0].first == 0);
+    REQUIRE(tasks[0].second == 3);
+    REQUIRE(tasks[1].first == 3);
+    REQUIRE(tasks[1].second == 6);
+    REQUIRE(tasks[2].first == 6);
+    REQUIRE(tasks[2].second == 10);
+
+    tasks = aare::split_task(0, 10, 1);
+    REQUIRE(tasks.size() == 1);
+    REQUIRE(tasks[0].first == 0);
+    REQUIRE(tasks[0].second == 10);
+
+    tasks = aare::split_task(0, 10, 10);
+    REQUIRE(tasks.size() == 10);
+    for (int i = 0; i < 10; i++){
+        REQUIRE(tasks[i].first == i);
+        REQUIRE(tasks[i].second == i+1);
+    }
+    
+
+
+}

tests/CMakeLists.txt

@@ -17,8 +17,8 @@ endif()
 list(APPEND CMAKE_MODULE_PATH ${Catch2_SOURCE_DIR}/extras)
 
 add_executable(tests test.cpp)
-target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
-
+target_link_libraries(tests PRIVATE Catch2::Catch2WithMain aare_core aare_compiler_flags)
+# target_compile_options(tests PRIVATE -fno-omit-frame-pointer -fsanitize=address)
 set_target_properties(tests PROPERTIES
         RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
         OUTPUT_NAME run_tests
@@ -34,7 +34,7 @@ set(TestSources
 target_sources(tests PRIVATE ${TestSources} )
 
 #Work around to remove, this is not the way to do it =) 
-target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
+# target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
 
 
 #configure a header to pass test file paths

tests/test.cpp

@@ -3,6 +3,7 @@
 #include <climits>
 #include <filesystem>
 #include <fstream>
+#include <fmt/core.h>
 
 TEST_CASE("Test suite can find data assets", "[.integration]") {
     auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
@@ -18,4 +19,20 @@ TEST_CASE("Test suite can open data assets", "[.integration]") {
 TEST_CASE("Test float32 and char8") {
     REQUIRE(sizeof(float) == 4);
     REQUIRE(CHAR_BIT == 8);
-}
+}
+
+/**
+ * Uncomment the following tests to verify that asan is working
+ */
+
+// TEST_CASE("trigger asan stack"){
+//     int arr[5] =  {1,2,3,4,5};
+//     int val = arr[7];
+//     fmt::print("val: {}\n", val);
+// }
+
+// TEST_CASE("trigger asan heap"){
+//     auto *ptr = new int[5];
+//     ptr[70] = 5;
+//     fmt::print("ptr: {}\n", ptr[70]);
+// }

tests/test_config.hpp.in

@@ -7,6 +7,6 @@ inline auto test_data_path(){
     if(const char* env_p = std::getenv("AARE_TEST_DATA")){
         return std::filesystem::path(env_p);
     }else{
-        throw std::runtime_error("AARE_TEST_DATA_PATH not set");
+        throw std::runtime_error("Path to test data: $AARE_TEST_DATA not set");
     }
 }