Dev/decode my302 (#254)

This PR adds support for decoding digital data from the my320 test chip.
- Added BitOffset (strong type)
- Expand 24 to 32 bit 
- Python bindings for decoding my302
- Improved docs

.gitea/workflows/cmake_build.yml

@@ -2,7 +2,6 @@ name: Build the package using cmake then documentation
 
 on:
   workflow_dispatch:
-  push:
 
     
 
@@ -16,12 +15,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        platform: [ubuntu-latest, ] 
-        python-version: ["3.12",]
+        python-version: ["3.12", ] 
 
     runs-on: ${{ matrix.platform }}
 
-    # The setup-miniconda action needs this to activate miniconda
+
     defaults:
       run:
         shell: "bash -l {0}"
@@ -35,13 +34,13 @@ jobs:
             sudo apt-get -y install cmake gcc g++
 
       - name: Get conda
-        uses: conda-incubator/setup-miniconda@v3.0.4
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
+          environment-file: etc/dev-env.yml
+          miniforge-version: latest
           channels: conda-forge
-
+          conda-remove-defaults: "true"
-      - name: Prepare
-        run: conda install doxygen sphinx=7.1.2 breathe pybind11 sphinx_rtd_theme furo nlohmann_json zeromq fmt numpy
 
       - name: Build library 
         run: |
@@ -56,3 +55,4 @@ jobs:
 
 
 
+

.gitea/workflows/rh8-native.yml

@@ -0,0 +1,36 @@
+name: Build on RHEL8
+
+on:
+  push:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+    container:
+      image: gitea.psi.ch/images/rhel8-developer-gitea-actions
+    steps:
+      # workaround until actions/checkout@v4 is available for RH8
+      # - uses: actions/checkout@v4
+      - name: Clone repository
+        run: |
+          echo Cloning ${{ github.ref_name }}
+          git clone https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git --branch=${{ github.ref_name }} .
+
+
+      - name: Install dependencies
+        run: |
+          dnf install -y cmake python3.12 python3.12-devel python3.12-pip
+
+      - name: Build library 
+        run: |
+            mkdir build && cd build
+            cmake .. -DAARE_PYTHON_BINDINGS=ON -DAARE_TESTS=ON -DPython_FIND_VIRTUALENV=FIRST
+            make -j 2 
+
+      - name: C++ unit tests
+        working-directory: ${{gitea.workspace}}/build
+        run: ctest

.gitea/workflows/rh9-native.yml

@@ -0,0 +1,31 @@
+name: Build on RHEL9
+
+on:
+  push:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+    container:
+      image: gitea.psi.ch/images/rhel9-developer-gitea-actions
+    steps:
+      - uses: actions/checkout@v4
+
+
+      - name: Install dependencies
+        run: |
+          dnf install -y cmake python3.12 python3.12-devel python3.12-pip
+
+      - name: Build library 
+        run: |
+            mkdir build && cd build
+            cmake .. -DAARE_PYTHON_BINDINGS=ON -DAARE_TESTS=ON
+            make -j 2 
+
+      - name: C++ unit tests
+        working-directory: ${{gitea.workspace}}/build
+        run: ctest

.github/workflows/build_and_deploy_conda.yml

@@ -1,16 +1,16 @@
 name: Build pkgs and deploy if on main
 
 on:
-  push:
+  release:
-    branches:
+      types:
-     - main
+        - published
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        platform: [ubuntu-latest] # macos-12, windows-2019]
         python-version: ["3.12",]
 
     runs-on: ${{ matrix.platform }}
@@ -24,13 +24,13 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Get conda
-        uses: conda-incubator/setup-miniconda@v3.0.4
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
+          environment-file: etc/dev-env.yml
+          miniforge-version: latest
           channels: conda-forge
-
+          conda-remove-defaults: "true"
-      - name: Prepare
-        run: conda install conda-build=24.9 conda-verify pytest anaconda-client
 
       - name: Enable upload
         run: conda config --set anaconda_upload yes

.github/workflows/build_conda.yml

@@ -24,13 +24,14 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Get conda
-        uses: conda-incubator/setup-miniconda@v3.0.4
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
+          environment-file: etc/dev-env.yml
+          miniforge-version: latest
           channels: conda-forge
+          conda-remove-defaults: "true"
 
-      - name: Prepare
-        run: conda install conda-build=24.9 conda-verify pytest anaconda-client
   
       - name: Disable upload
         run: conda config --set anaconda_upload no

.github/workflows/build_wheel.yml

@@ -0,0 +1,64 @@
+name: Build wheel
+
+on:
+    workflow_dispatch:
+    pull_request:
+    push:
+      branches:
+        - main
+    release:
+      types:
+        - published
+  
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest,]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        run: pipx run cibuildwheel==2.23.0
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+    
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build sdist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: cibw-sdist
+        path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+        id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+    # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+        - uses: actions/download-artifact@v4
+          with:
+            # unpacks all CIBW artifacts into dist/
+            pattern: cibw-*
+            path: dist
+            merge-multiple: true
+
+        - uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/build_with_docs.yml

@@ -3,8 +3,14 @@ name: Build the package using cmake then documentation
 on:
   workflow_dispatch:
   push:
+  pull_request:
+  release:
+    types:
+      - published
     
-
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Debug
 
 permissions:
   contents: read
@@ -16,12 +22,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        platform: [ubuntu-latest, macos-latest] 
         python-version: ["3.12",]
 
     runs-on: ${{ matrix.platform }}
 
-    # The setup-miniconda action needs this to activate miniconda
     defaults:
       run:
         shell: "bash -l {0}"
@@ -30,23 +35,28 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Get conda
-        uses: conda-incubator/setup-miniconda@v3.0.4
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
+          environment-file: etc/dev-env.yml
+          miniforge-version: latest
           channels: conda-forge
+          conda-remove-defaults: "true"
 
-      - name: Prepare
+      - name: Build library and docs
-        run: conda install doxygen sphinx=7.1.2 breathe pybind11 sphinx_rtd_theme furo nlohmann_json zeromq fmt numpy
-
-      - name: Build library 
         run: |
             mkdir build 
             cd build 
-            cmake .. -DAARE_SYSTEM_LIBRARIES=ON -DAARE_DOCS=ON 
+            cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DAARE_SYSTEM_LIBRARIES=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_DOCS=ON -DAARE_TESTS=ON
-            make -j 2 
+            make -j 4 
             make docs
 
+      - name: C++ unit tests
+        working-directory: ${{github.workspace}}/build
+        run: ctest -C ${{env.BUILD_TYPE}} -j4
+
       - name: Upload static files as artifact
+        if: matrix.platform == 'ubuntu-latest'
         id: deployment
         uses: actions/upload-pages-artifact@v3 
         with:
@@ -57,7 +67,7 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-latest
     needs: build
-    if: github.ref == 'refs/heads/main'
+    if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'workflow_dispatch' )
     steps:
       - name: Deploy to GitHub Pages
         id: deployment

.gitignore

@@ -17,7 +17,8 @@ Testing/
 ctbDict.cpp
 ctbDict.h
 
-
+wheelhouse/
+dist/
 
 *.pyc
 */__pycache__/*

CMakeLists.txt

@@ -1,16 +1,30 @@
-cmake_minimum_required(VERSION 3.14)
+# SPDX-License-Identifier: MPL-2.0
+cmake_minimum_required(VERSION 3.15)
 
 project(aare 
-    VERSION 1.0.0
     DESCRIPTION "Data processing library for PSI detectors"
     HOMEPAGE_URL "https://github.com/slsdetectorgroup/aare"
     LANGUAGES C CXX 
 )
 
+# Read VERSION file into project version
+set(VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/VERSION")
+file(READ "${VERSION_FILE}" VERSION_CONTENT)
+string(STRIP "${VERSION_CONTENT}" PROJECT_VERSION_STRING)
+set(PROJECT_VERSION ${PROJECT_VERSION_STRING})
+
 set(CMAKE_CXX_STANDARD 17) 
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
+execute_process(
+        COMMAND git log -1 --format=%h
+        WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+        OUTPUT_VARIABLE GIT_HASH
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+message(STATUS "Building from git hash: ${GIT_HASH}")
+
 if (${CMAKE_VERSION} VERSION_GREATER "3.24")
     cmake_policy(SET CMP0135 NEW) #Fetch content download timestamp
 endif()
@@ -64,19 +78,12 @@ if(AARE_SYSTEM_LIBRARIES)
     # on conda-forge
 endif()
 
-if(AARE_VERBOSE)
-    add_compile_definitions(AARE_VERBOSE)
-endif()
-
-if(AARE_CUSTOM_ASSERT)
-    add_compile_definitions(AARE_CUSTOM_ASSERT)
-endif()
-
 if(AARE_BENCHMARKS)
     add_subdirectory(benchmarks)
 endif()
 
 
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(AARE_FETCH_LMFIT)
@@ -104,6 +111,7 @@ if(AARE_FETCH_LMFIT)
         )
     endif()
     
+
     #Disable what we don't need from lmfit
     set(BUILD_TESTING OFF CACHE BOOL "")
     set(LMFIT_CPPTEST OFF CACHE BOOL "")
@@ -317,13 +325,10 @@ if(AARE_ASAN)
     )
 endif()
 
-
-
-
-
 if(AARE_TESTS)
     enable_testing()
     add_subdirectory(tests)
+    target_compile_definitions(tests PRIVATE AARE_TESTS)
 endif()
 
 ###------------------------------------------------------------------------------MAIN LIBRARY
@@ -343,9 +348,12 @@ set(PUBLICHEADERS
     include/aare/File.hpp
     include/aare/Fit.hpp
     include/aare/FileInterface.hpp
+    include/aare/FilePtr.hpp
     include/aare/Frame.hpp
     include/aare/GainMap.hpp
-    include/aare/geo_helpers.hpp
+    include/aare/DetectorGeometry.hpp
+    include/aare/JungfrauDataFile.hpp
+    include/aare/logger.hpp
     include/aare/NDArray.hpp
     include/aare/NDView.hpp
     include/aare/NumpyFile.hpp
@@ -361,31 +369,37 @@ set(PUBLICHEADERS
 
 
 set(SourceFiles
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/calibration.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/DetectorGeometry.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/FilePtr.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Fit.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolator.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolator.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/ifstream_helpers.cpp
 ) 
 
-
 add_library(aare_core STATIC ${SourceFiles})
 target_include_directories(aare_core PUBLIC 
     "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>" 
     "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>" 
 )
 
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
 target_link_libraries(
     aare_core 
     PUBLIC 
@@ -394,26 +408,42 @@ target_link_libraries(
     ${STD_FS_LIB} # from helpers.cmake
     PRIVATE 
     aare_compiler_flags 
+    Threads::Threads
     $<BUILD_INTERFACE:lmfit>
 
 )
 
+set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+if(AARE_TESTS)
+    target_compile_definitions(aare_core PRIVATE AARE_TESTS)
+endif()
+if(AARE_VERBOSE)
+    target_compile_definitions(aare_core PUBLIC AARE_VERBOSE)
+    target_compile_definitions(aare_core PUBLIC AARE_LOG_LEVEL=aare::logDEBUG5)
+else()
+    target_compile_definitions(aare_core PUBLIC AARE_LOG_LEVEL=aare::logERROR)
+endif()
+
+if(AARE_CUSTOM_ASSERT)
+    target_compile_definitions(aare_core PUBLIC AARE_CUSTOM_ASSERT)
+endif()
+
 set_target_properties(aare_core PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
     PUBLIC_HEADER "${PUBLICHEADERS}"
 )
 
-if (AARE_PYTHON_BINDINGS)
-    set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON)
-endif()
-
 if(AARE_TESTS)
     set(TestSources
             ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/calibration.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/geo_helpers.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/DetectorGeometry.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolation.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
@@ -422,10 +452,13 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Cluster.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/CalculateEta.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFile.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinderMT.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Pedestal.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.test.cpp
 
         )
@@ -433,11 +466,6 @@ if(AARE_TESTS)
 endif()
 
 
-
-###------------------------------------------------------------------------------------------
-###------------------------------------------------------------------------------------------
-
-
 if(AARE_MASTER_PROJECT)
     install(TARGETS aare_core aare_compiler_flags 
         EXPORT "${TARGETS_EXPORT_NAME}"
@@ -447,7 +475,6 @@ if(AARE_MASTER_PROJECT)
     )
 endif()
 
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_INSTALL_RPATH $ORIGIN)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 

LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

README.md

@@ -1,6 +1,14 @@
 # aare
 Data analysis library for PSI hybrid detectors
 
+## Documentation 
+
+Detailed documentation including installation can be found in [Documentation](https://slsdetectorgroup.github.io/aare/)
+
+## License
+
+This project is licensed under the MPL-2.0 license.
+See the LICENSE file or https://www.mozilla.org/en-US/MPL/ for details.
 
 ## Build and install
 

RELEASE.md

@@ -0,0 +1,113 @@
+# Release notes
+
+## head
+
+### New Features:
+
+- Expanding 24 to 32 bit data
+- Decoding digital data from Mythen 302
+
+
+### 2025.11.21
+
+### New Features: 
+
+- Added SPDX-License-Identifier: MPL-2.0 to source files
+- Calculate Eta3 supports all cluster types 
+- interpolation class supports using cross eta3x3 and eta3x3 on full cluster as well as eta2x2 on full cluster
+- interpolation class has option to calculate the rosenblatt transform 
+- reduction operations to reduce Clusters of general size to 2x2 or 3x3 clusters 
+- `max_sum_2x2` including index of subcluster with highest energy is now available from Python API 
+- interpolation supports bilinear interpolation of eta values for more fine grained transformed uniform coordinates
+- Interpolation is documented 
+
+- Added tell to ClusterFile. Returns position in bytes for debugging
+
+### Resolved Features: 
+
+- calculate_eta coincides with theoretical definition
+
+### Bugfixes: 
+
+- eta calculation assumes correct photon center 
+- eta transformation to uniform coordinates starts at 0
+- Bug in interpolation 
+- File supports reading new master json file format (multiple ROI's not supported yet)
+
+
+### API Changes: 
+
+- ClusterFinder for 2x2 Cluster disabled 
+- eta stores corner as enum class cTopLeft, cTopRight, BottomLeft, cBottomRight indicating 2x2 subcluster with largest energy relative to cluster center 
+- max_sum_2x2 returns corner as index 
+
+### 2025.8.22
+
+Features:
+
+- Apply calibration works in G0 if passes a 2D calibration and pedestal
+- count pixels that switch
+- calculate pedestal (also g0 version)
+- NDArray::view() needs an lvalue to reduce issues with the view outliving the array
+
+
+Bugfixes:
+
+- Now using glibc 2.17 in conda builds (was using the host)
+- Fixed shifted pixels in clusters close to the edge of a frame
+
+### 2025.7.18
+
+Features:
+
+- Cluster finder now works with 5x5, 7x7 and 9x9 clusters
+- Added ClusterVector::empty() member
+- Added apply_calibration function for Jungfrau data
+
+Bugfixes:
+- Fixed reading RawFiles with ROI fully excluding some sub files. 
+- Decoding of MH02 files placed the pixels in wrong position
+- Removed unused file: ClusterFile.cpp 
+
+
+### 2025.5.22
+
+Features:
+
+- Added scurve fitting
+
+Bugfixes:
+
+- Fixed crash when opening raw files with large number of data files
+
+## Download, Documentation & Support 
+
+### Download
+
+The Source Code: 
+https://github.com/slsdetectorgroup/aare
+
+
+### Documentation 
+
+
+Documentation including installation details: 
+https://github.com/slsdetectorgroup/aare 
+
+
+### Support
+
+
+erik.frojdh@psi.ch \
+alice.mazzoleni@psi.ch \
+dhanya.thattil@psi.ch
+
+
+
+
+
+
+
+
+
+

VERSION

@@ -0,0 +1 @@
+2025.11.21

benchmarks/CMakeLists.txt

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: MPL-2.0
 
 include(FetchContent)
 
@@ -15,7 +16,7 @@ FetchContent_MakeAvailable(benchmark)
 
 add_executable(benchmarks)
 
-target_sources(benchmarks PRIVATE ndarray_benchmark.cpp calculateeta_benchmark.cpp)
+target_sources(benchmarks PRIVATE ndarray_benchmark.cpp calculateeta_benchmark.cpp reduce_benchmark.cpp)
 
 # Link Google Benchmark and other necessary libraries
 target_link_libraries(benchmarks PRIVATE benchmark::benchmark aare_core aare_compiler_flags)

benchmarks/calculateeta_benchmark.cpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #include "aare/CalculateEta.hpp"
 #include "aare/ClusterFile.hpp"
 #include <benchmark/benchmark.h>
@@ -8,6 +9,7 @@ class ClusterFixture : public benchmark::Fixture {
   public:
     Cluster<int, 2, 2> cluster_2x2{};
     Cluster<int, 3, 3> cluster_3x3{};
+    Cluster<int, 4, 4> cluster_4x4{};
 
   private:
     using benchmark::Fixture::SetUp;
@@ -26,6 +28,13 @@ class ClusterFixture : public benchmark::Fixture {
 
         cluster_3x3.x = 0;
         cluster_3x3.y = 0;
+
+        int temp_data3[16] = {1, 2,  3,  4,  5,  6,  7,  8,
+                              9, 10, 11, 12, 13, 14, 15, 16};
+        std::copy(std::begin(temp_data3), std::end(temp_data3),
+                  std::begin(cluster_4x4.data));
+        cluster_4x4.x = 0;
+        cluster_4x4.y = 0;
     }
 
     // void TearDown(::benchmark::State& state) {
@@ -41,8 +50,8 @@ BENCHMARK_F(ClusterFixture, Calculate2x2Eta)(benchmark::State &st) {
 }
 
 // almost takes double the time
-BENCHMARK_F(ClusterFixture,
+BENCHMARK_F(ClusterFixture, CalculateGeneralEtaFor2x2Cluster)
-            CalculateGeneralEtaFor2x2Cluster)(benchmark::State &st) {
+(benchmark::State &st) {
     for (auto _ : st) {
         // This code gets timed
         Eta2 eta = calculate_eta2<int, 2, 2>(cluster_2x2);
@@ -59,12 +68,37 @@ BENCHMARK_F(ClusterFixture, Calculate3x3Eta)(benchmark::State &st) {
 }
 
 // almost takes double the time
-BENCHMARK_F(ClusterFixture,
+BENCHMARK_F(ClusterFixture, CalculateGeneralEtaFor3x3Cluster)
-            CalculateGeneralEtaFor3x3Cluster)(benchmark::State &st) {
+(benchmark::State &st) {
     for (auto _ : st) {
         // This code gets timed
         Eta2 eta = calculate_eta2<int, 3, 3>(cluster_3x3);
         benchmark::DoNotOptimize(eta);
     }
 }
+
+BENCHMARK_F(ClusterFixture, Calculate2x2Etawithreduction)
+(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        auto reduced_cluster = reduce_to_2x2(cluster_4x4);
+        Eta2 eta = calculate_eta2(reduced_cluster);
+        auto reduced_cluster_from_3x3 = reduce_to_2x2(cluster_3x3);
+        Eta2 eta2 = calculate_eta2(reduced_cluster_from_3x3);
+        benchmark::DoNotOptimize(eta);
+        benchmark::DoNotOptimize(eta2);
+    }
+}
+
+BENCHMARK_F(ClusterFixture, Calculate2x2Etawithoutreduction)
+(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        Eta2 eta = calculate_eta2(cluster_4x4);
+        Eta2 eta2 = calculate_eta2(cluster_3x3);
+        benchmark::DoNotOptimize(eta);
+        benchmark::DoNotOptimize(eta2);
+    }
+}
+
 // BENCHMARK_MAIN();

benchmarks/ndarray_benchmark.cpp

@@ -1,136 +1,133 @@
-#include <benchmark/benchmark.h>
+// SPDX-License-Identifier: MPL-2.0
 #include "aare/NDArray.hpp"
-
+#include <benchmark/benchmark.h>
 
 using aare::NDArray;
 
 constexpr ssize_t size = 1024;
 class TwoArrays : public benchmark::Fixture {
-public:
+  public:
-    NDArray<int,2> a{{size,size},0};
+    NDArray<int, 2> a{{size, size}, 0};
-    NDArray<int,2> b{{size,size},0};
+    NDArray<int, 2> b{{size, size}, 0};
-  void SetUp(::benchmark::State& state) {
+    void SetUp(::benchmark::State &state) {
-    for(uint32_t i = 0; i < size; i++){
+        for (uint32_t i = 0; i < size; i++) {
-      for(uint32_t j = 0; j < size; j++){
+            for (uint32_t j = 0; j < size; j++) {
-        a(i, j)= i*j+1;
+                a(i, j) = i * j + 1;
-        b(i, j)= i*j+1;
+                b(i, j) = i * j + 1;
-      }
+            }
+        }
     }
-  }
 
-  // void TearDown(::benchmark::State& state) {
+    // void TearDown(::benchmark::State& state) {
-  // }
+    // }
 };
 
-
+BENCHMARK_F(TwoArrays, AddWithOperator)(benchmark::State &st) {
-
+    for (auto _ : st) {
-
+        // This code gets timed
-BENCHMARK_F(TwoArrays, AddWithOperator)(benchmark::State& st) {
+        NDArray<int, 2> res = a + b;
-   for (auto _ : st) {
+        benchmark::DoNotOptimize(res);
-    // This code gets timed
+    }
-    NDArray<int,2> res = a+b;
+}
-    benchmark::DoNotOptimize(res);
+BENCHMARK_F(TwoArrays, AddWithIndex)(benchmark::State &st) {
-  }
+    for (auto _ : st) {
-}
+        // This code gets timed
-BENCHMARK_F(TwoArrays, AddWithIndex)(benchmark::State& st) {
+        NDArray<int, 2> res(a.shape());
-   for (auto _ : st) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-    // This code gets timed
+            res(i) = a(i) + b(i);
-    NDArray<int,2> res(a.shape());
+        }
-    for (uint32_t i = 0; i < a.size(); i++) {
+        benchmark::DoNotOptimize(res);
-        res(i) = a(i) + b(i);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
-BENCHMARK_F(TwoArrays, SubtractWithOperator)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, SubtractWithOperator)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res = a-b;
+        NDArray<int, 2> res = a - b;
-    benchmark::DoNotOptimize(res);
+        benchmark::DoNotOptimize(res);
-  }
+    }
-}
+}
-BENCHMARK_F(TwoArrays, SubtractWithIndex)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, SubtractWithIndex)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res(a.shape());
+        NDArray<int, 2> res(a.shape());
-    for (uint32_t i = 0; i < a.size(); i++) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-        res(i) = a(i) - b(i);
+            res(i) = a(i) - b(i);
+        }
+        benchmark::DoNotOptimize(res);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
-BENCHMARK_F(TwoArrays, MultiplyWithOperator)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, MultiplyWithOperator)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res = a*b;
+        NDArray<int, 2> res = a * b;
-    benchmark::DoNotOptimize(res);
+        benchmark::DoNotOptimize(res);
-  }
+    }
-}
+}
-BENCHMARK_F(TwoArrays, MultiplyWithIndex)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, MultiplyWithIndex)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res(a.shape());
+        NDArray<int, 2> res(a.shape());
-    for (uint32_t i = 0; i < a.size(); i++) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-        res(i) = a(i) * b(i);
+            res(i) = a(i) * b(i);
+        }
+        benchmark::DoNotOptimize(res);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
-BENCHMARK_F(TwoArrays, DivideWithOperator)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, DivideWithOperator)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res = a/b;
+        NDArray<int, 2> res = a / b;
-    benchmark::DoNotOptimize(res);
+        benchmark::DoNotOptimize(res);
-  }
+    }
-}
+}
-BENCHMARK_F(TwoArrays, DivideWithIndex)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, DivideWithIndex)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res(a.shape());
+        NDArray<int, 2> res(a.shape());
-    for (uint32_t i = 0; i < a.size(); i++) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-        res(i) = a(i) / b(i);
+            res(i) = a(i) / b(i);
+        }
+        benchmark::DoNotOptimize(res);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
-BENCHMARK_F(TwoArrays, FourAddWithOperator)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, FourAddWithOperator)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res = a+b+a+b;
+        NDArray<int, 2> res = a + b + a + b;
-    benchmark::DoNotOptimize(res);
+        benchmark::DoNotOptimize(res);
-  }
+    }
-}
+}
-BENCHMARK_F(TwoArrays, FourAddWithIndex)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, FourAddWithIndex)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res(a.shape());
+        NDArray<int, 2> res(a.shape());
-    for (uint32_t i = 0; i < a.size(); i++) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-        res(i) = a(i) + b(i) + a(i) + b(i);
+            res(i) = a(i) + b(i) + a(i) + b(i);
+        }
+        benchmark::DoNotOptimize(res);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
-BENCHMARK_F(TwoArrays, MultiplyAddDivideWithOperator)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, MultiplyAddDivideWithOperator)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res = a*a+b/a;
+        NDArray<int, 2> res = a * a + b / a;
-    benchmark::DoNotOptimize(res);
+        benchmark::DoNotOptimize(res);
-  }
+    }
-}
+}
-BENCHMARK_F(TwoArrays, MultiplyAddDivideWithIndex)(benchmark::State& st) {
+BENCHMARK_F(TwoArrays, MultiplyAddDivideWithIndex)(benchmark::State &st) {
-   for (auto _ : st) {
+    for (auto _ : st) {
-    // This code gets timed
+        // This code gets timed
-    NDArray<int,2> res(a.shape());
+        NDArray<int, 2> res(a.shape());
-    for (uint32_t i = 0; i < a.size(); i++) {
+        for (uint32_t i = 0; i < a.size(); i++) {
-        res(i) = a(i) * a(i) + b(i) / a(i);
+            res(i) = a(i) * a(i) + b(i) / a(i);
+        }
+        benchmark::DoNotOptimize(res);
     }
-    benchmark::DoNotOptimize(res);
-  }
 }
 
 BENCHMARK_MAIN();

benchmarks/reduce_benchmark.cpp

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: MPL-2.0
+#include "aare/Cluster.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace aare;
+
+class ClustersForReduceFixture : public benchmark::Fixture {
+  public:
+    Cluster<int, 5, 5> cluster_5x5{};
+    Cluster<int, 3, 3> cluster_3x3{};
+
+  private:
+    using benchmark::Fixture::SetUp;
+
+    void SetUp([[maybe_unused]] const benchmark::State &state) override {
+        int temp_data[25] = {1, 1, 1, 1, 1, 1, 1, 2, 1, 1,
+                             1, 2, 3, 1, 2, 1, 1, 1, 1, 2};
+        std::copy(std::begin(temp_data), std::end(temp_data),
+                  std::begin(cluster_5x5.data));
+
+        cluster_5x5.x = 5;
+        cluster_5x5.y = 5;
+
+        int temp_data2[9] = {1, 1, 1, 2, 3, 1, 2, 2, 1};
+        std::copy(std::begin(temp_data2), std::end(temp_data2),
+                  std::begin(cluster_3x3.data));
+
+        cluster_3x3.x = 5;
+        cluster_3x3.y = 5;
+    }
+
+    // void TearDown(::benchmark::State& state) {
+    // }
+};
+
+template <typename T>
+Cluster<T, 3, 3, uint16_t> reduce_to_3x3(const Cluster<T, 5, 5, uint16_t> &c) {
+    Cluster<T, 3, 3, uint16_t> result;
+
+    // Write out the sums in the hope that the compiler can optimize this
+    std::array<T, 9> sum_3x3_subclusters;
+
+    // Write out the sums in the hope that the compiler can optimize this
+    sum_3x3_subclusters[0] = c.data[0] + c.data[1] + c.data[2] + c.data[5] +
+                             c.data[6] + c.data[7] + c.data[10] + c.data[11] +
+                             c.data[12];
+    sum_3x3_subclusters[1] = c.data[1] + c.data[2] + c.data[3] + c.data[6] +
+                             c.data[7] + c.data[8] + c.data[11] + c.data[12] +
+                             c.data[13];
+    sum_3x3_subclusters[2] = c.data[2] + c.data[3] + c.data[4] + c.data[7] +
+                             c.data[8] + c.data[9] + c.data[12] + c.data[13] +
+                             c.data[14];
+    sum_3x3_subclusters[3] = c.data[5] + c.data[6] + c.data[7] + c.data[10] +
+                             c.data[11] + c.data[12] + c.data[15] + c.data[16] +
+                             c.data[17];
+    sum_3x3_subclusters[4] = c.data[6] + c.data[7] + c.data[8] + c.data[11] +
+                             c.data[12] + c.data[13] + c.data[16] + c.data[17] +
+                             c.data[18];
+    sum_3x3_subclusters[5] = c.data[7] + c.data[8] + c.data[9] + c.data[12] +
+                             c.data[13] + c.data[14] + c.data[17] + c.data[18] +
+                             c.data[19];
+    sum_3x3_subclusters[6] = c.data[10] + c.data[11] + c.data[12] + c.data[15] +
+                             c.data[16] + c.data[17] + c.data[20] + c.data[21] +
+                             c.data[22];
+    sum_3x3_subclusters[7] = c.data[11] + c.data[12] + c.data[13] + c.data[16] +
+                             c.data[17] + c.data[18] + c.data[21] + c.data[22] +
+                             c.data[23];
+    sum_3x3_subclusters[8] = c.data[12] + c.data[13] + c.data[14] + c.data[17] +
+                             c.data[18] + c.data[19] + c.data[22] + c.data[23] +
+                             c.data[24];
+
+    auto index = std::max_element(sum_3x3_subclusters.begin(),
+                                  sum_3x3_subclusters.end()) -
+                 sum_3x3_subclusters.begin();
+
+    switch (index) {
+    case 0:
+        result.x = c.x - 1;
+        result.y = c.y + 1;
+        result.data = {c.data[0], c.data[1],  c.data[2],  c.data[5], c.data[6],
+                       c.data[7], c.data[10], c.data[11], c.data[12]};
+        break;
+    case 1:
+        result.x = c.x;
+        result.y = c.y + 1;
+        result.data = {c.data[1], c.data[2],  c.data[3],  c.data[6], c.data[7],
+                       c.data[8], c.data[11], c.data[12], c.data[13]};
+        break;
+    case 2:
+        result.x = c.x + 1;
+        result.y = c.y + 1;
+        result.data = {c.data[2], c.data[3],  c.data[4],  c.data[7], c.data[8],
+                       c.data[9], c.data[12], c.data[13], c.data[14]};
+        break;
+    case 3:
+        result.x = c.x - 1;
+        result.y = c.y;
+        result.data = {c.data[5],  c.data[6],  c.data[7],
+                       c.data[10], c.data[11], c.data[12],
+                       c.data[15], c.data[16], c.data[17]};
+        break;
+    case 4:
+        result.x = c.x + 1;
+        result.y = c.y;
+        result.data = {c.data[6],  c.data[7],  c.data[8],
+                       c.data[11], c.data[12], c.data[13],
+                       c.data[16], c.data[17], c.data[18]};
+        break;
+    case 5:
+        result.x = c.x + 1;
+        result.y = c.y;
+        result.data = {c.data[7],  c.data[8],  c.data[9],
+                       c.data[12], c.data[13], c.data[14],
+                       c.data[17], c.data[18], c.data[19]};
+        break;
+    case 6:
+        result.x = c.x + 1;
+        result.y = c.y - 1;
+        result.data = {c.data[10], c.data[11], c.data[12],
+                       c.data[15], c.data[16], c.data[17],
+                       c.data[20], c.data[21], c.data[22]};
+        break;
+    case 7:
+        result.x = c.x + 1;
+        result.y = c.y - 1;
+        result.data = {c.data[11], c.data[12], c.data[13],
+                       c.data[16], c.data[17], c.data[18],
+                       c.data[21], c.data[22], c.data[23]};
+        break;
+    case 8:
+        result.x = c.x + 1;
+        result.y = c.y - 1;
+        result.data = {c.data[12], c.data[13], c.data[14],
+                       c.data[17], c.data[18], c.data[19],
+                       c.data[22], c.data[23], c.data[24]};
+        break;
+    }
+    return result;
+}
+
+BENCHMARK_F(ClustersForReduceFixture, Reduce2x2)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        benchmark::DoNotOptimize(reduce_to_2x2<int, 3, 3, uint16_t>(
+            cluster_3x3)); // make sure compiler evaluates the expression
+    }
+}
+
+BENCHMARK_F(ClustersForReduceFixture, SpecificReduce2x2)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        benchmark::DoNotOptimize(reduce_to_2x2<int>(cluster_3x3));
+    }
+}
+
+BENCHMARK_F(ClustersForReduceFixture, Reduce3x3)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        benchmark::DoNotOptimize(
+            reduce_to_3x3<int, 5, 5, uint16_t>(cluster_5x5));
+    }
+}
+
+BENCHMARK_F(ClustersForReduceFixture, SpecificReduce3x3)(benchmark::State &st) {
+    for (auto _ : st) {
+        // This code gets timed
+        benchmark::DoNotOptimize(reduce_to_3x3<int>(cluster_5x5));
+    }
+}

conda-recipe/conda_build_config.yaml

@@ -1,28 +1,16 @@
 python:
     - 3.11
-    - 3.11
-    - 3.11
-    - 3.12
-    - 3.12
     - 3.12
     - 3.13
 
+c_compiler:
+  - gcc                        # [linux]
 
+c_stdlib:
+  - sysroot                    # [linux]
 
-numpy: 
+cxx_compiler:
-    - 1.26
+  - gxx                        # [linux]
-    - 2.0
-    - 2.1
-    - 1.26
-    - 2.0
-    - 2.1
-    - 2.1
 
-
+c_stdlib_version:              # [linux]
-zip_keys:
+  - 2.17                       # [linux]
-    - python
-    - numpy
-
-pin_run_as_build:
-  numpy: x.x
-  python: x.x

conda-recipe/meta.yaml

@@ -1,9 +1,10 @@
+source:
+  path: ../
+
+{% set version = load_file_regex(load_file = 'VERSION', regex_pattern = '(\d+(?:\.\d+)*(?:[\+\w\.]+))').group(1) %}
 package:
   name: aare
-  version: 2025.4.1 #TODO! how to not duplicate this?
+  version: {{version}}
-
-
-
 
 source:
   path: ..
@@ -11,44 +12,42 @@ source:
 build:
   number: 0
   script:
-    - unset CMAKE_GENERATOR && {{ PYTHON }} -m pip install . -vv  # [not win]
+    - unset CMAKE_GENERATOR && {{ PYTHON }} -m pip install . -vv 
-    - {{ PYTHON }} -m pip install . -vv  # [win]
 
 requirements:
   build:
-    - python {{python}}
+    - {{ compiler('c') }}
-    - numpy {{ numpy }}
+    - {{ stdlib("c") }}
     - {{ compiler('cxx') }}
-
-
-  host:
     - cmake
     - ninja
-    - python {{python}}
+
-    - numpy {{ numpy }}
+  host:
+    - python
     - pip
+    - numpy=2.1
     - scikit-build-core
     - pybind11 >=2.13.0
-    - fmt
+    - matplotlib # needed in host to solve the environment for run
-    - zeromq
-    - nlohmann_json
-    - catch2
 
   run:
-    - python {{python}}
+    - python
-    - numpy {{ numpy }}
+    - {{ pin_compatible('numpy') }}
+    - matplotlib
+    
 
 
 test:
   imports:
     - aare
-  # requires:
+  requires:
-  #   - pytest
+    - pytest
-  # source_files:
+    - boost-histogram 
-  #   - tests
+  source_files:
-  # commands:
+    - python/tests
-  #   - pytest tests
+  commands:
+    - python -m pytest python/tests
 
 about:
-  summary: An example project built with pybind11 and scikit-build.
+  license: SPDX-License-Identifier MPL-2.0
-  # license_file: LICENSE
+  summary: Data analysis library for hybrid pixel detectors from PSI 

docs/CMakeLists.txt

@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: MPL-2.0
 find_package(Doxygen REQUIRED)
 find_package(Sphinx REQUIRED)
 
@@ -11,15 +12,19 @@ set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src)
 set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR})
 
 
-file(GLOB SPHINX_SOURCE_FILES CONFIGURE_DEPENDS "src/*.rst")
+file(GLOB_RECURSE SPHINX_SOURCE_FILES
+     CONFIGURE_DEPENDS
+     RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/src"
+     "${CMAKE_CURRENT_SOURCE_DIR}/src/*.rst"
+)
 
+foreach(relpath IN LISTS SPHINX_SOURCE_FILES)
+    set(src  "${CMAKE_CURRENT_SOURCE_DIR}/src/${relpath}")
+    set(dst  "${SPHINX_BUILD}/src/${relpath}")
 
-
+    message(STATUS "Copying ${src} to ${dst}")
-foreach(filename ${SPHINX_SOURCE_FILES})
+    configure_file("${src}" "${dst}" COPYONLY)
-    get_filename_component(fname ${filename} NAME)
+endforeach()
-    message(STATUS "Copying ${filename} to ${SPHINX_BUILD}/src/${fname}")
-    configure_file(${filename} "${SPHINX_BUILD}/src/${fname}")  
-endforeach(filename ${SPHINX_SOURCE_FILES})
 
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in"
@@ -27,6 +32,8 @@ configure_file(
     @ONLY
 )
 
+file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/figures"
+     DESTINATION "${SPHINX_BUILD}") 
 
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/static/extra.css"
@@ -45,12 +52,3 @@ add_custom_target(
     COMMENT "Generating documentation with Sphinx"
 )
 
-add_custom_target(
-    rst
-    COMMAND ${SPHINX_EXECUTABLE} -a -b html
-    -Dbreathe_projects.aare=${CMAKE_CURRENT_BINARY_DIR}/xml
-    -c "${SPHINX_BUILD}"
-    ${SPHINX_BUILD}/src 
-    ${SPHINX_BUILD}/html
-    COMMENT "Generating documentation with Sphinx"
-)

docs/Doxyfile.in

@@ -886,7 +886,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = */docs/* */tests/* */python/* */manual */slsDetectorServers/* */libs/* */integrationTests *README* */slsDetectorGui/* */ctbGui/* */slsDetectorCalibration/* *TobiSchluter*
+EXCLUDE_PATTERNS       = *build* */docs/* */tests/* *.test.cpp* */python/* */manual */slsDetectorServers/* */libs/* */integrationTests *README* *_deps* *TobiSchluter*
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the

docs/src/Cluster.rst

@@ -0,0 +1,15 @@
+Cluster
+========
+
+.. doxygenstruct:: aare::Cluster
+   :members:
+   :undoc-members:
+   :private-members:
+
+
+**Free Functions:** 
+
+.. doxygenfunction:: aare::reduce_to_3x3(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>&)
+   
+.. doxygenfunction:: aare::reduce_to_2x2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>&)
+   

docs/src/ClusterFile.rst

@@ -5,3 +5,4 @@ ClusterFile
    :members:
    :undoc-members:
    :private-members:
+

docs/src/ClusterVector.rst

@@ -4,3 +4,19 @@ ClusterVector
 .. doxygenclass:: aare::ClusterVector
    :members:
    :undoc-members:
+   :private-members:
+
+
+
+.. doxygenclass:: aare::ClusterVector< Cluster< T, ClusterSizeX, ClusterSizeY, CoordType > > 
+   :members:
+   :undoc-members:
+   :private-members:
+   
+
+**Free Functions:** 
+
+.. doxygenfunction:: aare::reduce_to_3x3(const ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>&)
+   
+.. doxygenfunction:: aare::reduce_to_2x2(const ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>&)
+   

docs/src/Interpolation.rst

@@ -0,0 +1,102 @@
+Interpolation
+==============
+
+Interpolation class for :math:`\eta` Interpolation. 
+
+The Interpolator class provides methods to interpolate the positions of photons based on their :math:`\eta` values. 
+
+.. warning:: 
+   The interpolation might lead to erroneous photon positions for clusters at the boarders of a frame. Make sure to filter out such cases. 
+
+:math:`\eta`-Functions: 
+---------------------------
+
+.. doxygenstruct:: aare::Eta2
+    :members: 
+    :undoc-members: 
+    :private-members: 
+
+.. note::
+    The corner value ``c`` is only relevant when one uses ``calculate_eta_2`` or ``calculate_full_eta2``. Otherwise its default value is ``cTopLeft``. 
+
+Supported are the following :math:`\eta`-functions: 
+
+
+.. image:: ../figures/Eta2x2.png
+    :target: ../figures/Eta2x2.png
+    :width: 650px
+    :align: center
+    :alt: Eta2x2
+
+.. math:: 
+    \begin{equation*}
+   {\color{blue}{\eta_x}} = \frac{Q_{1,1}}{Q_{1,0} + Q_{1,1}} \quad \quad 
+   {\color{green}{\eta_y}} = \frac{Q_{1,1}}{Q_{0,1} + Q_{1,1}}
+   \end{equation*}
+
+
+.. doxygenfunction:: aare::calculate_eta2(const ClusterVector<ClusterType>&)
+
+.. doxygenfunction:: aare::calculate_eta2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>&) 
+
+.. image:: ../figures/Eta2x2Full.png
+    :target: ../figures/Eta2x2Full.png
+    :width: 650px
+    :align: center
+    :alt: Eta2x2 Full 
+
+.. math:: 
+    \begin{equation*}
+        {\color{blue}{\eta_x}}  = \frac{Q_{0,1} + Q_{1,1}}{\sum_i^{2}\sum_j^{2}Q_{i,j}} \quad \quad
+        {\textcolor{green}{\eta_y}} = \frac{Q_{1,0} + Q_{1,1}}{\sum_i^{2}\sum_j^{2}Q_{i,j}}
+    \end{equation*} 
+
+
+.. doxygenfunction:: aare::calculate_full_eta2(const ClusterVector<ClusterType>&)
+
+.. doxygenfunction:: aare::calculate_full_eta2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>&)
+
+.. image:: ../figures/Eta3x3.png
+    :target: ../figures/Eta3x3.png
+    :width: 650px
+    :align: center
+    :alt: Eta3x3
+
+.. math::
+    \begin{equation*}
+         {\color{blue}{\eta_x}} = \frac{\sum_{i}^{3} Q_{i,2} - \sum_{i}^{3} Q_{i,0}}{\sum_{i}^{3}\sum_{j}^{3} Q_{i,j}} \quad \quad
+        {\color{green}{\eta_y}} = \frac{\sum_{j}^{3} Q_{2,j} - \sum_{j}^{3} Q_{0,j}}{\sum_{i}^{3}\sum_{j}^{3} Q_{i,j}}
+    \end{equation*}
+
+.. doxygenfunction:: aare::calculate_eta3(const ClusterVector<Cluster<T, 3,3, CoordType>>&)
+
+.. doxygenfunction:: aare::calculate_eta3(const Cluster<T, 3, 3, CoordType>&)
+
+.. image:: ../figures/Eta3x3Cross.png
+    :target: ../figures/Eta3x3Cross.png
+    :width: 650px
+    :align: center
+    :alt: Cross Eta3x3
+
+.. math:: 
+    \begin{equation*}
+       {\color{blue}{\eta_x}} = \frac{Q_{1,2} - Q_{1,0}}{Q_{1,0} +  Q_{1,1} + Q_{1,0}} \quad \quad
+        {\color{green}{\eta_y}} = \frac{Q_{0,2} - Q_{0,1}}{Q_{0,1} +  Q_{1,1} + Q_{1,2}}
+    \end{equation*}
+
+.. doxygenfunction:: aare::calculate_cross_eta3(const ClusterVector<Cluster<T, 3,3, CoordType>>&)
+
+.. doxygenfunction:: aare::calculate_cross_eta3(const Cluster<T, 3, 3, CoordType>&)
+
+Interpolation class: 
+---------------------
+
+.. Warning:: 
+    Make sure to use the same :math:`\eta`-function during interpolation as given by the joint :math:`\eta`-distribution passed to the constructor. 
+
+.. doxygenclass:: aare::Interpolator
+   :members:
+   :undoc-members:
+   :private-members:
+
+

docs/src/JungfrauDataFile.rst

@@ -0,0 +1,25 @@
+JungfrauDataFile
+==================
+
+JungfrauDataFile is a class to read the .dat files that are produced by Aldo's receiver. 
+It is mostly used for calibration. 
+
+The structure of the file is:
+
+* JungfrauDataHeader
+* Binary data (256x256, 256x1024 or 512x1024)
+* JungfrauDataHeader
+* ...
+
+There is no metadata indicating number of frames or the size of the image, but this
+will be infered by this reader. 
+
+.. doxygenstruct:: aare::JungfrauDataHeader
+   :members:
+   :undoc-members:
+   :private-members:
+
+.. doxygenclass:: aare::JungfrauDataFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/Philosophy.rst

@@ -0,0 +1,47 @@
+****************
+Philosophy
+****************
+
+
+Fast code with a simple interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Aare should be fast and efficient, but also easy to use. We strive to keep a simple interface that feels intuitive. 
+Internally we use C++ for performance and the ability to integrate the library in other programs, but we see most 
+users using the Python interface. 
+
+Live at head
+~~~~~~~~~~~~~~~~~~
+
+As a user of the library you should be able to, and is expected to, use the latest version. Bug fixes will rarely be backported
+to older releases. By upgrading frequently you will benefit from the latest features and minimize the effort to maintain your scripts/code 
+by doing several small upgrades instead of one big upgrade.
+
+API
+~~~~~~~~~~~~~~~~~~
+
+We aim to keep the API stable and only break it for good reasons. But specially now in the early stages of development
+the API will change. On those occasions it will be clearly stated in the release notes. However, the norm should be a 
+backward compatible API.
+
+Documentation
+~~~~~~~~~~~~~~~~~~
+
+Being a library it is important to have a well documented API. We use Doxygen to generate the C++ documentation
+and Sphinx for the Python part. Breathe is used to integrate the two into one Sphinx html site. The documentation is built
+automatically on release by the CI and published to GitHub pages. In addition to the generated API documentation,
+certain classes might need more descriptions of the usage. This is then placed in the .rst files in the docs/src directory.
+
+.. attention::
+
+    The code should be well documented, but using descriptive names is more important. In the same spirit
+    if a function is called `getNumberOfFrames()` you don't need to write a comment saying that it gets the 
+    number of frames.
+
+
+Dependencies
+~~~~~~~~~~~~~~~~~~
+
+Deployment in the scientific community is often tricky. Either due to old OS versions or the lack of package managers. 
+We strive to keep the dependencies to a minimum and will vendor some libraries to simplify deployment even though it comes
+at a cost of build time.

docs/src/Requirements.rst

@@ -2,18 +2,21 @@ Requirements
 ==============================================
 
 - C++17 compiler (gcc 8/clang 7)
-- CMake 3.14+
+- CMake 3.15+
 
 **Internally used libraries**
 
 .. note ::
 
-    These can also be picked up from the system/conda environment by specifying:
+    To save compile time some of the dependencies can also be picked up from the system/conda environment by specifying:
     -DAARE_SYSTEM_LIBRARIES=ON during the cmake configuration.
 
-- pybind11
+To simplify deployment we build and statically link a few libraries.
+
 - fmt
+- lmfit - https://jugit.fz-juelich.de/mlz/lmfit
 - nlohmann_json
+- pybind11
 - ZeroMQ
 
 **Extra dependencies for building documentation**

docs/src/Tests.rst

@@ -0,0 +1,47 @@
+****************
+Tests
+****************
+
+We test the code both from C++ and Python. By default only tests that does not require additional data are run.
+
+C++
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    mkdir build
+    cd build
+    cmake .. -DAARE_TESTS=ON
+    make -j 4
+
+    export AARE_TEST_DATA=/path/to/test/data
+    ./run_test [.with-data] #or using ctest, [.with-data] is the option to include tests needing data
+
+
+
+Python
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+    #From the root dir of the library
+    python -m pytest python/tests --with-data # passing --with-data will run the tests needing data
+
+
+
+Getting the test data
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. attention ::
+
+    The tests needing the test data are not run by default. To make the data available, you need to set the environment variable
+    AARE_TEST_DATA to the path of the test data directory. Then pass either [.with-data] for the C++ tests or --files for Python
+
+The image files needed for the test are large and are not included in the repository. They are stored
+using GIT LFS in a separate repository. To get the test data, you need to clone the repository.
+To do this, you need to have GIT LFS installed. You can find instructions on how to install it here: https://git-lfs.github.com/
+Once you have GIT LFS installed, you can clone the repository like any normal repo using:
+
+.. code-block:: bash
+    
+    git clone https://gitea.psi.ch/detectors/aare-test-data.git

docs/src/Workflow.rst

@@ -0,0 +1,86 @@
+****************
+Workflow
+****************
+
+This page describes how we develop aare. 
+
+GitHub centric
+~~~~~~~~~~~~~~~~~~
+
+We use GitHub for all development. Issues and pull requests provide a platform for collaboration as well
+as a record of the development process. Even if we discuss things in person, we record the outcome in an issue.
+If a particular implementation is chosen over another, the reason should be recorded in the pull request.
+
+
+Branches
+~~~~~~~~~~~~~~~~~~
+
+We aim for an as lightweight branching strategy as possible. Short-lived feature branches are merged back into main. 
+The main branch is expected to always be in a releasable state. A release is simply a tag on main which provides a
+reference and triggers the CI to build the release artifacts (conda, pypi etc.). For large features consider merging
+smaller chunks into main as they are completed, rather than waiting for the entire feature to be finished. Worst case 
+make sure your feature branch merges with main regularly to avoid large merge conflicts later on.
+
+.. note::
+
+    The main branch is expected to always work. Feel free to pull from main instead of sticking to a 
+    release
+
+
+Releases
+~~~~~~~~~~~~~~~~~~
+
+Release early, release often. As soon as "enough" new features have been implemented, a release is created.
+A release should not be a big thing, rather a routine part of development that does not require any special person or
+unfamiliar steps.
+
+
+
+Checklists for deployment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Feature:**
+
+#. Create a new issue for the feature (label feature)
+#. Create a new branch from main.
+#. Implement the feature including test and documentation
+#. Add the feature to RELEASE.md under head
+#. Create a pull request linked to the issue
+#. Code is reviewed by at least one other person
+#. Once approved, the branch is merged into main
+
+
+**BugFix:**
+
+Essentially the same as for a feature, if possible start with
+a failing test that demonstrates the bug.
+
+#. Create a new issue for the bug (label bug)
+#. Create a new branch from main.
+#. **Write a test that fails for the bug**
+#. Implement the fix
+#. **Run the test to ensure it passes**
+#. Add the bugfix to RELEASE.md under head
+#. Create a pull request linked to the issue.
+#. Code is reviewed by at least one other person
+#. Once approved, the branch is merged into main
+
+**Release:**
+
+#. Once "enough" new features have been implemented, a release is created
+#. Update RELEASE.md with the tag of the release and verify that it is complete
+#. Create the release in GitHub describing the new features and bug fixes
+#. CI makes magic
+
+
+**Update documentation only:**
+
+.. attention::
+
+    It's possible to update the documentation without changing the code, but take
+    care since the docs will reflect the code in main and not the latest release.
+
+#. Create a PR to main with the documentation changes
+#. Create a pull request linked to the issue.
+#. Code is reviewed by at least one other person
+#. Once merged you can manually trigger the CI workflow for documentation

docs/src/algorithm.rst

@@ -0,0 +1,5 @@
+algorithm
+=============
+
+.. doxygenfile:: algorithm.hpp
+

docs/src/index.rst

@@ -20,37 +20,35 @@ AARE
     Requirements
     Consume
 
-
-
-
 .. toctree::
     :caption: Python API
-    :maxdepth: 1
+    :maxdepth: 3
-    
+    :hidden:
-    pyFile
-    pyCtbRawFile
-    pyClusterFile
-    pyClusterVector
-    pyRawFile
-    pyRawMasterFile
-    pyVarClusterFinder
 
+    pycalibration
+    python/cluster/index
+    python/file/index
     pyFit
     
 
+
 .. toctree::
     :caption: C++ API
     :maxdepth: 1
     
+    algorithm
     NDArray
     NDView
     Frame
     File
     Dtype
+    Cluster
     ClusterFinder
     ClusterFinderMT
     ClusterFile
     ClusterVector
+    Interpolation
+    JungfrauDataFile
     Pedestal
     RawFile
     RawSubFile
@@ -59,4 +57,10 @@ AARE
 
 
 
+.. toctree::
+    :caption: Developer
+    :maxdepth: 3
     
+    Philosophy
+    Workflow
+    Tests

docs/src/pyCtbRawFile.rst

@@ -1,11 +0,0 @@
-
-CtbRawFile
-============
-
-.. py:currentmodule:: aare
-
-.. autoclass:: CtbRawFile
-    :members:
-    :undoc-members:
-    :show-inheritance:
-    :inherited-members:

docs/src/pycalibration.rst

@@ -0,0 +1,40 @@
+
+Calibration
+==============
+
+Functions for applying calibration to data.
+
+.. code-block:: python
+
+    import aare
+
+    # Load calibration data for a single JF module (512x1024 pixels)
+    calibration = aare.load_calibration('path/to/calibration/file.bin')
+
+    raw_data = ...  # Load your raw data here
+    pedestal = ...  # Load your pedestal data here
+
+    # Apply calibration to raw data to convert from raw ADC values to keV
+    data = aare.apply_calibration(raw_data, pd=pedestal, cal=calibration)
+
+    # If you pass a 2D pedestal and calibration only G0 will be used for the conversion
+    # Pixels that switched to G1 or G2 will be set to 0
+    data = aare.apply_calibration(raw_data, pd=pedestal[0], cal=calibration[0])
+
+    
+
+.. py:currentmodule:: aare
+
+.. autofunction:: apply_calibration
+
+.. autofunction:: load_calibration
+
+.. autofunction:: calculate_pedestal
+
+.. autofunction:: calculate_pedestal_float
+
+.. autofunction:: calculate_pedestal_g0
+
+.. autofunction:: calculate_pedestal_g0_float
+
+.. autofunction:: count_switching_pixels

docs/src/python/cluster/index.rst

@@ -0,0 +1,11 @@
+Cluster & Interpolation
+==========================
+
+.. toctree::
+   :caption: Cluster & Interpolation
+   :maxdepth: 1
+
+   pyCluster
+   pyClusterVector
+   pyInterpolation
+   pyVarClusterFinder

docs/src/python/cluster/pyCluster.rst

@@ -0,0 +1,23 @@
+Cluster
+========
+
+.. py:currentmodule:: aare
+
+.. autoclass:: Cluster
+    :members:
+    :undoc-members:
+    :inherited-members:
+
+
+Below is the API of a cluster of size :math:`3\times 3` and type ``int`` but all variants share the same API.
+
+    .. autoclass::  aare._aare.Cluster3x3i
+        :special-members: __init__
+        :members:
+        :undoc-members:
+        :show-inheritance:
+        :inherited-members:
+
+.. note:: 
+    More functions can be found in the :ref:`ClusterVector <py_clustervector>` documentation. Generally apply functions directly on the ``ClusterVector`` instead of looping over individual clusters.
+

docs/src/python/cluster/pyClusterVector.rst

@@ -1,9 +1,13 @@
+.. _py_clustervector:
+    
 ClusterVector
 ================
 
 The ClusterVector, holds clusters from the ClusterFinder. Since it is templated
-in C++  we use a suffix indicating the data type in python. The suffix is
+in C++  we use a suffix indicating the type of cluster it holds. The suffix follows
-``_i`` for integer, ``_f`` for float, and ``_d`` for double.
+the same pattern as for ClusterFile i.e. ``ClusterVector_Cluster3x3i``
+for a vector holding 3x3 integer clusters.
+
 
 At the moment the functionality from python is limited and it is not supported
 to push_back clusters to the vector. The intended use case is to pass it to 
@@ -26,8 +30,29 @@ C++ functions that support the ClusterVector or to view it as a numpy array.
 
 .. py:currentmodule:: aare
 
-.. autoclass:: ClusterVector_i
+.. autoclass:: ClusterVector
+    :members:
+    :undoc-members:
+    :inherited-members:
+
+Below is the API of the ClusterVector_Cluster3x3i but all variants share the same API.
+
+.. autoclass::  aare._aare.ClusterVector_Cluster3x3i
+    :special-members: __init__
     :members:
     :undoc-members:
     :show-inheritance:
     :inherited-members:
+
+
+**Free Functions:** 
+
+.. autofunction:: reduce_to_3x3
+   :noindex:
+
+   Reduce a single Cluster to 3x3 by taking the 3x3 subcluster with highest photon energy.
+
+.. autofunction:: reduce_to_2x2
+   :noindex:
+
+   Reduce a single Cluster to 2x2 by taking the 2x2 subcluster with highest photon energy.

docs/src/python/cluster/pyInterpolation.rst

@@ -0,0 +1,94 @@
+Interpolation 
+==============
+
+Interpolation class for :math:`\eta` Interpolation. 
+
+The Interpolator class provides methods to interpolate the positions of photons based on their :math:`\eta` values. 
+
+.. warning:: 
+   The interpolation might lead to erroneous photon positions for clusters at the boarders of a frame. Make sure to filter out such cases. 
+
+Below is an example of the Eta class of type ``double``. Supported are ``Etaf`` of type ``float`` and ``Etai`` of type ``int``. 
+
+.. autoclass:: aare._aare.Etad
+    :members:
+    :private-members:
+
+.. note::
+    The corner value ``c`` is only relevant when one uses ``calculate_eta_2`` or ``calculate_full_eta2``. Otherwise its default value is ``cTopLeft``. 
+
+Supported are the following :math:`\eta`-functions: 
+
+.. py:currentmodule:: aare
+
+.. image:: ../../../figures/Eta2x2.png
+    :target: ../../../figures/Eta2x2.png
+    :width: 650px
+    :align: center
+    :alt: Eta2x2
+
+.. math:: 
+    \begin{equation*}
+   {\color{blue}{\eta_x}} = \frac{Q_{1,1}}{Q_{1,0} + Q_{1,1}} \quad \quad 
+   {\color{green}{\eta_y}} = \frac{Q_{1,1}}{Q_{0,1} + Q_{1,1}}
+   \end{equation*}    
+
+.. autofunction:: calculate_eta2
+
+.. image:: ../../../figures/Eta2x2Full.png
+    :target: ../../../figures/Eta2x2Full.png
+    :width: 650px
+    :align: center
+    :alt: Eta2x2 Full 
+
+.. math:: 
+    \begin{equation*}
+        {\color{blue}{\eta_x}}  = \frac{Q_{0,1} + Q_{1,1}}{\sum_i^{2}\sum_j^{2}Q_{i,j}} \quad \quad
+        {\textcolor{green}{\eta_y}} = \frac{Q_{1,0} + Q_{1,1}}{\sum_i^{2}\sum_j^{2}Q_{i,j}}
+    \end{equation*} 
+
+.. autofunction:: calculate_full_eta2
+
+.. image:: ../../../figures/Eta3x3.png
+    :target: ../../../figures/Eta3x3.png
+    :width: 650px
+    :align: center
+    :alt: Eta3x3
+
+.. math::
+    \begin{equation*}
+         {\color{blue}{\eta_x}} = \frac{\sum_{i}^{3} Q_{i,2} - \sum_{i}^{3} Q_{i,0}}{\sum_{i}^{3}\sum_{j}^{3} Q_{i,j}} \quad \quad
+        {\color{green}{\eta_y}} = \frac{\sum_{j}^{3} Q_{2,j} - \sum_{j}^{3} Q_{0,j}}{\sum_{i}^{3}\sum_{j}^{3} Q_{i,j}}
+    \end{equation*}
+
+.. autofunction:: calculate_eta3
+
+.. image:: ../../../figures/Eta3x3Cross.png
+    :target: ../../../figures/Eta3x3Cross.png
+    :width: 650px
+    :align: center
+    :alt: Cross Eta3x3
+
+.. math:: 
+    \begin{equation*}
+       {\color{blue}{\eta_x}} = \frac{Q_{1,2} - Q_{1,0}}{Q_{1,0} +  Q_{1,1} + Q_{1,0}} \quad \quad
+        {\color{green}{\eta_y}} = \frac{Q_{0,2} - Q_{0,1}}{Q_{0,1} +  Q_{1,1} + Q_{1,2}}
+    \end{equation*}
+
+.. autofunction:: calculate_cross_eta3
+
+
+Interpolation class for :math:`\eta`-Interpolation 
+----------------------------------------------------
+
+.. Warning:: 
+    Make sure to use the same :math:`\eta`-function during interpolation as given by the joint :math:`\eta`-distribution passed to the constructor. 
+
+.. py:currentmodule:: aare
+
+.. autoclass:: Interpolator
+    :special-members: __init__
+    :members:
+    :undoc-members:
+    :inherited-members:
+

docs/src/python/file/index.rst

@@ -0,0 +1,14 @@
+File I/O
+===================
+
+.. toctree::
+   :caption: File I/O
+   :maxdepth: 1
+
+   pyClusterFile
+   pyCtbRawFile
+   pyFile
+   pyJungfrauDataFile
+   pyRawFile
+   pyRawMasterFile
+   pyTransform

docs/src/python/file/pyClusterFile.rst

@@ -0,0 +1,26 @@
+
+ClusterFile
+============
+
+
+The :class:`ClusterFile` class is the main interface to read and write clusters in aare. Unfortunately the
+old file format does not include metadata like the cluster size and the data type. This means that the
+user has to know this information from other sources. Specifying the wrong cluster size or data type
+will lead to garbage data being read. 
+
+.. py:currentmodule:: aare
+
+.. autoclass:: ClusterFile
+    :members:
+    :undoc-members:
+    :inherited-members:
+
+
+Below is the API of the ClusterFile_Cluster3x3i but all variants share the same API.
+
+.. autoclass:: aare._aare.ClusterFile_Cluster3x3i
+    :special-members: __init__
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/python/file/pyCtbRawFile.rst

@@ -0,0 +1,25 @@
+
+CtbRawFile
+============
+
+Read analog, digital and transceiver samples from a raw file containing
+data from the Chip Test Board. Uses :mod:`aare.transform` to decode the 
+data into a format that the user can work with. 
+
+.. code:: python
+
+    import aare
+    from aare.transform import Mythen302Transform
+    my302 = Mythen302Transform(offset = 4)
+
+    with aare.CtbRawFile(fname, transform = my302) as f:
+    for header, data in f:
+       #do something with the data
+
+.. py:currentmodule:: aare
+
+.. autoclass:: CtbRawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/python/file/pyJungfrauDataFile.rst

@@ -1,10 +1,9 @@
-
+JungfrauDataFile
-ClusterFile
+===================
-============
 
 .. py:currentmodule:: aare
 
-.. autoclass:: ClusterFile
+.. autoclass:: JungfrauDataFile
     :members:
     :undoc-members:
     :show-inheritance:

docs/src/python/file/pyTransform.rst

@@ -0,0 +1,27 @@
+Transform
+===================
+
+The transform module takes data read by :class:`aare.CtbRawFile` and decodes it
+to a useful image format. Depending on detector it supports both analog
+and digital samples. 
+
+For convenience the following transform objects are defined with a short name
+
+.. code:: python
+
+    moench05 = Moench05Transform()
+    moench05_1g = Moench05Transform1g()
+    moench05_old = Moench05TransformOld()
+    matterhorn02 = Matterhorn02Transform()
+    adc_sar_04_64to16 = AdcSar04Transform64to16()
+    adc_sar_05_64to16 = AdcSar05Transform64to16()
+
+.. py:currentmodule:: aare
+
+.. automodule:: aare.transform
+    :members:
+    :undoc-members:
+    :private-members:
+    :special-members: __call__
+    :show-inheritance:
+    :inherited-members:

etc/add_license.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import argparse
+import fnmatch
+import os
+from pathlib import Path
+
+CPP_PATTERNS = ["*.h", "*.hpp", "*.cpp"]
+PY_PATTERNS = ["*.py"]
+CMAKE_PATTERNS = ["CMakeLists.txt"]
+
+FILE_PATTERNS = CPP_PATTERNS + PY_PATTERNS + CMAKE_PATTERNS
+LICENSE_TEXT = "SPDX-License-Identifier: MPL-2.0"
+
+
+def get_comment_prefix(filename: str) -> str | None:
+    if any(fnmatch.fnmatch(filename, p) for p in CPP_PATTERNS):
+        return "// "
+    if any(fnmatch.fnmatch(filename, p) for p in (PY_PATTERNS + CMAKE_PATTERNS)):
+        return "# "
+    return None
+
+
+def matches_pattern(filename: str) -> bool:
+    return any(fnmatch.fnmatch(filename, p) for p in FILE_PATTERNS)
+
+
+def process_file(filepath: Path) -> bool:
+    filename = filepath.name
+    prefix = get_comment_prefix(filename)
+    if not prefix:
+        return False
+
+    license_line = f"{prefix}{LICENSE_TEXT}\n"
+
+    try:
+        with filepath.open("r", encoding="utf-8") as f:
+            lines = f.readlines()
+    except Exception as e:
+        print(f"⚠️ Error reading {filepath}: {e}")
+        return False
+
+    # Skip if SPDX already present anywhere in the file
+    if any("SPDX-License-Identifier" in line for line in lines):
+        return False
+
+    insert_index = 0
+
+    # For Python, keep shebang on the very first line
+    if filename.endswith(".py") and lines:
+        if lines[0].startswith("#!"):
+            insert_index = 1
+
+    lines.insert(insert_index, license_line)
+
+    try:
+        with filepath.open("w", encoding="utf-8") as f:
+            f.writelines(lines)
+    except Exception as e:
+        print(f"⚠️ Error writing {filepath}: {e}")
+        return False
+
+    return True
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Add SPDX-License-Identifier: MPL-2.0 to source files."
+    )
+    parser.add_argument(
+        "path",
+        help="Root directory to recursively process "
+             "(*.h, *.cpp, *.py, and CMakeLists.txt).",
+    )
+
+    args = parser.parse_args()
+
+    root_path = Path(args.path).expanduser().resolve()
+
+    if not root_path.exists():
+        print(f"Error: Path does not exist: {root_path}")
+        raise SystemExit(1)
+
+    if not root_path.is_dir():
+        print(f"Error: Path is not a directory: {root_path}")
+        raise SystemExit(1)
+
+    print(f"Processing directory: {root_path}")
+    modified = 0
+
+    for dirpath, _, files in os.walk(root_path):
+        dirpath = Path(dirpath)
+        for name in files:
+            if matches_pattern(name):
+                fullpath = dirpath / name
+                if process_file(fullpath):
+                    print(f"✔ Added SPDX: {fullpath}")
+                    modified += 1
+
+    print(f"\nDone. Updated {modified} file(s).")
+
+
+if __name__ == "__main__":
+    main()

etc/dev-env.yml

@@ -0,0 +1,18 @@
+name: dev-environment
+channels:
+  - conda-forge
+dependencies:
+  - anaconda-client
+  - catch2
+  - conda-build
+  - doxygen 
+  - sphinx
+  - breathe 
+  - sphinx_rtd_theme 
+  - furo 
+  - zeromq 
+  - pybind11
+  - numpy
+  - matplotlib
+  - nlohmann_json
+

etc/update_version.py

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: MPL-2.0
+# Copyright (C) 2021 Contributors to the Aare Package
+"""
+Script to update VERSION file with semantic versioning if provided as an argument, or with 0.0.0 if no argument is provided.
+"""
+
+import sys
+import os
+import re
+from datetime import datetime
+from pathlib import Path
+
+from packaging.version import Version, InvalidVersion
+
+
+SCRIPT_DIR = Path(__file__).absolute().parent.parent
+
+def is_integer(value): 
+    try:
+        int(value)
+    except ValueError:
+        return False
+    else:
+        return True
+
+
+def get_version():
+
+    # Check at least one argument is passed
+    if len(sys.argv) < 2:
+        version =   datetime.today().strftime('%Y.%-m.%-d')
+    else: 
+        version = sys.argv[1]
+
+    try:
+        v = Version(version)  # normalize check if version follows PEP 440 specification
+        
+        version_normalized = version.replace("-", ".")
+
+        version_normalized =  re.sub(r'0*(\d+)', lambda m : str(int(m.group(0))), version_normalized) #remove leading zeros 
+
+        return version_normalized 
+
+    except InvalidVersion as e:
+        print(f"Invalid version {version}. Version format must follow semantic versioning format of python PEP 440 version identification specification.")
+        sys.exit(1)
+    
+
+def write_version_to_file(version):
+    version_file_path =  SCRIPT_DIR/"VERSION"
+    print(version_file_path)
+    with open(version_file_path, "w") as version_file:
+        version_file.write(version)
+    print(f"Version {version} written to VERSION file.")
+
+# Main script
+if __name__ == "__main__":
+
+    version = get_version()
+    write_version_to_file(version)

include/aare/ArrayExpr.hpp

@@ -1,22 +1,26 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
-#include <cstdint> //int64_t
+#include "aare/defs.hpp"
-#include <cstddef> //size_t
 #include <array>
-
 #include <cassert>
+#include <cstddef>
+#include <cstdint>
+
 namespace aare {
 
-template <typename E, int64_t Ndim> class ArrayExpr {
+template <typename E, ssize_t Ndim> class ArrayExpr {
   public:
     static constexpr bool is_leaf = false;
 
     auto operator[](size_t i) const { return static_cast<E const &>(*this)[i]; }
     auto operator()(size_t i) const { return static_cast<E const &>(*this)[i]; }
     auto size() const { return static_cast<E const &>(*this).size(); }
-    std::array<int64_t, Ndim> shape() const { return static_cast<E const &>(*this).shape(); }
+    std::array<ssize_t, Ndim> shape() const {
+        return static_cast<E const &>(*this).shape();
+    }
 };
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
 class ArrayAdd : public ArrayExpr<ArrayAdd<A, B, Ndim>, Ndim> {
     const A &arr1_;
     const B &arr2_;
@@ -27,10 +31,10 @@ class ArrayAdd : public ArrayExpr<ArrayAdd<A, B, Ndim>, Ndim> {
     }
     auto operator[](int i) const { return arr1_[i] + arr2_[i]; }
     size_t size() const { return arr1_.size(); }
-    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+    std::array<ssize_t, Ndim> shape() const { return arr1_.shape(); }
 };
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
 class ArraySub : public ArrayExpr<ArraySub<A, B, Ndim>, Ndim> {
     const A &arr1_;
     const B &arr2_;
@@ -41,11 +45,11 @@ class ArraySub : public ArrayExpr<ArraySub<A, B, Ndim>, Ndim> {
     }
     auto operator[](int i) const { return arr1_[i] - arr2_[i]; }
     size_t size() const { return arr1_.size(); }
-    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+    std::array<ssize_t, Ndim> shape() const { return arr1_.shape(); }
 };
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
-class ArrayMul : public ArrayExpr<ArrayMul<A, B, Ndim>,Ndim> {
+class ArrayMul : public ArrayExpr<ArrayMul<A, B, Ndim>, Ndim> {
     const A &arr1_;
     const B &arr2_;
 
@@ -55,10 +59,10 @@ class ArrayMul : public ArrayExpr<ArrayMul<A, B, Ndim>,Ndim> {
     }
     auto operator[](int i) const { return arr1_[i] * arr2_[i]; }
     size_t size() const { return arr1_.size(); }
-    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+    std::array<ssize_t, Ndim> shape() const { return arr1_.shape(); }
 };
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
 class ArrayDiv : public ArrayExpr<ArrayDiv<A, B, Ndim>, Ndim> {
     const A &arr1_;
     const B &arr2_;
@@ -69,31 +73,27 @@ class ArrayDiv : public ArrayExpr<ArrayDiv<A, B, Ndim>, Ndim> {
     }
     auto operator[](int i) const { return arr1_[i] / arr2_[i]; }
     size_t size() const { return arr1_.size(); }
-    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+    std::array<ssize_t, Ndim> shape() const { return arr1_.shape(); }
 };
 
-
+template <typename A, typename B, ssize_t Ndim>
-
-template <typename A, typename B, int64_t Ndim>
 auto operator+(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
     return ArrayAdd<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
 }
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
-auto operator-(const ArrayExpr<A,Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
+auto operator-(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
     return ArraySub<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
 }
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
 auto operator*(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
     return ArrayMul<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
 }
 
-template <typename A, typename B, int64_t Ndim>
+template <typename A, typename B, ssize_t Ndim>
 auto operator/(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
     return ArrayDiv<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
 }
 
-
-
 } // namespace aare

include/aare/CalculateEta.hpp

@@ -1,19 +1,14 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include "aare/Cluster.hpp"
 #include "aare/ClusterVector.hpp"
 #include "aare/NDArray.hpp"
+#include "aare/defs.hpp"
 
 namespace aare {
 
-typedef enum {
+enum class pixel : int {
-    cBottomLeft = 0,
-    cBottomRight = 1,
-    cTopLeft = 2,
-    cTopRight = 3
-} corner;
-
-typedef enum {
     pBottomLeft = 0,
     pBottom = 1,
     pBottomRight = 2,
@@ -23,100 +18,429 @@ typedef enum {
     pTopLeft = 6,
     pTop = 7,
     pTopRight = 8
-} pixel;
+};
 
+// TODO: better to have sum after x,y
+/**
+ * eta struct
+ */
 template <typename T> struct Eta2 {
-    double x;
+    /// @brief eta in x direction
-    double y;
+    double x{};
-    int c;
+    /// @brief eta in y direction
-    T sum;
+    double y{};
+    /// @brief index of subcluster given as corner relative to cluster center
+    corner c{0};
+    /// @brief photon energy (cluster sum)
+    T sum{};
 };
 
 /**
- * @brief Calculate the eta2 values for all clusters in a Clsutervector
+ * @brief Calculate the eta2 values for all clusters in a ClusterVector
  */
 template <typename ClusterType,
           typename = std::enable_if_t<is_cluster_v<ClusterType>>>
-NDArray<double, 2> calculate_eta2(const ClusterVector<ClusterType> &clusters) {
+std::vector<Eta2<typename ClusterType::value_type>>
-    NDArray<double, 2> eta2({static_cast<int64_t>(clusters.size()), 2});
+calculate_eta2(const ClusterVector<ClusterType> &clusters) {
+
+    std::vector<Eta2<typename ClusterType::value_type>> eta2{};
+    eta2.reserve(clusters.size());
 
     for (size_t i = 0; i < clusters.size(); i++) {
-        auto e = calculate_eta2(clusters.at(i));
+        auto e = calculate_eta2(clusters[i]);
-        eta2(i, 0) = e.x;
+        eta2.push_back(e);
-        eta2(i, 1) = e.y;
     }
 
     return eta2;
 }
 
+/**
+ * @brief Calculate the full eta2 values for all clusters in a ClusterVector
+ */
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+std::vector<Eta2<typename ClusterType::value_type>>
+calculate_full_eta2(const ClusterVector<ClusterType> &clusters) {
+    std::vector<Eta2<typename ClusterType::value_type>> eta2{};
+    eta2.reserve(clusters.size());
+
+    for (size_t i = 0; i < clusters.size(); i++) {
+        auto e = calculate_full_eta2(clusters[i]);
+        eta2.push_back(e);
+    }
+
+    return eta2;
+}
+
+/**
+ * @brief Calculate eta3 for all 3x3 clusters in a ClusterVector
+ */
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+std::vector<Eta2<typename ClusterType::value_type>>
+calculate_eta3(const ClusterVector<ClusterType> &clusters) {
+    std::vector<Eta2<typename ClusterType::value_type>> eta2{};
+    eta2.reserve(clusters.size());
+
+    for (size_t i = 0; i < clusters.size(); i++) {
+        auto e = calculate_eta3(clusters[i]);
+        eta2.push_back(e);
+    }
+
+    return eta2;
+}
+
+/**
+ * @brief Calculate cross eta3 for all 3x3 clusters in a ClusterVector
+ */
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+std::vector<Eta2<typename ClusterType::value_type>>
+calculate_cross_eta3(const ClusterVector<ClusterType> &clusters) {
+    std::vector<Eta2<typename ClusterType::value_type>> eta2{};
+    eta2.reserve(clusters.size());
+
+    for (size_t i = 0; i < clusters.size(); i++) {
+        auto e = calculate_cross_eta3(clusters[i]);
+        eta2.push_back(e);
+    }
+
+    return eta2;
+}
+
+/**
+ * @brief helper function to calculate eta2 x and y values
+ * @param eta reference to the Eta2 object to update
+ * @param left_x value of the left pixel
+ * @param right_x value of the right pixel
+ * @param bottom_y value of the bottom pixel
+ * @param top_y value of the top pixel
+ */
+template <typename T>
+inline void calculate_eta2(Eta2<T> &eta, const T left_x, const T right_x,
+                           const T bottom_y, const T top_y) {
+    if ((right_x + left_x) != 0)
+        eta.x = static_cast<double>(right_x) /
+                static_cast<double>(right_x + left_x); // between (0,1) the
+                                                       // closer to zero left
+                                                       // value probably larger
+    if ((top_y + bottom_y) != 0)
+        eta.y = static_cast<double>(top_y) /
+                static_cast<double>(top_y + bottom_y); // between (0,1) the
+                                                       // closer to zero bottom
+                                                       // value probably larger
+}
+
 /**
  * @brief Calculate the eta2 values for a generic sized cluster and return them
- * in a Eta2 struct containing etay, etax and the index of the respective 2x2
+ * in a Eta2 struct containing etay, etax and the index (as corner) of the
- * subcluster.
+ * respective 2x2 subcluster relative to the cluster center.
  */
 template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
-          typename CoordType>
+          typename CoordType = uint16_t>
 Eta2<T>
 calculate_eta2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &cl) {
+
+    static_assert(ClusterSizeX > 1 && ClusterSizeY > 1);
     Eta2<T> eta{};
 
+    size_t cluster_center_index =
+        (ClusterSizeX / 2) + (ClusterSizeY / 2) * ClusterSizeX;
+
     auto max_sum = cl.max_sum_2x2();
-    eta.sum = max_sum.first;
+    eta.sum = max_sum.sum;
-    auto c = max_sum.second;
+    corner c = max_sum.index;
 
-    size_t index_bottom_left_max_2x2_subcluster =
+    // subcluster top right from center
-        (int(c / (ClusterSizeX - 1))) * ClusterSizeX + c % (ClusterSizeX - 1);
+    switch (c) {
+    case (corner::cTopLeft):
+        calculate_eta2(eta, cl.data[cluster_center_index - 1],
+                       cl.data[cluster_center_index],
+                       cl.data[cluster_center_index - ClusterSizeX],
+                       cl.data[cluster_center_index]);
+        // dx = -1
+        // dy = -1
+        break;
+    case (corner::cTopRight):
+        calculate_eta2(eta, cl.data[cluster_center_index],
+                       cl.data[cluster_center_index + 1],
+                       cl.data[cluster_center_index - ClusterSizeX],
+                       cl.data[cluster_center_index]);
+        // dx = 0
+        // dy = -1
+        break;
+    case (corner::cBottomLeft):
+        calculate_eta2(eta, cl.data[cluster_center_index - 1],
+                       cl.data[cluster_center_index],
+                       cl.data[cluster_center_index],
+                       cl.data[cluster_center_index + ClusterSizeX]);
+        // dx = -1
+        // dy = 0
+        break;
+    case (corner::cBottomRight):
+        calculate_eta2(eta, cl.data[cluster_center_index],
+                       cl.data[cluster_center_index + 1],
+                       cl.data[cluster_center_index],
+                       cl.data[cluster_center_index + ClusterSizeX]);
+        // dx = 0
+        // dy = 0
+        break;
+    }
 
-    if ((cl.data[index_bottom_left_max_2x2_subcluster] +
+    eta.c = c;
-         cl.data[index_bottom_left_max_2x2_subcluster + 1]) != 0)
-        eta.x = static_cast<double>(
-                    cl.data[index_bottom_left_max_2x2_subcluster + 1]) /
-                static_cast<double>(
-                    (cl.data[index_bottom_left_max_2x2_subcluster] +
-                     cl.data[index_bottom_left_max_2x2_subcluster + 1]));
 
-    if ((cl.data[index_bottom_left_max_2x2_subcluster] +
-         cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]) != 0)
-        eta.y =
-            static_cast<double>(
-                cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]) /
-            static_cast<double>(
-                (cl.data[index_bottom_left_max_2x2_subcluster] +
-                 cl.data[index_bottom_left_max_2x2_subcluster + ClusterSizeX]));
-
-    eta.c = c; // TODO only supported for 2x2 and 3x3 clusters -> at least no
-               // underyling enum class
     return eta;
 }
 
-// calculates Eta3 for 3x3 cluster based on code from analyze_cluster
+/**
-// TODO only supported for 3x3 Clusters
+ * @brief Calculate the eta2 values for a generic sized cluster and return them
-template <typename T> Eta2<T> calculate_eta3(const Cluster<T, 3, 3> &cl) {
+ * in a Eta2 struct containing etay, etax and the index (as corner) of the
+ * respective 2x2 subcluster relative to the cluster center.
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+Eta2<T> calculate_full_eta2(
+    const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &cl) {
+
+    static_assert(ClusterSizeX > 1 && ClusterSizeY > 1);
+    Eta2<T> eta{};
+
+    constexpr size_t cluster_center_index =
+        (ClusterSizeX / 2) + (ClusterSizeY / 2) * ClusterSizeX;
+
+    auto max_sum = cl.max_sum_2x2();
+    eta.sum = max_sum.sum;
+    corner c = max_sum.index;
+
+    // subcluster top right from center
+    switch (c) {
+    case (corner::cTopLeft):
+        if (eta.sum != 0) {
+            eta.x = static_cast<double>(
+                        cl.data[cluster_center_index] +
+                        cl.data[cluster_center_index - ClusterSizeX]) /
+                    static_cast<double>(eta.sum);
+
+            eta.y = static_cast<double>(cl.data[cluster_center_index - 1] +
+                                        cl.data[cluster_center_index]) /
+                    static_cast<double>(eta.sum);
+        }
+        // dx = -1
+        // dy = -1
+        break;
+    case (corner::cTopRight):
+        if (eta.sum != 0) {
+            eta.x = static_cast<double>(
+                        cl.data[cluster_center_index + 1] +
+                        cl.data[cluster_center_index - ClusterSizeX + 1]) /
+                    static_cast<double>(eta.sum);
+            eta.y = static_cast<double>(cl.data[cluster_center_index] +
+                                        cl.data[cluster_center_index + 1]) /
+                    static_cast<double>(eta.sum);
+        }
+        // dx = 0
+        // dy = -1
+        break;
+    case (corner::cBottomLeft):
+        if (eta.sum != 0) {
+            eta.x = static_cast<double>(
+                        cl.data[cluster_center_index] +
+                        cl.data[cluster_center_index + ClusterSizeX]) /
+                    static_cast<double>(eta.sum);
+            eta.y = static_cast<double>(
+                        cl.data[cluster_center_index + ClusterSizeX] +
+                        cl.data[cluster_center_index + ClusterSizeX - 1]) /
+                    static_cast<double>(eta.sum);
+        }
+        // dx = -1
+        // dy = 0
+        break;
+    case (corner::cBottomRight):
+        if (eta.sum != 0) {
+            eta.x = static_cast<double>(
+                        cl.data[cluster_center_index + 1] +
+                        cl.data[cluster_center_index + ClusterSizeX + 1]) /
+                    static_cast<double>(eta.sum);
+            eta.y = static_cast<double>(
+                        cl.data[cluster_center_index + ClusterSizeX] +
+                        cl.data[cluster_center_index + ClusterSizeX + 1]) /
+                    static_cast<double>(eta.sum);
+        }
+        // dx = 0
+        // dy = 0
+        break;
+    }
+
+    eta.c = c;
+
+    return eta;
+}
+
+template <typename T>
+Eta2<T> calculate_eta2(const Cluster<T, 2, 2, uint16_t> &cl) {
+    Eta2<T> eta{};
+
+    // TODO: maybe have as member function of cluster
+    const uint8_t photon_hit_index =
+        std::max_element(cl.data.begin(), cl.data.end()) - cl.data.begin();
+
+    eta.c = static_cast<corner>(3 - photon_hit_index);
+
+    switch (eta.c) {
+    case corner::cTopLeft:
+        calculate_eta2(eta, cl.data[2], cl.data[3], cl.data[1], cl.data[3]);
+        break;
+    case corner::cTopRight:
+        calculate_eta2(eta, cl.data[2], cl.data[3], cl.data[0], cl.data[2]);
+        break;
+    case corner::cBottomLeft:
+        calculate_eta2(eta, cl.data[0], cl.data[1], cl.data[1], cl.data[3]);
+        break;
+    case corner::cBottomRight:
+        calculate_eta2(eta, cl.data[0], cl.data[1], cl.data[0], cl.data[2]);
+        break;
+    }
+
+    eta.sum = cl.sum();
+
+    return eta;
+}
+
+template <typename T>
+Eta2<T> calculate_full_eta2(const Cluster<T, 2, 2, uint16_t> &cl) {
 
     Eta2<T> eta{};
 
-    T sum = 0;
+    eta.sum = cl.sum();
 
-    std::for_each(std::begin(cl.data), std::end(cl.data),
+    const uint8_t photon_hit_index =
-                  [&sum](T x) { sum += x; });
+        std::max_element(cl.data.begin(), cl.data.end()) - cl.data.begin();
 
-    eta.sum = sum;
+    eta.c = static_cast<corner>(3 - photon_hit_index);
 
-    eta.c = corner::cBottomLeft;
+    if (eta.sum != 0) {
+        eta.x = static_cast<double>(cl.data[1] + cl.data[3]) /
+                static_cast<double>(eta.sum);
+        eta.y = static_cast<double>(cl.data[2] + cl.data[3]) /
+                static_cast<double>(eta.sum);
+    }
+
+    return eta;
+}
+
+// TODO generalize
+template <typename T>
+Eta2<T> calculate_eta2(const Cluster<T, 1, 2, uint16_t> &cl) {
+    Eta2<T> eta{};
+
+    eta.x = 0;
+    eta.y = static_cast<double>(cl.data[1]) / cl.data[0];
+    eta.sum = cl.sum();
+}
+
+template <typename T>
+Eta2<T> calculate_eta2(const Cluster<T, 2, 1, uint16_t> &cl) {
+    Eta2<T> eta{};
+
+    eta.x = static_cast<double>(cl.data[1]) / cl.data[0];
+    eta.y = 0;
+    eta.sum = cl.sum();
+}
+
+/**
+ * @brief calculates cross Eta3 for 3x3 cluster
+ * cross Eta3 calculates the eta by taking into account only the cross pixels
+ * {top, bottom, left, right, center}
+ */
+template <typename T, typename CoordType = uint16_t>
+Eta2<T> calculate_cross_eta3(const Cluster<T, 3, 3, CoordType> &cl) {
+
+    Eta2<T> eta{};
+
+    T photon_energy = cl.sum();
+
+    eta.sum = photon_energy;
 
     if ((cl.data[3] + cl.data[4] + cl.data[5]) != 0)
 
-        eta.x = static_cast<double>(-cl.data[3] + cl.data[3 + 2]) /
+        eta.x =
+            static_cast<double>(-cl.data[3] + cl.data[3 + 2]) /
 
-                (cl.data[3] + cl.data[4] + cl.data[5]);
+            static_cast<double>(cl.data[3] + cl.data[4] + cl.data[5]); // (-1,1)
 
     if ((cl.data[1] + cl.data[4] + cl.data[7]) != 0)
 
         eta.y = static_cast<double>(-cl.data[1] + cl.data[2 * 3 + 1]) /
 
-                (cl.data[1] + cl.data[4] + cl.data[7]);
+                static_cast<double>(cl.data[1] + cl.data[4] + cl.data[7]);
 
     return eta;
 }
 
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = uint16_t>
+Eta2<T> calculate_cross_eta3(
+    const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &cl) {
+
+    static_assert(ClusterSizeX > 2 && ClusterSizeY > 2,
+                  "calculate_eta3 only defined for clusters larger than 2x2");
+
+    if constexpr (ClusterSizeX != 3 || ClusterSizeY != 3) {
+        auto reduced_cluster = reduce_cluster_to_3x3(cl);
+        return calculate_cross_eta3(reduced_cluster);
+    } else {
+        return calculate_cross_eta3(cl);
+    }
+}
+
+/**
+ * @brief calculates Eta3 for 3x3 cluster
+ * It calculates the eta by taking into account all pixels in the 3x3 cluster
+ */
+template <typename T, typename CoordType = uint16_t>
+Eta2<T> calculate_eta3(const Cluster<T, 3, 3, CoordType> &cl) {
+
+    Eta2<T> eta{};
+
+    T photon_energy = cl.sum();
+
+    eta.sum = photon_energy;
+
+    // TODO: how do we handle potential arithmetic overflows? - T could be
+    // uint16
+    if (photon_energy != 0) {
+        std::array<T, 2> column_sums{
+            static_cast<T>(cl.data[0] + cl.data[3] + cl.data[6]),
+            static_cast<T>(cl.data[2] + cl.data[5] + cl.data[8])};
+
+        eta.x = static_cast<double>(-column_sums[0] + column_sums[1]) /
+                static_cast<double>(photon_energy);
+
+        std::array<T, 2> row_sums{
+            static_cast<T>(cl.data[0] + cl.data[1] + cl.data[2]),
+            static_cast<T>(cl.data[6] + cl.data[7] + cl.data[8])};
+
+        eta.y = static_cast<double>(-row_sums[0] + row_sums[1]) /
+                static_cast<double>(photon_energy);
+    }
+
+    return eta;
+}
+
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = uint16_t>
+Eta2<T>
+calculate_eta3(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &cl) {
+
+    static_assert(ClusterSizeX > 2 && ClusterSizeY > 2,
+                  "calculate_eta3 only defined for clusters larger than 2x2");
+
+    if constexpr (ClusterSizeX != 3 || ClusterSizeY != 3) {
+        auto reduced_cluster = reduce_cluster_to_3x3(cl);
+        return calculate_eta3(reduced_cluster);
+    } else {
+        return calculate_eta3(cl);
+    }
+}
+
 } // namespace aare

include/aare/CircularFifo.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include <chrono>
@@ -17,7 +18,8 @@ template <class ItemType> class CircularFifo {
 
   public:
     CircularFifo() : CircularFifo(100){};
-    CircularFifo(uint32_t size) : fifo_size(size), free_slots(size + 1), filled_slots(size + 1) {
+    CircularFifo(uint32_t size)
+        : fifo_size(size), free_slots(size + 1), filled_slots(size + 1) {
 
         // TODO! how do we deal with alignment for writing? alignas???
         // Do we give the user a chance to provide memory locations?
@@ -55,7 +57,8 @@ template <class ItemType> class CircularFifo {
 
     bool try_pop_free(ItemType &v) { return free_slots.read(v); }
 
-    ItemType pop_value(std::chrono::nanoseconds wait, std::atomic<bool> &stopped) {
+    ItemType pop_value(std::chrono::nanoseconds wait,
+                       std::atomic<bool> &stopped) {
         ItemType v;
         while (!filled_slots.read(v) && !stopped) {
             std::this_thread::sleep_for(wait);

include/aare/Cluster.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 
 /************************************************
  * @file Cluster.hpp
@@ -8,6 +9,7 @@
 
 #pragma once
 
+#include "defs.hpp"
 #include <algorithm>
 #include <array>
 #include <cstdint>
@@ -16,82 +18,214 @@
 
 namespace aare {
 
+// requires clause c++20 maybe update
+
+/**
+ * @brief Cluster struct
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = uint16_t>
+struct Cluster {
+
+    static_assert(std::is_arithmetic_v<T>, "T needs to be an arithmetic type");
+    static_assert(std::is_integral_v<CoordType>,
+                  "CoordType needs to be an integral type");
+    static_assert(ClusterSizeX > 0 && ClusterSizeY > 0,
+                  "Cluster sizes must be bigger than zero");
+
+    /// @brief Cluster center x coordinate (in pixel coordinates)
+    CoordType x;
+    /// @brief Cluster center y coordinate (in pixel coordinates)
+    CoordType y;
+    /// @brief Cluster data stored in row-major order starting from top-left
+    std::array<T, ClusterSizeX * ClusterSizeY> data;
+
+    static constexpr uint8_t cluster_size_x = ClusterSizeX;
+    static constexpr uint8_t cluster_size_y = ClusterSizeY;
+    using value_type = T;
+    using coord_type = CoordType;
+
+    /**
+     * @brief Sum of all elements in the cluster
+     */
+    T sum() const { return std::accumulate(data.begin(), data.end(), T{}); }
+
+    // TODO: handle 1 dimensional clusters
+    /**
+     * @brief sum of 2x2 subcluster with highest energy
+     * @return photon energy of subcluster, 2x2 subcluster index relative to
+     * cluster center
+     */
+    Sum_index_pair<T, corner> max_sum_2x2() const {
+
+        if constexpr (cluster_size_x == 3 && cluster_size_y == 3) {
+            std::array<T, 4> sum_2x2_subclusters;
+            sum_2x2_subclusters[0] = data[0] + data[1] + data[3] + data[4];
+            sum_2x2_subclusters[1] = data[1] + data[2] + data[4] + data[5];
+            sum_2x2_subclusters[2] = data[3] + data[4] + data[6] + data[7];
+            sum_2x2_subclusters[3] = data[4] + data[5] + data[7] + data[8];
+            int index = std::max_element(sum_2x2_subclusters.begin(),
+                                         sum_2x2_subclusters.end()) -
+                        sum_2x2_subclusters.begin();
+            return Sum_index_pair<T, corner>{sum_2x2_subclusters[index],
+                                             corner{index}};
+        } else if constexpr (cluster_size_x == 2 && cluster_size_y == 2) {
+            return Sum_index_pair<T, corner>{
+                data[0] + data[1] + data[2] + data[3], corner{0}};
+        } else {
+            constexpr size_t cluster_center_index =
+                (ClusterSizeX / 2) + (ClusterSizeY / 2) * ClusterSizeX;
+
+            std::array<T, 4> sum_2x2_subcluster{0};
+            // subcluster top left from center
+            sum_2x2_subcluster[0] =
+                data[cluster_center_index] + data[cluster_center_index - 1] +
+                data[cluster_center_index - ClusterSizeX] +
+                data[cluster_center_index - 1 - ClusterSizeX];
+            // subcluster top right from center
+            if (ClusterSizeX > 2) {
+                sum_2x2_subcluster[1] =
+                    data[cluster_center_index] +
+                    data[cluster_center_index + 1] +
+                    data[cluster_center_index - ClusterSizeX] +
+                    data[cluster_center_index - ClusterSizeX + 1];
+            }
+            // subcluster bottom left from center
+            if (ClusterSizeY > 2) {
+                sum_2x2_subcluster[2] =
+                    data[cluster_center_index] +
+                    data[cluster_center_index - 1] +
+                    data[cluster_center_index + ClusterSizeX] +
+                    data[cluster_center_index + ClusterSizeX - 1];
+            }
+            // subcluster bottom right from center
+            if (ClusterSizeX > 2 && ClusterSizeY > 2) {
+                sum_2x2_subcluster[3] =
+                    data[cluster_center_index] +
+                    data[cluster_center_index + 1] +
+                    data[cluster_center_index + ClusterSizeX] +
+                    data[cluster_center_index + ClusterSizeX + 1];
+            }
+
+            int index = std::max_element(sum_2x2_subcluster.begin(),
+                                         sum_2x2_subcluster.end()) -
+                        sum_2x2_subcluster.begin();
+            return Sum_index_pair<T, corner>{sum_2x2_subcluster[index],
+                                             corner{index}};
+        }
+    }
+};
+
+/**
+ * @brief Reduce a cluster to a 2x2 cluster by selecting the 2x2 block with the
+ * highest sum.
+ * @param c Cluster to reduce
+ * @return reduced cluster
+ * @note The cluster is filled using row major ordering starting at the top-left
+ * (thus for a max subcluster in the top left cornern the photon hit is at
+ * the fourth position)
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType = uint16_t>
+Cluster<T, 2, 2, CoordType>
+reduce_to_2x2(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &c) {
+
+    static_assert(ClusterSizeX >= 2 && ClusterSizeY >= 2,
+                  "Cluster sizes must be at least 2x2 for reduction to 2x2");
+
+    Cluster<T, 2, 2, CoordType> result{};
+
+    auto [sum, index] = c.max_sum_2x2();
+
+    constexpr int16_t cluster_center_index =
+        (ClusterSizeX / 2) + (ClusterSizeY / 2) * ClusterSizeX;
+
+    int16_t index_top_left_max_2x2_subcluster = cluster_center_index;
+    switch (index) {
+    case corner::cTopLeft:
+        index_top_left_max_2x2_subcluster -= (ClusterSizeX + 1);
+        break;
+    case corner::cTopRight:
+        index_top_left_max_2x2_subcluster -= ClusterSizeX;
+        break;
+    case corner::cBottomLeft:
+        index_top_left_max_2x2_subcluster -= 1;
+        break;
+    case corner::cBottomRight:
+        // no change needed
+        break;
+    }
+
+    result.x = c.x;
+    result.y = c.y;
+
+    result.data = {
+        c.data[index_top_left_max_2x2_subcluster],
+        c.data[index_top_left_max_2x2_subcluster + 1],
+        c.data[index_top_left_max_2x2_subcluster + ClusterSizeX],
+        c.data[index_top_left_max_2x2_subcluster + ClusterSizeX + 1]};
+
+    return result;
+}
+
+template <typename T>
+Cluster<T, 2, 2, uint16_t> reduce_to_2x2(const Cluster<T, 3, 3, uint16_t> &c) {
+    Cluster<T, 2, 2, uint16_t> result{};
+
+    auto [s, i] = c.max_sum_2x2();
+    result.x = c.x;
+    result.y = c.y;
+    switch (i) {
+    case corner::cTopLeft:
+        result.data = {c.data[0], c.data[1], c.data[3], c.data[4]};
+        break;
+    case corner::cTopRight:
+        result.data = {c.data[1], c.data[2], c.data[4], c.data[5]};
+        break;
+    case corner::cBottomLeft:
+        result.data = {c.data[3], c.data[4], c.data[6], c.data[7]};
+        break;
+    case corner::cBottomRight:
+        result.data = {c.data[4], c.data[5], c.data[7], c.data[8]};
+        break;
+    }
+
+    return result;
+}
+
+/**
+ * @brief Reduce a cluster to a 3x3 cluster
+ * @param c Cluster to reduce
+ * @return reduced cluster
+ */
 template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
           typename CoordType = int16_t>
-constexpr bool is_valid_cluster =
+Cluster<T, 3, 3, CoordType>
-    std::is_arithmetic_v<T> && std::is_integral_v<CoordType> &&
+reduce_to_3x3(const Cluster<T, ClusterSizeX, ClusterSizeY, CoordType> &c) {
-    (ClusterSizeX > 0) && (ClusterSizeY > 0);
 
-// requires clause c++20 maybe update
+    static_assert(ClusterSizeX >= 3 && ClusterSizeY >= 3,
-template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+                  "Cluster sizes must be at least 3x3 for reduction to 3x3");
-          typename CoordType = int16_t,
-          typename Enable = std::enable_if_t<
-              is_valid_cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>>
-struct Cluster {
-    CoordType x;
-    CoordType y;
-    T data[ClusterSizeX * ClusterSizeY];
 
-    T sum() const {
+    Cluster<T, 3, 3, CoordType> result{};
-        return std::accumulate(data, data + ClusterSizeX * ClusterSizeY, 0);
-    }
 
-    std::pair<T, int> max_sum_2x2() const {
+    int16_t cluster_center_index =
+        (ClusterSizeX / 2) + (ClusterSizeY / 2) * ClusterSizeX;
 
-        constexpr size_t num_2x2_subclusters =
+    result.x = c.x;
-            (ClusterSizeX - 1) * (ClusterSizeY - 1);
+    result.y = c.y;
 
-        std::array<T, num_2x2_subclusters> sum_2x2_subcluster;
+    result.data = {c.data[cluster_center_index - ClusterSizeX - 1],
-        for (size_t i = 0; i < ClusterSizeY - 1; ++i) {
+                   c.data[cluster_center_index - ClusterSizeX],
-            for (size_t j = 0; j < ClusterSizeX - 1; ++j)
+                   c.data[cluster_center_index - ClusterSizeX + 1],
-                sum_2x2_subcluster[i * (ClusterSizeX - 1) + j] =
+                   c.data[cluster_center_index - 1],
-                    data[i * ClusterSizeX + j] +
+                   c.data[cluster_center_index],
-                    data[i * ClusterSizeX + j + 1] +
+                   c.data[cluster_center_index + 1],
-                    data[(i + 1) * ClusterSizeX + j] +
+                   c.data[cluster_center_index + ClusterSizeX - 1],
-                    data[(i + 1) * ClusterSizeX + j + 1];
+                   c.data[cluster_center_index + ClusterSizeX],
-        }
+                   c.data[cluster_center_index + ClusterSizeX + 1]};
 
-        int index = std::max_element(sum_2x2_subcluster.begin(),
+    return result;
-                                     sum_2x2_subcluster.end()) -
+}
-                    sum_2x2_subcluster.begin();
-        return std::make_pair(sum_2x2_subcluster[index], index);
-    }
-};
-
-// Specialization for 2x2 clusters (only one sum exists)
-template <typename T> struct Cluster<T, 2, 2, int16_t> {
-    int16_t x;
-    int16_t y;
-    T data[4];
-
-    T sum() const { return std::accumulate(data, data + 4, 0); }
-
-    std::pair<T, int> max_sum_2x2() const {
-        return std::make_pair(data[0] + data[1] + data[2] + data[3],
-                              0); // Only one possible 2x2 sum
-    }
-};
-
-// Specialization for 3x3 clusters
-template <typename T> struct Cluster<T, 3, 3, int16_t> {
-    int16_t x;
-    int16_t y;
-    T data[9];
-
-    T sum() const { return std::accumulate(data, data + 9, 0); }
-
-    std::pair<T, int> max_sum_2x2() const {
-        std::array<T, 4> sum_2x2_subclusters;
-        sum_2x2_subclusters[0] = data[0] + data[1] + data[3] + data[4];
-        sum_2x2_subclusters[1] = data[1] + data[2] + data[4] + data[5];
-        sum_2x2_subclusters[2] = data[3] + data[4] + data[6] + data[7];
-        sum_2x2_subclusters[3] = data[4] + data[5] + data[7] + data[8];
-        int index = std::max_element(sum_2x2_subclusters.begin(),
-                                     sum_2x2_subclusters.end()) -
-                    sum_2x2_subclusters.begin();
-        return std::make_pair(sum_2x2_subclusters[index], index);
-    }
-};
 
 // Type Traits for is_cluster_type
 template <typename T>
@@ -102,20 +236,4 @@ struct is_cluster<Cluster<T, X, Y, CoordType>> : std::true_type {}; // Cluster
 
 template <typename T> constexpr bool is_cluster_v = is_cluster<T>::value;
 
-template <typename ClusterType,
-          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
-struct extract_template_arguments; // Forward declaration
-
-// helper struct to extract template argument
-template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
-          typename CoordType>
-struct extract_template_arguments<
-    Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
-
-    using value_type = T;
-    static constexpr int cluster_size_x = ClusterSizeX;
-    static constexpr int cluster_size_y = ClusterSizeY;
-    using coordtype = CoordType;
-};
-
 } // namespace aare

include/aare/ClusterCollector.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include <atomic>
 #include <thread>
@@ -5,6 +6,7 @@
 #include "aare/ClusterFinderMT.hpp"
 #include "aare/ClusterVector.hpp"
 #include "aare/ProducerConsumerQueue.hpp"
+#include "aare/defs.hpp"
 
 namespace aare {
 
@@ -37,7 +39,11 @@ class ClusterCollector {
   public:
     ClusterCollector(ClusterFinderMT<ClusterType, uint16_t, double> *source) {
         m_source = source->sink();
-        m_thread = std::thread(&ClusterCollector::process, this);
+        m_thread =
+            std::thread(&ClusterCollector::process,
+                        this); // only one process does that so why isnt it
+                               // automatically written to m_cluster in collect
+                               // - instead of writing first to m_sink?
     }
     void stop() {
         m_stop_requested = true;

include/aare/ClusterFile.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include "aare/Cluster.hpp"
@@ -5,6 +6,8 @@
 #include "aare/GainMap.hpp"
 #include "aare/NDArray.hpp"
 #include "aare/defs.hpp"
+#include "aare/logger.hpp"
+
 #include <filesystem>
 #include <fstream>
 #include <optional>
@@ -12,7 +15,7 @@
 namespace aare {
 
 /*
-Binary cluster file. Expects data to be layed out as:
+Binary cluster file. Expects data to be laid out as:
 int32_t frame_number
 uint32_t number_of_clusters
 int16_t x, int16_t y, int32_t data[9] x number_of_clusters
@@ -39,14 +42,15 @@ template <typename ClusterType,
           typename Enable = std::enable_if_t<is_cluster_v<ClusterType>>>
 class ClusterFile {
     FILE *fp{};
+    const std::string m_filename{};
     uint32_t m_num_left{};    /*Number of photons left in frame*/
     size_t m_chunk_size{};    /*Number of clusters to read at a time*/
-    const std::string m_mode; /*Mode to open the file in*/
+    std::string m_mode;       /*Mode to open the file in*/
     std::optional<ROI> m_roi; /*Region of interest, will be applied if set*/
     std::optional<NDArray<int32_t, 2>>
         m_noise_map; /*Noise map to cut photons, will be applied if set*/
-    std::optional<GainMap> m_gain_map; /*Gain map to apply to the clusters, will
+    std::optional<InvertedGainMap> m_gain_map; /*Gain map to apply to the
-                                          be applied if set*/
+                                          clusters, will be applied if set*/
 
   public:
     /**
@@ -59,26 +63,81 @@ class ClusterFile {
      * @throws std::runtime_error if the file could not be opened
      */
     ClusterFile(const std::filesystem::path &fname, size_t chunk_size = 1000,
-                const std::string &mode = "r");
+                const std::string &mode = "r")
 
-    ~ClusterFile();
+        : m_filename(fname.string()), m_chunk_size(chunk_size), m_mode(mode) {
+
+        if (mode == "r") {
+            fp = fopen(m_filename.c_str(), "rb");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for reading: " +
+                                         m_filename);
+            }
+        } else if (mode == "w") {
+            fp = fopen(m_filename.c_str(), "wb");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for writing: " +
+                                         m_filename);
+            }
+        } else if (mode == "a") {
+            fp = fopen(m_filename.c_str(), "ab");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for appending: " +
+                                         m_filename);
+            }
+        } else {
+            throw std::runtime_error("Unsupported mode: " + mode);
+        }
+    }
+
+    ~ClusterFile() { close(); }
 
     /**
-     * @brief Read n_clusters clusters from the file discarding frame numbers.
+     * @brief Read n_clusters clusters from the file discarding
-     * If EOF is reached the returned vector will have less than n_clusters
+     * frame numbers. If EOF is reached the returned vector will
-     * clusters
+     * have less than n_clusters clusters
      */
-    ClusterVector<ClusterType> read_clusters(size_t n_clusters);
+    ClusterVector<ClusterType> read_clusters(size_t n_clusters) {
+        if (m_mode != "r") {
+            throw std::runtime_error("File not opened for reading");
+        }
+        if (m_noise_map || m_roi) {
+            return read_clusters_with_cut(n_clusters);
+        } else {
+            return read_clusters_without_cut(n_clusters);
+        }
+    }
 
     /**
-     * @brief Read a single frame from the file and return the clusters. The
+     * @brief Read a single frame from the file and return the
-     * cluster vector will have the frame number set.
+     * clusters. The cluster vector will have the frame number
-     * @throws std::runtime_error if the file is not opened for reading or the
+     * set.
-     * file pointer not at the beginning of a frame
+     * @throws std::runtime_error if the file is not opened for
+     * reading or the file pointer not at the beginning of a
+     * frame
      */
-    ClusterVector<ClusterType> read_frame();
+    ClusterVector<ClusterType> read_frame() {
+        if (m_mode != "r") {
+            throw std::runtime_error(LOCATION + "File not opened for reading");
+        }
+        if (m_noise_map || m_roi) {
+            return read_frame_with_cut();
+        } else {
+            return read_frame_without_cut();
+        }
+    }
 
-    void write_frame(const ClusterVector<ClusterType> &clusters);
+    void write_frame(const ClusterVector<ClusterType> &clusters) {
+        if (m_mode != "w" && m_mode != "a") {
+            throw std::runtime_error("File not opened for writing");
+        }
+
+        int32_t frame_number = clusters.frame_number();
+        fwrite(&frame_number, sizeof(frame_number), 1, fp);
+        uint32_t n_clusters = clusters.size();
+        fwrite(&n_clusters, sizeof(n_clusters), 1, fp);
+        fwrite(clusters.data(), clusters.item_size(), clusters.size(), fp);
+    }
 
     /**
      * @brief Return the chunk size
@@ -86,34 +145,94 @@ class ClusterFile {
     size_t chunk_size() const { return m_chunk_size; }
 
     /**
-     * @brief Set the region of interest to use when reading clusters. If set
+     * @brief Set the region of interest to use when reading
-     * only clusters within the ROI will be read.
+     * clusters. If set only clusters within the ROI will be
+     * read.
      */
-    void set_roi(ROI roi);
+    void set_roi(ROI roi) { m_roi = roi; }
 
     /**
-     * @brief Set the noise map to use when reading clusters. If set clusters
+     * @brief Set the noise map to use when reading clusters. If
-     * below the noise level will be discarded. Selection criteria one of:
+     * set clusters below the noise level will be discarded.
-     * Central pixel above noise, highest 2x2 sum above 2 * noise, total sum
+     * Selection criteria one of: Central pixel above noise,
-     * above 3 * noise.
+     * highest 2x2 sum above 2 * noise, total sum above 3 *
+     * noise.
      */
-    void set_noise_map(const NDView<int32_t, 2> noise_map);
+    void set_noise_map(const NDView<int32_t, 2> noise_map) {
+        m_noise_map = NDArray<int32_t, 2>(noise_map);
+    }
 
     /**
      * @brief Set the gain map to use when reading clusters. If set the gain map
      * will be applied to the clusters that pass ROI and noise_map selection.
+     * The gain map is expected to be in ADU/energy.
      */
-    void set_gain_map(const NDView<double, 2> gain_map);
+    void set_gain_map(const NDView<double, 2> gain_map) {
+        m_gain_map = InvertedGainMap(gain_map);
+    }
 
-    void set_gain_map(const GainMap &gain_map);
+    void set_gain_map(const InvertedGainMap &gain_map) {
+        m_gain_map = gain_map;
+    }
 
-    void set_gain_map(const GainMap &&gain_map);
+    void set_gain_map(const InvertedGainMap &&gain_map) {
+        m_gain_map = gain_map;
+    }
 
     /**
-     * @brief Close the file. If not closed the file will be closed in the
+     * @brief Close the file. If not closed the file will be
-     * destructor
+     * closed in the destructor
      */
-    void close();
+    void close() {
+        if (fp) {
+            fclose(fp);
+            fp = nullptr;
+        }
+    }
+
+    /**
+     * @brief Return the current position in the file (bytes)
+     */
+    int64_t tell() {
+        if (!fp) {
+            throw std::runtime_error(LOCATION + "File not opened");
+        }
+        return ftell(fp);
+    }
+
+    /** @brief Open the file in specific mode
+     *
+     */
+    void open(const std::string &mode) {
+        if (fp) {
+            close();
+        }
+
+        if (mode == "r") {
+            fp = fopen(m_filename.c_str(), "rb");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for reading: " +
+                                         m_filename);
+            }
+            m_mode = "r";
+        } else if (mode == "w") {
+            fp = fopen(m_filename.c_str(), "wb");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for writing: " +
+                                         m_filename);
+            }
+            m_mode = "w";
+        } else if (mode == "a") {
+            fp = fopen(m_filename.c_str(), "ab");
+            if (!fp) {
+                throw std::runtime_error("Could not open file for appending: " +
+                                         m_filename);
+            }
+            m_mode = "a";
+        } else {
+            throw std::runtime_error("Unsupported mode: " + mode);
+        }
+    }
 
   private:
     ClusterVector<ClusterType> read_clusters_with_cut(size_t n_clusters);
@@ -124,103 +243,6 @@ class ClusterFile {
     ClusterType read_one_cluster();
 };
 
-template <typename ClusterType, typename Enable>
-ClusterFile<ClusterType, Enable>::ClusterFile(
-    const std::filesystem::path &fname, size_t chunk_size,
-    const std::string &mode)
-    : m_chunk_size(chunk_size), m_mode(mode) {
-
-    if (mode == "r") {
-        fp = fopen(fname.c_str(), "rb");
-        if (!fp) {
-            throw std::runtime_error("Could not open file for reading: " +
-                                     fname.string());
-        }
-    } else if (mode == "w") {
-        fp = fopen(fname.c_str(), "wb");
-        if (!fp) {
-            throw std::runtime_error("Could not open file for writing: " +
-                                     fname.string());
-        }
-    } else if (mode == "a") {
-        fp = fopen(fname.c_str(), "ab");
-        if (!fp) {
-            throw std::runtime_error("Could not open file for appending: " +
-                                     fname.string());
-        }
-    } else {
-        throw std::runtime_error("Unsupported mode: " + mode);
-    }
-}
-
-template <typename ClusterType, typename Enable>
-ClusterFile<ClusterType, Enable>::~ClusterFile() {
-    close();
-}
-
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::close() {
-    if (fp) {
-        fclose(fp);
-        fp = nullptr;
-    }
-}
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::set_roi(ROI roi) {
-    m_roi = roi;
-}
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::set_noise_map(
-    const NDView<int32_t, 2> noise_map) {
-    m_noise_map = NDArray<int32_t, 2>(noise_map);
-}
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::set_gain_map(
-    const NDView<double, 2> gain_map) {
-    m_gain_map = GainMap(gain_map);
-}
-
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::set_gain_map(const GainMap &gain_map) {
-    m_gain_map = gain_map;
-}
-
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::set_gain_map(const GainMap &&gain_map) {
-    m_gain_map = gain_map;
-}
-
-// TODO generally supported for all clsuter types
-template <typename ClusterType, typename Enable>
-void ClusterFile<ClusterType, Enable>::write_frame(
-    const ClusterVector<ClusterType> &clusters) {
-    if (m_mode != "w" && m_mode != "a") {
-        throw std::runtime_error("File not opened for writing");
-    }
-    if (!(clusters.cluster_size_x() == 3) &&
-        !(clusters.cluster_size_y() == 3)) {
-        throw std::runtime_error("Only 3x3 clusters are supported");
-    }
-    int32_t frame_number = clusters.frame_number();
-    fwrite(&frame_number, sizeof(frame_number), 1, fp);
-    uint32_t n_clusters = clusters.size();
-    fwrite(&n_clusters, sizeof(n_clusters), 1, fp);
-    fwrite(clusters.data(), clusters.item_size(), clusters.size(), fp);
-}
-
-template <typename ClusterType, typename Enable>
-ClusterVector<ClusterType>
-ClusterFile<ClusterType, Enable>::read_clusters(size_t n_clusters) {
-    if (m_mode != "r") {
-        throw std::runtime_error("File not opened for reading");
-    }
-    if (m_noise_map || m_roi) {
-        return read_clusters_with_cut(n_clusters);
-    } else {
-        return read_clusters_without_cut(n_clusters);
-    }
-}
-
 template <typename ClusterType, typename Enable>
 ClusterVector<ClusterType>
 ClusterFile<ClusterType, Enable>::read_clusters_without_cut(size_t n_clusters) {
@@ -229,25 +251,24 @@ ClusterFile<ClusterType, Enable>::read_clusters_without_cut(size_t n_clusters) {
     }
 
     ClusterVector<ClusterType> clusters(n_clusters);
+    clusters.resize(n_clusters);
 
     int32_t iframe = 0; // frame number needs to be 4 bytes!
     size_t nph_read = 0;
     uint32_t nn = m_num_left;
     uint32_t nph = m_num_left; // number of clusters in frame needs to be 4
 
-    // auto buf = reinterpret_cast<Cluster3x3 *>(clusters.data());
     auto buf = clusters.data();
     // if there are photons left from previous frame read them first
     if (nph) {
         if (nph > n_clusters) {
-            // if we have more photons left in the frame then photons to read we
+            // if we have more photons left in the frame then photons to
-            // read directly the requested number
+            // read we read directly the requested number
             nn = n_clusters;
         } else {
             nn = nph;
         }
-        nph_read += fread((buf + nph_read * clusters.item_size()),
+        nph_read += fread((buf + nph_read), clusters.item_size(), nn, fp);
-                          clusters.item_size(), nn, fp);
         m_num_left = nph - nn; // write back the number of photons left
     }
 
@@ -262,8 +283,8 @@ ClusterFile<ClusterType, Enable>::read_clusters_without_cut(size_t n_clusters) {
                 else
                     nn = nph;
 
-                nph_read += fread((buf + nph_read * clusters.item_size()),
+                nph_read +=
-                                  clusters.item_size(), nn, fp);
+                    fread((buf + nph_read), clusters.item_size(), nn, fp);
                 m_num_left = nph - nn;
             }
             if (nph_read >= n_clusters)
@@ -271,7 +292,7 @@ ClusterFile<ClusterType, Enable>::read_clusters_without_cut(size_t n_clusters) {
         }
     }
 
-    // Resize the vector to the number of clusters.
+    // Resize the vector to the number o f clusters.
     // No new allocation, only change bounds.
     clusters.resize(nph_read);
     if (m_gain_map)
@@ -308,8 +329,8 @@ ClusterFile<ClusterType, Enable>::read_clusters_with_cut(size_t n_clusters) {
         while (fread(&frame_number, sizeof(frame_number), 1, fp)) {
             if (fread(&m_num_left, sizeof(m_num_left), 1, fp)) {
                 clusters.set_frame_number(
-                    frame_number); // cluster vector will hold the last frame
+                    frame_number); // cluster vector will hold the last
-                                   // number
+                                   // frame number
                 while (m_num_left && clusters.size() < n_clusters) {
                     ClusterType c = read_one_cluster();
                     if (is_selected(c)) {
@@ -340,31 +361,24 @@ ClusterType ClusterFile<ClusterType, Enable>::read_one_cluster() {
     return c;
 }
 
-template <typename ClusterType, typename Enable>
-ClusterVector<ClusterType> ClusterFile<ClusterType, Enable>::read_frame() {
-    if (m_mode != "r") {
-        throw std::runtime_error(LOCATION + "File not opened for reading");
-    }
-    if (m_noise_map || m_roi) {
-        return read_frame_with_cut();
-    } else {
-        return read_frame_without_cut();
-    }
-}
-
 template <typename ClusterType, typename Enable>
 ClusterVector<ClusterType>
 ClusterFile<ClusterType, Enable>::read_frame_without_cut() {
     if (m_mode != "r") {
-        throw std::runtime_error("File not opened for reading");
+        throw std::runtime_error(LOCATION + "File not opened for reading");
     }
     if (m_num_left) {
         throw std::runtime_error(
-            "There are still photons left in the last frame");
+            LOCATION + "There are still photons left in the last frame");
     }
     int32_t frame_number;
     if (fread(&frame_number, sizeof(frame_number), 1, fp) != 1) {
-        throw std::runtime_error(LOCATION + "Could not read frame number");
+        if (feof(fp))
+            throw std::runtime_error(LOCATION + "Unexpected end of file");
+        else if (ferror(fp))
+            throw std::runtime_error(LOCATION + "Error reading from file");
+
+        throw std::runtime_error(LOCATION + "Unexpected error (not feof or ferror) when reading frame number");
     }
 
     int32_t n_clusters; // Saved as 32bit integer in the cluster file
@@ -373,14 +387,20 @@ ClusterFile<ClusterType, Enable>::read_frame_without_cut() {
                                  "Could not read number of clusters");
     }
 
+    LOG(logDEBUG1) << "Reading " << n_clusters << " clusters from frame "
+                   << frame_number;
+
     ClusterVector<ClusterType> clusters(n_clusters);
     clusters.set_frame_number(frame_number);
+    clusters.resize(n_clusters);
+
+    LOG(logDEBUG1) << "clusters.item_size(): " << clusters.item_size();
 
     if (fread(clusters.data(), clusters.item_size(), n_clusters, fp) !=
         static_cast<size_t>(n_clusters)) {
         throw std::runtime_error(LOCATION + "Could not read clusters");
     }
-    clusters.resize(n_clusters);
+
     if (m_gain_map)
         m_gain_map->apply_gain_map(clusters);
     return clusters;
@@ -428,18 +448,14 @@ bool ClusterFile<ClusterType, Enable>::is_selected(ClusterType &cl) {
         }
     }
 
-    auto cluster_size_x = extract_template_arguments<
-        std::remove_reference_t<decltype(cl)>>::cluster_size_x;
-    auto cluster_size_y = extract_template_arguments<
-        std::remove_reference_t<decltype(cl)>>::cluster_size_y;
-
     size_t cluster_center_index =
-        (cluster_size_x / 2) + (cluster_size_y / 2) * cluster_size_x;
+        (ClusterType::cluster_size_x / 2) +
+        (ClusterType::cluster_size_y / 2) * ClusterType::cluster_size_x;
 
     if (m_noise_map) {
         auto sum_1x1 = cl.data[cluster_center_index]; // central pixel
-        auto sum_2x2 = cl.max_sum_2x2().first; // highest sum of 2x2 subclusters
+        auto sum_2x2 = cl.max_sum_2x2().sum; // highest sum of 2x2 subclusters
-        auto total_sum = cl.sum();             // sum of all pixels
+        auto total_sum = cl.sum();           // sum of all pixels
 
         auto noise =
             (*m_noise_map)(cl.y, cl.x); // TODO! check if this is correct

include/aare/ClusterFileSink.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include <atomic>
 #include <filesystem>
@@ -10,7 +11,8 @@
 namespace aare {
 
 template <typename ClusterType,
-          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>,
+          typename = std::enable_if_t<no_2x2_cluster<ClusterType>::value>>
 class ClusterFileSink {
     ProducerConsumerQueue<ClusterVector<ClusterType>> *m_source;
     std::atomic<bool> m_stop_requested{false};
@@ -21,7 +23,7 @@ class ClusterFileSink {
 
     void process() {
         m_stopped = false;
-        fmt::print("ClusterFileSink started\n");
+        LOG(logDEBUG) << "ClusterFileSink started";
         while (!m_stop_requested || !m_source->isEmpty()) {
             if (ClusterVector<ClusterType> *clusters = m_source->frontPtr();
                 clusters != nullptr) {
@@ -41,13 +43,16 @@ class ClusterFileSink {
                 std::this_thread::sleep_for(m_default_wait);
             }
         }
-        fmt::print("ClusterFileSink stopped\n");
+        LOG(logDEBUG) << "ClusterFileSink stopped";
         m_stopped = true;
     }
 
   public:
     ClusterFileSink(ClusterFinderMT<ClusterType, uint16_t, double> *source,
                     const std::filesystem::path &fname) {
+        LOG(logDEBUG) << "ClusterFileSink: "
+                      << "source: " << source->sink()
+                      << ", file: " << fname.string();
         m_source = source->sink();
         m_thread = std::thread(&ClusterFileSink::process, this);
         m_file.open(fname, std::ios::binary);

include/aare/ClusterFileV2.hpp

@@ -1,154 +0,0 @@
-#pragma once
-#include "aare/core/defs.hpp"
-#include <filesystem>
-#include <fmt/format.h>
-#include <string>
-
-namespace aare {
-struct ClusterHeader {
-    int32_t frame_number;
-    int32_t n_clusters;
-    std::string to_string() const {
-        return "frame_number: " + std::to_string(frame_number) +
-               ", n_clusters: " + std::to_string(n_clusters);
-    }
-};
-
-struct ClusterV2_ {
-    int16_t x;
-    int16_t y;
-    std::array<int32_t, 9> data;
-    std::string to_string(bool detailed = false) const {
-        if (detailed) {
-            std::string data_str = "[";
-            for (auto &d : data) {
-                data_str += std::to_string(d) + ", ";
-            }
-            data_str += "]";
-            return "x: " + std::to_string(x) + ", y: " + std::to_string(y) +
-                   ", data: " + data_str;
-        }
-        return "x: " + std::to_string(x) + ", y: " + std::to_string(y);
-    }
-};
-
-struct ClusterV2 {
-    ClusterV2_ cluster;
-    int32_t frame_number;
-    std::string to_string() const {
-        return "frame_number: " + std::to_string(frame_number) + ", " +
-               cluster.to_string();
-    }
-};
-
-/**
- * @brief
- * important not: fp always points to the clusters header and does not point to
- * individual clusters
- *
- */
-class ClusterFileV2 {
-    std::filesystem::path m_fpath;
-    std::string m_mode;
-    FILE *fp{nullptr};
-
-    void check_open() {
-        if (!fp)
-            throw std::runtime_error(
-                fmt::format("File: {} not open", m_fpath.string()));
-    }
-
-  public:
-    ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode)
-        : m_fpath(fpath), m_mode(mode) {
-        if (m_mode != "r" && m_mode != "w")
-            throw std::invalid_argument("mode must be 'r' or 'w'");
-        if (m_mode == "r" && !std::filesystem::exists(m_fpath))
-            throw std::invalid_argument("File does not exist");
-        if (mode == "r") {
-            fp = fopen(fpath.string().c_str(), "rb");
-        } else if (mode == "w") {
-            if (std::filesystem::exists(fpath)) {
-                fp = fopen(fpath.string().c_str(), "r+b");
-            } else {
-                fp = fopen(fpath.string().c_str(), "wb");
-            }
-        }
-        if (fp == nullptr) {
-            throw std::runtime_error("Failed to open file");
-        }
-    }
-    ~ClusterFileV2() { close(); }
-    std::vector<ClusterV2> read() {
-        check_open();
-
-        ClusterHeader header;
-        fread(&header, sizeof(ClusterHeader), 1, fp);
-        std::vector<ClusterV2_> clusters_(header.n_clusters);
-        fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp);
-        std::vector<ClusterV2> clusters;
-        for (auto &c : clusters_) {
-            ClusterV2 cluster;
-            cluster.cluster = std::move(c);
-            cluster.frame_number = header.frame_number;
-            clusters.push_back(cluster);
-        }
-
-        return clusters;
-    }
-    std::vector<std::vector<ClusterV2>> read(int n_frames) {
-        std::vector<std::vector<ClusterV2>> clusters;
-        for (int i = 0; i < n_frames; i++) {
-            clusters.push_back(read());
-        }
-        return clusters;
-    }
-
-    size_t write(std::vector<ClusterV2> const &clusters) {
-        check_open();
-        if (m_mode != "w")
-            throw std::runtime_error("File not opened in write mode");
-        if (clusters.empty())
-            return 0;
-
-        ClusterHeader header;
-        header.frame_number = clusters[0].frame_number;
-        header.n_clusters = clusters.size();
-        fwrite(&header, sizeof(ClusterHeader), 1, fp);
-        for (auto &c : clusters) {
-            fwrite(&c.cluster, sizeof(ClusterV2_), 1, fp);
-        }
-        return clusters.size();
-    }
-
-    size_t write(std::vector<std::vector<ClusterV2>> const &clusters) {
-        check_open();
-        if (m_mode != "w")
-            throw std::runtime_error("File not opened in write mode");
-
-        size_t n_clusters = 0;
-        for (auto &c : clusters) {
-            n_clusters += write(c);
-        }
-        return n_clusters;
-    }
-
-    int seek_to_begin() { return fseek(fp, 0, SEEK_SET); }
-    int seek_to_end() { return fseek(fp, 0, SEEK_END); }
-
-    int32_t frame_number() {
-        auto pos = ftell(fp);
-        ClusterHeader header;
-        fread(&header, sizeof(ClusterHeader), 1, fp);
-        fseek(fp, pos, SEEK_SET);
-        return header.frame_number;
-    }
-
-    void close() {
-        if (fp) {
-            fclose(fp);
-            fp = nullptr;
-        }
-    }
-};
-} // namespace aare

include/aare/ClusterFinder.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/ClusterFile.hpp"
 #include "aare/ClusterVector.hpp"
@@ -10,8 +11,16 @@
 
 namespace aare {
 
+template <typename ClusterType,
+          typename = std::enable_if_t<is_cluster_v<ClusterType>>>
+struct no_2x2_cluster {
+    constexpr static bool value =
+        ClusterType::cluster_size_x > 2 && ClusterType::cluster_size_y > 2;
+};
+
 template <typename ClusterType = Cluster<int32_t, 3, 3>,
-          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
+          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double,
+          typename = std::enable_if_t<no_2x2_cluster<ClusterType>::value>>
 class ClusterFinder {
     Shape<2> m_image_size;
     const PEDESTAL_TYPE m_nSigma;
@@ -20,11 +29,9 @@ class ClusterFinder {
     Pedestal<PEDESTAL_TYPE> m_pedestal;
     ClusterVector<ClusterType> m_clusters;
 
-    static const uint8_t ClusterSizeX =
+    static const uint8_t ClusterSizeX = ClusterType::cluster_size_x;
-        extract_template_arguments<ClusterType>::cluster_size_x;
+    static const uint8_t ClusterSizeY = ClusterType::cluster_size_y;
-    static const uint8_t ClusterSizeY =
+    using CT = typename ClusterType::value_type;
-        extract_template_arguments<ClusterType>::cluster_size_x;
-    using CT = typename extract_template_arguments<ClusterType>::value_type;
 
   public:
     /**
@@ -40,7 +47,11 @@ class ClusterFinder {
         : m_image_size(image_size), m_nSigma(nSigma),
           c2(sqrt((ClusterSizeY + 1) / 2 * (ClusterSizeX + 1) / 2)),
           c3(sqrt(ClusterSizeX * ClusterSizeY)),
-          m_pedestal(image_size[0], image_size[1]), m_clusters(capacity) {};
+          m_pedestal(image_size[0], image_size[1]), m_clusters(capacity) {
+        LOG(logDEBUG) << "ClusterFinder: "
+                      << "image_size: " << image_size[0] << "x" << image_size[1]
+                      << ", nSigma: " << nSigma << ", capacity: " << capacity;
+    }
 
     void push_pedestal_frame(NDView<FRAME_TYPE, 2> frame) {
         m_pedestal.push(frame);
@@ -79,7 +90,6 @@ class ClusterFinder {
         int has_center_pixel_y = ClusterSizeY % 2;
 
         m_clusters.set_frame_number(frame_number);
-        std::vector<CT> cluster_data(ClusterSizeX * ClusterSizeY);
         for (int iy = 0; iy < frame.shape(0); iy++) {
             for (int ix = 0; ix < frame.shape(1); ix++) {
 
@@ -126,35 +136,31 @@ class ClusterFinder {
 
                 // Store cluster
                 if (value == max) {
-                    // Zero out the cluster data
+                    ClusterType cluster{};
-                    std::fill(cluster_data.begin(), cluster_data.end(), 0);
+                    cluster.x = ix;
+                    cluster.y = iy;
 
                     // Fill the cluster data since we have a photon to store
                     // It's worth redoing the look since most of the time we
                     // don't have a photon
                     int i = 0;
                     for (int ir = -dy; ir < dy + has_center_pixel_y; ir++) {
-                        for (int ic = -dx; ic < dx + has_center_pixel_y; ic++) {
+                        for (int ic = -dx; ic < dx + has_center_pixel_x; ic++) {
                             if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
                                 iy + ir >= 0 && iy + ir < frame.shape(0)) {
                                 CT tmp =
                                     static_cast<CT>(frame(iy + ir, ix + ic)) -
                                     static_cast<CT>(
                                         m_pedestal.mean(iy + ir, ix + ic));
-                                cluster_data[i] =
+                                cluster.data[i] =
                                     tmp; // Watch for out of bounds access
-                                i++;
                             }
+                            i++;
                         }
                     }
 
-                    ClusterType new_cluster{};
-                    new_cluster.x = ix;
-                    new_cluster.y = iy;
-                    std::copy(cluster_data.begin(), cluster_data.end(),
-                              new_cluster.data);
                     // Add the cluster to the output ClusterVector
-                    m_clusters.push_back(new_cluster);
+                    m_clusters.push_back(cluster);
                 }
             }
         }

include/aare/ClusterFinderMT.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include <atomic>
 #include <cstdint>
@@ -8,6 +9,7 @@
 #include "aare/ClusterFinder.hpp"
 #include "aare/NDArray.hpp"
 #include "aare/ProducerConsumerQueue.hpp"
+#include "aare/logger.hpp"
 
 namespace aare {
 
@@ -31,10 +33,12 @@ struct FrameWrapper {
  * @tparam CT type of the cluster data
  */
 template <typename ClusterType = Cluster<int32_t, 3, 3>,
-          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
+          typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double,
+          typename = std::enable_if_t<no_2x2_cluster<ClusterType>::value>>
 class ClusterFinderMT {
 
-    using CT = typename extract_template_arguments<ClusterType>::value_type;
+  protected:
+    using CT = typename ClusterType::value_type;
     size_t m_current_thread{0};
     size_t m_n_threads{0};
     using Finder = ClusterFinder<ClusterType, FRAME_TYPE, PEDESTAL_TYPE>;
@@ -50,6 +54,7 @@ class ClusterFinderMT {
     std::thread m_collect_thread;
     std::chrono::milliseconds m_default_wait{1};
 
+  private:
     std::atomic<bool> m_stop_requested{false};
     std::atomic<bool> m_processing_threads_stopped{true};
 
@@ -120,6 +125,13 @@ class ClusterFinderMT {
     ClusterFinderMT(Shape<2> image_size, PEDESTAL_TYPE nSigma = 5.0,
                     size_t capacity = 2000, size_t n_threads = 3)
         : m_n_threads(n_threads) {
+
+        LOG(logDEBUG1) << "ClusterFinderMT: "
+                       << "image_size: " << image_size[0] << "x"
+                       << image_size[1] << ", nSigma: " << nSigma
+                       << ", capacity: " << capacity
+                       << ", n_threads: " << n_threads;
+
         for (size_t i = 0; i < n_threads; i++) {
             m_cluster_finders.push_back(
                 std::make_unique<

include/aare/ClusterVector.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Cluster.hpp" //TODO maybe store in seperate file !!!
 #include <algorithm>
@@ -18,256 +19,6 @@ template <typename ClusterType,
           typename = std::enable_if_t<is_cluster_v<ClusterType>>>
 class ClusterVector; // Forward declaration
 
-/**
- * @brief ClusterVector is a container for clusters of various sizes. It uses a
- * contiguous memory buffer to store the clusters. It is templated on the data
- * type and the coordinate type of the clusters.
- * @note push_back can invalidate pointers to elements in the container
- * @warning ClusterVector is currently move only to catch unintended copies, but
- * this might change since there are probably use cases where copying is needed.
- * @tparam T data type of the pixels in the cluster
- * @tparam CoordType data type of the x and y coordinates of the cluster
- * (normally int16_t)
- */
-#if 0
-template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
-          typename CoordType>
-class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
-
-    std::byte *m_data{};
-    size_t m_size{0};
-    size_t m_capacity;
-    uint64_t m_frame_number{0}; // TODO! Check frame number size and type
-    /** 
-    Format string used in the python bindings to create a numpy
-    array from the buffer
-    = - native byte order
-    h - short
-    d - double
-    i - int
-    */
-    constexpr static char m_fmt_base[] = "=h:x:\nh:y:\n({},{}){}:data:";
-
-  public:
-    using value_type = T;
-    using ClusterType = Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>;
-
-    /**
-     * @brief Construct a new ClusterVector object
-     * @param capacity initial capacity of the buffer in number of clusters
-     * @param frame_number frame number of the clusters. Default is 0, which is
-     * also used to indicate that the clusters come from many frames
-     */
-    ClusterVector(size_t capacity = 1024, uint64_t frame_number = 0)
-        : m_capacity(capacity), m_frame_number(frame_number) {
-        allocate_buffer(m_capacity);
-    }
-
-    ~ClusterVector() { delete[] m_data; }
-
-    // Move constructor
-    ClusterVector(ClusterVector &&other) noexcept
-        : m_data(other.m_data), m_size(other.m_size),
-          m_capacity(other.m_capacity), m_frame_number(other.m_frame_number) {
-        other.m_data = nullptr;
-        other.m_size = 0;
-        other.m_capacity = 0;
-    }
-
-    // Move assignment operator
-    ClusterVector &operator=(ClusterVector &&other) noexcept {
-        if (this != &other) {
-            delete[] m_data;
-            m_data = other.m_data;
-            m_size = other.m_size;
-            m_capacity = other.m_capacity;
-            m_frame_number = other.m_frame_number;
-            other.m_data = nullptr;
-            other.m_size = 0;
-            other.m_capacity = 0;
-            other.m_frame_number = 0;
-        }
-        return *this;
-    }
-
-    /**
-     * @brief Reserve space for at least capacity clusters
-     * @param capacity number of clusters to reserve space for
-     * @note If capacity is less than the current capacity, the function does
-     * nothing.
-     */
-    void reserve(size_t capacity) {
-        if (capacity > m_capacity) {
-            allocate_buffer(capacity);
-        }
-    }
-
-    /**
-     * @brief Add a cluster to the vector
-     */
-    void push_back(const ClusterType &cluster) {
-        if (m_size == m_capacity) {
-            allocate_buffer(m_capacity * 2);
-        }
-        std::byte *ptr = element_ptr(m_size);
-        *reinterpret_cast<CoordType *>(ptr) = cluster.x;
-        ptr += sizeof(CoordType);
-        *reinterpret_cast<CoordType *>(ptr) = cluster.y;
-        ptr += sizeof(CoordType);
-
-        std::memcpy(ptr, cluster.data, ClusterSizeX * ClusterSizeY * sizeof(T));
-
-        m_size++;
-    }
-
-    ClusterVector &operator+=(const ClusterVector &other) {
-        if (m_size + other.m_size > m_capacity) {
-            allocate_buffer(m_capacity + other.m_size);
-        }
-        std::copy(other.m_data, other.m_data + other.m_size * item_size(),
-                  m_data + m_size * item_size());
-        m_size += other.m_size;
-        return *this;
-    }
-
-    /**
-     * @brief Sum the pixels in each cluster
-     * @return std::vector<T> vector of sums for each cluster
-     */
-    /*
-    std::vector<T> sum() {
-        std::vector<T> sums(m_size);
-        const size_t stride = item_size();
-        const size_t n_pixels = ClusterSizeX * ClusterSizeY;
-        std::byte *ptr = m_data + 2 * sizeof(CoordType); // skip x and y
-
-        for (size_t i = 0; i < m_size; i++) {
-            sums[i] =
-                std::accumulate(reinterpret_cast<T *>(ptr),
-                                reinterpret_cast<T *>(ptr) + n_pixels, T{});
-            ptr += stride;
-        }
-        return sums;
-    }
-    */
-
-    /**
-     * @brief Sum the pixels in the 2x2 subcluster with the biggest pixel sum in
-     * each cluster
-     * @return std::vector<T> vector of sums for each cluster
-     */ //TODO if underlying container is a vector use std::for_each
-    /*
-    std::vector<T> sum_2x2() {
-        std::vector<T> sums_2x2(m_size);
-
-        for (size_t i = 0; i < m_size; i++) {
-            sums_2x2[i] = at(i).max_sum_2x2;
-        }
-        return sums_2x2;
-    }
-    */
-
-    /**
-     * @brief Return the number of clusters in the vector
-     */
-    size_t size() const { return m_size; }
-
-    uint8_t cluster_size_x() const { return ClusterSizeX; }
-
-    uint8_t cluster_size_y() const { return ClusterSizeY; }
-
-    /**
-     * @brief Return the capacity of the buffer in number of clusters. This is
-     * the number of clusters that can be stored in the current buffer without
-     * reallocation.
-     */
-    size_t capacity() const { return m_capacity; }
-
-    /**
-     * @brief Return the size in bytes of a single cluster
-     */
-    size_t item_size() const {
-        return 2 * sizeof(CoordType) + ClusterSizeX * ClusterSizeY * sizeof(T);
-    }
-
-    /**
-     * @brief Return the offset in bytes for the i-th cluster
-     */
-    size_t element_offset(size_t i) const { return item_size() * i; }
-
-    /**
-     * @brief Return a pointer to the i-th cluster
-     */
-    std::byte *element_ptr(size_t i) { return m_data + element_offset(i); }
-
-    /**
-     * @brief Return a pointer to the i-th cluster
-     */
-    const std::byte *element_ptr(size_t i) const {
-        return m_data + element_offset(i);
-    }
-
-    std::byte *data() { return m_data; }
-    std::byte const *data() const { return m_data; }
-
-    /**
-     * @brief Return a reference to the i-th cluster casted to type V
-     * @tparam V type of the cluster
-     */
-    ClusterType &at(size_t i) {
-        return *reinterpret_cast<ClusterType *>(element_ptr(i));
-    }
-
-    const ClusterType &at(size_t i) const {
-        return *reinterpret_cast<const ClusterType *>(element_ptr(i));
-    }
-
-    template <typename V> const V &at(size_t i) const {
-        return *reinterpret_cast<const V *>(element_ptr(i));
-    }
-
-    const std::string_view fmt_base() const {
-        // TODO! how do we match on coord_t?
-        return m_fmt_base;
-    }
-
-    /** 
-     * @brief Return the frame number of the clusters. 0 is used to indicate
-     * that the clusters come from many frames
-     */
-    uint64_t frame_number() const { return m_frame_number; }
-
-    void set_frame_number(uint64_t frame_number) {
-        m_frame_number = frame_number;
-    }
-
-    /** 
-     * @brief Resize the vector to contain new_size clusters. If new_size is
-     * greater than the current capacity, a new buffer is allocated. If the size
-     * is smaller no memory is freed, size is just updated.
-     * @param new_size new size of the vector
-     * @warning The additional clusters are not initialized
-     */
-    void resize(size_t new_size) {
-        // TODO! Should we initialize the new clusters?
-        if (new_size > m_capacity) {
-            allocate_buffer(new_size);
-        }
-        m_size = new_size;
-    }
-
-  private:
-    void allocate_buffer(size_t new_capacity) {
-        size_t num_bytes = item_size() * new_capacity;
-        std::byte *new_data = new std::byte[num_bytes]{};
-        std::copy(m_data, m_data + item_size() * m_size, new_data);
-        delete[] m_data;
-        m_data = new_data;
-        m_capacity = new_capacity;
-    }
-};
-#endif
-
 /**
  * @brief ClusterVector is a container for clusters of various sizes. It
  * uses a contiguous memory buffer to store the clusters. It is templated on
@@ -278,14 +29,14 @@ class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
  * needed.
  * @tparam T data type of the pixels in the cluster
  * @tparam CoordType data type of the x and y coordinates of the cluster
- * (normally int16_t)
+ * (normally uint16_t)
  */
 template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
           typename CoordType>
 class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
 
     std::vector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> m_data{};
-    uint64_t m_frame_number{0}; // TODO! Check frame number size and type
+    int32_t m_frame_number{0}; // TODO! Check frame number size and type
 
   public:
     using value_type = T;
@@ -319,6 +70,35 @@ class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
         return *this;
     }
 
+    /**
+     * @brief Sum the pixels in each cluster
+     * @return std::vector<T> vector of sums for each cluster
+     */
+    std::vector<T> sum() {
+        std::vector<T> sums(m_data.size());
+
+        std::transform(
+            m_data.begin(), m_data.end(), sums.begin(),
+            [](const ClusterType &cluster) { return cluster.sum(); });
+
+        return sums;
+    }
+
+    /**
+     * @brief Sum the pixels in the 2x2 subcluster with the biggest pixel sum in
+     * each cluster
+     * @return vector of sums index pairs for each cluster
+     */
+    std::vector<Sum_index_pair<T, corner>> sum_2x2() {
+        std::vector<Sum_index_pair<T, corner>> sums_2x2(m_data.size());
+
+        std::transform(
+            m_data.begin(), m_data.end(), sums_2x2.begin(),
+            [](const ClusterType &cluster) { return cluster.max_sum_2x2(); });
+
+        return sums_2x2;
+    }
+
     /**
      * @brief Reserve space for at least capacity clusters
      * @param capacity number of clusters to reserve space for
@@ -342,6 +122,11 @@ class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
      */
     size_t size() const { return m_data.size(); }
 
+    /**
+     * @brief Check if the vector is empty
+     */
+    bool empty() const { return m_data.empty(); }
+
     uint8_t cluster_size_x() const { return ClusterSizeX; }
 
     uint8_t cluster_size_y() const { return ClusterSizeY; }
@@ -353,15 +138,16 @@ class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
      */
     size_t capacity() const { return m_data.capacity(); }
 
-    const auto begin() const { return m_data.begin(); }
+    auto begin() const { return m_data.begin(); }
 
-    const auto end() const { return m_data.end(); }
+    auto end() const { return m_data.end(); }
 
     /**
      * @brief Return the size in bytes of a single cluster
      */
     size_t item_size() const {
-        return 2 * sizeof(CoordType) + ClusterSizeX * ClusterSizeY * sizeof(T);
+        return sizeof(ClusterType); // 2 * sizeof(CoordType) + ClusterSizeX *
+                                    // ClusterSizeY * sizeof(T);
     }
 
     ClusterType *data() { return m_data.data(); }
@@ -371,19 +157,57 @@ class ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>> {
      * @brief Return a reference to the i-th cluster casted to type V
      * @tparam V type of the cluster
      */
-    ClusterType &at(size_t i) { return m_data[i]; }
+    ClusterType &operator[](size_t i) { return m_data[i]; }
 
-    const ClusterType &at(size_t i) const { return m_data[i]; }
+    const ClusterType &operator[](size_t i) const { return m_data[i]; }
 
     /**
      * @brief Return the frame number of the clusters. 0 is used to indicate
      * that the clusters come from many frames
      */
-    uint64_t frame_number() const { return m_frame_number; }
+    int32_t frame_number() const { return m_frame_number; }
 
-    void set_frame_number(uint64_t frame_number) {
+    void set_frame_number(int32_t frame_number) {
         m_frame_number = frame_number;
     }
 };
 
+/**
+ * @brief Reduce a cluster to a 2x2 cluster by selecting the 2x2 block with the
+ * highest sum.
+ * @param cv Clustervector containing clusters to reduce
+ * @return Clustervector with reduced clusters
+ * @note The cluster is filled using row major ordering starting at the top-left
+ * (thus for a max subcluster in the top left cornern the photon hit is at
+ * the fourth position)
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+ClusterVector<Cluster<T, 2, 2, CoordType>> reduce_to_2x2(
+    const ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>
+        &cv) {
+    ClusterVector<Cluster<T, 2, 2, CoordType>> result;
+    for (const auto &c : cv) {
+        result.push_back(reduce_to_2x2(c));
+    }
+    return result;
+}
+
+/**
+ * @brief Reduce a cluster to a 3x3 cluster
+ * @param cv Clustervector containing clusters to reduce
+ * @return Clustervector with reduced clusters
+ */
+template <typename T, uint8_t ClusterSizeX, uint8_t ClusterSizeY,
+          typename CoordType>
+ClusterVector<Cluster<T, 3, 3, CoordType>> reduce_to_3x3(
+    const ClusterVector<Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>>
+        &cv) {
+    ClusterVector<Cluster<T, 3, 3, CoordType>> result;
+    for (const auto &c : cv) {
+        result.push_back(reduce_to_3x3(c));
+    }
+    return result;
+}
+
 } // namespace aare

include/aare/CtbRawFile.hpp

@@ -1,27 +1,28 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include "aare/FileInterface.hpp"
-#include "aare/RawMasterFile.hpp"
 #include "aare/Frame.hpp"
+#include "aare/RawMasterFile.hpp"
 
 #include <filesystem>
 #include <fstream>
 
-namespace aare{
+namespace aare {
 
-
+class CtbRawFile {
-class CtbRawFile{
     RawMasterFile m_master;
     std::ifstream m_file;
     size_t m_current_frame{0};
     size_t m_current_subfile{0};
     size_t m_num_subfiles{0};
-public:
+
+  public:
     CtbRawFile(const std::filesystem::path &fname);
 
-    void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
+    void read_into(std::byte *image_buf, DetectorHeader *header = nullptr);
-    void seek(size_t frame_index);          //!< seek to the given frame index
+    void seek(size_t frame_index); //!< seek to the given frame index
-    size_t tell() const;                    //!< get the frame index of the file pointer
+    size_t tell() const;           //!< get the frame index of the file pointer
 
     // in the specific class we can expose more functionality
 
@@ -29,13 +30,13 @@ public:
     size_t frames_in_file() const;
 
     RawMasterFile master() const;
-private:
+
+  private:
     void find_subfiles();
     size_t sub_file_index(size_t frame_index) const {
         return frame_index / m_master.max_frames_per_file();
     }
     void open_data_file(size_t subfile_index);
-
 };
 
-}
+} // namespace aare

include/aare/DetectorGeometry.hpp

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+#include "aare/RawMasterFile.hpp" //ROI refactor away
+#include "aare/defs.hpp"
+namespace aare {
+
+struct ModuleConfig {
+    int module_gap_row{};
+    int module_gap_col{};
+
+    bool operator==(const ModuleConfig &other) const {
+        if (module_gap_col != other.module_gap_col)
+            return false;
+        if (module_gap_row != other.module_gap_row)
+            return false;
+        return true;
+    }
+};
+
+/**
+ * @brief Class to hold the geometry of a module. Where pixel 0 is located and
+ * the size of the module
+ */
+struct ModuleGeometry {
+    int origin_x{};
+    int origin_y{};
+    int height{};
+    int width{};
+    int row_index{};
+    int col_index{};
+};
+
+/**
+ * @brief Class to hold the geometry of a detector. Number of modules, their
+ * size and where pixel 0 for each module is located
+ */
+class DetectorGeometry {
+  public:
+    DetectorGeometry(const xy &geometry, const ssize_t module_pixels_x,
+                     const ssize_t module_pixels_y,
+                     const xy udp_interfaces_per_module = xy{1, 1},
+                     const bool quad = false);
+
+    ~DetectorGeometry() = default;
+
+    /**
+     * @brief Update the detector geometry given a region of interest
+     *
+     * @param roi
+     * @return DetectorGeometry
+     */
+    void update_geometry_with_roi(ROI roi);
+
+    size_t n_modules() const;
+
+    size_t n_modules_in_roi() const;
+
+    size_t pixels_x() const;
+    size_t pixels_y() const;
+
+    size_t modules_x() const;
+    size_t modules_y() const;
+
+    const std::vector<ssize_t> &get_modules_in_roi() const;
+
+    ssize_t get_modules_in_roi(const size_t index) const;
+
+    const std::vector<ModuleGeometry> &get_module_geometries() const;
+
+    const ModuleGeometry &get_module_geometries(const size_t index) const;
+
+  private:
+    size_t m_modules_x{};
+    size_t m_modules_y{};
+    size_t m_pixels_x{};
+    size_t m_pixels_y{};
+    static constexpr ModuleConfig cfg{0, 0};
+    std::vector<ModuleGeometry> module_geometries{};
+    std::vector<ssize_t> modules_in_roi{};
+};
+
+} // namespace aare

include/aare/Dtype.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include <cstdint>
 #include <map>
@@ -6,31 +7,37 @@
 
 namespace aare {
 
-// The format descriptor is a single character that specifies the type of the data
+// The format descriptor is a single character that specifies the type of the
+// data
 // - python documentation: https://docs.python.org/3/c-api/arg.html#numbers
-// - py::format_descriptor<T>::format() (in pybind11) does not return the same format as
+// - py::format_descriptor<T>::format() (in pybind11) does not return the same
+// format as
 //   written in python.org documentation.
-// - numpy also doesn't use the same format. and also numpy associates the format
+// - numpy also doesn't use the same format. and also numpy associates the
-//   with variable bitdepth types. (e.g. long is int64 on linux64 and int32 on win64)
+// format
-//   https://numpy.org/doc/stable/reference/arrays.scalars.html
+//   with variable bitdepth types. (e.g. long is int64 on linux64 and int32 on
+//   win64) https://numpy.org/doc/stable/reference/arrays.scalars.html
 //
 // github issue discussing this:
 // https://github.com/pybind/pybind11/issues/1908#issuecomment-658358767
 //
-// [IN LINUX] the difference is for int64 (long) and uint64 (unsigned long). The format
+// [IN LINUX] the difference is for int64 (long) and uint64 (unsigned long). The
-// descriptor is 'q' and 'Q' respectively and in the documentation it is 'l' and 'k'.
+// format descriptor is 'q' and 'Q' respectively and in the documentation it is
+// 'l' and 'k'.
 
 // in practice numpy doesn't seem to care when reading buffer info: the library
 // interprets 'q' or 'l' as int64 and 'Q' or 'L' as uint64.
-// for this reason we decided to use the same format descriptor as pybind to avoid
+// for this reason we decided to use the same format descriptor as pybind to
-// any further discrepancies.
+// avoid any further discrepancies.
 
 // in the following order:
 // int8, uint8, int16, uint16, int32, uint32, int64, uint64, float, double
-const char DTYPE_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i', 'I', 'q', 'Q', 'f', 'd'};
+const char DTYPE_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i',
+                                 'I', 'q', 'Q', 'f', 'd'};
 
 // on linux64 & apple
-const char NUMPY_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'f', 'd'};
+const char NUMPY_FORMAT_DSC[] = {'b', 'B', 'h', 'H', 'i',
+                                 'I', 'l', 'L', 'f', 'd'};
 /**
  * @brief enum class to define the endianess of the system
  */
@@ -52,12 +59,29 @@ enum class endian {
  */
 class Dtype {
   public:
-    enum TypeIndex { INT8, UINT8, INT16, UINT16, INT32, UINT32, INT64, UINT64, FLOAT, DOUBLE, ERROR, NONE };
+    enum TypeIndex {
+        INT8,
+        UINT8,
+        INT16,
+        UINT16,
+        INT32,
+        UINT32,
+        INT64,
+        UINT64,
+        FLOAT,
+        DOUBLE,
+        ERROR,
+        NONE
+    };
 
     uint8_t bitdepth() const;
     size_t bytes() const;
-    std::string format_descr() const { return std::string(1, DTYPE_FORMAT_DSC[static_cast<int>(m_type)]); }
+    std::string format_descr() const {
-    std::string numpy_descr() const { return std::string(1, NUMPY_FORMAT_DSC[static_cast<int>(m_type)]); }
+        return std::string(1, DTYPE_FORMAT_DSC[static_cast<int>(m_type)]);
+    }
+    std::string numpy_descr() const {
+        return std::string(1, NUMPY_FORMAT_DSC[static_cast<int>(m_type)]);
+    }
 
     explicit Dtype(const std::type_info &t);
     explicit Dtype(std::string_view sv);

include/aare/File.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/FileInterface.hpp"
 #include <memory>
@@ -6,11 +7,11 @@ namespace aare {
 
 /**
  * @brief RAII File class for reading, and in the future potentially writing
- * image files in various formats. Minimal generic interface. For specail fuctions
+ * image files in various formats. Minimal generic interface. For specail
- * plase use the RawFile or NumpyFile classes directly.
+ * fuctions plase use the RawFile or NumpyFile classes directly. Wraps
- * Wraps FileInterface to abstract the underlying file format
+ * FileInterface to abstract the underlying file format
- * @note **frame_number** refers the the frame number sent by the detector while **frame_index**
+ * @note **frame_number** refers the the frame number sent by the detector while
- * is the position of the frame in the file
+ * **frame_index** is the position of the frame in the file
  */
 class File {
     std::unique_ptr<FileInterface> file_impl;
@@ -25,42 +26,46 @@ class File {
      * @throws std::invalid_argument if the file mode is not supported
      *
      */
-    File(const std::filesystem::path &fname, const std::string &mode="r", const FileConfig &cfg = {});
+    File(const std::filesystem::path &fname, const std::string &mode = "r",
+         const FileConfig &cfg = {});
 
-     /**Since the object is responsible for managing the file we disable copy construction */
+    /**Since the object is responsible for managing the file we disable copy
+     * construction */
     File(File const &other) = delete;
 
     /**The same goes for copy assignment */
-    File& operator=(File const &other) = delete;
+    File &operator=(File const &other) = delete;
 
     File(File &&other) noexcept;
-    File& operator=(File &&other) noexcept;
+    File &operator=(File &&other) noexcept;
     ~File() = default;
 
     // void close();                             //!< close the file
 
-    Frame read_frame();                       //!< read one frame from the file at the current position
+    Frame
-    Frame read_frame(size_t frame_index);     //!< read one frame at the position given by frame number
+    read_frame(); //!< read one frame from the file at the current position
-    std::vector<Frame> read_n(size_t n_frames); //!< read n_frames from the file at the current position
+    Frame read_frame(size_t frame_index); //!< read one frame at the position
+                                          //!< given by frame number
+    std::vector<Frame> read_n(size_t n_frames); //!< read n_frames from the file
+                                                //!< at the current position
 
     void read_into(std::byte *image_buf);
     void read_into(std::byte *image_buf, size_t n_frames);
 
-    size_t frame_number();                    //!< get the frame number at the current position
+    size_t frame_number(); //!< get the frame number at the current position
-    size_t frame_number(size_t frame_index);  //!< get the frame number at the given frame index
+    size_t frame_number(
+        size_t frame_index); //!< get the frame number at the given frame index
     size_t bytes_per_frame() const;
     size_t pixels_per_frame() const;
     size_t bytes_per_pixel() const;
     size_t bitdepth() const;
-    void seek(size_t frame_index);          //!< seek to the given frame index
+    void seek(size_t frame_index); //!< seek to the given frame index
-    size_t tell() const;                     //!< get the frame index of the file pointer
+    size_t tell() const;           //!< get the frame index of the file pointer
     size_t total_frames() const;
     size_t rows() const;
     size_t cols() const;
 
     DetectorType detector_type() const;
-
-    
 };
 
 } // namespace aare

include/aare/FileInterface.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Dtype.hpp"
 #include "aare/Frame.hpp"
@@ -20,8 +21,10 @@ struct FileConfig {
     uint64_t rows{};
     uint64_t cols{};
     bool operator==(const FileConfig &other) const {
-        return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry &&
+        return dtype == other.dtype && rows == other.rows &&
-               detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file;
+               cols == other.cols && geometry == other.geometry &&
+               detector_type == other.detector_type &&
+               max_frames_per_file == other.max_frames_per_file;
     }
     bool operator!=(const FileConfig &other) const { return !(*this == other); }
 
@@ -32,8 +35,11 @@ struct FileConfig {
     int max_frames_per_file{};
     size_t total_frames{};
     std::string to_string() const {
-        return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) +
+        return "{ dtype: " + dtype.to_string() +
-               ", geometry: " + geometry.to_string() + ", detector_type: " + ToString(detector_type) +
+               ", rows: " + std::to_string(rows) +
+               ", cols: " + std::to_string(cols) +
+               ", geometry: " + geometry.to_string() +
+               ", detector_type: " + ToString(detector_type) +
                ", max_frames_per_file: " + std::to_string(max_frames_per_file) +
                ", total_frames: " + std::to_string(total_frames) + " }";
     }
@@ -42,7 +48,8 @@ struct FileConfig {
 /**
  * @brief FileInterface class to define the interface for file operations
  * @note parent class for NumpyFile and RawFile
- * @note all functions are pure virtual and must be implemented by the derived classes
+ * @note all functions are pure virtual and must be implemented by the derived
+ * classes
  */
 class FileInterface {
   public:
@@ -64,17 +71,20 @@ class FileInterface {
      * @param n_frames number of frames to read
      * @return vector of frames
      */
-    virtual std::vector<Frame> read_n(size_t n_frames) = 0; // Is this the right interface?
+    virtual std::vector<Frame>
+    read_n(size_t n_frames) = 0; // Is this the right interface?
 
     /**
-     * @brief read one frame from the file at the current position and store it in the provided buffer
+     * @brief read one frame from the file at the current position and store it
+     * in the provided buffer
      * @param image_buf buffer to store the frame
      * @return void
      */
     virtual void read_into(std::byte *image_buf) = 0;
 
     /**
-     * @brief read n_frames from the file at the current position and store them in the provided buffer
+     * @brief read n_frames from the file at the current position and store them
+     * in the provided buffer
      * @param image_buf buffer to store the frames
      * @param n_frames number of frames to read
      * @return void
@@ -134,7 +144,6 @@ class FileInterface {
      */
     virtual size_t bitdepth() const = 0;
 
-
     virtual DetectorType detector_type() const = 0;
 
     // function to query the data type of the file

include/aare/FilePtr.hpp

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+#include <cstdio>
+#include <filesystem>
+
+namespace aare {
+
+/**
+ * \brief RAII wrapper for FILE pointer
+ */
+class FilePtr {
+    FILE *fp_{nullptr};
+
+  public:
+    FilePtr() = default;
+    FilePtr(const std::filesystem::path &fname, const std::string &mode);
+    FilePtr(const FilePtr &) = delete;            // we don't want a copy
+    FilePtr &operator=(const FilePtr &) = delete; // since we handle a resource
+    FilePtr(FilePtr &&other);
+    FilePtr &operator=(FilePtr &&other);
+    FILE *get();
+    ssize_t tell();
+    void seek(ssize_t offset, int whence = SEEK_SET) {
+        if (fseek(fp_, offset, whence) != 0)
+            throw std::runtime_error("Error seeking in file");
+    }
+    std::string error_msg();
+    ~FilePtr();
+};
+
+} // namespace aare

include/aare/Fit.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include <cmath>
@@ -15,15 +16,27 @@ NDArray<double, 1> gaus(NDView<double, 1> x, NDView<double, 1> par);
 double pol1(const double x, const double *par);
 NDArray<double, 1> pol1(NDView<double, 1> x, NDView<double, 1> par);
 
-} // namespace func
+double scurve(const double x, const double *par);
+NDArray<double, 1> scurve(NDView<double, 1> x, NDView<double, 1> par);
 
+double scurve2(const double x, const double *par);
+NDArray<double, 1> scurve2(NDView<double, 1> x, NDView<double, 1> par);
+
+} // namespace func
 
 /**
  * @brief Estimate the initial parameters for a Gaussian fit
  */
-std::array<double, 3> gaus_init_par(const NDView<double, 1> x, const NDView<double, 1> y);
+std::array<double, 3> gaus_init_par(const NDView<double, 1> x,
+                                    const NDView<double, 1> y);
 
-std::array<double, 2> pol1_init_par(const NDView<double, 1> x, const NDView<double, 1> y);
+std::array<double, 2> pol1_init_par(const NDView<double, 1> x,
+                                    const NDView<double, 1> y);
+
+std::array<double, 6> scurve_init_par(const NDView<double, 1> x,
+                                      const NDView<double, 1> y);
+std::array<double, 6> scurve2_init_par(const NDView<double, 1> x,
+                                       const NDView<double, 1> y);
 
 static constexpr int DEFAULT_NUM_THREADS = 4;
 
@@ -34,46 +47,41 @@ static constexpr int DEFAULT_NUM_THREADS = 4;
  */
 NDArray<double, 1> fit_gaus(NDView<double, 1> x, NDView<double, 1> y);
 
-
 /**
  * @brief Fit a 1D Gaussian to each pixel. Data layout [row, col, values]
  * @param x x values
- * @param y y vales, layout [row, col, values]
+ * @param y y values, layout [row, col, values]
  * @param n_threads number of threads to use
  */
 
 NDArray<double, 3> fit_gaus(NDView<double, 1> x, NDView<double, 3> y,
                             int n_threads = DEFAULT_NUM_THREADS);
 
-
-
-
 /**
  * @brief Fit a 1D Gaussian with error estimates
  * @param x x values
- * @param y y vales, layout [row, col, values]
+ * @param y y values, layout [row, col, values]
  * @param y_err error in y, layout [row, col, values]
  * @param par_out output parameters
  * @param par_err_out output error parameters
  */
 void fit_gaus(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
               NDView<double, 1> par_out, NDView<double, 1> par_err_out,
-              double& chi2);
+              double &chi2);
 
 /**
  * @brief Fit a 1D Gaussian to each pixel with error estimates. Data layout
  * [row, col, values]
  * @param x x values
- * @param y y vales, layout [row, col, values]
+ * @param y y values, layout [row, col, values]
  * @param y_err error in y, layout [row, col, values]
  * @param par_out output parameters, layout [row, col, values]
  * @param par_err_out output parameter errors, layout [row, col, values]
  * @param n_threads number of threads to use
  */
 void fit_gaus(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
-              NDView<double, 3> par_out, NDView<double, 3> par_err_out, NDView<double, 2> chi2_out,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out,
-              int n_threads = DEFAULT_NUM_THREADS
+              NDView<double, 2> chi2_out, int n_threads = DEFAULT_NUM_THREADS);
-              );
 
 NDArray<double, 1> fit_pol1(NDView<double, 1> x, NDView<double, 1> y);
 
@@ -81,12 +89,33 @@ NDArray<double, 3> fit_pol1(NDView<double, 1> x, NDView<double, 3> y,
                             int n_threads = DEFAULT_NUM_THREADS);
 
 void fit_pol1(NDView<double, 1> x, NDView<double, 1> y, NDView<double, 1> y_err,
-              NDView<double, 1> par_out, NDView<double, 1> par_err_out, double& chi2);
+              NDView<double, 1> par_out, NDView<double, 1> par_err_out,
+              double &chi2);
 
 // TODO! not sure we need to offer the different version in C++
 void fit_pol1(NDView<double, 1> x, NDView<double, 3> y, NDView<double, 3> y_err,
-              NDView<double, 3> par_out, NDView<double, 3> par_err_out,NDView<double, 2> chi2_out,
+              NDView<double, 3> par_out, NDView<double, 3> par_err_out,
-              int n_threads = DEFAULT_NUM_THREADS);
+              NDView<double, 2> chi2_out, int n_threads = DEFAULT_NUM_THREADS);
 
+NDArray<double, 1> fit_scurve(NDView<double, 1> x, NDView<double, 1> y);
+NDArray<double, 3> fit_scurve(NDView<double, 1> x, NDView<double, 3> y,
+                              int n_threads);
+void fit_scurve(NDView<double, 1> x, NDView<double, 1> y,
+                NDView<double, 1> y_err, NDView<double, 1> par_out,
+                NDView<double, 1> par_err_out, double &chi2);
+void fit_scurve(NDView<double, 1> x, NDView<double, 3> y,
+                NDView<double, 3> y_err, NDView<double, 3> par_out,
+                NDView<double, 3> par_err_out, NDView<double, 2> chi2_out,
+                int n_threads);
 
+NDArray<double, 1> fit_scurve2(NDView<double, 1> x, NDView<double, 1> y);
+NDArray<double, 3> fit_scurve2(NDView<double, 1> x, NDView<double, 3> y,
+                               int n_threads);
+void fit_scurve2(NDView<double, 1> x, NDView<double, 1> y,
+                 NDView<double, 1> y_err, NDView<double, 1> par_out,
+                 NDView<double, 1> par_err_out, double &chi2);
+void fit_scurve2(NDView<double, 1> x, NDView<double, 3> y,
+                 NDView<double, 3> y_err, NDView<double, 3> par_out,
+                 NDView<double, 3> par_err_out, NDView<double, 2> chi2_out,
+                 int n_threads);
 } // namespace aare

include/aare/Frame.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Dtype.hpp"
 #include "aare/NDArray.hpp"
@@ -19,7 +20,7 @@ class Frame {
     uint32_t m_cols;
     Dtype m_dtype;
     std::byte *m_data;
-    //TODO! Add frame number?
+    // TODO! Add frame number?
 
   public:
     /**
@@ -39,7 +40,7 @@ class Frame {
      * @param dtype data type of the pixels
      */
     Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype);
-    ~Frame(){ delete[] m_data; };
+    ~Frame() { delete[] m_data; };
 
     /** @warning Copy is disabled to ensure performance when passing
      * frames around. Can discuss enabling it.
@@ -52,7 +53,6 @@ class Frame {
     Frame &operator=(Frame &&other) noexcept;
     Frame(Frame &&other) noexcept;
 
-
     Frame clone() const; //<- Explicit copy
 
     uint32_t rows() const;
@@ -93,7 +93,7 @@ class Frame {
         if (row >= m_rows || col >= m_cols) {
             throw std::out_of_range("Invalid row or column index");
         }
-        //TODO! add tests then reimplement using pixel_ptr
+        // TODO! add tests then reimplement using pixel_ptr
         T data;
         std::memcpy(&data, m_data + (row * m_cols + col) * m_dtype.bytes(),
                     m_dtype.bytes());
@@ -106,9 +106,9 @@ class Frame {
      * @tparam T type of the pixels
      * @return NDView<T, 2>
      */
-    template <typename T> NDView<T, 2> view() {
+    template <typename T> NDView<T, 2> view() & {
-        std::array<int64_t, 2> shape = {static_cast<int64_t>(m_rows),
+        std::array<ssize_t, 2> shape = {static_cast<ssize_t>(m_rows),
-                                        static_cast<int64_t>(m_cols)};
+                                        static_cast<ssize_t>(m_cols)};
         T *data = reinterpret_cast<T *>(m_data);
         return NDView<T, 2>(data, shape);
     }

include/aare/GainMap.hpp

@@ -1,6 +1,8 @@
+// SPDX-License-Identifier: MPL-2.0
 /************************************************
- * @file ApplyGainMap.hpp
+ * @file GainMap.hpp
- * @short function to apply gain map of image size to a vector of clusters
+ * @short function to apply gain map of image size to a vector of clusters -
+ *note stored gainmap is inverted for efficient aaplication to images
  ***********************************************/
 
 #pragma once
@@ -12,14 +14,21 @@
 
 namespace aare {
 
-class GainMap {
+class InvertedGainMap {
 
   public:
-    explicit GainMap(const NDArray<double, 2> &gain_map)
+    explicit InvertedGainMap(const NDArray<double, 2> &gain_map)
-        : m_gain_map(gain_map) {};
+        : m_gain_map(gain_map) {
+        for (auto &item : m_gain_map) {
+            item = 1.0 / item;
+        }
+    };
 
-    explicit GainMap(const NDView<double, 2> gain_map) {
+    explicit InvertedGainMap(const NDView<double, 2> gain_map) {
         m_gain_map = NDArray<double, 2>(gain_map);
+        for (auto &item : m_gain_map) {
+            item = 1.0 / item;
+        }
     }
 
     template <typename ClusterType,
@@ -34,19 +43,21 @@ class GainMap {
         int64_t index_cluster_center_x = ClusterSizeX / 2;
         int64_t index_cluster_center_y = ClusterSizeY / 2;
         for (size_t i = 0; i < clustervec.size(); i++) {
-            auto &cl = clustervec.at(i);
+            auto &cl = clustervec[i];
 
             if (cl.x > 0 && cl.y > 0 && cl.x < m_gain_map.shape(1) - 1 &&
                 cl.y < m_gain_map.shape(0) - 1) {
                 for (size_t j = 0; j < ClusterSizeX * ClusterSizeY; j++) {
                     size_t x = cl.x + j % ClusterSizeX - index_cluster_center_x;
                     size_t y = cl.y + j / ClusterSizeX - index_cluster_center_y;
-                    cl.data[j] = cl.data[j] * static_cast<T>(m_gain_map(y, x));
+                    cl.data[j] = static_cast<T>(
+                        static_cast<double>(cl.data[j]) *
+                        m_gain_map(
+                            y, x)); // cast after conversion to keep precision
                 }
             } else {
-                memset(cl.data, 0,
+                // clear edge clusters
-                       ClusterSizeX * ClusterSizeY *
+                cl.data.fill(0);
-                           sizeof(T)); // clear edge clusters
             }
         }
     }

include/aare/Interpolator.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include "aare/CalculateEta.hpp"
@@ -17,7 +18,10 @@ struct Photon {
 };
 
 class Interpolator {
+    // marginal CDF of eta_x (if rosenblatt applied), conditional
+    // CDF of eta_x conditioned on eta_y
     NDArray<double, 3> m_ietax;
+    // conditional CDF of eta_y conditioned on eta_x
     NDArray<double, 3> m_ietay;
 
     NDArray<double, 1> m_etabinsx;
@@ -25,108 +29,210 @@ class Interpolator {
     NDArray<double, 1> m_energy_bins;
 
   public:
+    /**
+     * @brief Constructor for the Interpolator class
+     * @param etacube joint distribution of etaX, etaY and photon energy
+     * @param xbins bin edges for etaX
+     * @param ybins bin edges for etaY
+     * @param ebins bin edges for photon energy
+     * @note note first dimension is etaX, second etaY, third photon energy
+     */
     Interpolator(NDView<double, 3> etacube, NDView<double, 1> xbins,
                  NDView<double, 1> ybins, NDView<double, 1> ebins);
+
+    /**
+     * @brief Constructor for the Interpolator class
+     * @param xbins bin edges for etaX
+     * @param ybins bin edges for etaY
+     * @param ebins bin edges for photon energy
+     */
+    Interpolator(NDView<double, 1> xbins, NDView<double, 1> ybins,
+                 NDView<double, 1> ebins);
+
+    /**
+     * @brief transforms the joint eta distribution of etaX and etaY to the two
+     * independant uniform distributions based on the Roseblatt transform for
+     * each energy level
+     * @param etacube joint distribution of etaX, etaY and photon energy
+     * @note note first dimension is etaX, second etaY, third photon energy
+     */
+    void rosenblatttransform(NDView<double, 3> etacube);
+
     NDArray<double, 3> get_ietax() { return m_ietax; }
     NDArray<double, 3> get_ietay() { return m_ietay; }
 
-    template <typename ClusterType,
+    /**
+     * @brief interpolates the cluster centers for all clusters to a better
+     * precision
+     * @tparam ClusterType Type of Clusters to interpolate
+     * @tparam Etafunction Function object that calculates desired eta default:
+     * calculate_eta2
+     * @return interpolated photons (photon positions are given as double but
+     * following row column format e.g. x=0, y=0 means top row and first column
+     * of frame)
+     */
+    template <auto EtaFunction = calculate_eta2, typename ClusterType,
               typename Eanble = std::enable_if_t<is_cluster_v<ClusterType>>>
     std::vector<Photon> interpolate(const ClusterVector<ClusterType> &clusters);
+
+  private:
+    /**
+     * @brief implements underlying interpolation logic based on EtaFunction
+     * Type
+     * @tparam EtaFunction Function object that calculates desired eta default:
+     * @param u: transformed photon position in x between [0,1]
+     * @param v: transformed photon position in y between [0,1]
+     * @param c: corner of eta
+     */
+    template <auto EtaFunction, typename ClusterType>
+    void interpolation_logic(Photon &photon, const double u, const double v,
+                             const corner c = corner::cTopLeft);
+
+    /**
+     *  @brief bilinear interpolation of the transformed eta values
+     * @param ix index of etaX bin
+     * @param iy index of etaY bin
+     * @param ie index of energy bin
+     * @return pair of interpolated transformed eta values (ietax, ietay)
+     */
+    template <typename T>
+    std::pair<double, double>
+    bilinear_interpolation(const size_t ix, const size_t iy, const size_t ie,
+                           const Eta2<T> &eta);
 };
 
-// TODO: generalize to support any clustertype!!! otherwise add std::enable_if_t
+template <typename T>
-// to only take Cluster2x2 and Cluster3x3
+std::pair<double, double>
-template <typename ClusterType, typename Enable>
+Interpolator::bilinear_interpolation(const size_t ix, const size_t iy,
+                                     const size_t ie, const Eta2<T> &eta) {
+    auto next_index_y = static_cast<ssize_t>(iy + 1) >= m_ietax.shape(1)
+                            ? m_ietax.shape(1) - 1
+                            : iy + 1;
+    auto next_index_x = static_cast<ssize_t>(ix + 1) >= m_ietax.shape(0)
+                            ? m_ietax.shape(0) - 1
+                            : ix + 1;
+
+    // bilinear interpolation
+    double ietax_interp_left = linear_interpolation(
+        {m_etabinsy(iy), m_etabinsy(iy + 1)},
+        {m_ietax(ix, iy, ie), m_ietax(ix, next_index_y, ie)}, eta.y);
+    double ietax_interp_right =
+        linear_interpolation({m_etabinsy(iy), m_etabinsy(iy + 1)},
+                             {m_ietax(next_index_x, iy, ie),
+                              m_ietax(next_index_x, next_index_y, ie)},
+                             eta.y);
+
+    // transformed photon position x between [0,1]
+    double ietax_interpolated =
+        linear_interpolation({m_etabinsx(ix), m_etabinsx(ix + 1)},
+                             {ietax_interp_left, ietax_interp_right}, eta.x);
+
+    double ietay_interp_left = linear_interpolation(
+        {m_etabinsx(ix), m_etabinsx(ix + 1)},
+        {m_ietay(ix, iy, ie), m_ietay(next_index_x, iy, ie)}, eta.x);
+    double ietay_interp_right =
+        linear_interpolation({m_etabinsx(ix), m_etabinsx(ix + 1)},
+                             {m_ietay(ix, next_index_y, ie),
+                              m_ietay(next_index_x, next_index_y, ie)},
+                             eta.x);
+
+    // transformed photon position y between [0,1]
+    double ietay_interpolated =
+        linear_interpolation({m_etabinsy(iy), m_etabinsy(iy + 1)},
+                             {ietay_interp_left, ietay_interp_right}, eta.y);
+
+    return {ietax_interpolated, ietay_interpolated};
+}
+
+template <auto EtaFunction, typename ClusterType, typename Enable>
 std::vector<Photon>
 Interpolator::interpolate(const ClusterVector<ClusterType> &clusters) {
     std::vector<Photon> photons;
     photons.reserve(clusters.size());
 
-    if (clusters.cluster_size_x() == 3 || clusters.cluster_size_y() == 3) {
+    for (const ClusterType &cluster : clusters) {
-        for (size_t i = 0; i < clusters.size(); i++) {
 
-            auto cluster = clusters.at(i);
+        auto eta = EtaFunction(cluster);
-            auto eta = calculate_eta2(cluster);
 
-            Photon photon;
+        Photon photon;
-            photon.x = cluster.x;
+        photon.x = cluster.x;
-            photon.y = cluster.y;
+        photon.y = cluster.y;
-            photon.energy = eta.sum;
+        photon.energy = static_cast<decltype(photon.energy)>(eta.sum);
 
-            // auto ie = nearest_index(m_energy_bins, photon.energy)-1;
+        // std::cout << "eta.x: " << eta.x << " eta.y: " << eta.y << std::endl;
-            // auto ix = nearest_index(m_etabinsx, eta.x)-1;
-            // auto iy = nearest_index(m_etabinsy, eta.y)-1;
-            // Finding the index of the last element that is smaller
-            // should work fine as long as we have many bins
-            auto ie = last_smaller(m_energy_bins, photon.energy);
-            auto ix = last_smaller(m_etabinsx, eta.x);
-            auto iy = last_smaller(m_etabinsy, eta.y);
 
-            // fmt::print("ex: {}, ix: {}, iy: {}\n", ie, ix, iy);
+        // Finding the index of the last element that is smaller
+        // should work fine as long as we have many bins
+        auto ie = last_smaller(m_energy_bins, photon.energy);
+        auto ix = last_smaller(m_etabinsx, eta.x);
+        auto iy = last_smaller(m_etabinsy, eta.y);
 
-            double dX, dY;
+        // std::cout << "ix: " << ix << " iy: " << iy << std::endl;
-            // cBottomLeft = 0,
-            // cBottomRight = 1,
-            // cTopLeft = 2,
-            // cTopRight = 3
-            switch (eta.c) {
-            case cTopLeft:
-                dX = -1.;
-                dY = 0;
-                break;
-            case cTopRight:;
-                dX = 0;
-                dY = 0;
-                break;
-            case cBottomLeft:
-                dX = -1.;
-                dY = -1.;
-                break;
-            case cBottomRight:
-                dX = 0.;
-                dY = -1.;
-                break;
-            }
-            photon.x += m_ietax(ix, iy, ie) * 2 + dX;
-            photon.y += m_ietay(ix, iy, ie) * 2 + dY;
-            photons.push_back(photon);
-        }
-    } else if (clusters.cluster_size_x() == 2 ||
-               clusters.cluster_size_y() == 2) {
-        for (size_t i = 0; i < clusters.size(); i++) {
-            auto cluster = clusters.at(i);
-            auto eta = calculate_eta2(cluster);
 
-            Photon photon;
+        // TODO: bilinear interpolation only works if all bins have a size > 1 -
-            photon.x = cluster.x;
+        // otherwise bilinear interpolation with zero values which skew the
-            photon.y = cluster.y;
+        // results
-            photon.energy = eta.sum;
+        // TODO: maybe trim the bins at the edges with zero values beforehand
+        // auto [ietax_interpolated, ietay_interpolated] =
+        // bilinear_interpolation(ix, iy, ie, eta);
 
-            // Now do some actual interpolation.
+        double ietax_interpolated = m_ietax(ix, iy, ie);
-            // Find which energy bin the cluster is in
+        double ietay_interpolated = m_ietay(ix, iy, ie);
-            //  auto ie = nearest_index(m_energy_bins, photon.energy)-1;
-            //  auto ix = nearest_index(m_etabinsx, eta.x)-1;
-            //  auto iy = nearest_index(m_etabinsy, eta.y)-1;
-            // Finding the index of the last element that is smaller
-            // should work fine as long as we have many bins
-            auto ie = last_smaller(m_energy_bins, photon.energy);
-            auto ix = last_smaller(m_etabinsx, eta.x);
-            auto iy = last_smaller(m_etabinsy, eta.y);
 
-            photon.x += m_ietax(ix, iy, ie) *
+        interpolation_logic<EtaFunction, ClusterType>(
-                        2; // eta goes between 0 and 1 but we could move the hit
+            photon, ietax_interpolated, ietay_interpolated, eta.c);
-                           // anywhere in the 2x2
-            photon.y += m_ietay(ix, iy, ie) * 2;
-            photons.push_back(photon);
-        }
 
-    } else {
+        photons.push_back(photon);
-        throw std::runtime_error(
-            "Only 3x3 and 2x2 clusters are supported for interpolation");
     }
 
     return photons;
 }
 
+template <auto EtaFunction, typename ClusterType>
+void Interpolator::interpolation_logic(Photon &photon, const double u,
+                                       const double v, const corner c) {
+
+    // std::cout << "u: " << u << " v: " << v << std::endl;
+
+    // TODO: try to call this with std::is_same_v and have it constexpr if
+    // possible
+    if (EtaFunction == &calculate_eta2<typename ClusterType::value_type,
+                                       ClusterType::cluster_size_x,
+                                       ClusterType::cluster_size_y,
+                                       typename ClusterType::coord_type> ||
+        EtaFunction == &calculate_full_eta2<typename ClusterType::value_type,
+                                            ClusterType::cluster_size_x,
+                                            ClusterType::cluster_size_y,
+                                            typename ClusterType::coord_type>) {
+        double dX{}, dY{};
+
+        // TODO: could also chaneg the sign of the eta calculation
+        switch (c) {
+        case corner::cTopLeft:
+            dX = -1.0;
+            dY = -1.0;
+            break;
+        case corner::cTopRight:;
+            dX = 0.0;
+            dY = -1.0;
+            break;
+        case corner::cBottomLeft:
+            dX = -1.0;
+            dY = 0.0;
+            break;
+        case corner::cBottomRight:
+            dX = 0.0;
+            dY = 0.0;
+            break;
+        }
+        photon.x = photon.x + 0.5 + u + dX; // use pixel center + 0.5
+        photon.y = photon.y + 0.5 + v +
+                   dY; // eta2 calculates the ratio between bottom and sum of
+                       // bottom and top  shift by 1 add eta value correctly
+    } else {
+        photon.x += u;
+        photon.y += v;
+    }
+}
+
 } // namespace aare

include/aare/JungfrauDataFile.hpp

@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+#include <cstdint>
+#include <filesystem>
+#include <vector>
+
+#include "aare/FileInterface.hpp"
+#include "aare/FilePtr.hpp"
+#include "aare/NDArray.hpp"
+#include "aare/defs.hpp"
+namespace aare {
+
+struct JungfrauDataHeader {
+    uint64_t framenum;
+    uint64_t bunchid;
+};
+
+class JungfrauDataFile : public FileInterface {
+
+    size_t m_rows{}; //!< number of rows in the image, from find_frame_size();
+    size_t
+        m_cols{}; //!< number of columns in the image, from find_frame_size();
+    size_t m_bytes_per_frame{}; //!< number of bytes per frame excluding header
+    size_t m_total_frames{}; //!< total number of frames in the series of files
+    size_t m_offset{}; //!< file index of the first file, allow starting at non
+                       //!< zero file
+    size_t m_current_file_index{};  //!< The index of the open file
+    size_t m_current_frame_index{}; //!< The index of the current frame (with
+                                    //!< reference to all files)
+
+    std::vector<size_t>
+        m_last_frame_in_file{};   //!< Used for seeking to the correct file
+    std::filesystem::path m_path; //!< path to the files
+    std::string m_base_name;      //!< base name used for formatting file names
+
+    FilePtr m_fp; //!< RAII wrapper for a FILE*
+
+    using pixel_type = uint16_t;
+    static constexpr size_t header_size = sizeof(JungfrauDataHeader);
+    static constexpr size_t n_digits_in_file_index =
+        6; //!< to format file names
+
+  public:
+    JungfrauDataFile(const std::filesystem::path &fname);
+
+    std::string base_name()
+        const; //!< get the base name of the file (without path and extension)
+    size_t bytes_per_frame() override;
+    size_t pixels_per_frame() override;
+    size_t bytes_per_pixel() const;
+    size_t bitdepth() const override;
+    void seek(size_t frame_index)
+        override; //!< seek to the given frame index (note not byte offset)
+    size_t tell() override; //!< get the frame index of the file pointer
+    size_t total_frames() const override;
+    size_t rows() const override;
+    size_t cols() const override;
+    std::array<ssize_t, 2> shape() const;
+    size_t n_files() const; //!< get the number of files in the series.
+
+    // Extra functions needed for FileInterface
+    Frame read_frame() override;
+    Frame read_frame(size_t frame_number) override;
+    std::vector<Frame> read_n(size_t n_frames = 0) override;
+    void read_into(std::byte *image_buf) override;
+    void read_into(std::byte *image_buf, size_t n_frames) override;
+    size_t frame_number(size_t frame_index) override;
+    DetectorType detector_type() const override;
+
+    /**
+     * @brief Read a single frame from the file into the given buffer.
+     * @param image_buf buffer to read the frame into. (Note the caller is
+     * responsible for allocating the buffer)
+     * @param header pointer to a JungfrauDataHeader or nullptr to skip header)
+     */
+    void read_into(std::byte *image_buf, JungfrauDataHeader *header = nullptr);
+
+    /**
+     * @brief Read a multiple frames from the file into the given buffer.
+     * @param image_buf buffer to read the frame into. (Note the caller is
+     * responsible for allocating the buffer)
+     * @param n_frames number of frames to read
+     * @param header pointer to a JungfrauDataHeader or nullptr to skip header)
+     */
+    void read_into(std::byte *image_buf, size_t n_frames,
+                   JungfrauDataHeader *header = nullptr);
+
+    /**
+     * @brief Read a single frame from the file into the given NDArray
+     * @param image NDArray to read the frame into.
+     */
+    void read_into(NDArray<uint16_t> *image,
+                   JungfrauDataHeader *header = nullptr);
+
+    JungfrauDataHeader read_header();
+    std::filesystem::path current_file() const {
+        return fpath(m_current_file_index + m_offset);
+    }
+
+  private:
+    /**
+     * @brief Find the size of the frame in the file. (256x256, 256x1024,
+     * 512x1024)
+     * @param fname path to the file
+     * @throws std::runtime_error if the file is empty or the size cannot be
+     * determined
+     */
+    void find_frame_size(const std::filesystem::path &fname);
+
+    void parse_fname(const std::filesystem::path &fname);
+    void scan_files();
+    void open_file(size_t file_index);
+    std::filesystem::path fpath(size_t frame_index) const;
+};
+
+} // namespace aare

include/aare/NDArray.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 /*
 Container holding image data, or a time series of image data in contigious
@@ -21,12 +22,11 @@ TODO! Add expression templates for operators
 
 namespace aare {
 
-
+template <typename T, ssize_t Ndim = 2>
-template <typename T, int64_t Ndim = 2>
 class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
-    std::array<int64_t, Ndim> shape_;
+    std::array<ssize_t, Ndim> shape_;
-    std::array<int64_t, Ndim> strides_;
+    std::array<ssize_t, Ndim> strides_;
-    size_t size_{};
+    size_t size_{}; //TODO! do we need to store size when we have shape?
     T *data_;
 
   public:
@@ -42,20 +42,18 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
      *
      * @param shape shape of the new NDArray
      */
-    explicit NDArray(std::array<int64_t, Ndim> shape)
+    explicit NDArray(std::array<ssize_t, Ndim> shape)
         : shape_(shape), strides_(c_strides<Ndim>(shape_)),
-          size_(std::accumulate(shape_.begin(), shape_.end(), 1,
+          size_(num_elements(shape_)),
-                                std::multiplies<>())),
           data_(new T[size_]) {}
 
-
     /**
      * @brief Construct a new NDArray object with a shape and value.
      *
      * @param shape shape of the new array
      * @param value value to initialize the array with
      */
-    NDArray(std::array<int64_t, Ndim> shape, T value) : NDArray(shape) {
+    NDArray(std::array<ssize_t, Ndim> shape, T value) : NDArray(shape) {
         this->operator=(value);
     }
 
@@ -69,8 +67,8 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
         std::copy(v.begin(), v.end(), begin());
     }
 
-    template<size_t Size>
+    template <size_t Size>
-    NDArray(const std::array<T, Size>& arr) : NDArray<T,1>({Size}) {
+    NDArray(const std::array<T, Size> &arr) : NDArray<T, 1>({Size}) {
         std::copy(arr.begin(), arr.end(), begin());
     }
 
@@ -79,7 +77,24 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
         : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)),
           size_(other.size_), data_(other.data_) {
         other.reset(); // TODO! is this necessary?
+    }
 
+
+    //Move constructor from an an array with Ndim + 1
+    template <ssize_t M, typename = std::enable_if_t<(M == Ndim + 1)>>
+    NDArray(NDArray<T, M> &&other) 
+        : shape_(drop_first_dim(other.shape())),
+          strides_(c_strides<Ndim>(shape_)), size_(num_elements(shape_)),
+          data_(other.data()) {
+
+            // For now only allow move if the size matches, to avoid unreachable data
+            // if the use case arises we can remove this check
+            if(size() != other.size()) {
+                data_ = nullptr; // avoid double free, other will clean up the memory in it's destructor
+                throw std::runtime_error(LOCATION +
+                                         "Size mismatch in move constructor of NDArray<T, Ndim-1>");
+            }
+        other.reset();
     }
 
     // Copy constructor
@@ -113,10 +128,10 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     NDArray &operator-=(const NDArray &other);
     NDArray &operator*=(const NDArray &other);
 
-    //Write directly to the data array, or create a new one
+    // Write directly to the data array, or create a new one
-    template<size_t Size>
+    template <size_t Size>
-    NDArray<T,1>& operator=(const std::array<T,Size> &other){
+    NDArray<T, 1> &operator=(const std::array<T, Size> &other) {
-        if(Size != size_){
+        if (Size != size_) {
             delete[] data_;
             size_ = Size;
             data_ = new T[size_];
@@ -157,11 +172,6 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
 
     NDArray &operator&=(const T & /*mask*/);
 
-  
-
-
-    
-
     void sqrt() {
         for (int i = 0; i < size_; ++i) {
             data_[i] = std::sqrt(data_[i]);
@@ -186,22 +196,22 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
     }
 
     // TODO! is int the right type for index?
-    T &operator()(int64_t i) { return data_[i]; }
+    T &operator()(ssize_t i) { return data_[i]; }
-    const T &operator()(int64_t i) const { return data_[i]; }
+    const T &operator()(ssize_t i) const { return data_[i]; }
 
-    T &operator[](int64_t i) { return data_[i]; }
+    T &operator[](ssize_t i) { return data_[i]; }
-    const T &operator[](int64_t i) const { return data_[i]; }
+    const T &operator[](ssize_t i) const { return data_[i]; }
 
     T *data() { return data_; }
     std::byte *buffer() { return reinterpret_cast<std::byte *>(data_); }
-    size_t size() const { return size_; }
+    ssize_t size() const { return static_cast<ssize_t>(size_); }
     size_t total_bytes() const { return size_ * sizeof(T); }
-    std::array<int64_t, Ndim> shape() const noexcept { return shape_; }
+    std::array<ssize_t, Ndim> shape() const noexcept { return shape_; }
-    int64_t shape(int64_t i) const noexcept { return shape_[i]; }
+    ssize_t shape(ssize_t i) const noexcept { return shape_[i]; }
-    std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
+    std::array<ssize_t, Ndim> strides() const noexcept { return strides_; }
     size_t bitdepth() const noexcept { return sizeof(T) * 8; }
 
-    std::array<int64_t, Ndim> byte_strides() const noexcept {
+    std::array<ssize_t, Ndim> byte_strides() const noexcept {
         auto byte_strides = strides_;
         for (auto &val : byte_strides)
             val *= sizeof(T);
@@ -228,7 +238,7 @@ class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
 };
 
 // Move assign
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &
 NDArray<T, Ndim>::operator=(NDArray<T, Ndim> &&other) noexcept {
     if (this != &other) {
@@ -242,7 +252,7 @@ NDArray<T, Ndim>::operator=(NDArray<T, Ndim> &&other) noexcept {
     return *this;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
@@ -254,7 +264,7 @@ NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const NDArray<T, Ndim> &other) {
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
@@ -266,7 +276,7 @@ NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const NDArray<T, Ndim> &other) {
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
@@ -278,14 +288,14 @@ NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const NDArray<T, Ndim> &other) {
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator&=(const T &mask) {
     for (auto it = begin(); it != end(); ++it)
         *it &= mask;
     return *this;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<bool, Ndim> NDArray<T, Ndim>::operator>(const NDArray &other) {
     if (shape_ == other.shape_) {
         NDArray<bool, Ndim> result{shape_};
@@ -297,7 +307,7 @@ NDArray<bool, Ndim> NDArray<T, Ndim>::operator>(const NDArray &other) {
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const NDArray<T, Ndim> &other) {
     if (this != &other) {
         delete[] data_;
@@ -310,7 +320,7 @@ NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const NDArray<T, Ndim> &other) {
     return *this;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 bool NDArray<T, Ndim>::operator==(const NDArray<T, Ndim> &other) const {
     if (shape_ != other.shape_)
         return false;
@@ -322,83 +332,74 @@ bool NDArray<T, Ndim>::operator==(const NDArray<T, Ndim> &other) const {
     return true;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 bool NDArray<T, Ndim>::operator!=(const NDArray<T, Ndim> &other) const {
     return !((*this) == other);
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator++() {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] += 1;
     return *this;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const T &value) {
     std::fill_n(data_, size_, value);
     return *this;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] += value;
     return *this;
 }
 
-
+template <typename T, ssize_t Ndim>
-
-
-template <typename T, int64_t Ndim>
 NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const T &value) {
     NDArray result = *this;
     result += value;
     return result;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] -= value;
     return *this;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const T &value) {
     NDArray result = *this;
     result -= value;
     return result;
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator/=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] /= value;
     return *this;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const T &value) {
     NDArray result = *this;
     result /= value;
     return result;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] *= value;
     return *this;
 }
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const T &value) {
     NDArray result = *this;
     result *= value;
     return result;
 }
-// template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
-//     if (shape_[0] < 20 && shape_[1] < 20)
-//         Print_all();
-//     else
-//         Print_some();
-// }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 std::ostream &operator<<(std::ostream &os, const NDArray<T, Ndim> &arr) {
     for (auto row = 0; row < arr.shape(0); ++row) {
         for (auto col = 0; col < arr.shape(1); ++col) {
@@ -410,7 +411,7 @@ std::ostream &operator<<(std::ostream &os, const NDArray<T, Ndim> &arr) {
     return os;
 }
 
-template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
+template <typename T, ssize_t Ndim> void NDArray<T, Ndim>::Print_all() {
     for (auto row = 0; row < shape_[0]; ++row) {
         for (auto col = 0; col < shape_[1]; ++col) {
             std::cout << std::setw(3);
@@ -419,7 +420,7 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
         std::cout << "\n";
     }
 }
-template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_some() {
+template <typename T, ssize_t Ndim> void NDArray<T, Ndim>::Print_some() {
     for (auto row = 0; row < 5; ++row) {
         for (auto col = 0; col < 5; ++col) {
             std::cout << std::setw(7);
@@ -429,7 +430,7 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_some() {
     }
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 void save(NDArray<T, Ndim> &img, std::string &pathname) {
     std::ofstream f;
     f.open(pathname, std::ios::binary);
@@ -437,9 +438,9 @@ void save(NDArray<T, Ndim> &img, std::string &pathname) {
     f.close();
 }
 
-template <typename T, int64_t Ndim>
+template <typename T, ssize_t Ndim>
 NDArray<T, Ndim> load(const std::string &pathname,
-                      std::array<int64_t, Ndim> shape) {
+                      std::array<ssize_t, Ndim> shape) {
     NDArray<T, Ndim> img{shape};
     std::ifstream f;
     f.open(pathname, std::ios::binary);
@@ -448,6 +449,23 @@ NDArray<T, Ndim> load(const std::string &pathname,
     return img;
 }
 
-
+template <typename RT, typename NT, typename DT, ssize_t Ndim>
+NDArray<RT, Ndim> safe_divide(const NDArray<NT, Ndim> &numerator,
+                              const NDArray<DT, Ndim> &denominator) {
+    if (numerator.shape() != denominator.shape()) {
+        throw std::runtime_error(
+            "Shapes of numerator and denominator must match");
+    }
+    NDArray<RT, Ndim> result(numerator.shape());
+    for (ssize_t i = 0; i < numerator.size(); ++i) {
+        if (denominator[i] != 0) {
+            result[i] =
+                static_cast<RT>(numerator[i]) / static_cast<RT>(denominator[i]);
+        } else {
+            result[i] = RT{0}; // or handle division by zero as needed
+        }
+    }
+    return result;
+}
 
 } // namespace aare

include/aare/NDView.hpp

@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
-#include "aare/defs.hpp"
 #include "aare/ArrayExpr.hpp"
+#include "aare/defs.hpp"
 
 #include <algorithm>
 #include <array>
@@ -14,10 +15,11 @@
 #include <vector>
 namespace aare {
 
-template <int64_t Ndim> using Shape = std::array<int64_t, Ndim>;
+template <ssize_t Ndim> using Shape = std::array<ssize_t, Ndim>;
 
 // TODO! fix mismatch between signed and unsigned
-template <int64_t Ndim> Shape<Ndim> make_shape(const std::vector<size_t> &shape) {
+template <ssize_t Ndim>
+Shape<Ndim> make_shape(const std::vector<size_t> &shape) {
     if (shape.size() != Ndim)
         throw std::runtime_error("Shape size mismatch");
     Shape<Ndim> arr;
@@ -25,67 +27,124 @@ template <int64_t Ndim> Shape<Ndim> make_shape(const std::vector<size_t> &shape)
     return arr;
 }
 
-template <int64_t Dim = 0, typename Strides> int64_t element_offset(const Strides & /*unused*/) { return 0; }
 
-template <int64_t Dim = 0, typename Strides, typename... Ix>
+/**
-int64_t element_offset(const Strides &strides, int64_t i, Ix... index) {
+ * @brief Helper function to drop the first dimension of a shape.
+ * This is useful when you want to create a 2D view from a 3D array.
+ * @param shape The shape to drop the first dimension from.
+ * @return A new shape with the first dimension dropped.
+ */
+template<size_t Ndim>
+Shape<Ndim-1> drop_first_dim(const Shape<Ndim> &shape) {
+    static_assert(Ndim > 1, "Cannot drop first dimension from a 1D shape");
+    Shape<Ndim - 1> new_shape;
+    std::copy(shape.begin() + 1, shape.end(), new_shape.begin());
+    return new_shape;
+}
+
+/**
+ * @brief Helper function when constructing NDArray/NDView. Calculates the number
+ * of elements in the resulting array from a shape.
+ * @param shape The shape to calculate the number of elements for.
+ * @return The number of elements in and NDArray/NDView of that shape.
+ */
+template <size_t Ndim>
+size_t num_elements(const Shape<Ndim> &shape) {
+    return std::accumulate(shape.begin(), shape.end(), 1,
+                           std::multiplies<size_t>());
+}
+
+template <ssize_t Dim = 0, typename Strides>
+ssize_t element_offset(const Strides & /*unused*/) {
+    return 0;
+}
+
+template <ssize_t Dim = 0, typename Strides, typename... Ix>
+ssize_t element_offset(const Strides &strides, ssize_t i, Ix... index) {
     return i * strides[Dim] + element_offset<Dim + 1>(strides, index...);
 }
 
-template <int64_t Ndim> std::array<int64_t, Ndim> c_strides(const std::array<int64_t, Ndim> &shape) {
+template <ssize_t Ndim>
-    std::array<int64_t, Ndim> strides{};
+std::array<ssize_t, Ndim> c_strides(const std::array<ssize_t, Ndim> &shape) {
+    std::array<ssize_t, Ndim> strides{};
     std::fill(strides.begin(), strides.end(), 1);
-    for (int64_t i = Ndim - 1; i > 0; --i) {
+    for (ssize_t i = Ndim - 1; i > 0; --i) {
         strides[i - 1] = strides[i] * shape[i];
     }
     return strides;
 }
 
-template <int64_t Ndim> std::array<int64_t, Ndim> make_array(const std::vector<int64_t> &vec) {
+template <ssize_t Ndim>
+std::array<ssize_t, Ndim> make_array(const std::vector<ssize_t> &vec) {
     assert(vec.size() == Ndim);
-    std::array<int64_t, Ndim> arr{};
+    std::array<ssize_t, Ndim> arr{};
     std::copy_n(vec.begin(), Ndim, arr.begin());
     return arr;
 }
 
-template <typename T, int64_t Ndim = 2> class NDView : public ArrayExpr<NDView<T, Ndim>, Ndim> {
+template <typename T, ssize_t Ndim = 2>
+class NDView : public ArrayExpr<NDView<T, Ndim>, Ndim> {
   public:
     NDView() = default;
     ~NDView() = default;
     NDView(const NDView &) = default;
     NDView(NDView &&) = default;
 
-    NDView(T *buffer, std::array<int64_t, Ndim> shape)
+    NDView(T *buffer, std::array<ssize_t, Ndim> shape)
         : buffer_(buffer), strides_(c_strides<Ndim>(shape)), shape_(shape),
-          size_(std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>())) {}
+          size_(std::accumulate(std::begin(shape), std::end(shape), 1,
+                                std::multiplies<>())) {}
                    
-    // NDView(T *buffer, const std::vector<int64_t> &shape)
+    template <typename... Ix>
-    //     : buffer_(buffer), strides_(c_strides<Ndim>(make_array<Ndim>(shape))), shape_(make_array<Ndim>(shape)),
+    std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
-    //       size_(std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>())) {}
-
-    template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
         return buffer_[element_offset(strides_, index...)];
     }
 
-    template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) const {
+    template <typename... Ix>
+    std::enable_if_t<sizeof...(Ix) == 1 && (Ndim > 1), NDView<T, Ndim - 1>> operator()(Ix... index) {
+        // return a view of the next dimension
+        std::array<ssize_t, Ndim - 1> new_shape{};
+        std::copy_n(shape_.begin() + 1, Ndim - 1, new_shape.begin());
+        return NDView<T, Ndim - 1>(&buffer_[element_offset(strides_, index...)],
+                                   new_shape);
+        
+    }
+
+    template <typename... Ix>
+    std::enable_if_t<sizeof...(Ix) == Ndim, const T &> operator()(Ix... index) const {
         return buffer_[element_offset(strides_, index...)];
     }
 
-    size_t size() const { return size_; }
+
+    ssize_t size() const { return static_cast<ssize_t>(size_); }
     size_t total_bytes() const { return size_ * sizeof(T); }
-    std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
+    std::array<ssize_t, Ndim> strides() const noexcept { return strides_; }
 
     T *begin() { return buffer_; }
     T *end() { return buffer_ + size_; }
     T const *begin() const { return buffer_; }
     T const *end() const { return buffer_ + size_; }
-    T &operator()(int64_t i) const { return buffer_[i]; }
+    
-    T &operator[](int64_t i) const { return buffer_[i]; }
+    
+
+
+
+    /**
+     * @brief Access element at index i.
+     */
+    T &operator[](ssize_t i)  { return buffer_[i]; }
+
+    /**
+     * @brief Access element at index i.
+     */
+    const T &operator[](ssize_t i) const { return buffer_[i]; }
 
     bool operator==(const NDView &other) const {
         if (size_ != other.size_)
             return false;
-        for (uint64_t i = 0; i != size_; ++i) {
+        if (shape_ != other.shape_)
+            return false;
+        for (size_t i = 0; i != size_; ++i) {
             if (buffer_[i] != other.buffer_[i])
                 return false;
         }
@@ -94,16 +153,21 @@ template <typename T, int64_t Ndim = 2> class NDView : public ArrayExpr<NDView<T
 
     NDView &operator+=(const T val) { return elemenwise(val, std::plus<T>()); }
     NDView &operator-=(const T val) { return elemenwise(val, std::minus<T>()); }
-    NDView &operator*=(const T val) { return elemenwise(val, std::multiplies<T>()); }
+    NDView &operator*=(const T val) {
-    NDView &operator/=(const T val) { return elemenwise(val, std::divides<T>()); }
+        return elemenwise(val, std::multiplies<T>());
+    }
+    NDView &operator/=(const T val) {
+        return elemenwise(val, std::divides<T>());
+    }
 
-    NDView &operator/=(const NDView &other) { return elemenwise(other, std::divides<T>()); }
+    NDView &operator/=(const NDView &other) {
+        return elemenwise(other, std::divides<T>());
+    }
 
-
+    template <size_t Size> NDView &operator=(const std::array<T, Size> &arr) {
-    template<size_t Size>
+        if (size() != static_cast<ssize_t>(arr.size()))
-    NDView& operator=(const std::array<T, Size> &arr) {
+            throw std::runtime_error(LOCATION +
-        if(size() != arr.size())
+                                     "Array and NDView size mismatch");
-            throw std::runtime_error(LOCATION + "Array and NDView size mismatch");
         std::copy(arr.begin(), arr.end(), begin());
         return *this;
     }
@@ -136,31 +200,51 @@ template <typename T, int64_t Ndim = 2> class NDView : public ArrayExpr<NDView<T
     }
 
     auto &shape() const { return shape_; }
-    auto shape(int64_t i) const { return shape_[i]; }
+    auto shape(ssize_t i) const { return shape_[i]; }
 
     T *data() { return buffer_; }
+    const T *data() const { return buffer_; }
     void print_all() const;
 
+    /**
+     * @brief Create a subview of a range of the first dimension. 
+     * This is useful for splitting a batches of frames in parallel processing.
+     * @param first The first index of the subview (inclusive).
+     * @param last The last index of the subview (exclusive).
+     * @return A new NDView that is a subview of the current view.
+     * @throws std::runtime_error if the range is invalid.
+     */
+    NDView sub_view(ssize_t first, ssize_t last) const {
+        if (first < 0 || last > shape_[0] || first >= last)
+            throw std::runtime_error(LOCATION + "Invalid sub_view range");
+        auto new_shape = shape_;
+        new_shape[0] = last - first;
+        return NDView(buffer_ + first * strides_[0], new_shape);
+    }
+
   private:
     T *buffer_{nullptr};
-    std::array<int64_t, Ndim> strides_{};
+    std::array<ssize_t, Ndim> strides_{};
-    std::array<int64_t, Ndim> shape_{};
+    std::array<ssize_t, Ndim> shape_{};
     uint64_t size_{};
 
-    template <class BinaryOperation> NDView &elemenwise(T val, BinaryOperation op) {
+    template <class BinaryOperation>
+    NDView &elemenwise(T val, BinaryOperation op) {
         for (uint64_t i = 0; i != size_; ++i) {
             buffer_[i] = op(buffer_[i], val);
         }
         return *this;
     }
-    template <class BinaryOperation> NDView &elemenwise(const NDView &other, BinaryOperation op) {
+    template <class BinaryOperation>
+    NDView &elemenwise(const NDView &other, BinaryOperation op) {
         for (uint64_t i = 0; i != size_; ++i) {
             buffer_[i] = op(buffer_[i], other.buffer_[i]);
         }
         return *this;
     }
 };
-template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
+
+template <typename T, ssize_t Ndim> void NDView<T, Ndim>::print_all() const {
     for (auto row = 0; row < shape_[0]; ++row) {
         for (auto col = 0; col < shape_[1]; ++col) {
             std::cout << std::setw(3);
@@ -170,9 +254,8 @@ template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
     }
 }
 
-
+template <typename T, ssize_t Ndim>
-template <typename T, int64_t Ndim>
+std::ostream &operator<<(std::ostream &os, const NDView<T, Ndim> &arr) {
-std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
     for (auto row = 0; row < arr.shape(0); ++row) {
         for (auto col = 0; col < arr.shape(1); ++col) {
             os << std::setw(3);
@@ -183,5 +266,8 @@ std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
     return os;
 }
 
+template <typename T> NDView<T, 1> make_view(std::vector<T> &vec) {
+    return NDView<T, 1>(vec.data(), {static_cast<ssize_t>(vec.size())});
+}
 
 } // namespace aare

include/aare/NumpyFile.hpp

@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Dtype.hpp"
-#include "aare/defs.hpp"
 #include "aare/FileInterface.hpp"
 #include "aare/NumpyHelpers.hpp"
-
+#include "aare/defs.hpp"
 
 #include <filesystem>
 #include <iostream>
@@ -11,13 +11,12 @@
 
 namespace aare {
 
-
-
 /**
  * @brief NumpyFile class to read and write numpy files
  * @note derived from FileInterface
  * @note implements all the pure virtual functions from FileInterface
- * @note documentation for the functions can also be found in the FileInterface class
+ * @note documentation for the functions can also be found in the FileInterface
+ * class
  */
 class NumpyFile : public FileInterface {
 
@@ -28,26 +27,35 @@ class NumpyFile : public FileInterface {
      * @param mode file mode (r, w)
      * @param cfg file configuration
      */
-    explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
+    explicit NumpyFile(const std::filesystem::path &fname,
+                       const std::string &mode = "r", FileConfig cfg = {});
 
     void write(Frame &frame);
     Frame read_frame() override { return get_frame(this->current_frame++); }
-    Frame read_frame(size_t frame_number) override { return get_frame(frame_number); }
+    Frame read_frame(size_t frame_number) override {
+        return get_frame(frame_number);
+    }
 
     std::vector<Frame> read_n(size_t n_frames) override;
-    void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
+    void read_into(std::byte *image_buf) override {
+        return get_frame_into(this->current_frame++, image_buf);
+    }
     void read_into(std::byte *image_buf, size_t n_frames) override;
     size_t frame_number(size_t frame_index) override { return frame_index; };
     size_t bytes_per_frame() override;
     size_t pixels_per_frame() override;
-    void seek(size_t frame_number) override { this->current_frame = frame_number; }
+    void seek(size_t frame_number) override {
+        this->current_frame = frame_number;
+    }
     size_t tell() override { return this->current_frame; }
     size_t total_frames() const override { return m_header.shape[0]; }
     size_t rows() const override { return m_header.shape[1]; }
     size_t cols() const override { return m_header.shape[2]; }
     size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
 
-    DetectorType detector_type() const override { return DetectorType::Unknown; }
+    DetectorType detector_type() const override {
+        return DetectorType::Unknown;
+    }
 
     /**
      * @brief get the data type of the numpy file
@@ -69,8 +77,9 @@ class NumpyFile : public FileInterface {
      */
     template <typename T, size_t NDim> NDArray<T, NDim> load() {
         NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
-        if (fseek(fp, static_cast<int64_t>(header_size), SEEK_SET)) {
+        if (fseek(fp, static_cast<long>(header_size), SEEK_SET)) {
-            throw std::runtime_error(LOCATION + "Error seeking to the start of the data");
+            throw std::runtime_error(LOCATION +
+                                     "Error seeking to the start of the data");
         }
         size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp);
         if (rc != static_cast<size_t>(arr.size())) {
@@ -78,16 +87,20 @@ class NumpyFile : public FileInterface {
         }
         return arr;
     }
-    template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &frame) {
+    template <typename A, typename TYPENAME, A Ndim>
+    void write(NDView<TYPENAME, Ndim> &frame) {
         write_impl(frame.data(), frame.total_bytes());
     }
-    template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &frame) {
+    template <typename A, typename TYPENAME, A Ndim>
+    void write(NDArray<TYPENAME, Ndim> &frame) {
         write_impl(frame.data(), frame.total_bytes());
     }
-    template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &&frame) {
+    template <typename A, typename TYPENAME, A Ndim>
+    void write(NDView<TYPENAME, Ndim> &&frame) {
         write_impl(frame.data(), frame.total_bytes());
     }
-    template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &&frame) {
+    template <typename A, typename TYPENAME, A Ndim>
+    void write(NDArray<TYPENAME, Ndim> &&frame) {
         write_impl(frame.data(), frame.total_bytes());
     }
 

include/aare/NumpyHelpers.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 
 #pragma once
 #include <algorithm>
@@ -40,15 +41,18 @@ bool parse_bool(const std::string &in);
 
 std::string get_value_from_map(const std::string &mapstr);
 
-std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys);
+std::unordered_map<std::string, std::string>
+parse_dict(std::string in, const std::vector<std::string> &keys);
 
-template <typename T, size_t N> bool in_array(T val, const std::array<T, N> &arr) {
+template <typename T, size_t N>
+bool in_array(T val, const std::array<T, N> &arr) {
     return std::find(std::begin(arr), std::end(arr), val) != std::end(arr);
 }
 bool is_digits(const std::string &str);
 
 aare::Dtype parse_descr(std::string typestring);
-size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header);
+size_t write_header(const std::filesystem::path &fname,
+                    const NumpyHeader &header);
 size_t write_header(std::ostream &out, const NumpyHeader &header);
 
 } // namespace NumpyHelpers

include/aare/Pedestal.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Frame.hpp"
 #include "aare/NDArray.hpp"
@@ -19,13 +20,13 @@ template <typename SUM_TYPE = double> class Pedestal {
     uint32_t m_samples;
     NDArray<uint32_t, 2> m_cur_samples;
 
-    //TODO! in case of int needs to be changed to uint64_t
+    // TODO! in case of int needs to be changed to uint64_t
     NDArray<SUM_TYPE, 2> m_sum;
     NDArray<SUM_TYPE, 2> m_sum2;
 
-    //Cache mean since it is used over and over in the ClusterFinder
+    // Cache mean since it is used over and over in the ClusterFinder
-    //This optimization is related to the access pattern of the ClusterFinder
+    // This optimization is related to the access pattern of the ClusterFinder
-    //Relies on having more reads than pushes to the pedestal
+    // Relies on having more reads than pushes to the pedestal
     NDArray<SUM_TYPE, 2> m_mean;
 
   public:
@@ -42,9 +43,7 @@ template <typename SUM_TYPE = double> class Pedestal {
     }
     ~Pedestal() = default;
 
-    NDArray<SUM_TYPE, 2> mean() {
+    NDArray<SUM_TYPE, 2> mean() { return m_mean; }
-        return m_mean;
-    }
 
     SUM_TYPE mean(const uint32_t row, const uint32_t col) const {
         return m_mean(row, col);
@@ -71,8 +70,6 @@ template <typename SUM_TYPE = double> class Pedestal {
         return variance_array;
     }
 
-    
-
     NDArray<SUM_TYPE, 2> std() {
         NDArray<SUM_TYPE, 2> standard_deviation_array({m_rows, m_cols});
         for (uint32_t i = 0; i < m_rows * m_cols; i++) {
@@ -83,8 +80,6 @@ template <typename SUM_TYPE = double> class Pedestal {
         return standard_deviation_array;
     }
 
-    
-
     void clear() {
         m_sum = 0;
         m_sum2 = 0;
@@ -92,8 +87,6 @@ template <typename SUM_TYPE = double> class Pedestal {
         m_mean = 0;
     }
 
-    
-
     void clear(const uint32_t row, const uint32_t col) {
         m_sum(row, col) = 0;
         m_sum2(row, col) = 0;
@@ -101,13 +94,11 @@ template <typename SUM_TYPE = double> class Pedestal {
         m_mean(row, col) = 0;
     }
 
-
-
     template <typename T> void push(NDView<T, 2> frame) {
         assert(frame.size() == m_rows * m_cols);
 
         // TODO! move away from m_rows, m_cols
-        if (frame.shape() != std::array<int64_t, 2>{m_rows, m_cols}) {
+        if (frame.shape() != std::array<ssize_t, 2>{m_rows, m_cols}) {
             throw std::runtime_error(
                 "Frame shape does not match pedestal shape");
         }
@@ -128,7 +119,7 @@ template <typename SUM_TYPE = double> class Pedestal {
         assert(frame.size() == m_rows * m_cols);
 
         // TODO! move away from m_rows, m_cols
-        if (frame.shape() != std::array<int64_t, 2>{m_rows, m_cols}) {
+        if (frame.shape() != std::array<ssize_t, 2>{m_rows, m_cols}) {
             throw std::runtime_error(
                 "Frame shape does not match pedestal shape");
         }
@@ -140,9 +131,6 @@ template <typename SUM_TYPE = double> class Pedestal {
         }
     }
 
-
-
-
     template <typename T> void push(Frame &frame) {
         assert(frame.rows() == static_cast<size_t>(m_rows) &&
                frame.cols() == static_cast<size_t>(m_cols));
@@ -170,7 +158,8 @@ template <typename SUM_TYPE = double> class Pedestal {
             m_sum(row, col) += val - m_sum(row, col) / m_samples;
             m_sum2(row, col) += val * val - m_sum2(row, col) / m_samples;
         }
-        //Since we just did a push we know that m_cur_samples(row, col) is at least 1
+        // Since we just did a push we know that m_cur_samples(row, col) is at
+        // least 1
         m_mean(row, col) = m_sum(row, col) / m_cur_samples(row, col);
     }
 
@@ -183,7 +172,8 @@ template <typename SUM_TYPE = double> class Pedestal {
             m_cur_samples(row, col)++;
         } else {
             m_sum(row, col) += val - m_sum(row, col) / m_cur_samples(row, col);
-            m_sum2(row, col) += val * val - m_sum2(row, col) / m_cur_samples(row, col);
+            m_sum2(row, col) +=
+                val * val - m_sum2(row, col) / m_cur_samples(row, col);
         }
     }
 
@@ -191,19 +181,16 @@ template <typename SUM_TYPE = double> class Pedestal {
      * @brief Update the mean of the pedestal. This is used after having done
      * push_no_update. It is not necessary to call this function after push.
      */
-    void update_mean(){
+    void update_mean() { m_mean = m_sum / m_cur_samples; }
-        m_mean = m_sum / m_cur_samples;
-    }
 
-    template<typename T>
+    template <typename T>
-    void push_fast(const uint32_t row, const uint32_t col, const T val_){
+    void push_fast(const uint32_t row, const uint32_t col, const T val_) {
-        //Assume we reached the steady state where all pixels have
+        // Assume we reached the steady state where all pixels have
-        //m_samples samples
+        // m_samples samples
         SUM_TYPE val = static_cast<SUM_TYPE>(val_);
         m_sum(row, col) += val - m_sum(row, col) / m_samples;
         m_sum2(row, col) += val * val - m_sum2(row, col) / m_samples;
         m_mean(row, col) = m_sum(row, col) / m_samples;
     }
-
 };
 } // namespace aare

include/aare/PixelMap.hpp

@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
-#include "aare/defs.hpp"
 #include "aare/NDArray.hpp"
+#include "aare/defs.hpp"
 
 namespace aare {
 
@@ -10,11 +11,11 @@ NDArray<ssize_t, 2> GenerateMoench05PixelMap();
 NDArray<ssize_t, 2> GenerateMoench05PixelMap1g();
 NDArray<ssize_t, 2> GenerateMoench05PixelMapOld();
 
-//Matterhorn02
+// Matterhorn02
-NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap();
+NDArray<ssize_t, 2> GenerateMH02SingleCounterPixelMap();
 NDArray<ssize_t, 3> GenerateMH02FourCounterPixelMap();
 
-//Eiger 
+// Eiger
-NDArray<ssize_t, 2>GenerateEigerFlipRowsPixelMap();
+NDArray<ssize_t, 2> GenerateEigerFlipRowsPixelMap();
 
 } // namespace aare

include/aare/ProducerConsumerQueue.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 /*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
@@ -18,9 +19,9 @@
 // @author Jordan DeLong (delong.j@fb.com)
 
 // Changes made by PSD Detector Group:
-// Copied: Line 34 constexpr std::size_t hardware_destructive_interference_size = 128; from folly/lang/Align.h
+// Copied: Line 34 constexpr std::size_t hardware_destructive_interference_size
-// Changed extension to .hpp
+// = 128; from folly/lang/Align.h Changed extension to .hpp Changed namespace to
-// Changed namespace to aare
+// aare
 
 #pragma once
 
@@ -45,15 +46,14 @@ template <class T> struct ProducerConsumerQueue {
     ProducerConsumerQueue(const ProducerConsumerQueue &) = delete;
     ProducerConsumerQueue &operator=(const ProducerConsumerQueue &) = delete;
 
-    
+    ProducerConsumerQueue(ProducerConsumerQueue &&other) {
-    ProducerConsumerQueue(ProducerConsumerQueue &&other){
         size_ = other.size_;
         records_ = other.records_;
         other.records_ = nullptr;
         readIndex_ = other.readIndex_.load(std::memory_order_acquire);
         writeIndex_ = other.writeIndex_.load(std::memory_order_acquire);
     }
-    ProducerConsumerQueue &operator=(ProducerConsumerQueue &&other){
+    ProducerConsumerQueue &operator=(ProducerConsumerQueue &&other) {
         size_ = other.size_;
         records_ = other.records_;
         other.records_ = nullptr;
@@ -62,15 +62,16 @@ template <class T> struct ProducerConsumerQueue {
         return *this;
     }
 
-    
+    ProducerConsumerQueue() : ProducerConsumerQueue(2){};
-    ProducerConsumerQueue():ProducerConsumerQueue(2){};
     // size must be >= 2.
     //
     // Also, note that the number of usable slots in the queue at any
     // given time is actually (size-1), so if you start with an empty queue,
     // isFull() will return true after size-1 insertions.
     explicit ProducerConsumerQueue(uint32_t size)
-        : size_(size), records_(static_cast<T *>(std::malloc(sizeof(T) * size))), readIndex_(0), writeIndex_(0) {
+        : size_(size),
+          records_(static_cast<T *>(std::malloc(sizeof(T) * size))),
+          readIndex_(0), writeIndex_(0) {
         assert(size >= 2);
         if (!records_) {
             throw std::bad_alloc();
@@ -154,7 +155,8 @@ template <class T> struct ProducerConsumerQueue {
     }
 
     bool isEmpty() const {
-        return readIndex_.load(std::memory_order_acquire) == writeIndex_.load(std::memory_order_acquire);
+        return readIndex_.load(std::memory_order_acquire) ==
+               writeIndex_.load(std::memory_order_acquire);
     }
 
     bool isFull() const {
@@ -175,7 +177,8 @@ template <class T> struct ProducerConsumerQueue {
     //   be removing items concurrently).
     // * It is undefined to call this from any other thread.
     size_t sizeGuess() const {
-        int ret = writeIndex_.load(std::memory_order_acquire) - readIndex_.load(std::memory_order_acquire);
+        int ret = writeIndex_.load(std::memory_order_acquire) -
+                  readIndex_.load(std::memory_order_acquire);
         if (ret < 0) {
             ret += size_;
         }
@@ -192,7 +195,7 @@ template <class T> struct ProducerConsumerQueue {
     // const uint32_t size_;
     uint32_t size_;
     // T *const records_;
-    T* records_;
+    T *records_;
 
     alignas(hardware_destructive_interference_size) AtomicIndex readIndex_;
     alignas(hardware_destructive_interference_size) AtomicIndex writeIndex_;

include/aare/RawFile.hpp

@@ -1,28 +1,20 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
+#include "aare/DetectorGeometry.hpp"
 #include "aare/FileInterface.hpp"
-#include "aare/RawMasterFile.hpp"
 #include "aare/Frame.hpp"
 #include "aare/NDArray.hpp" //for pixel map
+#include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
 
+#ifdef AARE_TESTS
+#include "../tests/friend_test.hpp"
+#endif
 
 #include <optional>
 
 namespace aare {
 
-struct ModuleConfig {
-    int module_gap_row{};
-    int module_gap_col{};
-
-    bool operator==(const ModuleConfig &other) const {
-        if (module_gap_col != other.module_gap_col)
-            return false;
-        if (module_gap_row != other.module_gap_row)
-            return false;
-        return true;
-    }
-};
-
 /**
  * @brief Class to read .raw files. The class will parse the master file
  * to find the correct geometry for the frames.
@@ -30,22 +22,12 @@ struct ModuleConfig {
  * Consider using that unless you need raw file specific functionality.
  */
 class RawFile : public FileInterface {
-    size_t n_subfiles{}; //f0,f1...fn
-    size_t n_subfile_parts{}; // d0,d1...dn
-    //TODO! move to vector of SubFile instead of pointers
-    std::vector<std::vector<RawSubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
-    // std::vector<xy> positions;
 
-    ModuleConfig cfg{0, 0};
+    std::vector<std::unique_ptr<RawSubFile>> m_subfiles;
 
     RawMasterFile m_master;
-
     size_t m_current_frame{};
 
-    // std::vector<ModuleGeometry> m_module_pixel_0;
-    // size_t m_rows{};
-    // size_t m_cols{};
-
     DetectorGeometry m_geometry;
 
   public:
@@ -56,7 +38,7 @@ class RawFile : public FileInterface {
 
      */
     RawFile(const std::filesystem::path &fname, const std::string &mode = "r");
-    virtual ~RawFile() override;
+    virtual ~RawFile() override = default;
 
     Frame read_frame() override;
     Frame read_frame(size_t frame_number) override;
@@ -64,10 +46,10 @@ class RawFile : public FileInterface {
     void read_into(std::byte *image_buf) override;
     void read_into(std::byte *image_buf, size_t n_frames) override;
 
-    //TODO! do we need to adapt the API? 
+    // TODO! do we need to adapt the API?
     void read_into(std::byte *image_buf, DetectorHeader *header);
-    void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header);
+    void read_into(std::byte *image_buf, size_t n_frames,
-
+                   DetectorHeader *header);
 
     size_t frame_number(size_t frame_index) override;
     size_t bytes_per_frame() override;
@@ -79,35 +61,14 @@ class RawFile : public FileInterface {
     size_t rows() const override;
     size_t cols() const override;
     size_t bitdepth() const override;
-    xy geometry();
+    size_t n_modules() const;
-    size_t n_mod() const;
+    size_t n_modules_in_roi() const;
+    xy geometry() const;
 
     RawMasterFile master() const;
 
-
-
     DetectorType detector_type() const override;
 
-  private:
-
-    /**
-     * @brief read the frame at the given frame index into the image buffer
-     * @param frame_number frame number to read
-     * @param image_buf buffer to store the frame
-     */
-
-    void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
-
-
-    /**
-     * @brief get the frame at the given frame index
-     * @param frame_number frame number to read
-     * @return Frame
-     */
-    Frame get_frame(size_t frame_index);
-
-
-
     /**
      * @brief read the header of the file
      * @param fname path to the data subfile
@@ -115,12 +76,24 @@ class RawFile : public FileInterface {
      */
     static DetectorHeader read_header(const std::filesystem::path &fname);
 
-    // void update_geometry_with_roi();
+  private:
-    int find_number_of_subfiles();
+    /**
+     * @brief read the frame at the given frame index into the image buffer
+     * @param frame_number frame number to read
+     * @param image_buf buffer to store the frame
+     */
+
+    void get_frame_into(size_t frame_index, std::byte *frame_buffer,
+                        DetectorHeader *header = nullptr);
+
+    /**
+     * @brief get the frame at the given frame index
+     * @param frame_number frame number to read
+     * @return Frame
+     */
+    Frame get_frame(size_t frame_index);
 
     void open_subfiles();
-    void find_geometry();
 };
 
-
 } // namespace aare

include/aare/RawMasterFile.hpp

@@ -1,5 +1,7 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/defs.hpp"
+#include <algorithm>
 #include <filesystem>
 #include <fmt/format.h>
 #include <fstream>
@@ -41,14 +43,16 @@ class RawFileNameComponents {
 
 class ScanParameters {
     bool m_enabled = false;
-    std::string m_dac;
+    DACIndex m_dac{};
     int m_start = 0;
     int m_stop = 0;
     int m_step = 0;
-    //TODO! add settleTime, requires string to time conversion
+    int64_t m_settleTime = 0; // [ns]
 
   public:
     ScanParameters(const std::string &par);
+    ScanParameters(const bool enabled, const DACIndex dac, const int start,
+                   const int stop, const int step, const int64_t settleTime);
     ScanParameters() = default;
     ScanParameters(const ScanParameters &) = default;
     ScanParameters &operator=(const ScanParameters &) = default;
@@ -56,12 +60,12 @@ class ScanParameters {
     int start() const;
     int stop() const;
     int step() const;
-    const std::string &dac() const;
+    DACIndex dac() const;
     bool enabled() const;
+    int64_t settleTime() const;
     void increment_stop();
 };
 
-
 /**
  * @brief Class for parsing a master file either in our .json format or the old
  * .raw format
@@ -78,8 +82,10 @@ class RawMasterFile {
     size_t m_pixels_y{};
     size_t m_pixels_x{};
     size_t m_bitdepth{};
+    uint8_t m_quad = 0;
 
     xy m_geometry{};
+    xy m_udp_interfaces_per_module{1, 1};
 
     size_t m_max_frames_per_file{};
     // uint32_t m_adc_mask{}; // TODO! implement reading
@@ -97,11 +103,10 @@ class RawMasterFile {
     std::optional<size_t> m_digital_samples;
     std::optional<size_t> m_transceiver_samples;
     std::optional<size_t> m_number_of_rows;
-    std::optional<uint8_t> m_quad;
+    std::optional<uint8_t> m_counter_mask;
 
     std::optional<ROI> m_roi;
 
-
   public:
     RawMasterFile(const std::filesystem::path &fpath);
 
@@ -117,26 +122,28 @@ class RawMasterFile {
     size_t max_frames_per_file() const;
     size_t bitdepth() const;
     size_t frame_padding() const;
+    xy udp_interfaces_per_module() const;
     const FrameDiscardPolicy &frame_discard_policy() const;
 
     size_t total_frames_expected() const;
     xy geometry() const;
+    size_t n_modules() const;
+    uint8_t quad() const;
 
     std::optional<size_t> analog_samples() const;
     std::optional<size_t> digital_samples() const;
     std::optional<size_t> transceiver_samples() const;
     std::optional<size_t> number_of_rows() const;
-    std::optional<uint8_t> quad() const;
+    std::optional<uint8_t> counter_mask() const;
-
 
     std::optional<ROI> roi() const;
 
-
     ScanParameters scan_parameters() const;
 
   private:
     void parse_json(const std::filesystem::path &fpath);
     void parse_raw(const std::filesystem::path &fpath);
+    void retrieve_geometry();
 };
 
 } // namespace aare

include/aare/RawSubFile.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 #include "aare/Frame.hpp"
 #include "aare/defs.hpp"
@@ -10,23 +11,34 @@
 namespace aare {
 
 /**
- * @brief Class to read a singe subfile written in .raw format. Used from RawFile to read
+ * @brief Class to read a singe subfile written in .raw format. Used from
- * the entire detector. Can be used directly to read part of the image.
+ * RawFile to read the entire detector. Can be used directly to read part of the
+ * image.
  */
 class RawSubFile {
   protected:
     std::ifstream m_file;
     DetectorType m_detector_type;
     size_t m_bitdepth;
-    std::filesystem::path m_fname;
+    std::filesystem::path m_path; //!< path to the subfile
+    std::string m_base_name;      //!< base name used for formatting file names
+    size_t m_offset{}; //!< file index of the first file, allow starting at non
+                       //!< zero file
+    size_t m_total_frames{}; //!< total number of frames in the series of files
     size_t m_rows{};
     size_t m_cols{};
     size_t m_bytes_per_frame{};
-    size_t n_frames{};
+
+    int m_module_index{};
+    size_t m_current_file_index{};  //!< The index of the open file
+    size_t m_current_frame_index{}; //!< The index of the current frame (with
+                                    //!< reference to all files)
+    std::vector<size_t>
+        m_last_frame_in_file{}; //!< Used for seeking to the correct file
+
     uint32_t m_pos_row{};
     uint32_t m_pos_col{};
 
-    
     std::optional<NDArray<ssize_t, 2>> m_pixel_map;
 
   public:
@@ -40,12 +52,14 @@ class RawSubFile {
      * @throws std::invalid_argument if the detector,type pair is not supported
      */
     RawSubFile(const std::filesystem::path &fname, DetectorType detector,
-               size_t rows, size_t cols, size_t bitdepth, uint32_t pos_row = 0, uint32_t pos_col = 0);
+               size_t rows, size_t cols, size_t bitdepth, uint32_t pos_row = 0,
+               uint32_t pos_col = 0);
 
     ~RawSubFile() = default;
     /**
      * @brief Seek to the given frame number
-     * @note Puts the file pointer at the start of the header, not the start of the data
+     * @note Puts the file pointer at the start of the header, not the start of
+     * the data
      * @param frame_index frame position in file to seek to
      * @throws std::runtime_error if the frame number is out of range
      */
@@ -53,6 +67,8 @@ class RawSubFile {
     size_t tell();
 
     void read_into(std::byte *image_buf, DetectorHeader *header = nullptr);
+    void read_into(std::byte *image_buf, size_t n_frames,
+                   DetectorHeader *header = nullptr);
     void get_part(std::byte *buffer, size_t frame_index);
 
     void read_header(DetectorHeader *header);
@@ -66,10 +82,15 @@ class RawSubFile {
     size_t pixels_per_frame() const { return m_rows * m_cols; }
     size_t bytes_per_pixel() const { return m_bitdepth / bits_per_byte; }
 
-private:
+    size_t frames_in_file() const { return m_total_frames; }
-  template <typename T>
-  void read_with_map(std::byte *image_buf);
 
+  private:
+    template <typename T> void read_with_map(std::byte *image_buf);
+
+    void parse_fname(const std::filesystem::path &fname);
+    void scan_files();
+    void open_file(size_t file_index);
+    std::filesystem::path fpath(size_t file_index) const;
 };
 
 } // namespace aare

include/aare/VarClusterFinder.hpp

@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include <algorithm>
@@ -28,7 +29,7 @@ template <typename T> class VarClusterFinder {
     };
 
   private:
-    const std::array<int64_t, 2> shape_;
+    const std::array<ssize_t, 2> shape_;
     NDView<T, 2> original_;
     NDArray<int, 2> labeled_;
     NDArray<int, 2> peripheral_labeled_;
@@ -38,11 +39,13 @@ template <typename T> class VarClusterFinder {
     bool use_noise_map = false;
     int peripheralThresholdFactor_ = 5;
     int current_label;
-    const std::array<int, 4> di{{0, -1, -1, -1}};              // row ### 8-neighbour by scaning from left to right
+    const std::array<int, 4> di{
-    const std::array<int, 4> dj{{-1, -1, 0, 1}};               // col ### 8-neighbour by scaning from top to bottom
+        {0, -1, -1, -1}}; // row ### 8-neighbour by scaning from left to right
+    const std::array<int, 4> dj{
+        {-1, -1, 0, 1}}; // col ### 8-neighbour by scaning from top to bottom
     const std::array<int, 8> di_{{0, 0, -1, 1, -1, 1, -1, 1}}; // row
     const std::array<int, 8> dj_{{-1, 1, 0, 0, 1, -1, -1, 1}}; // col
-    std::map<int, int> child;                                  // heirachy: key: child; val: parent
+    std::map<int, int> child; // heirachy: key: child; val: parent
     std::unordered_map<int, Hit> h_size;
     std::vector<Hit> hits;
     // std::vector<std::vector<int16_t>> row
@@ -50,7 +53,8 @@ template <typename T> class VarClusterFinder {
 
   public:
     VarClusterFinder(Shape<2> shape, T threshold)
-        : shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) {
+        : shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0),
+          binary_(shape), threshold_(threshold) {
         hits.reserve(2000);
     }
 
@@ -60,7 +64,9 @@ template <typename T> class VarClusterFinder {
         noiseMap = noise_map;
         use_noise_map = true;
     }
-    void set_peripheralThresholdFactor(int factor) { peripheralThresholdFactor_ = factor; }
+    void set_peripheralThresholdFactor(int factor) {
+        peripheralThresholdFactor_ = factor;
+    }
     void find_clusters(NDView<T, 2> img);
     void find_clusters_X(NDView<T, 2> img);
     void rec_FillHit(int clusterIndex, int i, int j);
@@ -144,7 +150,8 @@ template <typename T> int VarClusterFinder<T>::check_neighbours(int i, int j) {
     }
 }
 
-template <typename T> void VarClusterFinder<T>::find_clusters(NDView<T, 2> img) {
+template <typename T>
+void VarClusterFinder<T>::find_clusters(NDView<T, 2> img) {
     original_ = img;
     labeled_ = 0;
     peripheral_labeled_ = 0;
@@ -156,7 +163,8 @@ template <typename T> void VarClusterFinder<T>::find_clusters(NDView<T, 2> img)
     store_clusters();
 }
 
-template <typename T> void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
+template <typename T>
+void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
     original_ = img;
     int clusterIndex = 0;
     for (int i = 0; i < shape_[0]; ++i) {
@@ -175,7 +183,8 @@ template <typename T> void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img
     h_size.clear();
 }
 
-template <typename T> void VarClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
+template <typename T>
+void VarClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
     // printf("original_(%d, %d)=%f\n", i, j, original_(i,j));
     // printf("h_size[%d].size=%d\n", clusterIndex, h_size[clusterIndex].size);
     if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE) {
@@ -203,11 +212,15 @@ template <typename T> void VarClusterFinder<T>::rec_FillHit(int clusterIndex, in
             } else {
                 // if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE){
                 //     h_size[clusterIndex].size += 1;
-                //     h_size[clusterIndex].rows[h_size[clusterIndex].size] = row;
+                //     h_size[clusterIndex].rows[h_size[clusterIndex].size] =
-                //     h_size[clusterIndex].cols[h_size[clusterIndex].size] = col;
+                //     row; h_size[clusterIndex].cols[h_size[clusterIndex].size]
-                //     h_size[clusterIndex].enes[h_size[clusterIndex].size] = original_(row, col);
+                //     = col;
+                //     h_size[clusterIndex].enes[h_size[clusterIndex].size] =
+                //     original_(row, col);
                 // }// ? weather to include peripheral pixels
-                original_(row, col) = 0; // remove peripheral pixels, to avoid potential influence for pedestal updating
+                original_(row, col) =
+                    0; // remove peripheral pixels, to avoid potential influence
+                       // for pedestal updating
             }
         }
     }
@@ -226,16 +239,16 @@ template <typename T> void VarClusterFinder<T>::single_pass(NDView<T, 2> img) {
 
 template <typename T> void VarClusterFinder<T>::first_pass() {
 
-    for (size_t i = 0; i < original_.size(); ++i) {
+    for (ssize_t i = 0; i < original_.size(); ++i) {
         if (use_noise_map)
-            threshold_ = 5 * noiseMap(i);
+            threshold_ = 5 * noiseMap[i];
-        binary_(i) = (original_(i) > threshold_);
+        binary_[i] = (original_[i] > threshold_);
     }
 
     for (int i = 0; i < shape_[0]; ++i) {
         for (int j = 0; j < shape_[1]; ++j) {
 
-            // do we have someting to process?
+            // do we have something to process?
             if (binary_(i, j)) {
                 auto tmp = check_neighbours(i, j);
                 if (tmp != 0) {
@@ -250,7 +263,7 @@ template <typename T> void VarClusterFinder<T>::first_pass() {
 
 template <typename T> void VarClusterFinder<T>::second_pass() {
 
-    for (size_t i = 0; i != labeled_.size(); ++i) {
+    for (ssize_t i = 0; i != labeled_.size(); ++i) {
         auto cl = labeled_(i);
         if (cl != 0) {
             auto it = child.find(cl);
@@ -275,8 +288,8 @@ template <typename T> void VarClusterFinder<T>::store_clusters() {
     for (int i = 0; i < shape_[0]; ++i) {
         for (int j = 0; j < shape_[1]; ++j) {
             if (labeled_(i, j) != 0 || false
-                // (i-1 >= 0 and labeled_(i-1, j) != 0) or // another circle of peripheral pixels
+                // (i-1 >= 0 and labeled_(i-1, j) != 0) or // another circle of
-                // (j-1 >= 0 and labeled_(i, j-1) != 0) or
+                // peripheral pixels (j-1 >= 0 and labeled_(i, j-1) != 0) or
                 // (i+1 < shape_[0] and labeled_(i+1, j) != 0) or
                 // (j+1 < shape_[1] and labeled_(i, j+1) != 0)
             ) {

include/aare/algorithm.hpp

@@ -1,55 +1,128 @@
+// SPDX-License-Identifier: MPL-2.0
 
 #pragma once
+#include <aare/NDArray.hpp>
 #include <algorithm>
 #include <array>
 #include <vector>
-#include <aare/NDArray.hpp>
 
 namespace aare {
 /**
- * @brief Find the index of the last element smaller than val
+ * @brief Index of the last element that is smaller than val.
- * assume a sorted array
+ * Requires a sorted array. Uses >= for ordering. If all elements
+ * are smaller it returns the last element and if all elements are
+ * larger it returns the first element.
+ * @param first iterator to the first element
+ * @param last iterator to the last element
+ * @param val value to compare
+ * @return index of the last element that is smaller than val
+ *
  */
 template <typename T>
-size_t last_smaller(const T* first, const T* last, T val) {
+size_t last_smaller(const T *first, const T *last, T val) {
-    for (auto iter = first+1; iter != last; ++iter) {
+    for (auto iter = first + 1; iter != last; ++iter) {
-        if (*iter > val) {
+        if (*iter >= val) {
-            return std::distance(first, iter-1);
+            return std::distance(first, iter - 1);
         }
     }
-    return std::distance(first, last-1);
+    return std::distance(first, last - 1);
 }
 
-template <typename T>
+template <typename T> size_t last_smaller(const NDArray<T, 1> &arr, T val) {
-size_t last_smaller(const NDArray<T, 1>& arr, T val) {
     return last_smaller(arr.begin(), arr.end(), val);
 }
 
+template <typename T> size_t last_smaller(const std::vector<T> &vec, T val) {
+    return last_smaller(vec.data(), vec.data() + vec.size(), val);
+}
 
+/**
+ * @brief Index of the first element that is larger than val.
+ * Requires a sorted array. Uses > for ordering. If all elements
+ * are larger it returns the first element and if all elements are
+ * smaller it returns the last element.
+ * @param first iterator to the first element
+ * @param last iterator to the last element
+ * @param val value to compare
+ * @return index of the first element that is larger than val
+ */
 template <typename T>
-size_t nearest_index(const T* first, const T* last, T val) {
+size_t first_larger(const T *first, const T *last, T val) {
-    auto iter = std::min_element(first, last,
+    for (auto iter = first; iter != last; ++iter) {
-    [val](T a, T b) {
+        if (*iter > val) {
+            return std::distance(first, iter);
+        }
+    }
+    return std::distance(first, last - 1);
+}
+
+template <typename T> size_t first_larger(const NDArray<T, 1> &arr, T val) {
+    return first_larger(arr.begin(), arr.end(), val);
+}
+
+template <typename T> size_t first_larger(const std::vector<T> &vec, T val) {
+    return first_larger(vec.data(), vec.data() + vec.size(), val);
+}
+
+/**
+ * @brief Index of the nearest element to val.
+ * Requires a sorted array. If there is no difference it takes the first
+ * element.
+ * @param first iterator to the first element
+ * @param last iterator to the last element
+ * @param val value to compare
+ * @return index of the nearest element
+ */
+template <typename T>
+size_t nearest_index(const T *first, const T *last, T val) {
+    auto iter = std::min_element(first, last, [val](T a, T b) {
         return std::abs(a - val) < std::abs(b - val);
     });
     return std::distance(first, iter);
 }
 
-template <typename T>
+template <typename T> size_t nearest_index(const NDArray<T, 1> &arr, T val) {
-size_t nearest_index(const NDArray<T, 1>& arr, T val) {
     return nearest_index(arr.begin(), arr.end(), val);
 }
 
-template <typename T>
+template <typename T> size_t nearest_index(const std::vector<T> &vec, T val) {
-size_t nearest_index(const std::vector<T>& vec, T val) {
+    return nearest_index(vec.data(), vec.data() + vec.size(), val);
-    return nearest_index(vec.data(), vec.data()+vec.size(), val);
 }
 
 template <typename T, size_t N>
-size_t nearest_index(const std::array<T,N>& arr, T val) {
+size_t nearest_index(const std::array<T, N> &arr, T val) {
-    return nearest_index(arr.data(), arr.data()+arr.size(), val);
+    return nearest_index(arr.data(), arr.data() + arr.size(), val);
 }
 
+template <typename T> std::vector<T> cumsum(const std::vector<T> &vec) {
+    std::vector<T> result(vec.size());
+    std::partial_sum(vec.begin(), vec.end(), result.begin());
+    return result;
+}
 
+template <typename Container> bool all_equal(const Container &c) {
+    if (!c.empty() &&
+        std::all_of(begin(c), end(c),
+                    [c](const typename Container::value_type &element) {
+                        return element == c.front();
+                    }))
+        return true;
+    return false;
+}
+
+/**
+ * linear interpolation
+ * @param bin_edge left and right bin edges
+ * @param bin_values function values at bin edges
+ * @param coord coordinate to interpolate at
+ * @return interpolated value at coord
+ */
+inline double linear_interpolation(const std::pair<double, double> &bin_edge,
+                                   const std::pair<double, double> &bin_values,
+                                   const double coord) {
+    const double bin_width = bin_edge.second - bin_edge.first;
+    return bin_values.first * (1 - (coord - bin_edge.first) / bin_width) +
+           bin_values.second * (coord - bin_edge.first) / bin_width;
+}
 
 } // namespace aare

include/aare/calibration.hpp

@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+
+#include "aare/NDArray.hpp"
+#include "aare/NDView.hpp"
+#include "aare/defs.hpp"
+#include "aare/utils/par.hpp"
+#include "aare/utils/task.hpp"
+#include <cstdint>
+#include <future>
+
+namespace aare {
+
+// Really try to convince the compile to inline this function
+// TODO! Clang?
+#if (defined(_MSC_VER) || defined(__INTEL_COMPILER))
+#define STRONG_INLINE __forceinline
+#else
+#define STRONG_INLINE inline
+#endif
+
+#if defined(__GNUC__)
+#define ALWAYS_INLINE __attribute__((always_inline)) inline
+#else
+#define ALWAYS_INLINE STRONG_INLINE
+#endif
+
+/**
+ * @brief Get the gain from the raw ADC value. In Jungfrau the gain is
+ * encoded in the left most 2 bits of the raw value.
+ * 00 -> gain 0
+ * 01 -> gain 1
+ * 11 -> gain 2
+ * @param raw the raw ADC value
+ * @return the gain as an integer
+ */
+ALWAYS_INLINE int get_gain(uint16_t raw) {
+    switch (raw >> 14) {
+    case 0:
+        return 0;
+    case 1:
+        return 1;
+    case 3:
+        return 2;
+    default:
+        return 0;
+    }
+}
+
+ALWAYS_INLINE uint16_t get_value(uint16_t raw) { return raw & ADC_MASK; }
+
+ALWAYS_INLINE std::pair<uint16_t, int16_t> get_value_and_gain(uint16_t raw) {
+    static_assert(
+        sizeof(std::pair<uint16_t, int16_t>) ==
+            sizeof(uint16_t) + sizeof(int16_t),
+        "Size of pair<uint16_t, int16_t> does not match expected size");
+    return {get_value(raw), get_gain(raw)};
+}
+
+template <class T>
+void apply_calibration_impl(NDView<T, 3> res, NDView<uint16_t, 3> raw_data,
+                            NDView<T, 3> ped, NDView<T, 3> cal, int start,
+                            int stop) {
+
+    for (int frame_nr = start; frame_nr != stop; ++frame_nr) {
+        for (int row = 0; row != raw_data.shape(1); ++row) {
+            for (int col = 0; col != raw_data.shape(2); ++col) {
+                auto [value, gain] =
+                    get_value_and_gain(raw_data(frame_nr, row, col));
+
+                // Using multiplication does not seem to speed up the code here
+                // ADU/keV is the standard unit for the calibration which
+                // means rewriting the formula is not worth it.
+                res(frame_nr, row, col) =
+                    (value - ped(gain, row, col)) / cal(gain, row, col);
+            }
+        }
+    }
+}
+
+template <class T>
+void apply_calibration_impl(NDView<T, 3> res, NDView<uint16_t, 3> raw_data,
+                            NDView<T, 2> ped, NDView<T, 2> cal, int start,
+                            int stop) {
+
+    for (int frame_nr = start; frame_nr != stop; ++frame_nr) {
+        for (int row = 0; row != raw_data.shape(1); ++row) {
+            for (int col = 0; col != raw_data.shape(2); ++col) {
+                auto [value, gain] =
+                    get_value_and_gain(raw_data(frame_nr, row, col));
+
+                // Using multiplication does not seem to speed up the code here
+                // ADU/keV is the standard unit for the calibration which
+                // means rewriting the formula is not worth it.
+
+                // Set the value to 0 if the gain is not 0
+                if (gain == 0)
+                    res(frame_nr, row, col) =
+                        (value - ped(row, col)) / cal(row, col);
+                else
+                    res(frame_nr, row, col) = 0;
+            }
+        }
+    }
+}
+
+template <class T, ssize_t Ndim = 3>
+void apply_calibration(NDView<T, 3> res, NDView<uint16_t, 3> raw_data,
+                       NDView<T, Ndim> ped, NDView<T, Ndim> cal,
+                       ssize_t n_threads = 4) {
+    std::vector<std::future<void>> futures;
+    futures.reserve(n_threads);
+    auto limits = split_task(0, raw_data.shape(0), n_threads);
+    for (const auto &lim : limits)
+        futures.push_back(std::async(
+            static_cast<void (*)(NDView<T, 3>, NDView<uint16_t, 3>,
+                                 NDView<T, Ndim>, NDView<T, Ndim>, int, int)>(
+                apply_calibration_impl),
+            res, raw_data, ped, cal, lim.first, lim.second));
+    for (auto &f : futures)
+        f.get();
+}
+
+template <bool only_gain0>
+std::pair<NDArray<size_t, 3>, NDArray<size_t, 3>>
+sum_and_count_per_gain(NDView<uint16_t, 3> raw_data) {
+    constexpr ssize_t num_gains = only_gain0 ? 1 : 3;
+    NDArray<size_t, 3> accumulator(
+        std::array<ssize_t, 3>{num_gains, raw_data.shape(1), raw_data.shape(2)},
+        0);
+    NDArray<size_t, 3> count(
+        std::array<ssize_t, 3>{num_gains, raw_data.shape(1), raw_data.shape(2)},
+        0);
+    for (int frame_nr = 0; frame_nr != raw_data.shape(0); ++frame_nr) {
+        for (int row = 0; row != raw_data.shape(1); ++row) {
+            for (int col = 0; col != raw_data.shape(2); ++col) {
+                auto [value, gain] =
+                    get_value_and_gain(raw_data(frame_nr, row, col));
+                if (gain != 0 && only_gain0)
+                    continue;
+                accumulator(gain, row, col) += value;
+                count(gain, row, col) += 1;
+            }
+        }
+    }
+
+    return {std::move(accumulator), std::move(count)};
+}
+
+template <typename T, bool only_gain0 = false>
+NDArray<T, 3 - static_cast<ssize_t>(only_gain0)>
+calculate_pedestal(NDView<uint16_t, 3> raw_data, ssize_t n_threads) {
+
+    constexpr ssize_t num_gains = only_gain0 ? 1 : 3;
+    std::vector<std::future<std::pair<NDArray<size_t, 3>, NDArray<size_t, 3>>>>
+        futures;
+    futures.reserve(n_threads);
+
+    auto subviews = make_subviews(raw_data, n_threads);
+
+    for (auto view : subviews) {
+        futures.push_back(std::async(
+            static_cast<std::pair<NDArray<size_t, 3>, NDArray<size_t, 3>> (*)(
+                NDView<uint16_t, 3>)>(&sum_and_count_per_gain<only_gain0>),
+            view));
+    }
+    Shape<3> shape{num_gains, raw_data.shape(1), raw_data.shape(2)};
+    NDArray<size_t, 3> accumulator(shape, 0);
+    NDArray<size_t, 3> count(shape, 0);
+
+    // Combine the results from the futures
+    for (auto &f : futures) {
+        auto [acc, cnt] = f.get();
+        accumulator += acc;
+        count += cnt;
+    }
+
+
+    // Will move to a NDArray<T, 3 - static_cast<ssize_t>(only_gain0)>
+    // if only_gain0 is true
+    return safe_divide<T>(accumulator, count);
+
+}
+
+/**
+ * @brief Count the number of switching pixels in the raw data.
+ * This function counts the number of pixels that switch between G1 and G2 gain.
+ * It returns an NDArray with the number of switching pixels per pixel.
+ * @param raw_data The NDView containing the raw data
+ * @return An NDArray with the number of switching pixels per pixel
+ */
+NDArray<int, 2> count_switching_pixels(NDView<uint16_t, 3> raw_data);
+
+/**
+ * @brief Count the number of switching pixels in the raw data.
+ * This function counts the number of pixels that switch between G1 and G2 gain.
+ * It returns an NDArray with the number of switching pixels per pixel.
+ * @param raw_data The NDView containing the raw data
+ * @param n_threads The number of threads to use for parallel processing
+ * @return An NDArray with the number of switching pixels per pixel
+ */
+NDArray<int, 2> count_switching_pixels(NDView<uint16_t, 3> raw_data,
+                                       ssize_t n_threads);
+
+template <typename T>
+auto calculate_pedestal_g0(NDView<uint16_t, 3> raw_data, ssize_t n_threads) {
+    return calculate_pedestal<T, true>(raw_data, n_threads);
+}
+
+} // namespace aare

include/aare/decode.hpp

@@ -1,13 +1,48 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
-
+#include "aare/defs.hpp"
-#include <cstdint>
 #include <aare/NDView.hpp>
+#include <cstdint>
+#include <vector>
 namespace aare {
 
 
 uint16_t adc_sar_05_decode64to16(uint64_t input);
 uint16_t adc_sar_04_decode64to16(uint64_t input);
-void adc_sar_05_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output);
+void adc_sar_05_decode64to16(NDView<uint64_t, 2> input,
-void adc_sar_04_decode64to16(NDView<uint64_t, 2> input, NDView<uint16_t,2> output);
+                             NDView<uint16_t, 2> output);
+void adc_sar_04_decode64to16(NDView<uint64_t, 2> input,
+                             NDView<uint16_t, 2> output);
+
+/**
+ * @brief Called with a 32 bit unsigned integer, shift by offset
+ * and then return the lower 24 bits as an 32 bit integer
+ * @param input 32-ibt input value
+ * @param offset (should be in range 0-7 to allow for full 24 bits)
+ * @return uint32_t 
+ */
+uint32_t mask32to24bits(uint32_t input, BitOffset offset={});
+
+/**
+ * @brief Expand 24 bit values in a 8bit buffer to 32bit unsigned integers
+ * Used for detectors with 24bit counters in combination with CTB
+ * 
+ * @param input View of the 24 bit data as uint8_t (no 24bit native data type exists)
+ * @param output Destination of the expanded data (32bit, unsigned) 
+ * @param offset Offset within the first byte to where the data starts (0-7 bits)
+ */
+void expand24to32bit(NDView<uint8_t,1> input, NDView<uint32_t,1> output, BitOffset offset={});
+
+/**
+ * @brief Apply custom weights to a 16-bit input value. Will sum up
+ * weights[i]**i for each bit i that is set in the input value.
+ * @throws std::out_of_range if weights.size() < 16
+ * @param input 16-bit input value
+ * @param weights vector of weights, size must be less than or equal to 16
+ */
+double apply_custom_weights(uint16_t input, const NDView<double, 1> weights);
+
+void apply_custom_weights(NDView<uint16_t, 1> input, NDView<double, 1> output,
+                          const NDView<double, 1> weights);
 
 } // namespace aare

include/aare/defs.hpp

@@ -1,18 +1,18 @@
+// SPDX-License-Identifier: MPL-2.0
 #pragma once
 
 #include "aare/Dtype.hpp"
 
 #include <array>
-#include <stdexcept>
 #include <cassert>
 #include <cstdint>
 #include <cstring>
+#include <stdexcept>
 #include <string>
 #include <string_view>
 #include <variant>
 #include <vector>
 
-
 /**
  * @brief LOCATION macro to get the current location in the code
  */
@@ -20,28 +20,24 @@
     std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) +      \
         ":" + std::string(__func__) + ":"
 
-
-
 #ifdef AARE_CUSTOM_ASSERT
-#define AARE_ASSERT(expr)\
+#define AARE_ASSERT(expr)                                                      \
-    if (expr)\
+    if (expr) {                                                                \
-        {}\
+    } else                                                                     \
-    else\
         aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n");
 #else
-#define AARE_ASSERT(cond)\
+#define AARE_ASSERT(cond)                                                      \
-    do { (void)sizeof(cond); } while(0)
+    do {                                                                       \
+        (void)sizeof(cond);                                                    \
+    } while (0)
 #endif
 
-
 namespace aare {
 
 inline constexpr size_t bits_per_byte = 8;
 
 void assert_failed(const std::string &msg);
 
-
-
 class DynamicCluster {
   public:
     int cluster_sizeX;
@@ -55,7 +51,7 @@ class DynamicCluster {
 
   public:
     DynamicCluster(int cluster_sizeX_, int cluster_sizeY_,
-            Dtype dt_ = Dtype(typeid(int32_t)))
+                   Dtype dt_ = Dtype(typeid(int32_t)))
         : cluster_sizeX(cluster_sizeX_), cluster_sizeY(cluster_sizeY_),
           dt(dt_) {
         m_data = new std::byte[cluster_sizeX * cluster_sizeY * dt.bytes()]{};
@@ -179,50 +175,22 @@ template <typename T> struct t_xy {
 };
 using xy = t_xy<uint32_t>;
 
+struct ROI {
+    ssize_t xmin{};
+    ssize_t xmax{};
+    ssize_t ymin{};
+    ssize_t ymax{};
 
-/**
+    ssize_t height() const { return ymax - ymin; }
- * @brief Class to hold the geometry of a module. Where pixel 0 is located and the size of the module
+    ssize_t width() const { return xmax - xmin; }
- */
+    bool contains(ssize_t x, ssize_t y) const {
-struct ModuleGeometry{
-    int origin_x{};
-    int origin_y{};
-    int height{};
-    int width{};
-    int row_index{};
-    int col_index{}; 
-};
-
-/**
- * @brief Class to hold the geometry of a detector. Number of modules, their size and where pixel 0 
- * for each module is located
- */
-struct DetectorGeometry{
-    int modules_x{};
-    int modules_y{};
-    int pixels_x{};
-    int pixels_y{};
-    int module_gap_row{};
-    int module_gap_col{};
-    std::vector<ModuleGeometry> module_pixel_0;
-};
-
-struct ROI{
-    int64_t xmin{};
-    int64_t xmax{};
-    int64_t ymin{};
-    int64_t ymax{};
-  
-    int64_t height() const { return ymax - ymin; }
-    int64_t width() const { return xmax - xmin; }
-    bool contains(int64_t x, int64_t y) const {
         return x >= xmin && x < xmax && y >= ymin && y < ymax;
     }
-  };
+};
 
+using dynamic_shape = std::vector<ssize_t>;
 
-using dynamic_shape = std::vector<int64_t>;
+// TODO! Can we uniform enums between the libraries?
-
-//TODO! Can we uniform enums between the libraries?
 
 /**
  * @brief Enum class to identify different detectors.
@@ -230,7 +198,7 @@ using dynamic_shape = std::vector<int64_t>;
  * Different spelling to avoid confusion with the slsDetectorPackage
  */
 enum class DetectorType {
-    //Standard detectors match the enum values from slsDetectorPackage
+    // Standard detectors match the enum values from slsDetectorPackage
     Generic,
     Eiger,
     Gotthard,
@@ -241,12 +209,142 @@ enum class DetectorType {
     Gotthard2,
     Xilinx_ChipTestBoard,
 
-    //Additional detectors used for defining processing. Variants of the standard ones.
+    // Additional detectors used for defining processing. Variants of the
-    Moench03=100,
+    // standard ones.
+    Moench03 = 100,
     Moench03_old,
     Unknown
 };
 
+/**
+ * @brief Enum class to define the Digital to Analog converter
+ * The values are the same as in slsDetectorPackage
+ */
+enum DACIndex {
+    DAC_0,
+    DAC_1,
+    DAC_2,
+    DAC_3,
+    DAC_4,
+    DAC_5,
+    DAC_6,
+    DAC_7,
+    DAC_8,
+    DAC_9,
+    DAC_10,
+    DAC_11,
+    DAC_12,
+    DAC_13,
+    DAC_14,
+    DAC_15,
+    DAC_16,
+    DAC_17,
+    VSVP,
+    VTRIM,
+    VRPREAMP,
+    VRSHAPER,
+    VSVN,
+    VTGSTV,
+    VCMP_LL,
+    VCMP_LR,
+    VCAL,
+    VCMP_RL,
+    RXB_RB,
+    RXB_LB,
+    VCMP_RR,
+    VCP,
+    VCN,
+    VISHAPER,
+    VTHRESHOLD,
+    IO_DELAY,
+    VREF_DS,
+    VOUT_CM,
+    VIN_CM,
+    VREF_COMP,
+    VB_COMP,
+    VDD_PROT,
+    VIN_COM,
+    VREF_PRECH,
+    VB_PIXBUF,
+    VB_DS,
+    VREF_H_ADC,
+    VB_COMP_FE,
+    VB_COMP_ADC,
+    VCOM_CDS,
+    VREF_RSTORE,
+    VB_OPA_1ST,
+    VREF_COMP_FE,
+    VCOM_ADC1,
+    VREF_L_ADC,
+    VREF_CDS,
+    VB_CS,
+    VB_OPA_FD,
+    VCOM_ADC2,
+    VCASSH,
+    VTH2,
+    VRSHAPER_N,
+    VIPRE_OUT,
+    VTH3,
+    VTH1,
+    VICIN,
+    VCAS,
+    VCAL_N,
+    VIPRE,
+    VCAL_P,
+    VDCSH,
+    VBP_COLBUF,
+    VB_SDA,
+    VCASC_SFP,
+    VIPRE_CDS,
+    IBIAS_SFP,
+    ADC_VPP,
+    HIGH_VOLTAGE,
+    TEMPERATURE_ADC,
+    TEMPERATURE_FPGA,
+    TEMPERATURE_FPGAEXT,
+    TEMPERATURE_10GE,
+    TEMPERATURE_DCDC,
+    TEMPERATURE_SODL,
+    TEMPERATURE_SODR,
+    TEMPERATURE_FPGA2,
+    TEMPERATURE_FPGA3,
+    TRIMBIT_SCAN,
+    V_POWER_A = 100,
+    V_POWER_B = 101,
+    V_POWER_C = 102,
+    V_POWER_D = 103,
+    V_POWER_IO = 104,
+    V_POWER_CHIP = 105,
+    I_POWER_A = 106,
+    I_POWER_B = 107,
+    I_POWER_C = 108,
+    I_POWER_D = 109,
+    I_POWER_IO = 110,
+    V_LIMIT = 111,
+    SLOW_ADC0 = 1000,
+    SLOW_ADC1,
+    SLOW_ADC2,
+    SLOW_ADC3,
+    SLOW_ADC4,
+    SLOW_ADC5,
+    SLOW_ADC6,
+    SLOW_ADC7,
+    SLOW_ADC_TEMP
+};
+
+// helper pair class to easily expose in python
+template <typename T1, typename T2> struct Sum_index_pair {
+    T1 sum;
+    T2 index;
+};
+
+enum class corner : int {
+    cTopLeft = 0,
+    cTopRight = 1,
+    cBottomLeft = 2,
+    cBottomRight = 3
+};
+
 enum class TimingMode { Auto, Trigger };
 enum class FrameDiscardPolicy { NoDiscard, Discard, DiscardPartial };
 
@@ -263,4 +361,26 @@ template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
 
 using DataTypeVariants = std::variant<uint16_t, uint32_t>;
 
+constexpr uint16_t ADC_MASK =
+    0x3FFF; // used to mask out the gain bits in Jungfrau
+
+/**
+ * @brief Convert a string to a DACIndex
+ * @param arg string representation of the dacIndex
+ * @return DACIndex
+ * @throw invalid argument error if the string does not match any DACIndex
+ */
+template <> DACIndex StringTo(const std::string &arg);
+
+class BitOffset{
+    uint8_t m_offset{};
+    public:
+    BitOffset() = default;
+    explicit BitOffset(uint32_t offset);
+    uint8_t value() const {return m_offset;}
+    bool operator==(const BitOffset& other) const;
+    bool operator<(const BitOffset& other) const;
+
+};
+
 } // namespace aare

include/aare/geo_helpers.hpp

@@ -1,16 +0,0 @@
-#pragma once
-#include "aare/defs.hpp"
-#include "aare/RawMasterFile.hpp" //ROI refactor away
-namespace aare{
-
-/**
- * @brief Update the detector geometry given a region of interest
- * 
- * @param geo 
- * @param roi 
- * @return DetectorGeometry 
- */
-DetectorGeometry update_geometry_with_roi(DetectorGeometry geo, ROI roi);
-
-
-} // namespace aare

include/aare/logger.hpp

@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+/*Utility to log to console*/
+
+#include <iostream>
+#include <sstream>
+#include <sys/time.h>
+
+namespace aare {
+
+#define RED      "\x1b[31m"
+#define GREEN    "\x1b[32m"
+#define YELLOW   "\x1b[33m"
+#define BLUE     "\x1b[34m"
+#define MAGENTA  "\x1b[35m"
+#define CYAN     "\x1b[36m"
+#define GRAY     "\x1b[37m"
+#define DARKGRAY "\x1b[30m"
+
+#define BG_BLACK   "\x1b[48;5;232m"
+#define BG_RED     "\x1b[41m"
+#define BG_GREEN   "\x1b[42m"
+#define BG_YELLOW  "\x1b[43m"
+#define BG_BLUE    "\x1b[44m"
+#define BG_MAGENTA "\x1b[45m"
+#define BG_CYAN    "\x1b[46m"
+#define RESET      "\x1b[0m"
+#define BOLD       "\x1b[1m"
+
+enum TLogLevel {
+    logERROR,
+    logWARNING,
+    logINFOBLUE,
+    logINFOGREEN,
+    logINFORED,
+    logINFOCYAN,
+    logINFOMAGENTA,
+    logINFO,
+    logDEBUG, // constructors, destructors etc. should still give too much
+              // output
+    logDEBUG1,
+    logDEBUG2,
+    logDEBUG3,
+    logDEBUG4,
+    logDEBUG5
+};
+
+// Compiler should optimize away anything below this value
+#ifndef AARE_LOG_LEVEL
+#define AARE_LOG_LEVEL                                                         \
+    "LOG LEVEL NOT SET IN CMAKE" // This is configured in the main
+                                 // CMakeLists.txt
+#endif
+
+#define __AT__                                                                 \
+    std::string(__FILE__) + std::string("::") + std::string(__func__) +        \
+        std::string("(): ")
+#define __SHORT_FORM_OF_FILE__                                                 \
+    (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
+#define __SHORT_AT__                                                           \
+    std::string(__SHORT_FORM_OF_FILE__) + std::string("::") +                  \
+        std::string(__func__) + std::string("(): ")
+
+class Logger {
+    std::ostringstream os;
+    TLogLevel m_level = AARE_LOG_LEVEL;
+
+  public:
+    Logger() = default;
+    explicit Logger(TLogLevel level) : m_level(level){};
+    ~Logger() {
+        // output in the destructor to allow for << syntax
+        os << RESET << '\n';
+        std::clog << os.str() << std::flush; // Single write
+    }
+
+    static TLogLevel &
+    ReportingLevel() { // singelton eeh TODO! Do we need a runtime option?
+        static TLogLevel reportingLevel = logDEBUG5;
+        return reportingLevel;
+    }
+
+    // Danger this buffer need as many elements as TLogLevel
+    static const char *Color(TLogLevel level) noexcept {
+        static const char *const colors[] = {
+            RED BOLD, YELLOW BOLD, BLUE,  GREEN, RED,   CYAN,  MAGENTA,
+            RESET,    RESET,       RESET, RESET, RESET, RESET, RESET};
+        // out of bounds
+        if (level < 0 || level >= sizeof(colors) / sizeof(colors[0])) {
+            return RESET;
+        }
+        return colors[level];
+    }
+
+    // Danger this buffer need as many elements as TLogLevel
+    static std::string ToString(TLogLevel level) {
+        static const char *const buffer[] = {
+            "ERROR",  "WARNING", "INFO",   "INFO",  "INFO",
+            "INFO",   "INFO",    "INFO",   "DEBUG", "DEBUG1",
+            "DEBUG2", "DEBUG3",  "DEBUG4", "DEBUG5"};
+        // out of bounds
+        if (level < 0 || level >= sizeof(buffer) / sizeof(buffer[0])) {
+            return "UNKNOWN";
+        }
+        return buffer[level];
+    }
+
+    std::ostringstream &Get() {
+        os << Color(m_level) << "- " << Timestamp() << " " << ToString(m_level)
+           << ": ";
+        return os;
+    }
+
+    static std::string Timestamp() {
+        constexpr size_t buffer_len = 12;
+        char buffer[buffer_len];
+        time_t t;
+        ::time(&t);
+        tm r;
+        strftime(buffer, buffer_len, "%X", localtime_r(&t, &r));
+        buffer[buffer_len - 1] = '\0';
+        struct timeval tv;
+        gettimeofday(&tv, nullptr);
+        constexpr size_t result_len = 100;
+        char result[result_len];
+        snprintf(result, result_len, "%s.%03ld", buffer,
+                 static_cast<long>(tv.tv_usec) / 1000);
+        result[result_len - 1] = '\0';
+        return result;
+    }
+};
+
+// TODO! Do we need to keep the runtime option?
+#define LOG(level)                                                             \
+    if (level > AARE_LOG_LEVEL)                                                \
+        ;                                                                      \
+    else if (level > aare::Logger::ReportingLevel())                           \
+        ;                                                                      \
+    else                                                                       \
+        aare::Logger(level).Get()
+
+} // namespace aare

include/aare/utils/ifstream_helpers.hpp

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+
+#include <fstream>
+#include <string>
+namespace aare {
+
+/**
+ * @brief Get the error message from an ifstream object
+ */
+std::string ifstream_error_msg(std::ifstream &ifs);
+
+} // namespace aare

include/aare/utils/par.hpp

@@ -1,18 +1,35 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
 #include <thread>
-#include <vector>
 #include <utility>
+#include <vector>
+
+#include "aare/utils/task.hpp"
 
 namespace aare {
 
-    template<typename F>
+template <typename F>
-    void RunInParallel(F func, const std::vector<std::pair<int, int>>& tasks) {
+void RunInParallel(F func, const std::vector<std::pair<int, int>> &tasks) {
-        // auto tasks = split_task(0, y.shape(0), n_threads);
+    // auto tasks = split_task(0, y.shape(0), n_threads);
-        std::vector<std::thread> threads;
+    std::vector<std::thread> threads;
-        for (auto &task : tasks) {
+    for (auto &task : tasks) {
-            threads.push_back(std::thread(func, task.first, task.second));
+        threads.push_back(std::thread(func, task.first, task.second));
-        }
-        for (auto &thread : threads) {
-            thread.join();
-        }
     }
+    for (auto &thread : threads) {
+        thread.join();
+    }
+}
+
+
+template <typename T>
+std::vector<NDView<T,3>> make_subviews(NDView<T, 3> &data, ssize_t n_threads) {
+    std::vector<NDView<T, 3>> subviews;
+    subviews.reserve(n_threads);
+    auto limits = split_task(0, data.shape(0), n_threads);
+    for (const auto &lim : limits) {
+        subviews.push_back(data.sub_view(lim.first, lim.second));
+    }
+    return subviews;
+}
+
 } // namespace aare

include/aare/utils/task.hpp

@@ -1,4 +1,5 @@
-
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
 #include <utility>
 #include <vector>