added links (#101)

Co-authored-by: froejdh_e <erik.frojdh@psi.ch>

.clang-format

@@ -0,0 +1,7 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+
+UseTab: Never
+ColumnLimit: 80
+AlignConsecutiveAssignments: false 
+AlignConsecutiveMacros: true

.github/workflows/build_docs.yml

@@ -0,0 +1,68 @@
+name: Build the package using cmake then documentation
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+     - main
+
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        python-version: ["3.12",]
+
+    runs-on: ${{ matrix.platform }}
+
+    # The setup-miniconda action needs this to activate miniconda
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get conda
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+
+      - name: Prepare
+        run: conda install doxygen sphinx=7.1.2 breathe pybind11 sphinx_rtd_theme furo nlohmann_json zeromq fmt numpy
+
+      - name: Build library 
+        run: |
+            mkdir build 
+            cd build 
+            cmake .. -DAARE_SYSTEM_LIBRARIES=ON -DAARE_DOCS=ON 
+            make -j 2 
+            make docs
+
+      - name: Upload static files as artifact
+        id: deployment
+        uses: actions/upload-pages-artifact@v3 
+        with:
+         path: build/docs/html/
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+
+
+
+

.github/workflows/build_pkg.yml

@@ -0,0 +1,43 @@
+name: Build packages but don't deploy
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+     - main
+     - developer
+
+    #run on PRs as well?
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        python-version: ["3.12",]
+
+    runs-on: ${{ matrix.platform }}
+
+    # The setup-miniconda action needs this to activate miniconda
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get conda
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+
+      - name: Prepare
+        run: conda install conda-build conda-verify pytest anaconda-client
+
+      - name: Build
+        env:
+          CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
+        run: conda build conda-recipe
+

.github/workflows/deploy.yml

@@ -0,0 +1,40 @@
+name: Deploy to slsdetectorgroup conda channel
+
+on:
+  workflow_dispatch
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, ] # macos-12, windows-2019]
+        python-version: ["3.12",]
+
+    runs-on: ${{ matrix.platform }}
+
+    # The setup-miniconda action needs this to activate miniconda
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get conda
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+
+      - name: Prepare
+        run: conda install conda-build conda-verify pytest anaconda-client
+
+      - name: Enable upload
+        run: conda config --set anaconda_upload yes
+
+      - name: Build
+        env:
+          CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }}
+        run: conda build conda-recipe --user slsdetectorgroup --token ${CONDA_TOKEN}
+

.gitignore

@@ -0,0 +1,24 @@
+install/
+.cproject
+.project
+bin/
+.settings
+*.aux
+*.log
+*.out
+*.toc
+*.o
+*.so
+.*
+build/
+RELEASE.txt
+Testing/
+
+ctbDict.cpp
+ctbDict.h
+
+
+
+*.pyc
+*/__pycache__/*
+

CMakeLists.txt

@@ -21,28 +21,35 @@ include(FetchContent)
 
 #Set default build type if none was specified
 include(cmake/helpers.cmake)
+
+
 default_build_type("Release")
-
-
+set_std_fs_lib()
+message(STATUS "Extra linking to fs lib:${STD_FS_LIB}")
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
-option(AARE_USE_WARNINGS "Enable warnings" ON)
+
+# General options
 option(AARE_PYTHON_BINDINGS "Build python bindings" ON)
-option(AARE_TESTS "Build tests" ON)
-option(AARE_EXAMPLES "Build examples" ON)
+option(AARE_TESTS "Build tests" OFF)
+option(AARE_BENCHMARKS "Build benchmarks" OFF)
+option(AARE_EXAMPLES "Build examples" OFF)
 option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
 option(AARE_DOCS "Build documentation" OFF)
+option(AARE_VERBOSE "Verbose output" OFF)
+option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
 
+
+# Configure which of the dependencies to use FetchContent for
 option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
 option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON)
 option(AARE_FETCH_CATCH "Use FetchContent to download catch2" ON)
 option(AARE_FETCH_JSON "Use FetchContent to download nlohmann::json" ON)
 option(AARE_FETCH_ZMQ "Use FetchContent to download libzmq" ON)
-option(ENABLE_DRAFTS "Enable zmq drafts (depends on gnutls or nss)" OFF)
 
 
-#Convenience option to use system libraries
+#Convenience option to use system libraries only (no FetchContent)
 option(AARE_SYSTEM_LIBRARIES "Use system libraries" OFF)
 if(AARE_SYSTEM_LIBRARIES)
     message(STATUS "Build using system libraries")
@@ -51,13 +58,29 @@ if(AARE_SYSTEM_LIBRARIES)
     set(AARE_FETCH_CATCH OFF CACHE BOOL "Disabled FetchContent for catch2" FORCE)
     set(AARE_FETCH_JSON OFF CACHE BOOL "Disabled FetchContent for nlohmann::json" FORCE)
     set(AARE_FETCH_ZMQ OFF CACHE BOOL "Disabled FetchContent for libzmq" FORCE)
+endif()
 
+if(AARE_VERBOSE)
+    add_compile_definitions(AARE_VERBOSE)
+endif()
+
+if(AARE_CUSTOM_ASSERT)
+    add_compile_definitions(AARE_CUSTOM_ASSERT)
+endif()
+
+if(AARE_BENCHMARKS)
+    add_subdirectory(benchmarks)
 endif()
 
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(AARE_FETCH_ZMQ)
+    # Fetchcontent_Declare is deprecated need to find a way to update this
+    # for now setting the policy to old is enough
+    if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30")
+        cmake_policy(SET CMP0169 OLD)
+    endif()
     FetchContent_Declare(
         libzmq
         GIT_REPOSITORY https://github.com/zeromq/libzmq.git
@@ -94,15 +117,40 @@ if (AARE_FETCH_FMT)
             GIT_PROGRESS    TRUE
             USES_TERMINAL_DOWNLOAD TRUE
     )
+    set(FMT_INSTALL ON CACHE BOOL "")
+    # set(FMT_CMAKE_DIR "")
     FetchContent_MakeAvailable(fmt)
     set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON)
+    install(TARGETS fmt 
+        EXPORT ${project}-targets
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
 else()
       find_package(fmt 6 REQUIRED)
 endif()
 
-#TODO! Add options for nlohmann_json as well
-find_package(nlohmann_json 3.11.3 REQUIRED)
 
+if (AARE_FETCH_JSON)
+    FetchContent_Declare(
+        json 
+        URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+    )
+    set(JSON_Install ON CACHE BOOL "")
+    FetchContent_MakeAvailable(json)
+    set(NLOHMANN_JSON_TARGET_NAME nlohmann_json)
+    
+    install(
+        TARGETS nlohmann_json
+        EXPORT "${TARGETS_EXPORT_NAME}"
+        
+    )
+    message(STATUS "target: ${NLOHMANN_JSON_TARGET_NAME}")
+else()
+    find_package(nlohmann_json 3.11.3 REQUIRED)
+endif()
 
 include(GNUInstallDirs)
 
@@ -181,32 +229,27 @@ else()
         INTERFACE 
             -Og 
             -ggdb3 
-            # -D_GLIBCXX_DEBUG # causes errors with boost
-            -D_GLIBCXX_DEBUG_PEDANTIC
     )
 
 endif()
 
-if(AARE_USE_WARNINGS)
-        target_compile_options(
-            aare_compiler_flags 
-            INTERFACE 
-                -Wall 
-                -Wextra 
-                -pedantic 
-                -Wshadow 
-                -Wnon-virtual-dtor
-                -Woverloaded-virtual
-                -Wdouble-promotion
-                -Wformat=2
-                -Wredundant-decls
-                -Wvla
-                -Wdouble-promotion
-                -Werror=return-type #important can cause segfault in optimzed builds
-            )
-endif()
-
-
+# Common flags for GCC and Clang
+target_compile_options(
+    aare_compiler_flags 
+    INTERFACE 
+        -Wall 
+        -Wextra 
+        -pedantic 
+        -Wshadow 
+        -Wnon-virtual-dtor
+        -Woverloaded-virtual
+        -Wdouble-promotion
+        -Wformat=2
+        -Wredundant-decls
+        -Wvla
+        -Wdouble-promotion
+        -Werror=return-type #important can cause segfault in optimzed builds
+    )
 
 endif() #GCC/Clang specific
 
@@ -226,34 +269,42 @@ endif()
 ###------------------------------------------------------------------------------------------
 
 set(PUBLICHEADERS
+    include/aare/ArrayExpr.hpp
     include/aare/ClusterFinder.hpp
+    include/aare/ClusterFile.hpp
+    include/aare/CtbRawFile.hpp
     include/aare/defs.hpp
     include/aare/Dtype.hpp
     include/aare/File.hpp
     include/aare/FileInterface.hpp
     include/aare/Frame.hpp
-    include/aare/json.hpp
     include/aare/NDArray.hpp
     include/aare/NDView.hpp
     include/aare/NumpyFile.hpp
     include/aare/NumpyHelpers.hpp
     include/aare/Pedestal.hpp
+    include/aare/PixelMap.hpp
     include/aare/RawFile.hpp
-    include/aare/SubFile.hpp
+    include/aare/RawMasterFile.hpp
+    include/aare/RawSubFile.hpp
     include/aare/VarClusterFinder.hpp
 
 )
 
 
 set(SourceFiles
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
 )
 
 
@@ -263,10 +314,17 @@ target_include_directories(aare_core PUBLIC
     "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
 )
 
-target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags nlohmann_json::nlohmann_json)
+target_link_libraries(
+    aare_core 
+    PUBLIC 
+    fmt::fmt 
+    nlohmann_json::nlohmann_json 
+    ${STD_FS_LIB} # from helpers.cmake
+    PRIVATE 
+    aare_compiler_flags 
+)
 
 set_target_properties(aare_core PROPERTIES
-    # ARCHIVE_OUTPUT_NAME SlsDetectorStatic
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
     PUBLIC_HEADER "${PUBLICHEADERS}"
 )
@@ -280,6 +338,7 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp
@@ -312,13 +371,13 @@ set(CMAKE_INSTALL_RPATH $ORIGIN)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 
 
-#Overall target to link to when using the library
-add_library(aare INTERFACE) 
-target_link_libraries(aare INTERFACE aare_core aare_compiler_flags)
-target_include_directories(aare INTERFACE 
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:include>
-)
+# #Overall target to link to when using the library
+# add_library(aare INTERFACE) 
+# target_link_libraries(aare INTERFACE aare_core aare_compiler_flags)
+# target_include_directories(aare INTERFACE 
+#     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+#     $<INSTALL_INTERFACE:include>
+# )
 
 # add_subdirectory(examples)
 
@@ -362,6 +421,6 @@ add_custom_target(
 
 if(AARE_MASTER_PROJECT)
     set(CMAKE_INSTALL_DIR "share/cmake/${PROJECT_NAME}")
-    set(PROJECT_LIBRARIES slsSupportShared slsDetectorShared slsReceiverShared)
+    set(PROJECT_LIBRARIES aare-core aare-compiler-flags )
     include(cmake/package_config.cmake)
 endif()

README.md

@@ -2,19 +2,70 @@
 Data analysis library for PSI hybrid detectors
 
 
-## Status
+## Build and install
 
-- [ ] Build with CMake on RH8
-- [ ] conda package
+Prerequisites
+- cmake >= 3.14
+- C++17 compiler (gcc >= 8)
+- python >= 3.10
+
+### Development install (for Python)
+
+```bash
+git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http...
+mkdir build
+cd build
+
+#configure using cmake
+cmake ../aare
+
+#build (replace 4 with the number of threads you want to use)
+make -j4 
+```
+
+Now you can use the Python module from your build directory
+
+```python
+import aare
+f = aare.File('Some/File/I/Want_to_open_master_0.json')
+```
+
+To run form other folders either add the path to your conda environment using conda-build or add it to your PYTHONPATH
 
 
-## Project structure 
+### Install using conda/mamba
 
-include/aare - public headers
+```bash
+#enable your env first!
+conda install aare=2024.10.29.dev0 -c slsdetectorgroup
+```
+
+### Install to a custom location and use in your project
+
+Working example in: https://github.com/slsdetectorgroup/aare-examples
+
+```bash
+#build and install aare 
+git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http...
+mkdir build
+cd build
+
+#configure using cmake
+cmake ../aare -DCMAKE_INSTALL_PREFIX=/where/to/put/aare
+
+#build (replace 4 with the number of threads you want to use)
+make -j4 
+
+#install
+make install
 
 
-## Open questions
+#Now configure your project
+ cmake .. -DCMAKE_PREFIX_PATH=SOME_PATH
+```
 
-- How many sub libraries? 
-- Where to place test data? This data is also needed for github actions...
-- What to return to numpy? Our NDArray or a numpy ndarray? Lifetime? 
+### Local build of conda pkgs
+
+```bash
+conda build . --variants="{python: [3.11, 3.12, 3.13]}"
+```

benchmarks/CMakeLists.txt

@@ -0,0 +1,11 @@
+find_package(benchmark REQUIRED)
+
+add_executable(ndarray_benchmark ndarray_benchmark.cpp)
+
+target_link_libraries(ndarray_benchmark benchmark::benchmark aare_core aare_compiler_flags)
+# target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
+
+set_target_properties(ndarray_benchmark PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+        # OUTPUT_NAME run_tests
+)

benchmarks/ndarray_benchmark.cpp

@@ -0,0 +1,136 @@
+#include <benchmark/benchmark.h>
+#include "aare/NDArray.hpp"
+
+
+using aare::NDArray;
+
+constexpr ssize_t size = 1024;
+class TwoArrays : public benchmark::Fixture {
+public:
+    NDArray<int,2> a{{size,size},0};
+    NDArray<int,2> b{{size,size},0};
+  void SetUp(::benchmark::State& state) {
+    for(uint32_t i = 0; i < size; i++){
+      for(uint32_t j = 0; j < size; j++){
+        a(i, j)= i*j+1;
+        b(i, j)= i*j+1;
+      }
+    }
+  }
+
+  // void TearDown(::benchmark::State& state) {
+  // }
+};
+
+
+
+
+BENCHMARK_F(TwoArrays, AddWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a+b;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, AddWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) + b(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_F(TwoArrays, SubtractWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a-b;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, SubtractWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) - b(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_F(TwoArrays, MultiplyWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a*b;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, MultiplyWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) * b(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_F(TwoArrays, DivideWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a/b;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, DivideWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) / b(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_F(TwoArrays, FourAddWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a+b+a+b;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, FourAddWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) + b(i) + a(i) + b(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_F(TwoArrays, MultiplyAddDivideWithOperator)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res = a*a+b/a;
+    benchmark::DoNotOptimize(res);
+  }
+}
+BENCHMARK_F(TwoArrays, MultiplyAddDivideWithIndex)(benchmark::State& st) {
+   for (auto _ : st) {
+    // This code gets timed
+    NDArray<int,2> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        res(i) = a(i) * a(i) + b(i) / a(i);
+    }
+    benchmark::DoNotOptimize(res);
+  }
+}
+
+BENCHMARK_MAIN();

cmake/helpers.cmake

@@ -4,3 +4,43 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   set(CMAKE_BUILD_TYPE ${val} CACHE STRING "Build type (default ${val})" FORCE)
 endif()
 endfunction()
+
+function(set_std_fs_lib)
+# from pybind11
+# Check if we need to add -lstdc++fs or -lc++fs or nothing
+if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD LESS 17)
+  set(STD_FS_NO_LIB_NEEDED TRUE)
+elseif(MSVC)
+  set(STD_FS_NO_LIB_NEEDED TRUE)
+else()
+  file(
+    WRITE ${CMAKE_CURRENT_BINARY_DIR}/main.cpp
+    "#include <filesystem>\nint main(int argc, char ** argv) {\n  std::filesystem::path p(argv[0]);\n  return p.string().length();\n}"
+  )
+  try_compile(
+    STD_FS_NO_LIB_NEEDED ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp
+    COMPILE_DEFINITIONS -std=c++17)
+  try_compile(
+    STD_FS_NEEDS_STDCXXFS ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp
+    COMPILE_DEFINITIONS -std=c++17
+    LINK_LIBRARIES stdc++fs)
+  try_compile(
+    STD_FS_NEEDS_CXXFS ${CMAKE_CURRENT_BINARY_DIR}
+    SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp
+    COMPILE_DEFINITIONS -std=c++17
+    LINK_LIBRARIES c++fs)
+endif()
+
+if(${STD_FS_NEEDS_STDCXXFS})
+  set(STD_FS_LIB stdc++fs PARENT_SCOPE)
+elseif(${STD_FS_NEEDS_CXXFS})
+  set(STD_FS_LIB c++fs PARENT_SCOPE)
+elseif(${STD_FS_NO_LIB_NEEDED})
+  set(STD_FS_LIB "" PARENT_SCOPE)
+else()
+  message(WARNING "Unknown C++17 compiler - not passing -lstdc++fs")
+  set(STD_FS_LIB "")
+endif()
+endfunction()

cmake/project-config.cmake.in

@@ -12,8 +12,10 @@ include(CMakeFindDependencyMacro)
 
 set(SLS_USE_HDF5 "@SLS_USE_HDF5@")
 
-
+# List dependencies
 find_dependency(Threads)
+find_dependency(fmt)
+find_dependency(nlohmann_json)
 
 # Add optional dependencies here
 if (SLS_USE_HDF5)

conda-recipe/build.sh

@@ -1,20 +0,0 @@
-
-
-mkdir build
-mkdir install
-cd build
-cmake .. \
-      -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
-      -DCMAKE_INSTALL_PREFIX=install \
-      -DAARE_SYSTEM_LIBRARIES=ON \
-      -DAARE_TESTS=ON \
-      -DAARE_PYTHON_BINDINGS=ON \
-      -DCMAKE_BUILD_TYPE=Release \
-
-     
-NCORES=$(getconf _NPROCESSORS_ONLN)
-echo "Building using: ${NCORES} cores"
-cmake --build . -- -j${NCORES}
-cmake --build . --target install
-
-# CTEST_OUTPUT_ON_FAILURE=1 ctest -j 1

conda-recipe/conda_build_config.yaml

@@ -0,0 +1,28 @@
+python:
+    - 3.11
+    - 3.11
+    - 3.11
+    - 3.12
+    - 3.12
+    - 3.12
+    - 3.13
+
+
+    
+numpy: 
+    - 1.26
+    - 2.0
+    - 2.1
+    - 1.26
+    - 2.0
+    - 2.1
+    - 2.1
+
+
+zip_keys:
+    - python
+    - numpy
+
+pin_run_as_build:
+  numpy: x.x
+  python: x.x

conda-recipe/copy_lib.sh

@@ -1,19 +0,0 @@
-mkdir -p $PREFIX/lib
-mkdir -p $PREFIX/bin
-mkdir -p $PREFIX/include/aare
-
-
-#Shared and static libraries
-cp build/install/lib/* $PREFIX/lib/
-
-#Binaries
-# cp build/install/bin/sls_detector_acquire $PREFIX/bin/.
-# cp build/install/bin/sls_detector_get $PREFIX/bin/.
-# cp build/install/bin/sls_detector_put $PREFIX/bin/.
-# cp build/install/bin/sls_detector_help $PREFIX/bin/.
-# cp build/install/bin/slsReceiver $PREFIX/bin/.
-# cp build/install/bin/slsMultiReceiver $PREFIX/bin/.
-
-
-cp build/install/include/aare/* $PREFIX/include/aare
-cp -rv build/install/share $PREFIX

conda-recipe/meta.yaml

@@ -1,104 +1,52 @@
-
-
 package:
-  name: aare_software
-  version: {{ environ.get('GIT_DESCRIBE_TAG', '') }}
+  name: aare
+  version: 2024.11.15.dev0 #TODO! how to not duplicate this?
+
 
 source:
-  - path: ..
+  path: ..
 
 build:
   number: 0
-  binary_relocation: True
-  rpaths:
-    - lib/
+  script:
+    - unset CMAKE_GENERATOR && {{ PYTHON }} -m pip install . -vv  # [not win]
+    - {{ PYTHON }} -m pip install . -vv  # [win]
 
 requirements:
   build:
-    - {{ compiler('c') }}
-    - {{compiler('cxx')}}
-    - cmake
-    # - qt 5.*
-    # - xorg-libx11
-    # - xorg-libice
-    # - xorg-libxext
-    # - xorg-libsm
-    # - xorg-libxau
-    # - xorg-libxrender
-    # - xorg-libxfixes
-    # - {{ cdt('mesa-libgl-devel') }}      # [linux]
-    # - {{ cdt('mesa-libegl-devel') }}     # [linux]
-    # - {{ cdt('mesa-dri-drivers') }}      # [linux]
-    # - {{ cdt('libselinux') }}            # [linux]
-    # - {{ cdt('libxdamage') }}            # [linux]
-    # - {{ cdt('libxxf86vm') }}            # [linux]
-    # - expat
+    - python {{python}}
+    - numpy {{ numpy }}
+    - {{ compiler('cxx') }}
+
 
   host:
-    # - libstdcxx-ng
-    # - libgcc-ng
-    # - xorg-libx11
-    # - xorg-libice
-    # - xorg-libxext
-    # - xorg-libsm
-    # - xorg-libxau
-    # - xorg-libxrender
-    # - xorg-libxfixes
-    # - expat
+    - cmake
+    - ninja
+    - python {{python}}
+    - numpy {{ numpy }}
+    - pip
+    - scikit-build-core
+    - pybind11 >=2.13.0
+    - fmt
+    - zeromq
+    - nlohmann_json
+    - catch2
 
   run:
-    # - libstdcxx-ng
-    # - libgcc-ng 
+    - python {{python}}
+    - numpy {{ numpy }}
 
 
-outputs:
-  - name: aarelib
-    script: copy_lib.sh
+test:
+  imports:
+    - aare
+  # requires:
+  #   - pytest
+  # source_files:
+  #   - tests
+  # commands:
+  #   - pytest tests
 
-    requirements:
-      build:
-        - {{ compiler('c') }}
-        - {{compiler('cxx')}}
-        - catch2
-        - zstd
-
-        # - libstdcxx-ng
-        # - libgcc-ng
-
-
-      run: 
-        # - libstdcxx-ng
-        # - libgcc-ng
-
-  # - name: aare
-
-  #   script: build_pylib.sh
-
-  #   requirements:
-  #     build:
-  #       - python
-  #       - {{ compiler('c') }}
-  #       - {{compiler('cxx')}}
-  #       - {{ pin_subpackage('slsdetlib', exact=True) }}
-  #       - setuptools
-  #       - pybind11=2.11
-
-  #     host:
-  #       - python
-  #       - {{ pin_subpackage('slsdetlib', exact=True) }}
-  #       - pybind11=2.11
-
-
-  #     run:
-  #       - libstdcxx-ng
-  #       - libgcc-ng
-  #       - python
-  #       - numpy
-  #       - {{ pin_subpackage('slsdetlib', exact=True) }}
-
-
-  #   test:
-  #     imports:
-  #       - slsdet
-
-  
+about:
+  summary: An example project built with pybind11 and scikit-build.
+  # license_file: LICENSE

docs/CMakeLists.txt

@@ -10,22 +10,36 @@ configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
 set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src)
 set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR})
 
-set(SPHINX_SOURCE_FILES
-  src/index.rst
-  src/NDArray.rst
-  src/NDView.rst
-  src/File.rst
-  src/Frame.rst
-  src/Dtype.rst
-  src/ClusterFinder.rst
-  src/Pedestal.rst
-  src/VarClusterFinder.rst
-  src/pyFile.rst
-)
+
+file(GLOB SPHINX_SOURCE_FILES CONFIGURE_DEPENDS "src/*.rst")
+# set(SPHINX_SOURCE_FILES
+#   src/index.rst
+#   src/Installation.rst
+#   src/Requirements.rst
+#   src/NDArray.rst
+#   src/NDView.rst
+#   src/File.rst
+#   src/Frame.rst
+#   src/Dtype.rst
+#   src/ClusterFinder.rst
+#   src/ClusterFile.rst
+#   src/Pedestal.rst
+#   src/RawFile.rst
+#   src/RawSubFile.rst
+#   src/RawMasterFile.rst
+#   src/VarClusterFinder.rst
+#   src/pyVarClusterFinder.rst
+#   src/pyFile.rst
+#   src/pyCtbRawFile.rst
+#   src/pyRawFile.rst
+#   src/pyRawMasterFile.rst
+# )
+
 
 foreach(filename ${SPHINX_SOURCE_FILES})
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${filename} 
-              "${SPHINX_BUILD}/${filename}")
+    get_filename_component(fname ${filename} NAME)
+    message(STATUS "Copying ${filename} to ${SPHINX_BUILD}/src/${fname}")
+    configure_file(${filename} "${SPHINX_BUILD}/src/${fname}")  
 endforeach(filename ${SPHINX_SOURCE_FILES})
 
 configure_file(

docs/Doxyfile.in

@@ -241,11 +241,7 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
 
-TCL_SUBST              =
 
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
@@ -1082,12 +1078,7 @@ VERBATIM_HEADERS       = YES
 
 ALPHABETICAL_INDEX     = YES
 
-# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
-# which the alphabetical index list will be split.
-# Minimum value: 1, maximum value: 20, default value: 5.
-# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
-COLS_IN_ALPHA_INDEX    = 5
 
 # In case all classes in a project start with a common prefix, all classes will
 # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
@@ -1216,14 +1207,6 @@ HTML_COLORSTYLE_SAT    = 100
 
 HTML_COLORSTYLE_GAMMA  = 80
 
-# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
-# page will contain the date and time when the page was generated. Setting this
-# to YES can help to show when doxygen was last run and thus if the
-# documentation is up to date.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_TIMESTAMP         = NO
 
 # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
 # documentation will contain a main index with vertical navigation menus that
@@ -1248,7 +1231,7 @@ HTML_DYNAMIC_SECTIONS  = NO
 # shown in the various tree structured indices initially; the user can expand
 # and collapse entries dynamically later on. Doxygen will expand the tree to
 # such a level that at most the specified number of entries are visible (unless
-# a fully collapsed tree already exceeds this amount). So setting the number of
+# a fully collapsed tree exceeds this amount). So setting the number of
 # entries 1 will produce a full collapsed tree by default. 0 is a special value
 # representing an infinite number of entries and will result in a full expanded
 # tree by default.
@@ -1503,16 +1486,6 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
-# generated for formulas are transparent PNGs. Transparent PNGs are not
-# supported properly for IE 6.0, but are supported on all modern browsers.
-#
-# Note that when changing this option you need to delete any form_*.png files in
-# the HTML output directory before the changes have effect.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-FORMULA_TRANSPARENT    = YES
 
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
 # https://www.mathjax.org) which uses client side Javascript for the rendering
@@ -1776,31 +1749,6 @@ PDF_HYPERLINKS         = YES
 
 USE_PDFLATEX           = YES
 
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
-# command to the generated LaTeX files. This will instruct LaTeX to keep running
-# if errors occur, instead of asking the user for help. This option is also used
-# when generating formulas in HTML.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_BATCHMODE        = NO
-
-# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
-# index chapters (such as File Index, Compound Index, etc.) in the output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_HIDE_INDICES     = NO
-
-# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
-# code with syntax highlighting in the LaTeX output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
@@ -1810,227 +1758,6 @@ LATEX_SOURCE_CODE      = NO
 
 LATEX_BIB_STYLE        = plain
 
-# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
-# page will contain the date and time when the page was generated. Setting this
-# to NO can help when comparing the output of multiple runs.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_TIMESTAMP        = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the RTF output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
-# RTF output is optimized for Word 97 and may not look too pretty with other RTF
-# readers/editors.
-# The default value is: NO.
-
-GENERATE_RTF           = NO
-
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: rtf.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_OUTPUT             = rtf
-
-# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
-# documents. This may be useful for small projects and may help to save some
-# trees in general.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-COMPACT_RTF            = NO
-
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
-# contain hyperlink fields. The RTF file will contain links (just like the HTML
-# output) instead of page references. This makes the output suitable for online
-# browsing using Word or some other Word compatible readers that support those
-# fields.
-#
-# Note: WordPad (write) and others do not support links.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_HYPERLINKS         = NO
-
-# Load stylesheet definitions from file. Syntax is similar to doxygen's config
-# file, i.e. a series of assignments. You only have to provide replacements,
-# missing definitions are set to their default value.
-#
-# See also section "Doxygen usage" for information on how to generate the
-# default style sheet that doxygen normally uses.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_STYLESHEET_FILE    =
-
-# Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's config file. A template extensions file can be generated
-# using doxygen -e rtf extensionFile.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_EXTENSIONS_FILE    =
-
-# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
-# with syntax highlighting in the RTF output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_SOURCE_CODE        = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the man page output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
-# classes and files.
-# The default value is: NO.
-
-GENERATE_MAN           = NO
-
-# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it. A directory man3 will be created inside the directory specified by
-# MAN_OUTPUT.
-# The default directory is: man.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_OUTPUT             = man
-
-# The MAN_EXTENSION tag determines the extension that is added to the generated
-# man pages. In case the manual section does not start with a number, the number
-# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
-# optional.
-# The default value is: .3.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_EXTENSION          = .3
-
-# The MAN_SUBDIR tag determines the name of the directory created within
-# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
-# MAN_EXTENSION with the initial . removed.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_SUBDIR             =
-
-# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
-# will generate one additional man file for each entity documented in the real
-# man page(s). These additional files only source the real man page, but without
-# them the man command would be unable to find the correct page.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_MAN is set to YES.
-
-MAN_LINKS              = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the XML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
-# captures the structure of the code including all documentation.
-# The default value is: NO.
-
-GENERATE_XML           = YES
-
-# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
-# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
-# it.
-# The default directory is: xml.
-# This tag requires that the tag GENERATE_XML is set to YES.
-
-XML_OUTPUT             = xml
-
-# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
-# listings (including syntax highlighting and cross-referencing information) to
-# the XML output. Note that enabling this will significantly increase the size
-# of the XML output.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_XML is set to YES.
-
-XML_PROGRAMLISTING     = YES
-
-#---------------------------------------------------------------------------
-# Configuration options related to the DOCBOOK output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
-# that can be used to generate PDF.
-# The default value is: NO.
-
-GENERATE_DOCBOOK       = NO
-
-# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
-# front of it.
-# The default directory is: docbook.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_OUTPUT         = docbook
-
-# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
-# program listings (including syntax highlighting and cross-referencing
-# information) to the DOCBOOK output. Note that enabling this will significantly
-# increase the size of the DOCBOOK output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_PROGRAMLISTING = NO
-
-#---------------------------------------------------------------------------
-# Configuration options for the AutoGen Definitions output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
-# the structure of the code including all documentation. Note that this feature
-# is still experimental and incomplete at the moment.
-# The default value is: NO.
-
-GENERATE_AUTOGEN_DEF   = NO
-
-#---------------------------------------------------------------------------
-# Configuration options related to the Perl module output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
-# file that captures the structure of the code including all documentation.
-#
-# Note that this feature is still experimental and incomplete at the moment.
-# The default value is: NO.
-
-GENERATE_PERLMOD       = NO
-
-# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
-# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
-# output from the Perl module output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_LATEX          = NO
-
-# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
-# formatted so it can be parsed by a human reader. This is useful if you want to
-# understand what is going on. On the other hand, if this tag is set to NO, the
-# size of the Perl module output will be much smaller and Perl will parse it
-# just the same.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_PRETTY         = YES
-
-# The names of the make variables in the generated doxyrules.make file are
-# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
-# so different doxyrules.make files included by the same Makefile don't
-# overwrite each other's variables.
-# This tag requires that the tag GENERATE_PERLMOD is set to YES.
-
-PERLMOD_MAKEVAR_PREFIX =
 
 #---------------------------------------------------------------------------
 # Configuration options related to the preprocessor
@@ -2162,321 +1889,29 @@ EXTERNAL_PAGES         = YES
 
 PERL_PATH              = /usr/bin/perl
 
+
 #---------------------------------------------------------------------------
-# Configuration options related to the dot tool
+# Configuration options related to the XML output
 #---------------------------------------------------------------------------
-
-# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
-# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
-# NO turns the diagrams off. Note that this option also works with HAVE_DOT
-# disabled, but it is recommended to install and use dot, since it yields more
-# powerful graphs.
-# The default value is: YES.
-
-CLASS_DIAGRAMS         = YES
-
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see:
-# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH            =
-
-# You can include diagrams made with dia in doxygen documentation. Doxygen will
-# then run dia to produce the diagram and insert it in the documentation. The
-# DIA_PATH tag allows you to specify the directory where the dia binary resides.
-# If left empty dia is assumed to be found in the default search path.
-
-DIA_PATH               =
-
-# If set to YES the inheritance and collaboration graphs will hide inheritance
-# and usage relations if the target is undocumented or is not a class.
-# The default value is: YES.
-
-HIDE_UNDOC_RELATIONS   = YES
-
-# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
-# available from the path. This tool is part of Graphviz (see:
-# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
-# Bell Labs. The other options in this section have no effect if this option is
-# set to NO
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
 # The default value is: NO.
 
-HAVE_DOT               = NO
+GENERATE_XML           = YES
 
-# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
-# to run in parallel. When set to 0 doxygen will base this on the number of
-# processors available in the system. You can set it explicitly to a value
-# larger than 0 to get control over the balance between CPU load and processing
-# speed.
-# Minimum value: 0, maximum value: 32, default value: 0.
-# This tag requires that the tag HAVE_DOT is set to YES.
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
 
-DOT_NUM_THREADS        = 0
+XML_OUTPUT             = xml
 
-# When you want a differently looking font in the dot files that doxygen
-# generates you can specify the font name using DOT_FONTNAME. You need to make
-# sure dot is able to find the font, which can be done by putting it in a
-# standard location or by setting the DOTFONTPATH environment variable or by
-# setting DOT_FONTPATH to the directory containing the font.
-# The default value is: Helvetica.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTNAME           = Helvetica
-
-# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
-# dot graphs.
-# Minimum value: 4, maximum value: 24, default value: 10.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTSIZE           = 10
-
-# By default doxygen will tell dot to use the default font as specified with
-# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
-# the path where dot can find it using this tag.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTPATH           =
-
-# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
-# each documented class showing the direct and indirect inheritance relations.
-# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
 # The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
 
-CLASS_GRAPH            = YES
-
-# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
-# graph for each documented class showing the direct and indirect implementation
-# dependencies (inheritance, containment, and class references variables) of the
-# class with other documented classes.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-COLLABORATION_GRAPH    = YES
-
-# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
-# groups, showing the direct groups dependencies.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GROUP_GRAPHS           = YES
-
-# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
-# collaboration diagrams in a style similar to the OMG's Unified Modeling
-# Language.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-UML_LOOK               = NO
-
-# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
-# class node. If there are many fields or methods and many nodes the graph may
-# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
-# number of items for each type to make the size more manageable. Set this to 0
-# for no limit. Note that the threshold may be exceeded by 50% before the limit
-# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
-# but if the number exceeds 15, the total amount of fields shown is limited to
-# 10.
-# Minimum value: 0, maximum value: 100, default value: 10.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-UML_LIMIT_NUM_FIELDS   = 10
-
-# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
-# collaboration graphs will show the relations between templates and their
-# instances.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-TEMPLATE_RELATIONS     = NO
-
-# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
-# YES then doxygen will generate a graph for each documented file showing the
-# direct and indirect include dependencies of the file with other documented
-# files.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INCLUDE_GRAPH          = YES
-
-# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
-# set to YES then doxygen will generate a graph for each documented file showing
-# the direct and indirect include dependencies of the file with other documented
-# files.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INCLUDED_BY_GRAPH      = YES
-
-# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
-# dependency graph for every global function or class method.
-#
-# Note that enabling this option will significantly increase the time of a run.
-# So in most cases it will be better to enable call graphs for selected
-# functions only using the \callgraph command. Disabling a call graph can be
-# accomplished by means of the command \hidecallgraph.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-CALL_GRAPH             = NO
-
-# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
-# dependency graph for every global function or class method.
-#
-# Note that enabling this option will significantly increase the time of a run.
-# So in most cases it will be better to enable caller graphs for selected
-# functions only using the \callergraph command. Disabling a caller graph can be
-# accomplished by means of the command \hidecallergraph.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-CALLER_GRAPH           = NO
-
-# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
-# hierarchy of all classes instead of a textual one.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GRAPHICAL_HIERARCHY    = YES
-
-# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
-# dependencies a directory has on other directories in a graphical way. The
-# dependency relations are determined by the #include relations between the
-# files in the directories.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DIRECTORY_GRAPH        = YES
-
-# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
-# generated by dot. For an explanation of the image formats see the section
-# output formats in the documentation of the dot tool (Graphviz (see:
-# http://www.graphviz.org/)).
-# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
-# to make the SVG files visible in IE 9+ (other browsers do not have this
-# requirement).
-# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo,
-# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
-# png:gdiplus:gdiplus.
-# The default value is: png.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_IMAGE_FORMAT       = png
-
-# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
-# enable generation of interactive SVG images that allow zooming and panning.
-#
-# Note that this requires a modern browser other than Internet Explorer. Tested
-# and working are Firefox, Chrome, Safari, and Opera.
-# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
-# the SVG files visible. Older versions of IE do not have SVG support.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-INTERACTIVE_SVG        = NO
-
-# The DOT_PATH tag can be used to specify the path where the dot tool can be
-# found. If left blank, it is assumed the dot tool can be found in the path.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_PATH               =
-
-# The DOTFILE_DIRS tag can be used to specify one or more directories that
-# contain dot files that are included in the documentation (see the \dotfile
-# command).
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOTFILE_DIRS           =
-
-# The MSCFILE_DIRS tag can be used to specify one or more directories that
-# contain msc files that are included in the documentation (see the \mscfile
-# command).
-
-MSCFILE_DIRS           =
-
-# The DIAFILE_DIRS tag can be used to specify one or more directories that
-# contain dia files that are included in the documentation (see the \diafile
-# command).
-
-DIAFILE_DIRS           =
-
-# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
-# path where java can find the plantuml.jar file. If left blank, it is assumed
-# PlantUML is not used or called during a preprocessing step. Doxygen will
-# generate a warning when it encounters a \startuml command in this case and
-# will not generate output for the diagram.
-
-PLANTUML_JAR_PATH      =
-
-# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
-# configuration file for plantuml.
-
-PLANTUML_CFG_FILE      =
-
-# When using plantuml, the specified paths are searched for files specified by
-# the !include statement in a plantuml block.
-
-PLANTUML_INCLUDE_PATH  =
-
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
-# that will be shown in the graph. If the number of nodes in a graph becomes
-# larger than this value, doxygen will truncate the graph, which is visualized
-# by representing a node as a red box. Note that doxygen if the number of direct
-# children of the root node in a graph is already larger than
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
-# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
-# Minimum value: 0, maximum value: 10000, default value: 50.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_GRAPH_MAX_NODES    = 50
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
-# generated by dot. A depth value of 3 means that only nodes reachable from the
-# root by following a path via at most 3 edges will be shown. Nodes that lay
-# further from the root node will be omitted. Note that setting this option to 1
-# or 2 may greatly reduce the computation time needed for large code bases. Also
-# note that the size of a graph can be further restricted by
-# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
-# Minimum value: 0, maximum value: 1000, default value: 0.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-MAX_DOT_GRAPH_DEPTH    = 0
-
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not seem
-# to support this out of the box.
-#
-# Warning: Depending on the platform used, enabling this option may lead to
-# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
-# read).
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_TRANSPARENT        = NO
-
-# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
-# files in one run (i.e. multiple -o and -T options on the command line). This
-# makes dot run faster, but since only newer versions of dot (>1.8.10) support
-# this, this feature is disabled by default.
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_MULTI_TARGETS      = NO
-
-# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
-# explaining the meaning of the various boxes and arrows in the dot generated
-# graphs.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-GENERATE_LEGEND        = YES
-
-# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
-# files that are used to generate the various graphs.
-# The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_CLEANUP            = YES
+XML_PROGRAMLISTING     = YES

docs/conf.py.in

@@ -12,10 +12,8 @@
 #
 import os
 import sys
-# sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath('../bin/'))
-#sys.path.insert(0, '/home/l_frojdh/sls/build/bin')
-#sys.path.insert(0, @CMAKE_CURRENT_BINARY_DIR@)
+sys.path.insert(0, os.path.abspath('..'))
+
 print(sys.path)
 
 # -- Project information -----------------------------------------------------

docs/src/ClusterFile.rst

@@ -0,0 +1,7 @@
+ClusterFile
+=============
+
+.. doxygenclass:: aare::ClusterFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/Consume.rst

@@ -0,0 +1,19 @@
+Use from C++
+========================
+
+There are a few different way to use aare in your C++ project. Which one you choose 
+depends on how you intend to work with the library and how you manage your dependencies.
+
+
+Install and use cmake with find_package(aare)
+-------------------------------------------------
+
+https://github.com/slsdetectorgroup/aare-examples
+
+.. include:: _install.rst
+
+
+Use as a submodule
+------------------- 
+
+Coming soon...

docs/src/File.rst

@@ -4,4 +4,5 @@ File
 
 .. doxygenclass:: aare::File
    :members:
-   :undoc-members:
+   :undoc-members:
+   :private-members:

docs/src/Frame.rst

@@ -4,4 +4,5 @@ Frame
 
 .. doxygenclass:: aare::Frame
    :members:
-   :undoc-members:
+   :undoc-members:
+   :private-members:

docs/src/Installation.rst

@@ -0,0 +1,106 @@
+****************
+Installation
+****************
+
+.. attention ::
+
+    - https://cliutils.gitlab.io/modern-cmake/README.html
+
+conda/mamaba
+~~~~~~~~~~~~~~~~~~
+
+This is the recommended way to install aare. Using a package manager makes it easy to 
+switch between versions and is (one of) the most convenient way to install up to date
+dependencies on older distributions.
+
+.. note ::
+
+    aare is developing rapidly. Check for the latest release by 
+    using: **conda search aare -c slsdetectorgroup**
+
+
+.. code-block:: bash
+
+    # Install a specific version: 
+    conda install aare=2024.11.11.dev0 -c slsdetectorgroup
+
+
+cmake build (development install)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are working on aare or want to test our a version that doesn't yet have
+a conda package. Build using cmake and then run from the build folder.
+
+.. code-block:: bash
+
+    git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http...
+    mkdir build
+    cd build
+
+    #configure using cmake
+    cmake ../aare
+
+    #build (replace 4 with the number of threads you want to use)
+    make -j4 
+
+
+    # add the build folder to your PYTHONPATH and then you should be able to
+    # import aare in python
+
+cmake build + install and use in your C++ project
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. warning ::
+
+    When building aare with default settings we also include fmt and nlohmann_json.
+    Installation to a custom location is highly recommended.
+
+
+.. note ::
+
+    It is also possible to install aare with conda and then use in your C++ project.
+
+.. include:: _install.rst
+
+
+cmake options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For detailed options see the CMakeLists.txt file in the root directory of the project.
+
+.. code-block:: bash
+
+    # usage (or edit with ccmake .)
+    cmake ../aare -DOPTION1=ON -DOPTION2=OFF
+
+
+**AARE_SYSTEM_LIBRARIES "Use system libraries" OFF**
+
+Use system libraries instead of using FetchContent to pull in dependencies. Default option is off.
+
+
+**AARE_PYTHON_BINDINGS "Build python bindings" ON**
+
+Build the Python bindings. Default option is on. 
+
+.. warning ::
+
+    If you have a newer system Python compared to the one in your virtual environment,
+    you might have to pass -DPython_FIND_VIRTUALENV=ONLY to cmake.
+
+**AARE_TESTS "Build tests" OFF**
+
+Build unit tests. Default option is off.
+
+**AARE_EXAMPLES "Build examples" OFF**
+
+**AARE_DOCS "Build documentation" OFF**
+
+Build documentation. Needs doxygen, sphinx and breathe. Default option is off.
+Requires a separate make docs.
+
+**AARE_VERBOSE "Verbose output" OFF**
+
+**AARE_CUSTOM_ASSERT "Use custom assert" OFF**
+
+Enable custom assert macro to check for errors. Default option is off.

docs/src/Pedestal.rst

@@ -4,4 +4,5 @@ Pedestal
 
 .. doxygenclass:: aare::Pedestal
    :members:
-   :undoc-members:
+   :undoc-members:
+   :private-members:

docs/src/RawFile.rst

@@ -0,0 +1,8 @@
+RawFile
+===============
+
+
+.. doxygenclass:: aare::RawFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/RawMasterFile.rst

@@ -0,0 +1,14 @@
+RawMasterFile
+===============
+
+
+.. doxygenclass:: aare::RawMasterFile
+   :members:
+   :undoc-members:
+   :private-members:
+
+
+.. doxygenclass:: aare::RawFileNameComponents
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/RawSubFile.rst

@@ -0,0 +1,8 @@
+RawSubFile
+===============
+
+
+.. doxygenclass:: aare::RawSubFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/Requirements.rst

@@ -0,0 +1,23 @@
+Requirements
+==============================================
+
+- C++17 compiler (gcc 8/clang 7)
+- CMake 3.14+
+
+**Internally used libraries**
+
+.. note ::
+
+    These can also be picked up from the system/conda environment by specifying:
+    -DAARE_SYSTEM_LIBRARIES=ON during the cmake configuration.
+
+- pybind11
+- fmt
+- nlohmann_json
+- ZeroMQ
+
+**Extra dependencies for building documentation**
+
+- Sphinx
+- Breathe
+- Doxygen

docs/src/_install.rst

@@ -0,0 +1,23 @@
+.. code-block:: bash
+
+    #build and install aare 
+    git clone git@github.com:slsdetectorgroup/aare.git --branch=developer #or using http...
+    mkdir build
+    cd build
+
+    #configure using cmake
+    cmake ../aare -DCMAKE_INSTALL_PREFIX=/where/to/put/aare
+
+    #build (replace 4 with the number of threads you want to use)
+    make -j4 
+
+    #install
+    make install
+
+    #Go to your project
+    cd /your/project/source
+
+    #Now configure your project
+    mkdir build
+    cd build
+    cmake .. -DCMAKE_PREFIX_PATH=SOME_PATH

docs/src/index.rst

@@ -2,8 +2,38 @@ AARE
 ==============================================
 
 .. note :: 
+    
+    **Examples:**
+
+    - `jupyter notebooks <https://github.com/slsdetectorgroup/aare-notebooks>`_
+    - `cmake+install <https://github.com/slsdetectorgroup/aare-examples>`_
+    - `git submodule <https://github.com/slsdetectorgroup/aare-submodule>`_
+
+
+
+
+.. toctree::
+    :caption: Installation
+    :maxdepth: 3
+    
+    Installation
+    Requirements
+    Consume
+
+
+
+
+.. toctree::
+    :caption: Python API
+    :maxdepth: 1
+    
+    pyFile
+    pyCtbRawFile
+    pyClusterFile
+    pyRawFile
+    pyRawMasterFile
+    pyVarClusterFinder
 
-    Hello
 
 .. toctree::
     :caption: C++ API
@@ -15,11 +45,13 @@ AARE
     File
     Dtype
     ClusterFinder
+    ClusterFile
     Pedestal
+    RawFile
+    RawSubFile
+    RawMasterFile
     VarClusterFinder
 
-.. toctree::
-    :caption: Python API
-    :maxdepth: 1
-    
-    pyFile
+
+
+

docs/src/pyClusterFile.rst

@@ -0,0 +1,11 @@
+
+ClusterFile
+============
+
+.. py:currentmodule:: aare
+
+.. autoclass:: ClusterFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyCtbRawFile.rst

@@ -0,0 +1,11 @@
+
+CtbRawFile
+============
+
+.. py:currentmodule:: aare
+
+.. autoclass:: CtbRawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyRawFile.rst

@@ -0,0 +1,10 @@
+RawFile
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: RawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyRawMasterFile.rst

@@ -0,0 +1,10 @@
+RawMasterFile
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: RawMasterFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyVarClusterFinder.rst

@@ -0,0 +1,10 @@
+VarClusterFinder
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: VarClusterFinder
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

include/aare/ArrayExpr.hpp

@@ -0,0 +1,99 @@
+#pragma once
+#include <cstdint> //int64_t
+#include <cstddef> //size_t
+#include <array> 
+
+#include <cassert>
+namespace aare {
+
+template <typename E, int64_t Ndim> class ArrayExpr {
+  public:
+    static constexpr bool is_leaf = false;
+
+    auto operator[](size_t i) const { return static_cast<E const &>(*this)[i]; }
+    auto operator()(size_t i) const { return static_cast<E const &>(*this)[i]; }
+    auto size() const { return static_cast<E const &>(*this).size(); }
+    std::array<int64_t, Ndim> shape() const { return static_cast<E const &>(*this).shape(); }
+};
+
+template <typename A, typename B, int64_t Ndim>
+class ArrayAdd : public ArrayExpr<ArrayAdd<A, B, Ndim>, Ndim> {
+    const A &arr1_;
+    const B &arr2_;
+
+  public:
+    ArrayAdd(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) {
+        assert(arr1.size() == arr2.size());
+    }
+    auto operator[](int i) const { return arr1_[i] + arr2_[i]; }
+    size_t size() const { return arr1_.size(); }
+    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+};
+
+template <typename A, typename B, int64_t Ndim>
+class ArraySub : public ArrayExpr<ArraySub<A, B, Ndim>, Ndim> {
+    const A &arr1_;
+    const B &arr2_;
+
+  public:
+    ArraySub(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) {
+        assert(arr1.size() == arr2.size());
+    }
+    auto operator[](int i) const { return arr1_[i] - arr2_[i]; }
+    size_t size() const { return arr1_.size(); }
+    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+};
+
+template <typename A, typename B, int64_t Ndim>
+class ArrayMul : public ArrayExpr<ArrayMul<A, B, Ndim>,Ndim> {
+    const A &arr1_;
+    const B &arr2_;
+
+  public:
+    ArrayMul(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) {
+        assert(arr1.size() == arr2.size());
+    }
+    auto operator[](int i) const { return arr1_[i] * arr2_[i]; }
+    size_t size() const { return arr1_.size(); }
+    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+};
+
+template <typename A, typename B, int64_t Ndim>
+class ArrayDiv : public ArrayExpr<ArrayDiv<A, B, Ndim>, Ndim> {
+    const A &arr1_;
+    const B &arr2_;
+
+  public:
+    ArrayDiv(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) {
+        assert(arr1.size() == arr2.size());
+    }
+    auto operator[](int i) const { return arr1_[i] / arr2_[i]; }
+    size_t size() const { return arr1_.size(); }
+    std::array<int64_t, Ndim> shape() const { return arr1_.shape(); }
+};
+
+
+
+template <typename A, typename B, int64_t Ndim>
+auto operator+(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
+    return ArrayAdd<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
+}
+
+template <typename A, typename B, int64_t Ndim>
+auto operator-(const ArrayExpr<A,Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
+    return ArraySub<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
+}
+
+template <typename A, typename B, int64_t Ndim>
+auto operator*(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
+    return ArrayMul<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
+}
+
+template <typename A, typename B, int64_t Ndim>
+auto operator/(const ArrayExpr<A, Ndim> &arr1, const ArrayExpr<B, Ndim> &arr2) {
+    return ArrayDiv<ArrayExpr<A, Ndim>, ArrayExpr<B, Ndim>, Ndim>(arr1, arr2);
+}
+
+
+
+} // namespace aare

include/aare/ClusterFile.hpp

@@ -0,0 +1,65 @@
+
+
+#include "aare/defs.hpp"
+#include <filesystem>
+#include <fstream>
+
+namespace aare {
+
+struct Cluster {
+    int16_t x;
+    int16_t y;
+    int32_t data[9];
+};
+
+typedef enum {
+    cBottomLeft = 0,
+    cBottomRight = 1,
+    cTopLeft = 2,
+    cTopRight = 3
+} corner;
+
+typedef enum {
+    pBottomLeft = 0,
+    pBottom = 1,
+    pBottomRight = 2,
+    pLeft = 3,
+    pCenter = 4,
+    pRight = 5,
+    pTopLeft = 6,
+    pTop = 7,
+    pTopRight = 8
+} pixel;
+
+struct ClusterAnalysis {
+    uint32_t c;
+    int32_t tot;
+    double etax;
+    double etay;
+};
+
+
+
+class ClusterFile {
+    FILE *fp{};
+    uint32_t m_num_left{};
+    size_t m_chunk_size{};
+
+  public:
+    ClusterFile(const std::filesystem::path &fname, size_t chunk_size = 1000);
+    std::vector<Cluster> read_clusters(size_t n_clusters);
+    std::vector<Cluster> read_frame(int32_t &out_fnum);
+    std::vector<Cluster>
+    read_cluster_with_cut(size_t n_clusters, double *noise_map, int nx, int ny);
+
+    int analyze_data(int32_t *data, int32_t *t2, int32_t *t3, char *quad,
+                 double *eta2x, double *eta2y, double *eta3x, double *eta3y);
+    int analyze_cluster(Cluster cl, int32_t *t2, int32_t *t3, char *quad,
+                    double *eta2x, double *eta2y, double *eta3x,
+                    double *eta3y);
+
+    size_t chunk_size() const { return m_chunk_size; }
+
+};
+
+} // namespace aare

include/aare/ClusterFinder.hpp

@@ -3,33 +3,60 @@
 #include "aare/NDArray.hpp"
 #include "aare/NDView.hpp"
 #include "aare/Pedestal.hpp"
+#include "aare/defs.hpp"
 #include <cstddef>
 
 namespace aare {
 
 /** enum to define the event types */
 enum eventType {
-    PEDESTAL,            /** pedestal */
-    NEIGHBOUR,           /** neighbour i.e. below threshold, but in the cluster of a photon */
-    PHOTON,              /** photon i.e. above threshold */
-    PHOTON_MAX,          /** maximum of a cluster satisfying the photon conditions */
-    NEGATIVE_PEDESTAL,   /** negative value, will not be accounted for as pedestal in order to
-                             avoid drift of the pedestal towards negative values */
+    PEDESTAL,   /** pedestal */
+    NEIGHBOUR,  /** neighbour i.e. below threshold, but in the cluster of a
+                   photon */
+    PHOTON,     /** photon i.e. above threshold */
+    PHOTON_MAX, /** maximum of a cluster satisfying the photon conditions */
+    NEGATIVE_PEDESTAL, /** negative value, will not be accounted for as pedestal
+                          in order to avoid drift of the pedestal towards
+                          negative values */
     UNDEFINED_EVENT = -1 /** undefined */
 };
 
+template <typename FRAME_TYPE = uint16_t, typename PEDESTAL_TYPE = double>
 class ClusterFinder {
+    Shape<2> m_image_size;
+    const int m_cluster_sizeX;
+    const int m_cluster_sizeY;
+    const double m_threshold;
+    const double m_nSigma;
+    const double c2;
+    const double c3;
+    Pedestal<PEDESTAL_TYPE> m_pedestal;
+
   public:
-    ClusterFinder(int cluster_sizeX, int cluster_sizeY, double nSigma = 5.0, double threshold = 0.0)
-        : m_cluster_sizeX(cluster_sizeX), m_cluster_sizeY(cluster_sizeY), m_threshold(threshold), m_nSigma(nSigma) {
+    ClusterFinder(Shape<2> image_size, Shape<2>cluster_size, double nSigma = 5.0,
+                  double threshold = 0.0)
+        : m_image_size(image_size), m_cluster_sizeX(cluster_size[0]), m_cluster_sizeY(cluster_size[1]),
+          m_threshold(threshold), m_nSigma(nSigma),
+          c2(sqrt((m_cluster_sizeY + 1) / 2 * (m_cluster_sizeX + 1) / 2)),
+          c3(sqrt(m_cluster_sizeX * m_cluster_sizeY)),
+          m_pedestal(image_size[0], image_size[1]) {
 
-        c2 = sqrt((cluster_sizeY + 1) / 2 * (cluster_sizeX + 1) / 2);
-        c3 = sqrt(cluster_sizeX * cluster_sizeY);
-    };
+              // c2 = sqrt((cluster_sizeY + 1) / 2 * (cluster_sizeX + 1) / 2);
+              // c3 = sqrt(cluster_sizeX * cluster_sizeY);
+          };
 
-    template <typename FRAME_TYPE, typename PEDESTAL_TYPE>
-    std::vector<Cluster> find_clusters_without_threshold(NDView<FRAME_TYPE, 2> frame, Pedestal<PEDESTAL_TYPE> &pedestal,
-                                                         bool late_update = false) {
+    void push_pedestal_frame(NDView<FRAME_TYPE, 2> frame) {
+        m_pedestal.push(frame);
+    }
+
+    NDArray<PEDESTAL_TYPE, 2> pedestal() {
+        return m_pedestal.mean();
+    }
+
+    std::vector<DynamicCluster>
+    find_clusters_without_threshold(NDView<FRAME_TYPE, 2> frame,
+                                    // Pedestal<PEDESTAL_TYPE> &pedestal,
+                                    bool late_update = false) {
         struct pedestal_update {
             int x;
             int y;
@@ -37,7 +64,7 @@ class ClusterFinder {
         };
         std::vector<pedestal_update> pedestal_updates;
 
-        std::vector<Cluster> clusters;
+        std::vector<DynamicCluster> clusters;
         std::vector<std::vector<eventType>> eventMask;
         for (int i = 0; i < frame.shape(0); i++) {
             eventMask.push_back(std::vector<eventType>(frame.shape(1)));
@@ -52,10 +79,14 @@ class ClusterFinder {
                 long double total = 0;
                 eventMask[iy][ix] = PEDESTAL;
 
-                for (short ir = -(m_cluster_sizeY / 2); ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                    for (short ic = -(m_cluster_sizeX / 2); ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                        if (ix + ic >= 0 && ix + ic < frame.shape(1) && iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                            val = frame(iy + ir, ix + ic) - pedestal.mean(iy + ir, ix + ic);
+                for (short ir = -(m_cluster_sizeY / 2);
+                     ir < (m_cluster_sizeY / 2) + 1; ir++) {
+                    for (short ic = -(m_cluster_sizeX / 2);
+                         ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                        if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
+                            iy + ir >= 0 && iy + ir < frame.shape(0)) {
+                            val = frame(iy + ir, ix + ic) -
+                                  m_pedestal.mean(iy + ir, ix + ic);
                             total += val;
                             if (val > max) {
                                 max = val;
@@ -63,9 +94,9 @@ class ClusterFinder {
                         }
                     }
                 }
-                auto rms = pedestal.standard_deviation(iy, ix);
+                auto rms = m_pedestal.std(iy, ix);
 
-                if (frame(iy, ix) - pedestal.mean(iy, ix) < -m_nSigma * rms) {
+                if (frame(iy, ix) - m_pedestal.mean(iy, ix) < -m_nSigma * rms) {
                     eventMask[iy][ix] = NEGATIVE_PEDESTAL;
                     continue;
                 } else if (max > m_nSigma * rms) {
@@ -73,26 +104,33 @@ class ClusterFinder {
 
                 } else if (total > c3 * m_nSigma * rms) {
                     eventMask[iy][ix] = PHOTON;
-                } else{
+                } else {
                     if (late_update) {
                         pedestal_updates.push_back({ix, iy, frame(iy, ix)});
                     } else {
-                        pedestal.push(iy, ix, frame(iy, ix));
+                        m_pedestal.push(iy, ix, frame(iy, ix));
                     }
                     continue;
                 }
-                if (eventMask[iy][ix] == PHOTON && (frame(iy, ix) - pedestal.mean(iy, ix)) >= max) {
+                if (eventMask[iy][ix] == PHOTON &&
+                    (frame(iy, ix) - m_pedestal.mean(iy, ix)) >= max) {
                     eventMask[iy][ix] = PHOTON_MAX;
-                    Cluster cluster(m_cluster_sizeX, m_cluster_sizeY, Dtype(typeid(PEDESTAL_TYPE)));
+                    DynamicCluster cluster(m_cluster_sizeX, m_cluster_sizeY,
+                                    Dtype(typeid(PEDESTAL_TYPE)));
                     cluster.x = ix;
                     cluster.y = iy;
                     short i = 0;
 
-                    for (short ir = -(m_cluster_sizeY / 2); ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                        for (short ic = -(m_cluster_sizeX / 2); ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                            if (ix + ic >= 0 && ix + ic < frame.shape(1) && iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                                PEDESTAL_TYPE tmp = static_cast<PEDESTAL_TYPE>(frame(iy + ir, ix + ic)) -
-                                                    pedestal.mean(iy + ir, ix + ic);
+                    for (short ir = -(m_cluster_sizeY / 2);
+                         ir < (m_cluster_sizeY / 2) + 1; ir++) {
+                        for (short ic = -(m_cluster_sizeX / 2);
+                             ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                            if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
+                                iy + ir >= 0 && iy + ir < frame.shape(0)) {
+                                PEDESTAL_TYPE tmp =
+                                    static_cast<PEDESTAL_TYPE>(
+                                        frame(iy + ir, ix + ic)) -
+                                    m_pedestal.mean(iy + ir, ix + ic);
                                 cluster.set<PEDESTAL_TYPE>(i, tmp);
                                 i++;
                             }
@@ -104,16 +142,18 @@ class ClusterFinder {
         }
         if (late_update) {
             for (auto &update : pedestal_updates) {
-                pedestal.push(update.y, update.x, update.value);
+                m_pedestal.push(update.y, update.x, update.value);
             }
         }
         return clusters;
     }
 
-    template <typename FRAME_TYPE, typename PEDESTAL_TYPE>
-    std::vector<Cluster> find_clusters_with_threshold(NDView<FRAME_TYPE, 2> frame, Pedestal<PEDESTAL_TYPE> &pedestal) {
+    // template <typename FRAME_TYPE, typename PEDESTAL_TYPE>
+    std::vector<DynamicCluster>
+    find_clusters_with_threshold(NDView<FRAME_TYPE, 2> frame,
+                                 Pedestal<PEDESTAL_TYPE> &pedestal) {
         assert(m_threshold > 0);
-        std::vector<Cluster> clusters;
+        std::vector<DynamicCluster> clusters;
         std::vector<std::vector<eventType>> eventMask;
         for (int i = 0; i < frame.shape(0); i++) {
             eventMask.push_back(std::vector<eventType>(frame.shape(1)));
@@ -123,7 +163,8 @@ class ClusterFinder {
         NDArray<FRAME_TYPE, 2> rest({frame.shape(0), frame.shape(1)});
         NDArray<int, 2> nph({frame.shape(0), frame.shape(1)});
         // convert to n photons
-        // nph = (frame-pedestal.mean()+0.5*m_threshold)/m_threshold; // can be optimized with expression templates?
+        // nph = (frame-pedestal.mean()+0.5*m_threshold)/m_threshold; // can be
+        // optimized with expression templates?
         for (int iy = 0; iy < frame.shape(0); iy++) {
             for (int ix = 0; ix < frame.shape(1); ix++) {
                 auto val = frame(iy, ix) - pedestal.mean(iy, ix);
@@ -145,10 +186,14 @@ class ClusterFinder {
                 }
                 eventMask[iy][ix] = NEIGHBOUR;
                 // iterate over cluster pixels around the current pixel (ix,iy)
-                for (short ir = -(m_cluster_sizeY / 2); ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                    for (short ic = -(m_cluster_sizeX / 2); ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                        if (ix + ic >= 0 && ix + ic < frame.shape(1) && iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                            auto val = frame(iy + ir, ix + ic) - pedestal.mean(iy + ir, ix + ic);
+                for (short ir = -(m_cluster_sizeY / 2);
+                     ir < (m_cluster_sizeY / 2) + 1; ir++) {
+                    for (short ic = -(m_cluster_sizeX / 2);
+                         ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                        if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
+                            iy + ir >= 0 && iy + ir < frame.shape(0)) {
+                            auto val = frame(iy + ir, ix + ic) -
+                                       pedestal.mean(iy + ir, ix + ic);
                             total += val;
                             if (val > max) {
                                 max = val;
@@ -157,7 +202,7 @@ class ClusterFinder {
                     }
                 }
 
-                auto rms = pedestal.standard_deviation(iy, ix);
+                auto rms = pedestal.std(iy, ix);
                 if (m_nSigma == 0) {
                     tthr = m_threshold;
                     tthr1 = m_threshold;
@@ -182,16 +227,22 @@ class ClusterFinder {
                     pedestal.push(iy, ix, frame(iy, ix));
                     continue;
                 }
-                if (eventMask[iy][ix] == PHOTON && frame(iy, ix) - pedestal.mean(iy, ix) >= max) {
+                if (eventMask[iy][ix] == PHOTON &&
+                    frame(iy, ix) - pedestal.mean(iy, ix) >= max) {
                     eventMask[iy][ix] = PHOTON_MAX;
-                    Cluster cluster(m_cluster_sizeX, m_cluster_sizeY, Dtype(typeid(FRAME_TYPE)));
+                    DynamicCluster cluster(m_cluster_sizeX, m_cluster_sizeY,
+                                    Dtype(typeid(FRAME_TYPE)));
                     cluster.x = ix;
                     cluster.y = iy;
                     short i = 0;
-                    for (short ir = -(m_cluster_sizeY / 2); ir < (m_cluster_sizeY / 2) + 1; ir++) {
-                        for (short ic = -(m_cluster_sizeX / 2); ic < (m_cluster_sizeX / 2) + 1; ic++) {
-                            if (ix + ic >= 0 && ix + ic < frame.shape(1) && iy + ir >= 0 && iy + ir < frame.shape(0)) {
-                                auto tmp = frame(iy + ir, ix + ic) - pedestal.mean(iy + ir, ix + ic);
+                    for (short ir = -(m_cluster_sizeY / 2);
+                         ir < (m_cluster_sizeY / 2) + 1; ir++) {
+                        for (short ic = -(m_cluster_sizeX / 2);
+                             ic < (m_cluster_sizeX / 2) + 1; ic++) {
+                            if (ix + ic >= 0 && ix + ic < frame.shape(1) &&
+                                iy + ir >= 0 && iy + ir < frame.shape(0)) {
+                                auto tmp = frame(iy + ir, ix + ic) -
+                                           pedestal.mean(iy + ir, ix + ic);
                                 cluster.set<FRAME_TYPE>(i, tmp);
                                 i++;
                             }
@@ -203,14 +254,6 @@ class ClusterFinder {
         }
         return clusters;
     }
-
-  protected:
-    int m_cluster_sizeX;
-    int m_cluster_sizeY;
-    double m_threshold;
-    double m_nSigma;
-    double c2;
-    double c3;
 };
 
 } // namespace aare

include/aare/CtbRawFile.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "aare/FileInterface.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/Frame.hpp"
+
+#include <filesystem>
+#include <fstream>
+
+namespace aare{
+
+
+class CtbRawFile{
+    RawMasterFile m_master;
+    std::ifstream m_file;
+    size_t m_current_frame{0};
+    size_t m_current_subfile{0};
+    size_t m_num_subfiles{0};
+public:
+    CtbRawFile(const std::filesystem::path &fname);
+
+    void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
+    void seek(size_t frame_index);          //!< seek to the given frame index
+    size_t tell() const;                    //!< get the frame index of the file pointer
+
+    // in the specific class we can expose more functionality
+
+    size_t image_size_in_bytes() const;
+    size_t frames_in_file() const;
+
+    RawMasterFile master() const;
+private:
+    void find_subfiles();
+    size_t sub_file_index(size_t frame_index) const {
+        return frame_index / m_master.max_frames_per_file();
+    }
+    void open_data_file(size_t subfile_index);
+
+};
+
+}

include/aare/File.hpp

@@ -1,18 +1,19 @@
 #pragma once
 #include "aare/FileInterface.hpp"
-
+#include <memory>
 
 namespace aare {
 
 /**
- * @brief RAII File class for reading and writing image files in various formats
- * wrapper on a FileInterface to abstract the underlying file format
- * @note documentation for each function is in the FileInterface class
+ * @brief RAII File class for reading, and in the future potentially writing 
+ * image files in various formats. Minimal generic interface. For specail fuctions
+ * plase use the RawFile or NumpyFile classes directly.
+ * Wraps FileInterface to abstract the underlying file format
+ * @note **frame_number** refers the the frame number sent by the detector while **frame_index**
+ * is the position of the frame in the file
  */
 class File {
-  private:
-    FileInterface *file_impl;
-    bool is_npy;
+    std::unique_ptr<FileInterface> file_impl;
 
   public:
     /**
@@ -24,37 +25,39 @@ class File {
      * @throws std::invalid_argument if the file mode is not supported
      *
      */
-    File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {});
-    void write(Frame &frame, sls_detector_header header = {});
-    Frame read();
-    Frame iread(size_t frame_number);
-    std::vector<Frame> read(size_t n_frames);
+    File(const std::filesystem::path &fname, const std::string &mode="r", const FileConfig &cfg = {});
+    
+     /**Since the object is responsible for managing the file we disable copy construction */
+    File(File const &other) = delete;     
+
+    /**The same goes for copy assignment */
+    File& operator=(File const &other) = delete;
+
+    File(File &&other) noexcept;
+    File& operator=(File &&other) noexcept;
+    ~File() = default;
+      
+    Frame read_frame();                       //!< read one frame from the file at the current position
+    Frame read_frame(size_t frame_index);     //!< read one frame at the position given by frame number
+    std::vector<Frame> read_n(size_t n_frames); //!< read n_frames from the file at the current position
+
     void read_into(std::byte *image_buf);
     void read_into(std::byte *image_buf, size_t n_frames);
-    size_t frame_number(size_t frame_index);
-    size_t bytes_per_frame();
-    size_t pixels_per_frame();
-    void seek(size_t frame_number);
-    size_t tell() const;
+    
+    size_t frame_number(size_t frame_index);  //!< get the frame number at the given frame index
+    size_t bytes_per_frame() const;           
+    size_t pixels_per_frame() const;    
+    size_t bytes_per_pixel() const;  
+    size_t bitdepth() const;
+    void seek(size_t frame_index);          //!< seek to the given frame index
+    size_t tell() const;                     //!< get the frame index of the file pointer
     size_t total_frames() const;
     size_t rows() const;
     size_t cols() const;
-    size_t bitdepth() const;
-    size_t bytes_per_pixel() const;
-    void set_total_frames(size_t total_frames);
+
     DetectorType detector_type() const;
-    xy geometry() const;
 
-    /**
-     * @brief Move constructor
-     * @param other File object to move from
-     */
-    File(File &&other) noexcept;
-
-    /**
-     * @brief destructor: will only delete the FileInterface object
-     */
-    ~File();
+    
 };
 
 } // namespace aare

include/aare/FileInterface.hpp

@@ -33,7 +33,7 @@ struct FileConfig {
     size_t total_frames{};
     std::string to_string() const {
         return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) +
-               ", geometry: " + geometry.to_string() + ", detector_type: " + toString(detector_type) +
+               ", geometry: " + geometry.to_string() + ", detector_type: " + ToString(detector_type) +
                ", max_frames_per_file: " + std::to_string(max_frames_per_file) +
                ", total_frames: " + std::to_string(total_frames) + " }";
     }
@@ -47,32 +47,24 @@ struct FileConfig {
 class FileInterface {
   public:
     /**
-     * @brief write a frame to the file
-     * @param frame frame to write
-     * @return void
-     * @throws std::runtime_error if the function is not implemented
-     */
-    // virtual void write(Frame &frame) = 0;
-
-    /**
-     * @brief write a vector of frames to the file
-     * @param frames vector of frames to write
-     * @return void
-     */
-    // virtual void write(std::vector<Frame> &frames) = 0;
-
-    /**
-     * @brief read one frame from the file at the current position
+     * @brief  one frame from the file at the current position
      * @return Frame
      */
-    virtual Frame read() = 0;
+    virtual Frame read_frame() = 0;
+
+    /**
+     * @brief read one frame from the file at the given frame number
+     * @param frame_number frame number to read
+     * @return frame
+     */
+    virtual Frame read_frame(size_t frame_number) = 0;
 
     /**
      * @brief read n_frames from the file at the current position
      * @param n_frames number of frames to read
      * @return vector of frames
      */
-    virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
+    virtual std::vector<Frame> read_n(size_t n_frames) = 0; // Is this the right interface?
 
     /**
      * @brief read one frame from the file at the current position and store it in the provided buffer
@@ -142,55 +134,28 @@ class FileInterface {
      */
     virtual size_t bitdepth() const = 0;
 
-    /**
-     * @brief read one frame from the file at the given frame number
-     * @param frame_number frame number to read
-     * @return frame
-     */
-    Frame iread(size_t frame_number) {
-        auto old_pos = tell();
-        seek(frame_number);
-        Frame tmp = read();
-        seek(old_pos);
-        return tmp;
-    };
 
-    /**
-     * @brief read n_frames from the file starting at the given frame number
-     * @param frame_number frame number to start reading from
-     * @param n_frames number of frames to read
-     * @return vector of frames
-     */
-    std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
-        auto old_pos = tell();
-        seek(frame_number);
-        std::vector<Frame> tmp = read(n_frames);
-        seek(old_pos);
-        return tmp;
-    }
-    DetectorType detector_type() const { return m_type; }
+    virtual DetectorType detector_type() const = 0;
 
     // function to query the data type of the file
     /*virtual DataType dtype = 0; */
 
     virtual ~FileInterface() = default;
 
-    void set_total_frames(size_t total_frames) { m_total_frames = total_frames; }
-
   protected:
     std::string m_mode{};
-    std::filesystem::path m_fname{};
-    std::filesystem::path m_base_path{};
-    std::string m_base_name{}, m_ext{};
-    int m_findex{};
-    size_t m_total_frames{};
-    size_t max_frames_per_file{};
-    std::string version{};
-    DetectorType m_type{DetectorType::Unknown};
-    size_t m_rows{};
-    size_t m_cols{};
-    size_t m_bitdepth{};
-    size_t current_frame{};
+    // std::filesystem::path m_fname{};
+    // std::filesystem::path m_base_path{};
+    // std::string m_base_name{}, m_ext{};
+    // int m_findex{};
+    // size_t m_total_frames{};
+    // size_t max_frames_per_file{};
+    // std::string version{};
+    // DetectorType m_type{DetectorType::Unknown};
+    // size_t m_rows{};
+    // size_t m_cols{};
+    // size_t m_bitdepth{};
+    // size_t current_frame{};
 };
 
 } // namespace aare

include/aare/Frame.hpp

@@ -11,31 +11,49 @@
 namespace aare {
 
 /**
- * @brief Frame class to represent a single frame of data
- * model class
- * should be able to work with streams coming from files or network
+ * @brief Frame class to represent a single frame of data. Not much more than a
+ * pointer and some info. Limited interface to accept frames from many sources.
  */
 class Frame {
     uint32_t m_rows;
     uint32_t m_cols;
     Dtype m_dtype;
     std::byte *m_data;
+    //TODO! Add frame number?
 
   public:
+    /**
+     * @brief Construct a new Frame
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param dtype data type of the pixels
+     * @note the data is initialized to zero
+     */
     Frame(uint32_t rows, uint32_t cols, Dtype dtype);
-    Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype);
-    ~Frame() noexcept;
 
-    // disable copy and assignment
-    Frame &operator=(const Frame &other)=delete;
-    Frame(const Frame &other)=delete;
+    /**
+     * @brief Construct a new Frame
+     * @param bytes pointer to the data to be copied into the frame
+     * @param rows number of rows
+     * @param cols number of columns
+     * @param dtype data type of the pixels
+     */
+    Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype);
+    ~Frame(){ delete[] m_data; };
+
+    /** @warning Copy is disabled to ensure performance when passing
+     * frames around. Can discuss enabling it.
+     *
+     */
+    Frame &operator=(const Frame &other) = delete;
+    Frame(const Frame &other) = delete;
 
     // enable move
     Frame &operator=(Frame &&other) noexcept;
     Frame(Frame &&other) noexcept;
 
-    // explicit copy
-    Frame copy() const;
+
+    Frame clone() const; //<- Explicit copy
 
     uint32_t rows() const;
     uint32_t cols() const;
@@ -45,32 +63,62 @@ class Frame {
     size_t bytes() const;
     std::byte *data() const;
 
-    std::byte *get(uint32_t row, uint32_t col);
+    /**
+     * @brief Get the pointer to the pixel at the given row and column
+     * @param row row index
+     * @param col column index
+     * @return pointer to the pixel
+     * @warning The user should cast the pointer to the appropriate type. Think
+     * twice if this is the function you want to use.
+     */
+    std::byte *pixel_ptr(uint32_t row, uint32_t col) const;
 
-    // TODO! can we, or even want to remove the template?
+    /**
+     * @brief Set the pixel at the given row and column to the given value
+     * @tparam T type of the value
+     * @param row row index
+     * @param col column index
+     * @param data value to set
+     */
     template <typename T> void set(uint32_t row, uint32_t col, T data) {
         assert(sizeof(T) == m_dtype.bytes());
         if (row >= m_rows || col >= m_cols) {
             throw std::out_of_range("Invalid row or column index");
         }
-        std::memcpy(m_data + (row * m_cols + col) * m_dtype.bytes(), &data, m_dtype.bytes());
+        std::memcpy(m_data + (row * m_cols + col) * m_dtype.bytes(), &data,
+                    m_dtype.bytes());
     }
-    template <typename T> T get_t(uint32_t row, uint32_t col) {
+    template <typename T> T get(uint32_t row, uint32_t col) {
         assert(sizeof(T) == m_dtype.bytes());
         if (row >= m_rows || col >= m_cols) {
             throw std::out_of_range("Invalid row or column index");
         }
+        //TODO! add tests then reimplement using pixel_ptr
         T data;
-        std::memcpy(&data, m_data + (row * m_cols + col) * m_dtype.bytes(), m_dtype.bytes());
+        std::memcpy(&data, m_data + (row * m_cols + col) * m_dtype.bytes(),
+                    m_dtype.bytes());
         return data;
     }
+    /**
+     * @brief Return an NDView of the frame. This is the preferred way to access
+     * data in the frame.
+     * 
+     * @tparam T type of the pixels
+     * @return NDView<T, 2> 
+     */
     template <typename T> NDView<T, 2> view() {
-        std::array<int64_t, 2> shape = {static_cast<int64_t>(m_rows), static_cast<int64_t>(m_cols)};
+        std::array<int64_t, 2> shape = {static_cast<int64_t>(m_rows),
+                                        static_cast<int64_t>(m_cols)};
         T *data = reinterpret_cast<T *>(m_data);
         return NDView<T, 2>(data, shape);
     }
 
-    template <typename T> NDArray<T> image() { return NDArray<T>(this->view<T>()); }
+    /** 
+     * @brief Copy the frame data into a new NDArray. This is a deep copy.
+     */
+    template <typename T> NDArray<T> image() {
+        return NDArray<T>(this->view<T>());
+    }
 };
 
 } // namespace aare

include/aare/NDArray.hpp

@@ -7,6 +7,7 @@ memory.
 TODO! Add expression templates for operators
 
 */
+#include "aare/ArrayExpr.hpp"
 #include "aare/NDView.hpp"
 
 #include <algorithm>
@@ -20,36 +21,75 @@ TODO! Add expression templates for operators
 
 namespace aare {
 
-template <typename T, int64_t Ndim = 2> class NDArray {
-  public:
-    NDArray() : shape_(), strides_(c_strides<Ndim>(shape_)), data_(nullptr){};
 
+template <typename T, int64_t Ndim = 2>
+class NDArray : public ArrayExpr<NDArray<T, Ndim>, Ndim> {
+    std::array<int64_t, Ndim> shape_;
+    std::array<int64_t, Ndim> strides_;
+    size_t size_{};
+    T *data_;
+
+  public:
+    /**
+     * @brief Default constructor. Will construct an empty NDArray.
+     *
+     */
+    NDArray() : shape_(), strides_(c_strides<Ndim>(shape_)), data_(nullptr) {};
+
+    /**
+     * @brief Construct a new NDArray object with a given shape.
+     * @note The data is uninitialized.
+     *
+     * @param shape shape of the new NDArray
+     */
     explicit NDArray(std::array<int64_t, Ndim> shape)
         : shape_(shape), strides_(c_strides<Ndim>(shape_)),
-          size_(std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<>())), data_(new T[size_]){};
+          size_(std::accumulate(shape_.begin(), shape_.end(), 1,
+                                std::multiplies<>())),
+          data_(new T[size_]) {}
 
-    NDArray(std::array<int64_t, Ndim> shape, T value) : NDArray(shape) { this->operator=(value); }
 
-    /* When constructing from a NDView we need to copy the data since
-    NDArray expect to own its data, and span is just a view*/
-    explicit NDArray(NDView<T, Ndim> span) : NDArray(span.shape()) {
-        std::copy(span.begin(), span.end(), begin());
-        // fmt::print("NDArray(NDView<T, Ndim> span)\n");
+    /**
+     * @brief Construct a new NDArray object with a shape and value.
+     *
+     * @param shape shape of the new array
+     * @param value value to initialize the array with
+     */
+    NDArray(std::array<int64_t, Ndim> shape, T value) : NDArray(shape) {
+        this->operator=(value);
+    }
+
+    /**
+     * @brief Construct a new NDArray object from a NDView.
+     * @note The data is copied from the view to the NDArray.
+     *
+     * @param v view of data to initialize the NDArray with
+     */
+    explicit NDArray(const NDView<T, Ndim> v) : NDArray(v.shape()) {
+        std::copy(v.begin(), v.end(), begin());
     }
 
     // Move constructor
     NDArray(NDArray &&other) noexcept
-        : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)), size_(other.size_), data_(other.data_) {
+        : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)),
+          size_(other.size_), data_(other.data_) {
+        other.reset(); // TODO! is this necessary?
 
-        other.reset();
-        // fmt::print("NDArray(NDArray &&other)\n");
     }
 
     // Copy constructor
     NDArray(const NDArray &other)
-        : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)), size_(other.size_), data_(new T[size_]) {
+        : shape_(other.shape_), strides_(c_strides<Ndim>(shape_)),
+          size_(other.size_), data_(new T[size_]) {
         std::copy(other.data_, other.data_ + size_, data_);
-        // fmt::print("NDArray(const NDArray &other)\n");
+    }
+
+    // Conversion operator from array expression to array
+    template <typename E>
+    NDArray(ArrayExpr<E, Ndim> &&expr) : NDArray(expr.shape()) {
+        for (int i = 0; i < size_; ++i) {
+            data_[i] = expr[i];
+        }
     }
 
     ~NDArray() { delete[] data_; }
@@ -61,15 +101,12 @@ template <typename T, int64_t Ndim = 2> class NDArray {
 
     NDArray &operator=(NDArray &&other) noexcept; // Move assign
     NDArray &operator=(const NDArray &other);     // Copy assign
-
-    NDArray operator+(const NDArray &other);
     NDArray &operator+=(const NDArray &other);
-    NDArray operator-(const NDArray &other);
     NDArray &operator-=(const NDArray &other);
-    NDArray operator*(const NDArray &other);
     NDArray &operator*=(const NDArray &other);
-    NDArray operator/(const NDArray &other);
+
     // NDArray& operator/=(const NDArray& other);
+
     template <typename V> NDArray &operator/=(const NDArray<V, Ndim> &other) {
         // check shape
         if (shape_ == other.shape()) {
@@ -106,38 +143,50 @@ template <typename T, int64_t Ndim = 2> class NDArray {
 
     NDArray &operator++(); // pre inc
 
-    template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
+    template <typename... Ix>
+    std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) {
         return data_[element_offset(strides_, index...)];
     }
 
-    template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) const {
+    template <typename... Ix>
+    std::enable_if_t<sizeof...(Ix) == Ndim, T &> operator()(Ix... index) const {
         return data_[element_offset(strides_, index...)];
     }
 
-    template <typename... Ix> std::enable_if_t<sizeof...(Ix) == Ndim, T> value(Ix... index) {
+    template <typename... Ix>
+    std::enable_if_t<sizeof...(Ix) == Ndim, T> value(Ix... index) {
         return data_[element_offset(strides_, index...)];
     }
 
+    // TODO! is int the right type for index?
     T &operator()(int i) { return data_[i]; }
     const T &operator()(int i) const { return data_[i]; }
 
+    T &operator[](int i) { return data_[i]; }
+    const T &operator[](int i) const { return data_[i]; }
+
     T *data() { return data_; }
     std::byte *buffer() { return reinterpret_cast<std::byte *>(data_); }
-    uint64_t size() const { return size_; }
+    size_t size() const { return size_; }
     size_t total_bytes() const { return size_ * sizeof(T); }
     std::array<int64_t, Ndim> shape() const noexcept { return shape_; }
     int64_t shape(int64_t i) const noexcept { return shape_[i]; }
     std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
     size_t bitdepth() const noexcept { return sizeof(T) * 8; }
+
     std::array<int64_t, Ndim> byte_strides() const noexcept {
         auto byte_strides = strides_;
         for (auto &val : byte_strides)
             val *= sizeof(T);
         return byte_strides;
-        // return strides_;
     }
 
-    NDView<T, Ndim> span() const { return NDView<T, Ndim>{data_, shape_}; }
+    /**
+     * @brief Create a view of the NDArray.
+     *
+     * @return NDView<T, Ndim>
+     */
+    NDView<T, Ndim> view() const { return NDView<T, Ndim>{data_, shape_}; }
 
     void Print();
     void Print_all();
@@ -149,16 +198,12 @@ template <typename T, int64_t Ndim = 2> class NDArray {
         std::fill(shape_.begin(), shape_.end(), 0);
         std::fill(strides_.begin(), strides_.end(), 0);
     }
-
-  private:
-    std::array<int64_t, Ndim> shape_;
-    std::array<int64_t, Ndim> strides_;
-    uint64_t size_{};
-    T *data_;
 };
 
 // Move assign
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(NDArray<T, Ndim> &&other) noexcept {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &
+NDArray<T, Ndim>::operator=(NDArray<T, Ndim> &&other) noexcept {
     if (this != &other) {
         delete[] data_;
         data_ = other.data_;
@@ -170,15 +215,11 @@ template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator
     return *this;
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const NDArray &other) {
-    NDArray result(*this);
-    result += other;
-    return result;
-}
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const NDArray<T, Ndim> &other) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
-        for (uint32_t i = 0; i < size_; ++i) {
+        for (size_t i = 0; i < size_; ++i) {
             data_[i] += other.data_[i];
         }
         return *this;
@@ -186,13 +227,8 @@ template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const NDArray &other) {
-    NDArray result{*this};
-    result -= other;
-    return result;
-}
-
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const NDArray<T, Ndim> &other) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
         for (uint32_t i = 0; i < size_; ++i) {
@@ -202,13 +238,9 @@ template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator
     }
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const NDArray &other) {
-    NDArray result = *this;
-    result *= other;
-    return result;
-}
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const NDArray<T, Ndim> &other) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const NDArray<T, Ndim> &other) {
     // check shape
     if (shape_ == other.shape_) {
         for (uint32_t i = 0; i < size_; ++i) {
@@ -219,36 +251,17 @@ template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const NDArray &other) {
-    NDArray result = *this;
-    result /= other;
-    return result;
-}
-
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator&=(const T &mask) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator&=(const T &mask) {
     for (auto it = begin(); it != end(); ++it)
         *it &= mask;
     return *this;
 }
 
-// template <typename T, int64_t Ndim>
-// NDArray<T, Ndim>& NDArray<T, Ndim>::operator/=(const NDArray<T, Ndim>&
-// other)
-// {
-//     //check shape
-//     if (shape_ == other.shape_) {
-//         for (int i = 0; i < size_; ++i) {
-//             data_[i] /= other.data_[i];
-//         }
-//         return *this;
-//     } else {
-//         throw(std::runtime_error("Shape of ImageDatas must match"));
-//     }
-// }
-
-template <typename T, int64_t Ndim> NDArray<bool, Ndim> NDArray<T, Ndim>::operator>(const NDArray &other) {
+template <typename T, int64_t Ndim>
+NDArray<bool, Ndim> NDArray<T, Ndim>::operator>(const NDArray &other) {
     if (shape_ == other.shape_) {
-        NDArray<bool> result{shape_};
+        NDArray<bool, Ndim> result{shape_};
         for (int i = 0; i < size_; ++i) {
             result(i) = (data_[i] > other.data_[i]);
         }
@@ -257,7 +270,8 @@ template <typename T, int64_t Ndim> NDArray<bool, Ndim> NDArray<T, Ndim>::operat
     throw(std::runtime_error("Shape of ImageDatas must match"));
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const NDArray<T, Ndim> &other) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const NDArray<T, Ndim> &other) {
     if (this != &other) {
         delete[] data_;
         shape_ = other.shape_;
@@ -269,7 +283,8 @@ template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator
     return *this;
 }
 
-template <typename T, int64_t Ndim> bool NDArray<T, Ndim>::operator==(const NDArray<T, Ndim> &other) const {
+template <typename T, int64_t Ndim>
+bool NDArray<T, Ndim>::operator==(const NDArray<T, Ndim> &other) const {
     if (shape_ != other.shape_)
         return false;
 
@@ -280,57 +295,68 @@ template <typename T, int64_t Ndim> bool NDArray<T, Ndim>::operator==(const NDAr
     return true;
 }
 
-template <typename T, int64_t Ndim> bool NDArray<T, Ndim>::operator!=(const NDArray<T, Ndim> &other) const {
+template <typename T, int64_t Ndim>
+bool NDArray<T, Ndim>::operator!=(const NDArray<T, Ndim> &other) const {
     return !((*this) == other);
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator++() {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator++() {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] += 1;
     return *this;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator=(const T &value) {
     std::fill_n(data_, size_, value);
     return *this;
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator+=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] += value;
     return *this;
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> NDArray<T, Ndim>::operator+(const T &value) {
     NDArray result = *this;
     result += value;
     return result;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator-=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] -= value;
     return *this;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> NDArray<T, Ndim>::operator-(const T &value) {
     NDArray result = *this;
     result -= value;
     return result;
 }
 
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator/=(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator/=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] /= value;
     return *this;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> NDArray<T, Ndim>::operator/(const T &value) {
     NDArray result = *this;
     result /= value;
     return result;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> &NDArray<T, Ndim>::operator*=(const T &value) {
     for (uint32_t i = 0; i < size_; ++i)
         data_[i] *= value;
     return *this;
 }
-template <typename T, int64_t Ndim> NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const T &value) {
+template <typename T, int64_t Ndim>
+NDArray<T, Ndim> NDArray<T, Ndim>::operator*(const T &value) {
     NDArray result = *this;
     result *= value;
     return result;
@@ -341,6 +367,19 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
     else
         Print_some();
 }
+
+template <typename T, int64_t Ndim>
+std::ostream &operator<<(std::ostream &os, const NDArray<T, Ndim> &arr) {
+    for (auto row = 0; row < arr.shape(0); ++row) {
+        for (auto col = 0; col < arr.shape(1); ++col) {
+            os << std::setw(3);
+            os << arr(row, col) << " ";
+        }
+        os << "\n";
+    }
+    return os;
+}
+
 template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
     for (auto row = 0; row < shape_[0]; ++row) {
         for (auto col = 0; col < shape_[1]; ++col) {
@@ -360,7 +399,8 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_some() {
     }
 }
 
-template <typename T, int64_t Ndim> void save(NDArray<T, Ndim> &img, std::string &pathname) {
+template <typename T, int64_t Ndim>
+void save(NDArray<T, Ndim> &img, std::string &pathname) {
     std::ofstream f;
     f.open(pathname, std::ios::binary);
     f.write(img.buffer(), img.size() * sizeof(T));
@@ -368,7 +408,8 @@ template <typename T, int64_t Ndim> void save(NDArray<T, Ndim> &img, std::string
 }
 
 template <typename T, int64_t Ndim>
-NDArray<T, Ndim> load(const std::string &pathname, std::array<int64_t, Ndim> shape) {
+NDArray<T, Ndim> load(const std::string &pathname,
+                      std::array<int64_t, Ndim> shape) {
     NDArray<T, Ndim> img{shape};
     std::ifstream f;
     f.open(pathname, std::ios::binary);

include/aare/NDView.hpp

@@ -1,4 +1,7 @@
 #pragma once
+
+#include "aare/ArrayExpr.hpp"
+
 #include <algorithm>
 #include <array>
 #include <cassert>
@@ -45,7 +48,7 @@ template <int64_t Ndim> std::array<int64_t, Ndim> make_array(const std::vector<i
     return arr;
 }
 
-template <typename T, int64_t Ndim = 2> class NDView {
+template <typename T, int64_t Ndim = 2> class NDView : public ArrayExpr<NDView<T, Ndim>, Ndim> {
   public:
     NDView() = default;
     ~NDView() = default;
@@ -68,12 +71,14 @@ template <typename T, int64_t Ndim = 2> class NDView {
         return buffer_[element_offset(strides_, index...)];
     }
 
-    uint64_t size() const { return size_; }
+    size_t size() const { return size_; }
     size_t total_bytes() const { return size_ * sizeof(T); }
     std::array<int64_t, Ndim> strides() const noexcept { return strides_; }
 
     T *begin() { return buffer_; }
     T *end() { return buffer_ + size_; }
+    T const *begin() const { return buffer_; }
+    T const *end() const { return buffer_ + size_; }
     T &operator()(int64_t i) const { return buffer_[i]; }
     T &operator[](int64_t i) const { return buffer_[i]; }
 
@@ -121,7 +126,7 @@ template <typename T, int64_t Ndim = 2> class NDView {
         return *this;
     }
 
-    auto &shape() { return shape_; }
+    auto &shape() const { return shape_; }
     auto shape(int64_t i) const { return shape_[i]; }
 
     T *data() { return buffer_; }
@@ -156,4 +161,18 @@ template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
     }
 }
 
+
+template <typename T, int64_t Ndim>
+std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
+    for (auto row = 0; row < arr.shape(0); ++row) {
+        for (auto col = 0; col < arr.shape(1); ++col) {
+            os << std::setw(3);
+            os << arr(row, col) << " ";
+        }
+        os << "\n";
+    }
+    return os;
+}
+
+
 } // namespace aare

include/aare/NumpyFile.hpp

@@ -31,9 +31,10 @@ class NumpyFile : public FileInterface {
     explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
 
     void write(Frame &frame);
-    Frame read() override { return get_frame(this->current_frame++); }
+    Frame read_frame() override { return get_frame(this->current_frame++); }
+    Frame read_frame(size_t frame_number) override { return get_frame(frame_number); }
 
-    std::vector<Frame> read(size_t n_frames) override;
+    std::vector<Frame> read_n(size_t n_frames) override;
     void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
     void read_into(std::byte *image_buf, size_t n_frames) override;
     size_t frame_number(size_t frame_index) override { return frame_index; };
@@ -46,6 +47,8 @@ class NumpyFile : public FileInterface {
     size_t cols() const override { return m_header.shape[2]; }
     size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
 
+    DetectorType detector_type() const override { return DetectorType::Unknown; }
+
     /**
      * @brief get the data type of the numpy file
      * @return DType
@@ -103,6 +106,10 @@ class NumpyFile : public FileInterface {
     size_t m_bytes_per_frame{};
     size_t m_pixels_per_frame{};
 
+    size_t m_cols;
+    size_t m_rows;
+    size_t m_bitdepth;
+
     void load_metadata();
     void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
     Frame get_frame(size_t frame_number);

include/aare/Pedestal.hpp

@@ -6,11 +6,27 @@
 
 namespace aare {
 
+/**
+ * @brief Calculate the pedestal of a series of frames. Can be used as
+ * standalone but mostly used in the ClusterFinder.
+ *
+ * @tparam SUM_TYPE type of the sum
+ */
 template <typename SUM_TYPE = double> class Pedestal {
+    uint32_t m_rows;
+    uint32_t m_cols;
+
+    uint32_t m_samples;
+    NDArray<uint32_t, 2> m_cur_samples;
+    NDArray<SUM_TYPE, 2> m_sum;
+    NDArray<SUM_TYPE, 2> m_sum2;
+
   public:
     Pedestal(uint32_t rows, uint32_t cols, uint32_t n_samples = 1000)
-        : m_rows(rows), m_cols(cols), m_freeze(false), m_samples(n_samples),           m_cur_samples(NDArray<uint32_t, 2>({rows, cols}, 0)),m_sum(NDArray<SUM_TYPE, 2>({rows, cols})),
- m_sum2(NDArray<SUM_TYPE, 2>({rows, cols})) {
+        : m_rows(rows), m_cols(cols), m_samples(n_samples),
+          m_cur_samples(NDArray<uint32_t, 2>({rows, cols}, 0)),
+          m_sum(NDArray<SUM_TYPE, 2>({rows, cols})),
+          m_sum2(NDArray<SUM_TYPE, 2>({rows, cols})) {
         assert(rows > 0 && cols > 0 && n_samples > 0);
         m_sum = 0;
         m_sum2 = 0;
@@ -25,44 +41,51 @@ template <typename SUM_TYPE = double> class Pedestal {
         return mean_array;
     }
 
-    NDArray<SUM_TYPE, 2> variance() {
-        NDArray<SUM_TYPE, 2> variance_array({m_rows, m_cols});
-        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
-            variance_array(i / m_cols, i % m_cols) = variance(i / m_cols, i % m_cols);
-        }
-        return variance_array;
-    }
-
-    NDArray<SUM_TYPE, 2> standard_deviation() {
-        NDArray<SUM_TYPE, 2> standard_deviation_array({m_rows, m_cols});
-        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
-            standard_deviation_array(i / m_cols, i % m_cols) = standard_deviation(i / m_cols, i % m_cols);
-        }
-
-        return standard_deviation_array;
-    }
-    void clear() {
-        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
-            clear(i / m_cols, i % m_cols);
-        }
-    }
-
-    /*
-     * index level operations
-     */
     SUM_TYPE mean(const uint32_t row, const uint32_t col) const {
         if (m_cur_samples(row, col) == 0) {
             return 0.0;
         }
         return m_sum(row, col) / m_cur_samples(row, col);
     }
+
+    NDArray<SUM_TYPE, 2> variance() {
+        NDArray<SUM_TYPE, 2> variance_array({m_rows, m_cols});
+        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
+            variance_array(i / m_cols, i % m_cols) =
+                variance(i / m_cols, i % m_cols);
+        }
+        return variance_array;
+    }
+
     SUM_TYPE variance(const uint32_t row, const uint32_t col) const {
         if (m_cur_samples(row, col) == 0) {
             return 0.0;
         }
-        return m_sum2(row, col) / m_cur_samples(row, col) - mean(row, col) * mean(row, col);
+        return m_sum2(row, col) / m_cur_samples(row, col) -
+               mean(row, col) * mean(row, col);
     }
-    SUM_TYPE standard_deviation(const uint32_t row, const uint32_t col) const { return std::sqrt(variance(row, col)); }
+
+    NDArray<SUM_TYPE, 2> std() {
+        NDArray<SUM_TYPE, 2> standard_deviation_array({m_rows, m_cols});
+        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
+            standard_deviation_array(i / m_cols, i % m_cols) =
+                std(i / m_cols, i % m_cols);
+        }
+
+        return standard_deviation_array;
+    }
+
+    SUM_TYPE std(const uint32_t row, const uint32_t col) const {
+        return std::sqrt(variance(row, col));
+    }
+
+    void clear() {
+        for (uint32_t i = 0; i < m_rows * m_cols; i++) {
+            clear(i / m_cols, i % m_cols);
+        }
+    }
+
+    
 
     void clear(const uint32_t row, const uint32_t col) {
         m_sum(row, col) = 0;
@@ -72,29 +95,42 @@ template <typename SUM_TYPE = double> class Pedestal {
     // frame level operations
     template <typename T> void push(NDView<T, 2> frame) {
         assert(frame.size() == m_rows * m_cols);
-        // TODO: test the effect of #pragma omp parallel for
-        for (uint32_t index = 0; index < m_rows * m_cols; index++) {
-            push<T>(index / m_cols, index % m_cols, frame(index));
+
+        // TODO! move away from m_rows, m_cols
+        if (frame.shape() != std::array<int64_t, 2>{m_rows, m_cols}) {
+            throw std::runtime_error(
+                "Frame shape does not match pedestal shape");
         }
+
+        for (uint32_t row = 0; row < m_rows; row++) {
+            for (uint32_t col = 0; col < m_cols; col++) {
+                push<T>(row, col, frame(row, col));
+            }
+        }
+
+        // // TODO: test the effect of #pragma omp parallel for
+        // for (uint32_t index = 0; index < m_rows * m_cols; index++) {
+        //     push<T>(index / m_cols, index % m_cols, frame(index));
+        // }
     }
     template <typename T> void push(Frame &frame) {
-        assert(frame.rows() == static_cast<size_t>(m_rows) && frame.cols() == static_cast<size_t>(m_cols));
+        assert(frame.rows() == static_cast<size_t>(m_rows) &&
+               frame.cols() == static_cast<size_t>(m_cols));
         push<T>(frame.view<T>());
     }
 
     // getter functions
-    inline uint32_t rows() const { return m_rows; }
-    inline uint32_t cols() const { return m_cols; }
-    inline uint32_t n_samples() const { return m_samples; }
-    inline NDArray<uint32_t, 2> cur_samples() const { return m_cur_samples; }
-    inline NDArray<SUM_TYPE, 2> get_sum() const { return m_sum; }
-    inline NDArray<SUM_TYPE, 2> get_sum2() const { return m_sum2; }
+    uint32_t rows() const { return m_rows; }
+    uint32_t cols() const { return m_cols; }
+    uint32_t n_samples() const { return m_samples; }
+    NDArray<uint32_t, 2> cur_samples() const { return m_cur_samples; }
+    NDArray<SUM_TYPE, 2> get_sum() const { return m_sum; }
+    NDArray<SUM_TYPE, 2> get_sum2() const { return m_sum2; }
 
-    // pixel level operations (should be refactored to allow users to implement their own pixel level operations)
-    template <typename T> inline void push(const uint32_t row, const uint32_t col, const T val_) {
-        if (m_freeze) {
-            return;
-        }
+    // pixel level operations (should be refactored to allow users to implement
+    // their own pixel level operations)
+    template <typename T>
+    void push(const uint32_t row, const uint32_t col, const T val_) {
         SUM_TYPE val = static_cast<SUM_TYPE>(val_);
         const uint32_t idx = index(row, col);
         if (m_cur_samples(idx) < m_samples) {
@@ -106,16 +142,8 @@ template <typename SUM_TYPE = double> class Pedestal {
             m_sum2(idx) += val * val - m_sum2(idx) / m_cur_samples(idx);
         }
     }
-    inline uint32_t index(const uint32_t row, const uint32_t col) const { return row * m_cols + col; };
-    void set_freeze(bool freeze) { m_freeze = freeze; }
-
-  private:
-    uint32_t m_rows;
-    uint32_t m_cols;
-    bool m_freeze;
-    uint32_t m_samples;
-    NDArray<uint32_t, 2> m_cur_samples;
-    NDArray<SUM_TYPE, 2> m_sum;
-    NDArray<SUM_TYPE, 2> m_sum2;
+    uint32_t index(const uint32_t row, const uint32_t col) const {
+        return row * m_cols + col;
+    };
 };
 } // namespace aare

include/aare/PixelMap.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "aare/defs.hpp"
+#include "aare/NDArray.hpp"
+
+namespace aare {
+
+NDArray<ssize_t, 2> GenerateMoench03PixelMap();
+NDArray<ssize_t, 2> GenerateMoench05PixelMap();
+
+//Matterhorn02
+NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap();
+NDArray<ssize_t, 3> GenerateMH02FourCounterPixelMap();
+
+//Eiger 
+NDArray<ssize_t, 2>GenerateEigerFlipRowsPixelMap();
+
+} // namespace aare

include/aare/RawFile.hpp

@@ -1,7 +1,12 @@
 #pragma once
-#include "aare/Frame.hpp"
 #include "aare/FileInterface.hpp"
-#include "aare/SubFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/Frame.hpp"
+#include "aare/NDArray.hpp" //for pixel map
+#include "aare/RawSubFile.hpp"
+
+
+#include <optional>
 
 namespace aare {
 
@@ -19,113 +24,76 @@ struct ModuleConfig {
 };
 
 /**
- * @brief RawFile class to read .raw and .json files
- * @note derived from FileInterface
- * @note documentation can also be found in the FileInterface class
+ * @brief Class to read .raw files. The class will parse the master file
+ * to find the correct geometry for the frames.
+ * @note A more generic interface is available in the aare::File class.
+ * Consider using that unless you need raw file specific functionality.
  */
 class RawFile : public FileInterface {
+    size_t n_subfiles{}; //f0,f1...fn
+    size_t n_subfile_parts{}; // d0,d1...dn
+    //TODO! move to vector of SubFile instead of pointers
+    std::vector<std::vector<RawSubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
+    std::vector<xy> positions;
+    std::vector<ModuleGeometry> m_module_pixel_0;
+    ModuleConfig cfg{0, 0};
+
+    RawMasterFile m_master;
+
+    size_t m_current_frame{};
+    size_t m_rows{};
+    size_t m_cols{};
+
   public:
     /**
      * @brief RawFile constructor
-     * @param fname path to the file
-     * @param mode file mode (r, w)
-     * @param cfg file configuration
-     */
-    explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r",
-                     const FileConfig &config = FileConfig{});
+     * @param fname path to the master file (.json)
+     * @param mode file mode (only "r" is supported at the moment)
 
-    /**
-     * @brief write function is not implemented for RawFile
-     * @param frame frame to write
      */
-    void write(Frame &frame, sls_detector_header header);
-    Frame read() override { return get_frame(this->current_frame++); };
-    std::vector<Frame> read(size_t n_frames) override;
-    void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
+    RawFile(const std::filesystem::path &fname, const std::string &mode = "r");
+    virtual ~RawFile() override;
+
+    Frame read_frame() override;
+    Frame read_frame(size_t frame_number) override;
+    std::vector<Frame> read_n(size_t n_frames) override;
+    void read_into(std::byte *image_buf) override;
     void read_into(std::byte *image_buf, size_t n_frames) override;
+
+    //TODO! do we need to adapt the API? 
+    void read_into(std::byte *image_buf, DetectorHeader *header);
+    void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header);
+
+
     size_t frame_number(size_t frame_index) override;
+    size_t bytes_per_frame() override;
+    size_t pixels_per_frame() override;
+    size_t bytes_per_pixel() const;
+    void seek(size_t frame_index) override;
+    size_t tell() override;
+    size_t total_frames() const override;
+    size_t rows() const override;
+    size_t cols() const override;
+    size_t bitdepth() const override;
+    xy geometry();
+    size_t n_mod() const;
+    
+    RawMasterFile master() const;
 
-    /**
-     * @brief get the number of bytess per frame
-     * @return size of one frame in bytes
-     */
-    size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
 
-    /**
-     * @brief get the number of pixels in the frame
-     * @return number of pixels
-     */
-    size_t pixels_per_frame() override { return m_rows * m_cols; }
 
-    // goto frame index
-    void seek(size_t frame_index) override {
-        // check if the frame number is greater than the total frames
-        // if frame_number == total_frames, then the next read will throw an error
-        if (frame_index > this->total_frames()) {
-            throw std::runtime_error(
-                fmt::format("frame number {} is greater than total frames {}", frame_index, m_total_frames));
-        }
-        this->current_frame = frame_index;
-    };
-
-    // return the position of the file pointer (in number of frames)
-    size_t tell() override { return this->current_frame; };
-
-    /**
-     * @brief check if the file is a master file
-     * @param fpath path to the file
-     */
-    static bool is_master_file(const std::filesystem::path &fpath);
-
-    /**
-     * @brief set the module gap row and column
-     * @param row gap between rows
-     * @param col gap between columns
-     */
-    inline void set_config(int row, int col) {
-        cfg.module_gap_row = row;
-        cfg.module_gap_col = col;
-    }
-    // TODO! Deal with fast quad and missing files
-
-    /**
-     * @brief get the number of subfiles for the RawFile
-     * @return number of subfiles
-     */
-    void find_number_of_subfiles();
-
-    /**
-     * @brief get the master file name path for the RawFile
-     * @return path to the master file
-     */
-    inline std::filesystem::path master_fname();
-    /**
-     * @brief get the data file name path for the RawFile with the given module id and file id
-     * @param mod_id module id
-     * @param file_id file id
-     * @return path to the data file
-     */
-    inline std::filesystem::path data_fname(size_t mod_id, size_t file_id);
-
-    /**
-     * @brief destructor: will delete the subfiles
-     */
-    ~RawFile() noexcept override;
-
-    size_t total_frames() const override { return m_total_frames; }
-    size_t rows() const override { return m_rows; }
-    size_t cols() const override { return m_cols; }
-    size_t bitdepth() const override { return m_bitdepth; }
-    xy geometry() { return m_geometry; }
+    DetectorType detector_type() const override;
 
   private:
-    void write_master_file();
+
     /**
      * @brief read the frame at the given frame index into the image buffer
      * @param frame_number frame number to read
      * @param image_buf buffer to store the frame
      */
-    void get_frame_into(size_t frame_index, std::byte *frame_buffer);
+
+    void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
+
 
     /**
      * @brief get the frame at the given frame index
@@ -134,53 +102,20 @@ class RawFile : public FileInterface {
      */
     Frame get_frame(size_t frame_index);
 
-    /**
-     * @brief parse the file name to get the extension, base name and index
-     */
-    void parse_fname();
 
-    /**
-     * @brief parse the metadata from the file
-     */
-    void parse_metadata();
-
-    /**
-     * @brief parse the metadata of a .raw file
-     */
-    void parse_raw_metadata();
-
-    /**
-     * @brief parse the metadata of a .json file
-     */
-    void parse_json_metadata();
-
-    /**
-     * @brief finds the geometry of the file
-     */
-    void find_geometry();
 
     /**
      * @brief read the header of the file
      * @param fname path to the data subfile
-     * @return sls_detector_header
+     * @return DetectorHeader
      */
-    static sls_detector_header read_header(const std::filesystem::path &fname);
+    static DetectorHeader read_header(const std::filesystem::path &fname);
+
+    void update_geometry_with_roi();
+    int find_number_of_subfiles();
 
-    /**
-     * @brief open the subfiles
-     */
     void open_subfiles();
-    void parse_config(const FileConfig &config);
-
-    size_t n_subfiles{};
-    size_t n_subfile_parts{};
-    std::vector<std::vector<SubFile *>> subfiles;
-    size_t subfile_rows{}, subfile_cols{};
-    xy m_geometry{};
-    std::vector<xy> positions;
-    ModuleConfig cfg{0, 0};
-    TimingMode timing_mode{};
-    bool quad{false};
+    void find_geometry();
 };
 
 } // namespace aare

include/aare/RawMasterFile.hpp

@@ -0,0 +1,151 @@
+#pragma once
+#include "aare/defs.hpp"
+#include <filesystem>
+#include <fmt/format.h>
+#include <fstream>
+#include <optional>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+namespace aare {
+
+/**
+ * @brief Implementation used in RawMasterFile to parse the file name
+ */
+class RawFileNameComponents {
+    std::filesystem::path m_base_path{};
+    std::string m_base_name{};
+    std::string m_ext{};
+    int m_file_index{}; // TODO! is this measurement_index?
+
+  public:
+    RawFileNameComponents(const std::filesystem::path &fname);
+
+    /// @brief Get the filename including path of the master file.
+    /// (i.e. what was passed in to the constructor))
+    std::filesystem::path master_fname() const;
+
+    /// @brief Get the filename including path of the data file.
+    /// @param mod_id module id run_d[module_id]_f0_0
+    /// @param file_id file id run_d0_f[file_id]_0
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
+
+    const std::filesystem::path &base_path() const;
+    const std::string &base_name() const;
+    const std::string &ext() const;
+    int file_index() const;
+};
+
+class ScanParameters {
+    bool m_enabled = false;
+    std::string m_dac;
+    int m_start = 0;
+    int m_stop = 0;
+    int m_step = 0;
+    //TODO! add settleTime, requires string to time conversion
+
+  public:
+    ScanParameters(const std::string &par);
+    ScanParameters() = default;
+    ScanParameters(const ScanParameters &) = default;
+    ScanParameters &operator=(const ScanParameters &) = default;
+    ScanParameters(ScanParameters &&) = default;
+    int start() const;
+    int stop() const;
+    int step() const;
+    const std::string &dac() const;
+    bool enabled() const;
+    void increment_stop();
+};
+
+
+struct ROI{
+  size_t xmin{};
+  size_t xmax{};
+  size_t ymin{};
+  size_t ymax{};
+
+  size_t height() const { return ymax - ymin; }
+  size_t width() const { return xmax - xmin; }
+}__attribute__((packed));
+
+
+/**
+ * @brief Class for parsing a master file either in our .json format or the old
+ * .raw format
+ */
+class RawMasterFile {
+    RawFileNameComponents m_fnc;
+    std::string m_version;
+    DetectorType m_type;
+    TimingMode m_timing_mode;
+
+    size_t m_image_size_in_bytes{};
+    size_t m_frames_in_file{};
+    size_t m_total_frames_expected{};
+    size_t m_pixels_y{};
+    size_t m_pixels_x{};
+    size_t m_bitdepth{};
+
+    xy m_geometry;
+
+    size_t m_max_frames_per_file{};
+    uint32_t m_adc_mask{};
+    FrameDiscardPolicy m_frame_discard_policy{};
+    size_t m_frame_padding{};
+
+    // TODO! should these be bool?
+    uint8_t m_analog_flag{};
+    uint8_t m_digital_flag{};
+    uint8_t m_transceiver_flag{};
+
+    ScanParameters m_scan_parameters;
+
+    std::optional<size_t> m_analog_samples;
+    std::optional<size_t> m_digital_samples;
+    std::optional<size_t> m_transceiver_samples;
+    std::optional<size_t> m_number_of_rows;
+    std::optional<uint8_t> m_quad;
+
+    std::optional<ROI> m_roi;
+
+
+  public:
+    RawMasterFile(const std::filesystem::path &fpath);
+
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
+
+    const std::string &version() const; //!< For example "7.2"
+    const DetectorType &detector_type() const;
+    const TimingMode &timing_mode() const;
+    size_t image_size_in_bytes() const;
+    size_t frames_in_file() const;
+    size_t pixels_y() const;
+    size_t pixels_x() const;
+    size_t max_frames_per_file() const;
+    size_t bitdepth() const;
+    size_t frame_padding() const;
+    const FrameDiscardPolicy &frame_discard_policy() const;
+
+    size_t total_frames_expected() const;
+    xy geometry() const;
+
+    std::optional<size_t> analog_samples() const;
+    std::optional<size_t> digital_samples() const;
+    std::optional<size_t> transceiver_samples() const;
+    std::optional<size_t> number_of_rows() const;
+    std::optional<uint8_t> quad() const;
+
+
+    std::optional<ROI> roi() const;
+
+
+    ScanParameters scan_parameters() const;
+
+  private:
+    void parse_json(const std::filesystem::path &fpath);
+    void parse_raw(const std::filesystem::path &fpath);
+};
+
+} // namespace aare

include/aare/RawSubFile.hpp

@@ -0,0 +1,71 @@
+#pragma once
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <optional>
+
+namespace aare {
+
+/**
+ * @brief Class to read a singe subfile written in .raw format. Used from RawFile to read
+ * the entire detector. Can be used directly to read part of the image.
+ */
+class RawSubFile {
+  protected:
+    std::ifstream m_file;
+    size_t m_bitdepth;
+    std::filesystem::path m_fname;
+    size_t m_rows{};
+    size_t m_cols{};
+    size_t m_bytes_per_frame{};
+    size_t n_frames{};
+    uint32_t m_pos_row{};
+    uint32_t m_pos_col{};
+ 
+    DetectorType m_detector_type;
+    std::optional<NDArray<ssize_t, 2>> m_pixel_map;
+
+  public:
+    /**
+     * @brief SubFile constructor
+     * @param fname path to the subfile
+     * @param detector detector type
+     * @param rows number of rows in the subfile
+     * @param cols number of columns in the subfile
+     * @param bitdepth bitdepth of the subfile
+     * @throws std::invalid_argument if the detector,type pair is not supported
+     */
+    RawSubFile(const std::filesystem::path &fname, DetectorType detector,
+               size_t rows, size_t cols, size_t bitdepth, uint32_t pos_row = 0, uint32_t pos_col = 0);
+
+    ~RawSubFile() = default;
+    /**
+     * @brief Seek to the given frame number
+     * @note Puts the file pointer at the start of the header, not the start of the data
+     * @param frame_index frame position in file to seek to
+     * @throws std::runtime_error if the frame number is out of range
+     */
+    void seek(size_t frame_index);
+    size_t tell();
+
+    void read_into(std::byte *image_buf, DetectorHeader *header = nullptr);
+    void get_part(std::byte *buffer, size_t frame_index);
+    
+    void read_header(DetectorHeader *header);
+    
+    size_t rows() const;
+    size_t cols() const;
+    
+    size_t frame_number(size_t frame_index);
+
+    size_t bytes_per_frame() const { return m_bytes_per_frame; }
+    size_t pixels_per_frame() const { return m_rows * m_cols; }
+    size_t bytes_per_pixel() const { return m_bitdepth / 8; }
+
+
+};
+
+} // namespace aare

include/aare/SubFile.hpp

@@ -1,77 +0,0 @@
-#pragma once
-#include "aare/Frame.hpp"
-#include "aare/defs.hpp"
-
-#include <cstdint>
-#include <filesystem>
-#include <map>
-#include <variant>
-
-namespace aare {
-
-/**
- * @brief Class to read a subfile from a RawFile
- */
-class SubFile {
-  public:
-    size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
-    /**
-     * @brief SubFile constructor
-     * @param fname path to the subfile
-     * @param detector detector type
-     * @param rows number of rows in the subfile
-     * @param cols number of columns in the subfile
-     * @param bitdepth bitdepth of the subfile
-     * @throws std::invalid_argument if the detector,type pair is not supported
-     */
-    SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
-            const std::string &mode = "r");
-
-    /**
-     * @brief read the subfile into a buffer
-     * @param buffer pointer to the buffer to read the data into
-     * @return number of bytes read
-     */
-    size_t read_impl_normal(std::byte *buffer);
-
-    /**
-     * @brief read the subfile into a buffer with the bytes flipped
-     * @param buffer pointer to the buffer to read the data into
-     * @return number of bytes read
-     */
-    template <typename DataType> size_t read_impl_flip(std::byte *buffer);
-
-    /**
-     * @brief read the subfile into a buffer with the bytes reordered
-     * @param buffer pointer to the buffer to read the data into
-     * @return number of bytes read
-     */
-    template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
-
-    /**
-     * @brief read the subfile into a buffer with the bytes reordered and flipped
-     * @param buffer pointer to the buffer to read the data into
-     * @param frame_number frame number to read
-     * @return number of bytes read
-     */
-    size_t get_part(std::byte *buffer, size_t frame_index);
-    size_t frame_number(size_t frame_index);
-
-    // TODO: define the inlines as variables and assign them in constructor
-    inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
-    inline size_t pixels_per_part() const { return m_rows * m_cols; }
-
-    ~SubFile();
-
-  protected:
-    FILE *fp = nullptr;
-    size_t m_bitdepth;
-    std::filesystem::path m_fname;
-    size_t m_rows{};
-    size_t m_cols{};
-    std::string m_mode;
-    size_t n_frames{};
-    int m_sub_file_index_{};
-};
-
-} // namespace aare

include/aare/VarClusterFinder.hpp

@@ -49,7 +49,7 @@ template <typename T> class VarClusterFinder {
     int check_neighbours(int i, int j);
 
   public:
-    VarClusterFinder(image_shape shape, T threshold)
+    VarClusterFinder(Shape<2> shape, T threshold)
         : shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) {
         hits.reserve(2000);
     }

include/aare/defs.hpp

@@ -14,15 +14,34 @@
 #include <variant>
 #include <vector>
 
+
 /**
  * @brief LOCATION macro to get the current location in the code
  */
-#define LOCATION std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + ":" + std::string(__func__) + ":"
+#define LOCATION                                                               \
+    std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) +      \
+        ":" + std::string(__func__) + ":"
+
+
+
+#ifdef AARE_CUSTOM_ASSERT
+#define AARE_ASSERT(expr)\
+    if (expr)\
+        {}\
+    else\
+        aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n");
+#else
+#define AARE_ASSERT(cond)\
+    do { (void)sizeof(cond); } while(0)
+#endif
 
 
 namespace aare {
 
-class Cluster {
+void assert_failed(const std::string &msg);
+
+
+class DynamicCluster {
   public:
     int cluster_sizeX;
     int cluster_sizeY;
@@ -34,50 +53,59 @@ class Cluster {
     std::byte *m_data;
 
   public:
-    Cluster(int cluster_sizeX_, int cluster_sizeY_, Dtype dt_ = Dtype(typeid(int32_t)))
-        : cluster_sizeX(cluster_sizeX_), cluster_sizeY(cluster_sizeY_), dt(dt_) {
+    DynamicCluster(int cluster_sizeX_, int cluster_sizeY_,
+            Dtype dt_ = Dtype(typeid(int32_t)))
+        : cluster_sizeX(cluster_sizeX_), cluster_sizeY(cluster_sizeY_),
+          dt(dt_) {
         m_data = new std::byte[cluster_sizeX * cluster_sizeY * dt.bytes()]{};
     }
-    Cluster() : Cluster(3, 3) {}
-    Cluster(const Cluster &other) : Cluster(other.cluster_sizeX, other.cluster_sizeY, other.dt) {
+    DynamicCluster() : DynamicCluster(3, 3) {}
+    DynamicCluster(const DynamicCluster &other)
+        : DynamicCluster(other.cluster_sizeX, other.cluster_sizeY, other.dt) {
         if (this == &other)
             return;
         x = other.x;
         y = other.y;
         memcpy(m_data, other.m_data, other.bytes());
     }
-    Cluster &operator=(const Cluster &other) {
+    DynamicCluster &operator=(const DynamicCluster &other) {
         if (this == &other)
             return *this;
-        this->~Cluster();
-        new (this) Cluster(other);
+        this->~DynamicCluster();
+        new (this) DynamicCluster(other);
         return *this;
     }
-    Cluster(Cluster &&other) noexcept
-        : cluster_sizeX(other.cluster_sizeX), cluster_sizeY(other.cluster_sizeY), x(other.x), y(other.y), dt(other.dt),
-          m_data(other.m_data) {
+    DynamicCluster(DynamicCluster &&other) noexcept
+        : cluster_sizeX(other.cluster_sizeX),
+          cluster_sizeY(other.cluster_sizeY), x(other.x), y(other.y),
+          dt(other.dt), m_data(other.m_data) {
         other.m_data = nullptr;
         other.dt = Dtype(Dtype::TypeIndex::ERROR);
     }
-    ~Cluster() { delete[] m_data; }
+    ~DynamicCluster() { delete[] m_data; }
     template <typename T> T get(int idx) {
-        (sizeof(T) == dt.bytes()) ? 0 : throw std::invalid_argument("[ERROR] Type size mismatch");
+        (sizeof(T) == dt.bytes())
+            ? 0
+            : throw std::invalid_argument("[ERROR] Type size mismatch");
         return *reinterpret_cast<T *>(m_data + idx * dt.bytes());
     }
     template <typename T> auto set(int idx, T val) {
-        (sizeof(T) == dt.bytes()) ? 0 : throw std::invalid_argument("[ERROR] Type size mismatch");
+        (sizeof(T) == dt.bytes())
+            ? 0
+            : throw std::invalid_argument("[ERROR] Type size mismatch");
         return memcpy(m_data + idx * dt.bytes(), &val, (size_t)dt.bytes());
     }
-    // auto x() const { return x; }
-    // auto y() const { return y; }
-    // auto x(int16_t x_) { return x = x_; }
-    // auto y(int16_t y_) { return y = y_; }
 
     template <typename T> std::string to_string() const {
-        (sizeof(T) == dt.bytes()) ? 0 : throw std::invalid_argument("[ERROR] Type size mismatch");
-        std::string s = "x: " + std::to_string(x) + " y: " + std::to_string(y) + "\nm_data: [";
+        (sizeof(T) == dt.bytes())
+            ? 0
+            : throw std::invalid_argument("[ERROR] Type size mismatch");
+        std::string s = "x: " + std::to_string(x) + " y: " + std::to_string(y) +
+                        "\nm_data: [";
         for (int i = 0; i < cluster_sizeX * cluster_sizeY; i++) {
-            s += std::to_string(*reinterpret_cast<T *>(m_data + i * dt.bytes())) + " ";
+            s += std::to_string(
+                     *reinterpret_cast<T *>(m_data + i * dt.bytes())) +
+                 " ";
         }
         s += "]";
         return s;
@@ -85,17 +113,19 @@ class Cluster {
     /**
      * @brief size of the cluster in bytes when saved to a file
      */
-    size_t size() const { return cluster_sizeX * cluster_sizeY ; }
+    size_t size() const { return cluster_sizeX * cluster_sizeY; }
     size_t bytes() const { return cluster_sizeX * cluster_sizeY * dt.bytes(); }
     auto begin() const { return m_data; }
-    auto end() const { return m_data + cluster_sizeX * cluster_sizeY * dt.bytes(); }
+    auto end() const {
+        return m_data + cluster_sizeX * cluster_sizeY * dt.bytes();
+    }
     std::byte *data() { return m_data; }
 };
 
 /**
  * @brief header contained in parts of frames
  */
-struct sls_detector_header {
+struct DetectorHeader {
     uint64_t frameNumber;
     uint32_t expLength;
     uint32_t packetNumber;
@@ -117,40 +147,74 @@ struct sls_detector_header {
         }
         packetMaskStr += "]";
 
-        return "frameNumber: " + std::to_string(frameNumber) + "\n" + "expLength: " + std::to_string(expLength) + "\n" +
-               "packetNumber: " + std::to_string(packetNumber) + "\n" + "bunchId: " + std::to_string(bunchId) + "\n" +
-               "timestamp: " + std::to_string(timestamp) + "\n" + "modId: " + std::to_string(modId) + "\n" +
-               "row: " + std::to_string(row) + "\n" + "column: " + std::to_string(column) + "\n" +
-               "reserved: " + std::to_string(reserved) + "\n" + "debug: " + std::to_string(debug) + "\n" +
-               "roundRNumber: " + std::to_string(roundRNumber) + "\n" + "detType: " + std::to_string(detType) + "\n" +
-               "version: " + std::to_string(version) + "\n" + "packetMask: " + packetMaskStr + "\n";
+        return "frameNumber: " + std::to_string(frameNumber) + "\n" +
+               "expLength: " + std::to_string(expLength) + "\n" +
+               "packetNumber: " + std::to_string(packetNumber) + "\n" +
+               "bunchId: " + std::to_string(bunchId) + "\n" +
+               "timestamp: " + std::to_string(timestamp) + "\n" +
+               "modId: " + std::to_string(modId) + "\n" +
+               "row: " + std::to_string(row) + "\n" +
+               "column: " + std::to_string(column) + "\n" +
+               "reserved: " + std::to_string(reserved) + "\n" +
+               "debug: " + std::to_string(debug) + "\n" +
+               "roundRNumber: " + std::to_string(roundRNumber) + "\n" +
+               "detType: " + std::to_string(detType) + "\n" +
+               "version: " + std::to_string(version) + "\n" +
+               "packetMask: " + packetMaskStr + "\n";
     }
 };
 
 template <typename T> struct t_xy {
     T row;
     T col;
-    bool operator==(const t_xy &other) const { return row == other.row && col == other.col; }
+    bool operator==(const t_xy &other) const {
+        return row == other.row && col == other.col;
+    }
     bool operator!=(const t_xy &other) const { return !(*this == other); }
-    std::string to_string() const { return "{ x: " + std::to_string(row) + " y: " + std::to_string(col) + " }"; }
+    std::string to_string() const {
+        return "{ x: " + std::to_string(row) + " y: " + std::to_string(col) +
+               " }";
+    }
 };
 using xy = t_xy<uint32_t>;
 
+
+struct ModuleGeometry{
+    int x{};
+    int y{};
+    int height{};
+    int width{};
+};
+
+
 using dynamic_shape = std::vector<int64_t>;
 
-enum class DetectorType { Jungfrau, Eiger, Mythen3, Moench, ChipTestBoard, Unknown };
+//TODO! Can we uniform enums between the libraries?
+enum class DetectorType {
+    Jungfrau,
+    Eiger,
+    Mythen3,
+    Moench,
+    Moench03,
+    Moench03_old,
+    ChipTestBoard,
+    Unknown
+};
 
 enum class TimingMode { Auto, Trigger };
+enum class FrameDiscardPolicy { NoDiscard, Discard, DiscardPartial };
 
 template <class T> T StringTo(const std::string &arg) { return T(arg); }
 
-template <class T> std::string toString(T arg) { return T(arg); }
+template <class T> std::string ToString(T arg) { return T(arg); }
 
 template <> DetectorType StringTo(const std::string & /*name*/);
-template <> std::string toString(DetectorType arg);
+template <> std::string ToString(DetectorType arg);
 
 template <> TimingMode StringTo(const std::string & /*mode*/);
 
+template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
+
 using DataTypeVariants = std::variant<uint16_t, uint32_t>;
 
 } // namespace aare

include/aare/json.hpp

@@ -1,68 +0,0 @@
-#pragma once
-#include <array>
-#include <map>
-#include <string>
-#include <vector>
-
-// helper functions to write json
-// append to string for better performance (not tested)
-
-namespace aare {
-
-/**
- * @brief write a digit to a string
- * takes key and value and outputs->"key": value,
- * @tparam T type of value (int, uint32_t, ...)
- * @param s string to append to
- * @param key key to write
- * @param value value to write
- * @return void
- * @note
- * - can't use concepts here because we are using c++17
- */
-template <typename T> inline void write_digit(std::string &s, const std::string &key, const T &value) {
-    s += "\"";
-    s += key;
-    s += "\": ";
-    s += std::to_string(value);
-    s += ", ";
-}
-inline void write_str(std::string &s, const std::string &key, const std::string &value) {
-    s += "\"";
-    s += key;
-    s += "\": \"";
-    s += value;
-    s += "\", ";
-}
-inline void write_map(std::string &s, const std::string &key, const std::map<std::string, std::string> &value) {
-    s += "\"";
-    s += key;
-    s += "\": {";
-    for (const auto &kv : value) {
-        write_str(s, kv.first, kv.second);
-    }
-    // remove last comma or trailing spaces
-    for (size_t i = s.size() - 1; i > 0; i--) {
-        if ((s[i] == ',') || (s[i] == ' ')) {
-            s.pop_back();
-        } else
-            break;
-    }
-    s += "}, ";
-}
-
-template <typename T, int N> void write_array(std::string &s, const std::string &key, const std::array<T, N> &value) {
-    s += "\"";
-    s += key;
-    s += "\": [";
-
-    for (size_t i = 0; i < N - 1; i++) {
-        s += std::to_string(value[i]);
-        s += ", ";
-    }
-    s += std::to_string(value[N - 1]);
-
-    s += "], ";
-}
-
-} // namespace aare

pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["scikit-build-core>=0.10", "pybind11", "numpy"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "aare"
+version = "2024.11.15.dev0"
+
+
+[tool.scikit-build]
+cmake.verbose = true
+
+[tool.scikit-build.cmake.define]
+AARE_PYTHON_BINDINGS = "ON"
+AARE_SYSTEM_LIBRARIES = "ON"

python/CMakeLists.txt

@@ -1,34 +1,50 @@
+
+find_package (Python 3.10 COMPONENTS Interpreter Development)
+
+# Download or find pybind11 depending on configuration
 if(AARE_FETCH_PYBIND11)
     FetchContent_Declare(
         pybind11
         GIT_REPOSITORY https://github.com/pybind/pybind11
-        GIT_TAG        v2.11.0
+        GIT_TAG        v2.13.0
     )
     FetchContent_MakeAvailable(pybind11)
 else()
-    find_package(pybind11 2.11 REQUIRED)
+    find_package(pybind11 2.13 REQUIRED)
 endif()
 
-
+# Add the compiled python extension
 pybind11_add_module(
-    _aare 
-    src/module.cpp
+    _aare           # name of the module
+    src/module.cpp  # source file
     )
+
 set_target_properties(_aare PROPERTIES
     LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
 )
 target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
 
+# List of python files to be copied to the build directory
 set( PYTHON_FILES
     aare/__init__.py
+    aare/CtbRawFile.py
+    aare/transform.py
+    aare/ScanParameters.py
 )
 
+# Copy the python files to the build directory
 foreach(FILE ${PYTHON_FILES})
-    configure_file( ${FILE}  
-    ${CMAKE_BINARY_DIR}/${FILE}  )
+    configure_file(${FILE} ${CMAKE_BINARY_DIR}/${FILE}  )
 endforeach(FILE ${PYTHON_FILES})
 
+set_target_properties(_aare PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/aare
+)
 
-configure_file( examples/play.py  
-    ${CMAKE_BINARY_DIR}/play.py 
-)
+
+
+# Copy the examples/scripts to the build directory
+configure_file(examples/play.py ${CMAKE_BINARY_DIR}/play.py)
+
+
+install(TARGETS _aare DESTINATION aare)

python/aare/CtbRawFile.py

@@ -0,0 +1,171 @@
+
+from . import _aare
+import numpy as np
+from .ScanParameters import ScanParameters
+class CtbRawFile(_aare.CtbRawFile):
+    """File reader for the CTB raw file format.
+    
+    Args:  
+            fname (pathlib.Path | str): Path to the file to be read.
+            transform (function): Function to apply to the data after reading it. 
+                The function should take a numpy array of type uint8 and return one
+                or several numpy arrays.
+    """
+    def __init__(self, fname, transform = None):
+        super().__init__(fname)
+        self.transform = transform
+
+
+    def read_frame(self, frame_index: int | None = None ) -> tuple:
+        """Read one frame from the file and then advance the file pointer.
+
+        .. note::
+
+            Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine
+            which frame to read unless frame_index is specified.
+
+        Args:
+            frame_index (int): If not None, seek to this frame before reading.
+
+        Returns:
+            tuple: header, data
+
+        Raises:
+            RuntimeError: If the file is at the end.
+        """
+        if frame_index is not None:
+            self.seek(frame_index)
+
+
+        header, data = super().read_frame()
+        if header.shape == (1,):
+            header = header[0]
+
+
+        if self.transform:
+            res = self.transform(data)
+            if isinstance(res, tuple):
+                return header, *res
+            else:
+                return header, res
+        else:
+            return header, data
+            
+    def read_n(self, n_frames:int) -> tuple:
+        """Read several frames from the file.
+
+        .. note::
+
+            Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine
+            where to start reading from.
+
+        Args:
+            n_frames (int): Number of frames to read.
+
+        Returns:
+            tuple: header, data
+
+        Raises:
+            RuntimeError: If EOF is reached.
+        """
+        # Do the first read to figure out what we have
+        tmp_header, tmp_data = self.read_frame()
+        
+        # Allocate arrays for
+        header = np.zeros(n_frames, dtype = tmp_header.dtype)
+        data = np.zeros((n_frames, *tmp_data.shape), dtype = tmp_data.dtype)
+        
+        # Copy the first frame
+        header[0] = tmp_header
+        data[0] = tmp_data
+
+        # Do the rest of the reading
+        for i in range(1, n_frames):
+            header[i], data[i] = self.read_frame()
+
+        return header, data
+
+    def read(self) -> tuple:
+        """Read the entire file.
+
+        Returns:
+            tuple: header, data
+        """
+        return self.read_n(self.frames_in_file)
+
+    def seek(self, frame_index:int) -> None:
+        """Seek to a specific frame in the file.
+
+        Args:
+            frame_index (int): Frame position in file to seek to.
+        """
+        super().seek(frame_index)
+
+    def tell() -> int:
+        """Return the current frame position in the file.
+
+        Returns:
+            int: Frame position in file.
+        """
+        return super().tell()
+    
+    @property
+    def scan_parameters(self):
+        """Return the scan parameters.
+
+        Returns:
+            ScanParameters: Scan parameters.
+        """
+        return ScanParameters(self.master.scan_parameters)
+
+    @property
+    def master(self):
+        """Return the master file.
+
+        Returns:
+            RawMasterFile: Master file.
+        """
+        return super().master()
+
+    @property
+    def image_size_in_bytes(self) -> int:
+        """Return the size of the image in bytes.
+
+        Returns:
+            int: Size of image in bytes.
+        """
+        return super().image_size_in_bytes
+
+    def __len__(self) -> int:
+        """Return the number of frames in the file.
+
+        Returns:
+            int: Number of frames in file.
+        """
+        return super().frames_in_file
+    
+    @property
+    def frames_in_file(self) -> int:
+        """Return the number of frames in the file.
+
+        Returns:
+            int: Number of frames in file.
+        """
+        return super().frames_in_file
+
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __iter__(self):
+        return self
+    
+    def __next__(self):
+        try:
+            return self.read_frame()
+        except RuntimeError:
+            # TODO! find a good way to check that we actually have the right exception
+            raise StopIteration
+

python/aare/ScanParameters.py

@@ -0,0 +1,16 @@
+from . import _aare
+
+class ScanParameters(_aare.ScanParameters):
+    def __init__(self, s):
+        super().__init__(s)
+        
+    def __iter__(self):
+        return [getattr(self, a) for a in ['start', 'stop', 'step']].__iter__()
+    
+    def __str__(self):
+        return f'ScanParameters({self.dac}: {self.start}, {self.stop}, {self.step})'
+    
+    def __repr__(self):
+        return self.__str__()
+    
+    

python/aare/__init__.py

@@ -1,3 +1,11 @@
+# Make the compiled classes that live in _aare available from aare.
+from . import _aare
 
 
-from _aare import File
+from ._aare import File, RawFile, RawMasterFile, RawSubFile
+from ._aare import Pedestal, ClusterFinder, VarClusterFinder
+from ._aare import DetectorType
+from ._aare import ClusterFile
+
+from .CtbRawFile import CtbRawFile
+from .ScanParameters import ScanParameters

python/aare/transform.py

@@ -0,0 +1,28 @@
+import numpy as np
+from . import _aare
+
+
+class Moench05Transform:
+    #Could be moved to C++ without changing the interface
+    def __init__(self):
+        self.pixel_map = _aare.GenerateMoench05PixelMap()
+
+    def __call__(self, data):
+        return np.take(data.view(np.uint16), self.pixel_map)
+
+
+class Matterhorn02Transform:
+    def __init__(self):
+        self.pixel_map = _aare.GenerateMH02FourCounterPixelMap()
+
+    def __call__(self, data):
+        counters = int(data.size / 48**2 / 2)
+        if counters == 1:
+            return np.take(data.view(np.uint16), self.pixel_map[0])
+        else:
+            return np.take(data.view(np.uint16), self.pixel_map[0:counters])
+
+
+#on import generate the pixel maps to avoid doing it every time
+moench05 = Moench05Transform()
+matterhorn02 = Matterhorn02Transform()

python/examples/play.py

@@ -1,14 +1,15 @@
 import matplotlib.pyplot as plt
 import numpy as np
 plt.ion()
-
-import aare
 from pathlib import Path
+from aare import ClusterFile
 
-p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
+base = Path('~/data/aare_test_data/clusters').expanduser()
 
-f = aare.File(p)
-frame = f.read_frame()
+f = ClusterFile(base / 'beam_En700eV_-40deg_300V_10us_d0_f0_100.clust')
+# f = ClusterFile(base / 'single_frame_97_clustrers.clust')
 
-fig, ax = plt.subplots()
-im = ax.imshow(frame, cmap='viridis')
+
+for i in range(10):
+    fn, cl = f.read_frame()
+    print(fn, cl.size)

python/src/cluster.hpp

@@ -0,0 +1,52 @@
+#include "aare/ClusterFinder.hpp"
+#include "aare/NDView.hpp"
+#include "aare/Pedestal.hpp"
+#include "np_helper.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+void define_cluster_finder_bindings(py::module &m) {
+    py::class_<ClusterFinder<uint16_t, double>>(m, "ClusterFinder")
+        .def(py::init<Shape<2>, Shape<2>>())
+        .def("push_pedestal_frame",
+             [](ClusterFinder<uint16_t, double> &self,
+                py::array_t<uint16_t> frame) {
+                 auto view = make_view_2d(frame);
+                 self.push_pedestal_frame(view);
+             })
+        .def("pedestal",
+             [](ClusterFinder<uint16_t, double> &self) {
+                 auto m = new NDArray<double, 2>{};
+                 *m = self.pedestal();
+                 return return_image_data(m);
+             })
+        .def("find_clusters_without_threshold",
+             [](ClusterFinder<uint16_t, double> &self,
+                py::array_t<uint16_t> frame) {
+                 auto view = make_view_2d(frame);
+                 auto clusters = self.find_clusters_without_threshold(view);
+                 return clusters;
+             });
+
+    py::class_<DynamicCluster>(m, "DynamicCluster", py::buffer_protocol())
+        .def(py::init<int, int, Dtype>())
+        .def("size", &DynamicCluster::size)
+        .def("begin", &DynamicCluster::begin)
+        .def("end", &DynamicCluster::end)
+        .def_readwrite("x", &DynamicCluster::x)
+        .def_readwrite("y", &DynamicCluster::y)
+        .def_buffer([](DynamicCluster &c) -> py::buffer_info {
+            return py::buffer_info(c.data(), c.dt.bytes(), c.dt.format_descr(),
+                                   1, {c.size()}, {c.dt.bytes()});
+        })
+
+        .def("__repr__", [](const DynamicCluster &a) {
+            return "<DynamicCluster: x: " + std::to_string(a.x) +
+                   ", y: " + std::to_string(a.y) + ">";
+        });
+}

python/src/cluster_file.hpp

@@ -0,0 +1,50 @@
+#include "aare/ClusterFile.hpp"
+#include "aare/defs.hpp"
+
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_cluster_file_io_bindings(py::module &m) {
+    PYBIND11_NUMPY_DTYPE(Cluster, x, y, data);
+
+    py::class_<ClusterFile>(m, "ClusterFile")
+        .def(py::init<const std::filesystem::path &, size_t>(), py::arg(), py::arg("chunk_size") = 1000)
+        .def("read_clusters",
+             [](ClusterFile &self, size_t n_clusters) {
+                 auto* vec = new std::vector<Cluster>(self.read_clusters(n_clusters));
+                 return return_vector(vec);
+             })
+        .def("read_frame",
+             [](ClusterFile &self) {
+                 int32_t frame_number;
+                 auto* vec = new std::vector<Cluster>(self.read_frame(frame_number));
+                 return py::make_tuple(frame_number, return_vector(vec));
+             })
+        .def("read_cluster_with_cut",
+             [](ClusterFile &self, size_t n_clusters, py::array_t<double> noise_map, int nx, int ny) {
+                 auto view = make_view_2d(noise_map);
+                 auto* vec = new std::vector<Cluster>(self.read_cluster_with_cut(n_clusters, view.data(), nx, ny));
+                 return return_vector(vec);
+             })
+        .def("__enter__", [](ClusterFile &self) { return &self; })
+        .def("__exit__", [](ClusterFile &self, py::args args) { return; })
+        .def("__iter__", [](ClusterFile &self) { return &self; })
+        .def("__next__", [](ClusterFile &self) {
+            auto vec =  new std::vector<Cluster>(self.read_clusters(self.chunk_size()));
+            if(vec->size() == 0) {
+                throw py::stop_iteration();
+            }
+            return return_vector(vec);
+        });
+
+}

python/src/file.hpp

@@ -1,40 +1,85 @@
-#include "aare/Frame.hpp"
+#include "aare/CtbRawFile.hpp"
 #include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/RawFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/RawSubFile.hpp"
+
 #include "aare/defs.hpp"
 // #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
 #include <filesystem>
-#include <pybind11/numpy.h>
 #include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 #include <pybind11/stl/filesystem.h>
 #include <string>
 
 namespace py = pybind11;
-using namespace::aare;
+using namespace ::aare;
 
 void define_file_io_bindings(py::module &m) {
-    py::class_<xy>(m, "xy")
-        .def(py::init<>())
-        .def(py::init<uint32_t, uint32_t>())
-        .def_readwrite("row", &xy::row)
-        .def_readwrite("col", &xy::col)
-        .def("__eq__", &xy::operator==)
-        .def("__ne__", &xy::operator!=)
-        .def("__repr__",
-             [](const xy &a) { return "<xy: row=" + std::to_string(a.row) + ", col=" + std::to_string(a.col) + ">"; });
 
 
+    py::enum_<DetectorType>(m, "DetectorType")
+        .value("Jungfrau", DetectorType::Jungfrau)
+        .value("Eiger", DetectorType::Eiger)
+        .value("Mythen3", DetectorType::Mythen3)
+        .value("Moench", DetectorType::Moench)
+        .value("Moench03", DetectorType::Moench03)
+        .value("Moench03_old", DetectorType::Moench03_old)
+        .value("ChipTestBoard", DetectorType::ChipTestBoard)
+        .value("Unknown", DetectorType::Unknown);
+
+
+    PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
+                         bunchId, timestamp, modId, row, column, reserved,
+                         debug, roundRNumber, detType, version, packetMask);
+
+    py::class_<CtbRawFile>(m, "CtbRawFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("read_frame",
+             [](CtbRawFile &self) {
+                 size_t image_size = self.image_size_in_bytes();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(1);
+                 shape.push_back(image_size);
+
+                 py::array_t<DetectorHeader> header(1);
+
+                 // always read bytes
+                 image = py::array_t<uint8_t>(shape);
+
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def("seek", &CtbRawFile::seek)
+        .def("tell", &CtbRawFile::tell)
+        .def("master", &CtbRawFile::master)
+
+        .def_property_readonly("image_size_in_bytes",
+                               &CtbRawFile::image_size_in_bytes)
+
+        .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
+
     py::class_<File>(m, "File")
-        .def(py::init([](const std::filesystem::path &fname) { return File(fname, "r", {}); }))
-        .def(
-            py::init([](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); }))
-        .def(py::init<const std::filesystem::path &, const std::string &, const FileConfig &>())
-        // .def("read", py::overload_cast<>(&File::read))
-        // .def("read", py::overload_cast<size_t>(&File::read))
-        .def("iread", py::overload_cast<size_t>(&File::iread),py::call_guard<py::gil_scoped_release>())
+        .def(py::init([](const std::filesystem::path &fname) {
+            return File(fname, "r", {});
+        }))
+        .def(py::init(
+            [](const std::filesystem::path &fname, const std::string &mode) {
+                return File(fname, mode, {});
+            }))
+        .def(py::init<const std::filesystem::path &, const std::string &,
+                      const FileConfig &>())
+
         .def("frame_number", &File::frame_number)
         .def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
         .def_property_readonly("pixels_per_frame", &File::pixels_per_frame)
@@ -44,15 +89,55 @@ void define_file_io_bindings(py::module &m) {
         .def_property_readonly("rows", &File::rows)
         .def_property_readonly("cols", &File::cols)
         .def_property_readonly("bitdepth", &File::bitdepth)
-        .def_property_readonly("detector_type", &File::detector_type)
-        .def_property_readonly("geometry", &File::geometry,
-                               py::call_guard<py::scoped_ostream_redirect, py::scoped_estream_redirect>())
-        // .def("set_total_frames", &File::set_total_frames)
-        .def("read_frame", [](File &self) {
+        .def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
+        .def_property_readonly(
+            "detector_type",
+            [](File &self) { return ToString(self.detector_type()); })
+        .def("read_frame",
+             [](File &self) {
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+             })
+        .def("read_frame",
+             [](File &self, size_t frame_number) {
+                 self.seek(frame_number);
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+             })
+        .def("read_n", [](File &self, size_t n_frames) {
             const uint8_t item_size = self.bytes_per_pixel();
             py::array image;
             std::vector<ssize_t> shape;
-            shape.reserve(2);
+            shape.reserve(3);
+            shape.push_back(n_frames);
             shape.push_back(self.rows());
             shape.push_back(self.cols());
             if (item_size == 1) {
@@ -62,7 +147,8 @@ void define_file_io_bindings(py::module &m) {
             } else if (item_size == 4) {
                 image = py::array_t<uint32_t>(shape);
             }
-            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
+            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
+                           n_frames);
             return image;
         });
 
@@ -78,37 +164,108 @@ void define_file_io_bindings(py::module &m) {
         .def_readwrite("dtype", &FileConfig::dtype)
         .def("__eq__", &FileConfig::operator==)
         .def("__ne__", &FileConfig::operator!=)
-        .def("__repr__", [](const FileConfig &a) { return "<FileConfig: " + a.to_string() + ">"; });
+        .def("__repr__", [](const FileConfig &a) {
+            return "<FileConfig: " + a.to_string() + ">";
+        });
+
+
+
+    py::class_<ScanParameters>(m, "ScanParameters")
+        .def(py::init<const std::string &>())
+        .def(py::init<const ScanParameters &>())
+
+        .def_property_readonly("enabled", &ScanParameters::enabled)
+        .def_property_readonly("dac", &ScanParameters::dac)
+        .def_property_readonly("start", &ScanParameters::start)
+        .def_property_readonly("stop", &ScanParameters::stop)
+        .def_property_readonly("step", &ScanParameters::step);
+
+
+    py::class_<ROI>(m, "ROI")
+        .def(py::init<>())
+        .def_readwrite("xmin", &ROI::xmin)
+        .def_readwrite("xmax", &ROI::xmax)
+        .def_readwrite("ymin", &ROI::ymin)
+        .def_readwrite("ymax", &ROI::ymax)
+        .def("__str__", [](const ROI& self){
+            return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax);
+        })
+        .def("__repr__", [](const ROI& self){
+            return fmt::format("<ROI: xmin: {} xmax: {} ymin: {} ymax: {}>", self.xmin, self.xmax, self.ymin, self.ymax);
+        })
+        .def("__iter__", [](const ROI &self) {
+            return py::make_iterator(&self.xmin, &self.ymax+1);
+        });
+
+    
+
+
+    py::class_<RawSubFile>(m, "RawSubFile")
+        .def(py::init<const std::filesystem::path &, DetectorType, size_t,
+                      size_t, size_t>())
+        .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame)
+        .def_property_readonly("pixels_per_frame",
+                               &RawSubFile::pixels_per_frame)
+        .def("seek", &RawSubFile::seek)
+        .def("tell", &RawSubFile::tell)
+        .def_property_readonly("rows", &RawSubFile::rows)
+        .def_property_readonly("cols", &RawSubFile::cols)
+        .def("read_frame",
+             [](RawSubFile &self) {
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols());
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+             });
+
 
     // py::class_<ClusterHeader>(m, "ClusterHeader")
     //     .def(py::init<>())
     //     .def_readwrite("frame_number", &ClusterHeader::frame_number)
     //     .def_readwrite("n_clusters", &ClusterHeader::n_clusters)
-    //     .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader: " + a.to_string() + ">"; });
+    //     .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader:
+    //     " + a.to_string() + ">"; });
 
     // py::class_<ClusterV2_>(m, "ClusterV2_")
     //     .def(py::init<>())
     //     .def_readwrite("x", &ClusterV2_::x)
     //     .def_readwrite("y", &ClusterV2_::y)
     //     .def_readwrite("data", &ClusterV2_::data)
-    //     .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " + a.to_string(false) + ">"; });
+    //     .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " +
+    //     a.to_string(false) + ">"; });
 
     // py::class_<ClusterV2>(m, "ClusterV2")
     //     .def(py::init<>())
     //     .def_readwrite("cluster", &ClusterV2::cluster)
     //     .def_readwrite("frame_number", &ClusterV2::frame_number)
-    //     .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " + a.to_string() + ">"; });
+    //     .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " +
+    //     a.to_string() + ">"; });
 
     // py::class_<ClusterFileV2>(m, "ClusterFileV2")
     //     .def(py::init<const std::filesystem::path &, const std::string &>())
     //     .def("read", py::overload_cast<>(&ClusterFileV2::read))
     //     .def("read", py::overload_cast<int>(&ClusterFileV2::read))
     //     .def("frame_number", &ClusterFileV2::frame_number)
-    //     .def("write", py::overload_cast<std::vector<ClusterV2> const &>(&ClusterFileV2::write))
-        
+    //     .def("write", py::overload_cast<std::vector<ClusterV2> const
+    //     &>(&ClusterFileV2::write))
+
     //     .def("close", &ClusterFileV2::close);
 
-    // m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int frame_number) {
+    // m.def("to_clustV2", [](std::vector<DynamicCluster> &clusters, const int
+    // frame_number) {
     //     std::vector<ClusterV2> clusters_;
     //     for (auto &c : clusters) {
     //         ClusterV2 cluster;

python/src/module.cpp

@@ -1,14 +1,27 @@
-
+//Files with bindings to the different classes
 #include "file.hpp"
+#include "raw_file.hpp"
+#include "raw_master_file.hpp"
+#include "var_cluster.hpp"
+#include "pixel_map.hpp"
+#include "pedestal.hpp"
+#include "cluster.hpp"
+#include "cluster_file.hpp"
 
+//Pybind stuff
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
 namespace py = pybind11;
 
 PYBIND11_MODULE(_aare, m) {
-
-    
-
     define_file_io_bindings(m);
+    define_raw_file_io_bindings(m);
+    define_raw_master_file_bindings(m);
+    define_var_cluster_finder_bindings(m);
+    define_pixel_map_bindings(m);
+    define_pedestal_bindings<double>(m, "Pedestal");
+    define_pedestal_bindings<float>(m, "Pedestal_float32");
+    define_cluster_finder_bindings(m);
+    define_cluster_file_io_bindings(m);
 }

python/src/np_helper.hpp

@@ -7,36 +7,37 @@
 
 #include "aare/Frame.hpp"
 #include "aare/NDArray.hpp"
+#include "aare/NDView.hpp"
 
 namespace py = pybind11;
 
 // Pass image data back to python as a numpy array
-// template <typename T, ssize_t Ndim>
-// py::array return_image_data(pl::ImageData<T, Ndim> *image) {
+template <typename T, int64_t Ndim>
+py::array return_image_data(aare::NDArray<T, Ndim> *image) {
 
-//     py::capsule free_when_done(image, [](void *f) {
-//         pl::ImageData<T, Ndim> *foo =
-//             reinterpret_cast<pl::ImageData<T, Ndim> *>(f);
-//         delete foo;
-//     });
+    py::capsule free_when_done(image, [](void *f) {
+        aare::NDArray<T, Ndim> *foo =
+            reinterpret_cast<aare::NDArray<T, Ndim> *>(f);
+        delete foo;
+    });
 
-//     return py::array_t<T>(
-//         image->shape(),        // shape
-//         image->byte_strides(), // C-style contiguous strides for double
-//         image->data(),         // the data pointer
-//         free_when_done);       // numpy array references this parent
-// }
+    return py::array_t<T>(
+        image->shape(),        // shape
+        image->byte_strides(), // C-style contiguous strides for double
+        image->data(),         // the data pointer
+        free_when_done);       // numpy array references this parent
+}
 
-// template <typename T> py::array return_vector(std::vector<T> *vec) {
-//     py::capsule free_when_done(vec, [](void *f) {
-//         std::vector<T> *foo = reinterpret_cast<std::vector<T> *>(f);
-//         delete foo;
-//     });
-//     return py::array_t<T>({vec->size()}, // shape
-//                           {sizeof(T)}, // C-style contiguous strides for double
-//                           vec->data(), // the data pointer
-//                           free_when_done); // numpy array references this parent
-// }
+template <typename T> py::array return_vector(std::vector<T> *vec) {
+    py::capsule free_when_done(vec, [](void *f) {
+        std::vector<T> *foo = reinterpret_cast<std::vector<T> *>(f);
+        delete foo;
+    });
+    return py::array_t<T>({vec->size()}, // shape
+                          {sizeof(T)}, // C-style contiguous strides for double
+                          vec->data(), // the data pointer
+                          free_when_done); // numpy array references this parent
+}
 
 // template <typename Reader> py::array do_read(Reader &r, size_t n_frames) {
 //     py::array image;
@@ -62,54 +63,54 @@ namespace py = pybind11;
 //     return image;
 // }
 
-py::array return_frame(pl::Frame *ptr) {
-    py::capsule free_when_done(ptr, [](void *f) {
-        pl::Frame *foo = reinterpret_cast<pl::Frame *>(f);
-        delete foo;
-    });
+// py::array return_frame(pl::Frame *ptr) {
+//     py::capsule free_when_done(ptr, [](void *f) {
+//         pl::Frame *foo = reinterpret_cast<pl::Frame *>(f);
+//         delete foo;
+//     });
 
-    const uint8_t item_size = ptr->bytes_per_pixel();
-    std::vector<ssize_t> shape;
-    for (auto val : ptr->shape())
-        if (val > 1)
-            shape.push_back(val);
+//     const uint8_t item_size = ptr->bytes_per_pixel();
+//     std::vector<ssize_t> shape;
+//     for (auto val : ptr->shape())
+//         if (val > 1)
+//             shape.push_back(val);
 
-    std::vector<ssize_t> strides;
-    if (shape.size() == 1)
-        strides.push_back(item_size);
-    else if (shape.size() == 2) {
-        strides.push_back(item_size * shape[1]);
-        strides.push_back(item_size);
-    }
+//     std::vector<ssize_t> strides;
+//     if (shape.size() == 1)
+//         strides.push_back(item_size);
+//     else if (shape.size() == 2) {
+//         strides.push_back(item_size * shape[1]);
+//         strides.push_back(item_size);
+//     }
 
-    if (item_size == 1)
-        return py::array_t<uint8_t>(
-            shape, strides,
-            reinterpret_cast<uint8_t *>(ptr->data()), free_when_done);
-    else if (item_size == 2)
-        return py::array_t<uint16_t>(shape, strides,
-                                     reinterpret_cast<uint16_t *>(ptr->data()),
-                                     free_when_done);
-    else if (item_size == 4)
-        return py::array_t<uint32_t>(shape, strides,
-                                     reinterpret_cast<uint32_t *>(ptr->data()),
-                                     free_when_done);
-    return {};
-}
+//     if (item_size == 1)
+//         return py::array_t<uint8_t>(
+//             shape, strides,
+//             reinterpret_cast<uint8_t *>(ptr->data()), free_when_done);
+//     else if (item_size == 2)
+//         return py::array_t<uint16_t>(shape, strides,
+//                                      reinterpret_cast<uint16_t *>(ptr->data()),
+//                                      free_when_done);
+//     else if (item_size == 4)
+//         return py::array_t<uint32_t>(shape, strides,
+//                                      reinterpret_cast<uint32_t *>(ptr->data()),
+//                                      free_when_done);
+//     return {};
+// }
 
 // todo rewrite generic
 template <class T, int Flags> auto get_shape_3d(py::array_t<T, Flags> arr) {
-    return pl::Shape<3>{arr.shape(0), arr.shape(1), arr.shape(2)};
+    return aare::Shape<3>{arr.shape(0), arr.shape(1), arr.shape(2)};
 }
 
-template <class T, int Flags> auto make_span_3d(py::array_t<T, Flags> arr) {
-    return pl::DataSpan<T, 3>(arr.mutable_data(), get_shape_3d<T, Flags>(arr));
+template <class T, int Flags> auto make_view_3d(py::array_t<T, Flags> arr) {
+    return aare::NDView<T, 3>(arr.mutable_data(), get_shape_3d<T, Flags>(arr));
 }
 
 template <class T, int Flags> auto get_shape_2d(py::array_t<T, Flags> arr) {
-    return pl::Shape<2>{arr.shape(0), arr.shape(1)};
+    return aare::Shape<2>{arr.shape(0), arr.shape(1)};
 }
 
-template <class T, int Flags> auto make_span_2d(py::array_t<T, Flags> arr) {
-    return pl::DataSpan<T, 2>(arr.mutable_data(), get_shape_2d<T, Flags>(arr));
+template <class T, int Flags> auto make_view_2d(py::array_t<T, Flags> arr) {
+    return aare::NDView<T, 2>(arr.mutable_data(), get_shape_2d<T, Flags>(arr));
 }

python/src/pedestal.hpp

@@ -0,0 +1,47 @@
+
+#include "aare/Pedestal.hpp"
+#include "np_helper.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+template <typename SUM_TYPE> void define_pedestal_bindings(py::module &m, const std::string &name) {
+    py::class_<Pedestal<SUM_TYPE>>(m, name.c_str())
+        .def(py::init<int, int, int>())
+        .def(py::init<int, int>())
+        .def("mean",
+             [](Pedestal<SUM_TYPE> &self) {
+                 auto m = new NDArray<SUM_TYPE, 2>{};
+                 *m = self.mean();
+                 return return_image_data(m);
+             })
+        .def("variance", [](Pedestal<SUM_TYPE> &self) {
+            auto m = new NDArray<SUM_TYPE, 2>{};
+            *m = self.variance();
+            return return_image_data(m);
+        })
+        .def("std", [](Pedestal<SUM_TYPE> &self) {
+            auto m = new NDArray<SUM_TYPE, 2>{};
+            *m = self.std();
+            return return_image_data(m);
+        })
+        .def("clear", py::overload_cast<>(&Pedestal<SUM_TYPE>::clear))
+        .def_property_readonly("rows", &Pedestal<SUM_TYPE>::rows)
+        .def_property_readonly("cols", &Pedestal<SUM_TYPE>::cols)
+        .def_property_readonly("n_samples", &Pedestal<SUM_TYPE>::n_samples)
+        .def_property_readonly("sum", &Pedestal<SUM_TYPE>::get_sum)
+        .def_property_readonly("sum2", &Pedestal<SUM_TYPE>::get_sum2)
+        .def("clone",
+             [&](Pedestal<SUM_TYPE> &pedestal) {
+                 return Pedestal<SUM_TYPE>(pedestal);
+             })
+        //TODO! add push for other data types
+        .def("push", [](Pedestal<SUM_TYPE> &pedestal, py::array_t<uint16_t> &f) {
+            auto v = make_view_2d(f);
+            pedestal.push(v);
+        });
+}

python/src/pixel_map.hpp

@@ -0,0 +1,33 @@
+#include "aare/PixelMap.hpp"
+#include "np_helper.hpp"
+
+
+#include <cstdint>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+
+namespace py = pybind11;
+using namespace::aare;
+
+
+void define_pixel_map_bindings(py::module &m) {
+    m.def("GenerateMoench03PixelMap", []() {
+        auto ptr = new NDArray<ssize_t,2>(GenerateMoench03PixelMap());
+        return return_image_data(ptr);
+    })
+    .def("GenerateMoench05PixelMap", []() {
+        auto ptr = new NDArray<ssize_t,2>(GenerateMoench05PixelMap());
+        return return_image_data(ptr);
+    })
+    .def("GenerateMH02SingleCounterPixelMap", []() {
+        auto ptr = new NDArray<ssize_t,2>(GenerateMH02SingleCounterPixelMap());
+        return return_image_data(ptr);
+    })
+    .def("GenerateMH02FourCounterPixelMap", []() {
+        auto ptr = new NDArray<ssize_t,3>(GenerateMH02FourCounterPixelMap());
+        return return_image_data(ptr);
+    });
+
+}

python/src/raw_file.hpp

@@ -0,0 +1,95 @@
+#include "aare/CtbRawFile.hpp"
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/RawFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/RawSubFile.hpp"
+
+#include "aare/defs.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_raw_file_io_bindings(py::module &m) {
+    py::class_<RawFile>(m, "RawFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("read_frame",
+             [](RawFile &self) {
+                 size_t image_size = self.bytes_per_frame();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+
+                 // return headers from all subfiles
+                 py::array_t<DetectorHeader> header(self.n_mod());
+
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def("read_n",
+             [](RawFile &self, size_t n_frames) {
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(3);
+                 shape.push_back(n_frames);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+
+                 // return headers from all subfiles
+                 py::array_t<DetectorHeader> header({self.n_mod(), n_frames});
+
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     n_frames, header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def("frame_number", &RawFile::frame_number)
+        .def_property_readonly("bytes_per_frame", &RawFile::bytes_per_frame)
+        .def_property_readonly("pixels_per_frame", &RawFile::pixels_per_frame)
+        .def_property_readonly("bytes_per_pixel", &RawFile::bytes_per_pixel)
+        .def("seek", &RawFile::seek, R"(
+            Seek to a frame index in file.
+            )")
+        .def("tell", &RawFile::tell, R"(
+            Return the current frame number.)")
+        .def_property_readonly("total_frames", &RawFile::total_frames)
+        .def_property_readonly("rows", &RawFile::rows)
+        .def_property_readonly("cols", &RawFile::cols)
+        .def_property_readonly("bitdepth", &RawFile::bitdepth)
+        .def_property_readonly("geometry", &RawFile::geometry)
+        .def_property_readonly("n_mod", &RawFile::n_mod)
+        .def_property_readonly("detector_type", &RawFile::detector_type)
+        .def_property_readonly("master", &RawFile::master);
+}

python/src/raw_master_file.hpp

@@ -0,0 +1,85 @@
+
+#include "aare/CtbRawFile.hpp"
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/RawFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/RawSubFile.hpp"
+
+#include "aare/defs.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_raw_master_file_bindings(py::module &m) {
+    py::class_<RawMasterFile>(m, "RawMasterFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("data_fname", &RawMasterFile::data_fname, R"(
+
+            Parameters
+            ------------
+                module_index : int 
+                        module index (d0, d1 .. dN)
+                file_index : int 
+                        file index (f0, f1 .. fN)
+
+            Returns
+            ----------
+                os.PathLike
+                        The name of the data file.
+
+            )")
+        .def_property_readonly("version", &RawMasterFile::version)
+        .def_property_readonly("detector_type", &RawMasterFile::detector_type)
+        .def_property_readonly("timing_mode", &RawMasterFile::timing_mode)
+        .def_property_readonly("image_size_in_bytes",
+                               &RawMasterFile::image_size_in_bytes)
+        .def_property_readonly("frames_in_file", &RawMasterFile::frames_in_file)
+        .def_property_readonly("pixels_y", &RawMasterFile::pixels_y)
+        .def_property_readonly("pixels_x", &RawMasterFile::pixels_x)
+        .def_property_readonly("max_frames_per_file",
+                               &RawMasterFile::max_frames_per_file)
+        .def_property_readonly("bitdepth", &RawMasterFile::bitdepth)
+        .def_property_readonly("frame_padding", &RawMasterFile::frame_padding)
+        .def_property_readonly("frame_discard_policy",
+                               &RawMasterFile::frame_discard_policy)
+
+        .def_property_readonly("total_frames_expected",
+                               &RawMasterFile::total_frames_expected)
+        .def_property_readonly("geometry", &RawMasterFile::geometry)
+        .def_property_readonly("analog_samples", &RawMasterFile::analog_samples, R"(
+            Number of analog samples
+
+            Returns
+            ----------
+                int | None
+                    The number of analog samples in the file (or None if not enabled)
+            )")
+        .def_property_readonly("digital_samples",
+                               &RawMasterFile::digital_samples,  R"(
+            Number of digital samples
+
+            Returns
+            ----------
+                int | None
+                    The number of digital samples in the file (or None if not enabled)
+            )")
+
+        .def_property_readonly("transceiver_samples",
+                               &RawMasterFile::transceiver_samples)
+        .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows)
+        .def_property_readonly("quad", &RawMasterFile::quad)
+        .def_property_readonly("scan_parameters",
+                               &RawMasterFile::scan_parameters)
+        .def_property_readonly("roi", &RawMasterFile::roi);
+}

python/src/var_cluster.hpp

@@ -0,0 +1,46 @@
+#include "aare/VarClusterFinder.hpp"
+#include "np_helper.hpp"
+// #include "aare/defs.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+// #include <filesystem>
+#include <pybind11/numpy.h>
+// #include <pybind11/iostream.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+// #include <pybind11/stl/filesystem.h>
+// #include <string>
+
+
+namespace py = pybind11;
+using namespace::aare;
+
+
+void define_var_cluster_finder_bindings(py::module &m) {
+    PYBIND11_NUMPY_DTYPE(VarClusterFinder<double>::Hit, size, row, col,
+                         reserved, energy, max);
+
+    py::class_<VarClusterFinder<double>>(m, "VarClusterFinder")
+        .def(py::init<Shape<2>, double>())
+        .def("labeled",
+             [](VarClusterFinder<double> &self) {
+                 auto ptr = new NDArray<int, 2>(self.labeled());
+                 return return_image_data(ptr);
+             })
+        .def("find_clusters",
+             [](VarClusterFinder<double> &self,
+                py::array_t<double, py::array::c_style | py::array::forcecast>
+                    img) {
+                 auto view = make_view_2d(img);
+                 self.find_clusters(view);
+             })
+        .def("steal_hits",
+             [](VarClusterFinder<double> &self) {
+                 auto ptr = new std::vector<VarClusterFinder<double>::Hit>(
+                     self.steal_hits());
+                 return return_vector(ptr);
+             })
+        .def("total_clusters", &VarClusterFinder<double>::total_clusters);
+
+}

src/ClusterFile.cpp

@@ -0,0 +1,312 @@
+#include "aare/ClusterFile.hpp"
+
+namespace aare {
+
+ClusterFile::ClusterFile(const std::filesystem::path &fname, size_t chunk_size): m_chunk_size(chunk_size) {
+    fp = fopen(fname.c_str(), "rb");
+    if (!fp) {
+        throw std::runtime_error("Could not open file: " + fname.string());
+    }
+}
+
+std::vector<Cluster> ClusterFile::read_clusters(size_t n_clusters) {
+    std::vector<Cluster> clusters(n_clusters);
+
+    int32_t iframe = 0; // frame number needs to be 4 bytes!
+    size_t nph_read = 0;
+    uint32_t nn = m_num_left;
+    uint32_t nph = m_num_left; // number of clusters in frame needs to be 4
+
+    auto buf = reinterpret_cast<Cluster *>(clusters.data());
+    // if there are photons left from previous frame read them first
+    if (nph) {
+        if (nph > n_clusters) {
+            // if we have more photons left in the frame then photons to read we
+            // read directly the requested number
+            nn = n_clusters;
+        } else {
+            nn = nph;
+        }
+        nph_read += fread((void *)(buf + nph_read), sizeof(Cluster), nn, fp);
+        m_num_left = nph - nn; // write back the number of photons left
+    }
+
+    if (nph_read < n_clusters) {
+        // keep on reading frames and photons until reaching n_clusters
+        while (fread(&iframe, sizeof(iframe), 1, fp)) {
+            // read number of clusters in frame
+            if (fread(&nph, sizeof(nph), 1, fp)) {
+                if (nph > (n_clusters - nph_read))
+                    nn = n_clusters - nph_read;
+                else
+                    nn = nph;
+
+                nph_read +=
+                    fread((void *)(buf + nph_read), sizeof(Cluster), nn, fp);
+                m_num_left = nph - nn;
+            }
+            if (nph_read >= n_clusters)
+                break;
+        }
+    }
+
+    // Resize the vector to the number of clusters.
+    // No new allocation, only change bounds.
+    clusters.resize(nph_read);
+    return clusters;
+}
+
+std::vector<Cluster> ClusterFile::read_frame(int32_t &out_fnum) {
+    if (m_num_left) {
+        throw std::runtime_error("There are still photons left in the last frame");
+    }
+
+    if (fread(&out_fnum, sizeof(out_fnum), 1, fp) != 1) {
+        throw std::runtime_error("Could not read frame number");
+    }
+
+    int n_clusters;
+    if (fread(&n_clusters, sizeof(n_clusters), 1, fp) != 1) {
+        throw std::runtime_error("Could not read number of clusters");
+    }
+    std::vector<Cluster> clusters(n_clusters);
+
+    if (fread(clusters.data(), sizeof(Cluster), n_clusters, fp) != n_clusters) {
+        throw std::runtime_error("Could not read clusters");
+    }
+    return clusters;
+
+}
+
+std::vector<Cluster> ClusterFile::read_cluster_with_cut(size_t n_clusters,
+                                                        double *noise_map,
+                                                        int nx, int ny) {
+
+    std::vector<Cluster> clusters(n_clusters);
+    // size_t read_clusters_with_cut(FILE *fp, size_t n_clusters, Cluster *buf,
+    //                               uint32_t *n_left, double *noise_map, int
+    //                               nx, int ny) {
+    int iframe = 0;
+    //     uint32_t nph = *n_left;
+    uint32_t nph = m_num_left;
+    //     uint32_t nn = *n_left;
+    uint32_t nn = m_num_left;
+    size_t nph_read = 0;
+
+    int32_t t2max, tot1;
+    int32_t tot3;
+    // Cluster *ptr = buf;
+    Cluster *ptr = clusters.data();
+    int good = 1;
+    double noise;
+    // read photons left from previous frame
+    if (noise_map)
+        printf("Using noise map\n");
+
+    if (nph) {
+        if (nph > n_clusters) {
+            // if we have more photons left in the frame then photons to
+            // read we read directly the requested number
+            nn = n_clusters;
+        } else {
+            nn = nph;
+        }
+        for (size_t iph = 0; iph < nn; iph++) {
+            // read photons 1 by 1
+            size_t n_read = fread((void *)(ptr), sizeof(Cluster), 1, fp);
+            if (n_read != 1) {
+                clusters.resize(nph_read);
+                return clusters;
+            }
+            // TODO! error handling on read
+            good = 1;
+            if (noise_map) {
+                if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 && ptr->y < ny) {
+                    tot1 = ptr->data[4];
+                    analyze_cluster(*ptr, &t2max, &tot3, NULL, NULL, NULL, NULL,
+                                    NULL);
+                    noise = noise_map[ptr->y * nx + ptr->x];
+                    if (tot1 > noise || t2max > 2 * noise || tot3 > 3 * noise) {
+                        ;
+                    } else {
+                        good = 0;
+                        printf("%d %d %f %d %d %d\n", ptr->x, ptr->y, noise,
+                               tot1, t2max, tot3);
+                    }
+                } else {
+                    printf("Bad pixel number %d %d\n", ptr->x, ptr->y);
+                    good = 0;
+                }
+            }
+            if (good) {
+                ptr++;
+                nph_read++;
+            }
+            (m_num_left)--;
+            if (nph_read >= n_clusters)
+                break;
+        }
+    }
+        if (nph_read < n_clusters) {
+    //         // keep on reading frames and photons until reaching n_clusters
+            while (fread(&iframe, sizeof(iframe), 1, fp)) {
+    //             // printf("%d\n",nph_read);
+
+                if (fread(&nph, sizeof(nph), 1, fp)) {
+    //                 // printf("** %d\n",nph);
+                    m_num_left = nph;
+                    for (size_t iph = 0; iph < nph; iph++) {
+    //                     // read photons 1 by 1
+                        size_t n_read =
+                            fread((void *)(ptr), sizeof(Cluster), 1, fp);
+                        if (n_read != 1) {
+                            clusters.resize(nph_read);
+                            return clusters;
+                            // return nph_read;
+                        }
+                        good = 1;
+                        if (noise_map) {
+                            if (ptr->x >= 0 && ptr->x < nx && ptr->y >= 0 &&
+                                ptr->y < ny) {
+                                tot1 = ptr->data[4];
+                                analyze_cluster(*ptr, &t2max, &tot3, NULL,
+                                NULL,
+                                                NULL, NULL, NULL);
+                                // noise = noise_map[ptr->y * nx + ptr->x];
+                                noise = noise_map[ptr->y + ny * ptr->x];
+    			    if (tot1 > noise || t2max > 2 * noise ||
+                                    tot3 > 3 * noise) {
+                                    ;
+                                } else
+                                    good = 0;
+                            } else {
+                                printf("Bad pixel number %d %d\n", ptr->x,
+                                ptr->y); good = 0;
+                            }
+                        }
+                        if (good) {
+                            ptr++;
+    			nph_read++;
+                        }
+    		    (m_num_left)--;
+                        if (nph_read >= n_clusters)
+                            break;
+                    }
+                }
+                if (nph_read >= n_clusters)
+                    break;
+            }
+        }
+        // printf("%d\n",nph_read);
+        clusters.resize(nph_read);
+        return clusters;
+
+}
+
+int ClusterFile::analyze_cluster(Cluster cl, int32_t *t2, int32_t *t3, char *quad,
+                    double *eta2x, double *eta2y, double *eta3x,
+                    double *eta3y) {
+
+    return analyze_data(cl.data, t2, t3, quad, eta2x, eta2y, eta3x, eta3y);
+}
+
+int ClusterFile::analyze_data(int32_t *data, int32_t *t2, int32_t *t3, char *quad,
+                 double *eta2x, double *eta2y, double *eta3x, double *eta3y) {
+
+    int ok = 1;
+
+    int32_t tot2[4];
+    int32_t t2max = 0;
+    char c = 0;
+    int32_t val, tot3;
+
+    tot3 = 0;
+    for (int i = 0; i < 4; i++)
+        tot2[i] = 0;
+
+    for (int ix = 0; ix < 3; ix++) {
+        for (int iy = 0; iy < 3; iy++) {
+            val = data[iy * 3 + ix];
+            //	printf ("%d ",data[iy * 3 + ix]);
+            tot3 += val;
+            if (ix <= 1 && iy <= 1)
+                tot2[cBottomLeft] += val;
+            if (ix >= 1 && iy <= 1)
+                tot2[cBottomRight] += val;
+            if (ix <= 1 && iy >= 1)
+                tot2[cTopLeft] += val;
+            if (ix >= 1 && iy >= 1)
+                tot2[cTopRight] += val;
+        }
+        //	printf ("\n");
+    }
+    // printf ("\n");
+
+    if (t2 || quad) {
+
+        t2max = tot2[0];
+        c = cBottomLeft;
+        for (int i = 1; i < 4; i++) {
+            if (tot2[i] > t2max) {
+                t2max = tot2[i];
+                c = i;
+            }
+        }
+	//printf("*** %d %d %d %d -- %d\n",tot2[0],tot2[1],tot2[2],tot2[3],t2max);
+        if (quad)
+            *quad = c;
+        if (t2)
+            *t2 = t2max;
+    }
+    if (t3)
+        *t3 = tot3;
+
+    if (eta2x || eta2y) {
+        if (eta2x)
+            *eta2x = 0;
+        if (eta2y)
+            *eta2y = 0;
+        switch (c) {
+        case cBottomLeft:
+            if (eta2x && (data[3] + data[4]) != 0)
+                *eta2x = (double)(data[4]) / (data[3] + data[4]);
+            if (eta2y && (data[1] + data[4]) != 0)
+                *eta2y = (double)(data[4]) / (data[1] + data[4]);
+            break;
+        case cBottomRight:
+            if (eta2x && (data[2] + data[5]) != 0)
+                *eta2x = (double)(data[5]) / (data[4] + data[5]);
+            if (eta2y && (data[1] + data[4]) != 0)
+                *eta2y = (double)(data[4]) / (data[1] + data[4]);
+            break;
+        case cTopLeft:
+            if (eta2x && (data[7] + data[4]) != 0)
+                *eta2x = (double)(data[4]) / (data[3] + data[4]);
+            if (eta2y && (data[7] + data[4]) != 0)
+                *eta2y = (double)(data[7]) / (data[7] + data[4]);
+            break;
+        case cTopRight:
+            if (eta2x && t2max != 0)
+                *eta2x = (double)(data[5]) / (data[5] + data[4]);
+            if (eta2y && t2max != 0)
+                *eta2y = (double)(data[7]) / (data[7] + data[4]);
+            break;
+        default:;
+        }
+    }
+
+    if (eta3x || eta3y) {
+        if (eta3x && (data[3] + data[4] + data[5]) != 0)
+            *eta3x = (double)(-data[3] + data[3 + 2]) /
+                     (data[3] + data[4] + data[5]);
+        if (eta3y && (data[1] + data[4] + data[7]) != 0)
+            *eta3y = (double)(-data[1] + data[2 * 3 + 1]) /
+                     (data[1] + data[4] + data[7]);
+    }
+
+    return ok;
+}
+
+
+
+} // namespace aare

src/ClusterFinder.test.cpp

@@ -7,53 +7,65 @@
 
 using namespace aare;
 
-class ClusterFinderUnitTest : public ClusterFinder {
-  public:
-    ClusterFinderUnitTest(int cluster_sizeX, int cluster_sizeY, double nSigma = 5.0, double threshold = 0.0)
-        : ClusterFinder(cluster_sizeX, cluster_sizeY, nSigma, threshold) {}
-    double get_c2() { return c2; }
-    double get_c3() { return c3; }
-    auto get_threshold() { return m_threshold; }
-    auto get_nSigma() { return m_nSigma; }
-    auto get_cluster_sizeX() { return m_cluster_sizeX; }
-    auto get_cluster_sizeY() { return m_cluster_sizeY; }
-};
+//TODO! Find a way to test the cluster finder
 
-TEST_CASE("test ClusterFinder constructor") {
-    ClusterFinderUnitTest cf(55, 100);
-    REQUIRE(cf.get_cluster_sizeX() == 55);
-    REQUIRE(cf.get_cluster_sizeY() == 100);
-    REQUIRE(cf.get_threshold() == 0.0);
-    REQUIRE(cf.get_nSigma() == 5.0);
-    double c2 = sqrt((100 + 1) / 2 * (55 + 1) / 2);
-    double c3 = sqrt(55 * 100);
-    // REQUIRE(compare_floats<double>(cf.get_c2(), c2));
-    // REQUIRE(compare_floats<double>(cf.get_c3(), c3));
-    REQUIRE_THAT(cf.get_c2(), Catch::Matchers::WithinRel(c2, 1e-9));
-    REQUIRE_THAT(cf.get_c3(), Catch::Matchers::WithinRel(c3, 1e-9));
+
+
+// class ClusterFinderUnitTest : public ClusterFinder {
+//   public:
+//     ClusterFinderUnitTest(int cluster_sizeX, int cluster_sizeY, double nSigma = 5.0, double threshold = 0.0)
+//         : ClusterFinder(cluster_sizeX, cluster_sizeY, nSigma, threshold) {}
+//     double get_c2() { return c2; }
+//     double get_c3() { return c3; }
+//     auto get_threshold() { return m_threshold; }
+//     auto get_nSigma() { return m_nSigma; }
+//     auto get_cluster_sizeX() { return m_cluster_sizeX; }
+//     auto get_cluster_sizeY() { return m_cluster_sizeY; }
+// };
+
+// TEST_CASE("test ClusterFinder constructor") {
+//     ClusterFinderUnitTest cf(55, 100);
+//     REQUIRE(cf.get_cluster_sizeX() == 55);
+//     REQUIRE(cf.get_cluster_sizeY() == 100);
+//     REQUIRE(cf.get_threshold() == 0.0);
+//     REQUIRE(cf.get_nSigma() == 5.0);
+//     double c2 = sqrt((100 + 1) / 2 * (55 + 1) / 2);
+//     double c3 = sqrt(55 * 100);
+//     // REQUIRE(compare_floats<double>(cf.get_c2(), c2));
+//     // REQUIRE(compare_floats<double>(cf.get_c3(), c3));
+//     REQUIRE_THAT(cf.get_c2(), Catch::Matchers::WithinRel(c2, 1e-9));
+//     REQUIRE_THAT(cf.get_c3(), Catch::Matchers::WithinRel(c3, 1e-9));
+// }
+
+TEST_CASE("Construct a cluster finder"){
+    ClusterFinder clusterFinder({400,400}, {3,3});
+    // REQUIRE(clusterFinder.get_cluster_sizeX() == 3);
+    // REQUIRE(clusterFinder.get_cluster_sizeY() == 3);
+    // REQUIRE(clusterFinder.get_threshold() == 1);
+    // REQUIRE(clusterFinder.get_nSigma() == 1);
 }
 
-TEST_CASE("test cluster finder") {
-    aare::Pedestal pedestal(10, 10, 5);
-    NDArray<double, 2> frame({10, 10});
-    frame = 0;
-    ClusterFinder clusterFinder(3, 3, 1, 1); // 3x3 cluster, 1 nSigma, 1 threshold
+// TEST_CASE("test cluster finder") {
+//     aare::Pedestal pedestal(10, 10, 5);
+//     NDArray<double, 2> frame({10, 10});
+//     frame = 0;
+//     ClusterFinder clusterFinder(3, 3, 1, 1); // 3x3 cluster, 1 nSigma, 1 threshold
 
-    auto clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
+//     auto clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
 
-    REQUIRE(clusters.size() == 0);
+//     REQUIRE(clusters.size() == 0);
 
-    frame(5, 5) = 10;
-    clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
-    REQUIRE(clusters.size() == 1);
-    REQUIRE(clusters[0].x == 5);
-    REQUIRE(clusters[0].y == 5);
-    for (int i = 0; i < 3; i++) {
-        for (int j = 0; j < 3; j++) {
-            if (i == 1 && j == 1)
-                REQUIRE(clusters[0].get<double>(i * 3 + j) == 10);
-            else
-                REQUIRE(clusters[0].get<double>(i * 3 + j) == 0);
-        }
-    }
-}
+//     frame(5, 5) = 10;
+//     clusters = clusterFinder.find_clusters_without_threshold(frame.span(), pedestal);
+//     REQUIRE(clusters.size() == 1);
+//     REQUIRE(clusters[0].x == 5);
+//     REQUIRE(clusters[0].y == 5);
+//     for (int i = 0; i < 3; i++) {
+//         for (int j = 0; j < 3; j++) {
+//             if (i == 1 && j == 1)
+//                 REQUIRE(clusters[0].get<double>(i * 3 + j) == 10);
+//             else
+//                 REQUIRE(clusters[0].get<double>(i * 3 + j) == 0);
+//         }
+//     }
+// }

src/CtbRawFile.cpp

@@ -0,0 +1,74 @@
+#include "aare/CtbRawFile.hpp"
+
+#include <fmt/format.h>
+namespace aare {
+
+CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) {
+    if (m_master.detector_type() != DetectorType::ChipTestBoard) {
+        throw std::runtime_error(LOCATION + "Not a Ctb file");
+    }
+
+    find_subfiles();
+
+    // open the first subfile
+    m_file.open(m_master.data_fname(0, 0), std::ios::binary);
+}
+
+void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) {
+    if(m_current_frame >= m_master.frames_in_file()){
+        throw std::runtime_error(LOCATION + "End of file reached");
+    }
+
+    if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){
+        open_data_file(m_current_subfile+1);
+    }
+    
+    if(header){
+        m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
+    }else{
+        m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
+    }
+
+    m_file.read(reinterpret_cast<char *>(image_buf), m_master.image_size_in_bytes());
+    m_current_frame++;
+}
+
+void CtbRawFile::seek(size_t frame_number) {
+    if (auto index = sub_file_index(frame_number); index != m_current_subfile) {
+        open_data_file(index);
+    }
+    size_t frame_number_in_file = frame_number % m_master.max_frames_per_file();
+    m_file.seekg((sizeof(DetectorHeader)+m_master.image_size_in_bytes()) * frame_number_in_file);
+    m_current_frame = frame_number;
+}
+
+size_t CtbRawFile::tell() const { return m_current_frame; }
+
+size_t CtbRawFile::image_size_in_bytes() const { return m_master.image_size_in_bytes(); }
+
+size_t CtbRawFile::frames_in_file() const { return m_master.frames_in_file(); }
+
+RawMasterFile CtbRawFile::master() const { return m_master; }
+
+void CtbRawFile::find_subfiles() {
+    // we can semi safely assume that there is only one module for CTB
+    while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles)))
+        m_num_subfiles++;
+
+    fmt::print("Found {} subfiles\n", m_num_subfiles);
+}
+
+void CtbRawFile::open_data_file(size_t subfile_index) {
+    if (subfile_index >= m_num_subfiles) {
+        throw std::runtime_error(LOCATION + "Subfile index out of range");
+    }
+    m_current_subfile = subfile_index;
+    m_file = std::ifstream(m_master.data_fname(0, subfile_index), std::ios::binary); // only one module for CTB
+    if (!m_file.is_open()) {
+        throw std::runtime_error(LOCATION + "Could not open data file");
+    }
+}
+
+
+
+} // namespace aare

src/File.cpp

@@ -6,66 +6,72 @@
 
 namespace aare {
 
-File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg)
-    : file_impl(nullptr), is_npy(true) {
-    if (mode != "r" && mode != "w" && mode != "a") {
-        throw std::invalid_argument("Unsupported file mode");
+File::File(const std::filesystem::path &fname, const std::string &mode,
+           const FileConfig &cfg)
+    : file_impl(nullptr) {
+    if (mode != "r") {
+        throw std::invalid_argument("At the moment only reading is supported");
     }
 
-    if ((mode == "r" || mode == "a") && !std::filesystem::exists(fname)) {
-        throw std::runtime_error(fmt::format("File does not exist: {}", fname.string()));
+    if ((mode == "r") && !std::filesystem::exists(fname)) {
+        throw std::runtime_error(
+            fmt::format("File does not exist: {}", fname.string()));
     }
 
+    // Assuming we are pointing at a master file? 
+    // TODO! How do we read raw files directly?
     if (fname.extension() == ".raw" || fname.extension() == ".json") {
-        // aare::logger::debug("Loading raw file");
-        file_impl = new RawFile(fname, mode, cfg);
-        is_npy = false;
+        // file_impl = new RawFile(fname, mode, cfg);
+        file_impl = std::make_unique<RawFile>(fname, mode);
     }
-    // check if extension is numpy
     else if (fname.extension() == ".npy") {
-        // aare::logger::debug("Loading numpy file");
-        file_impl = new NumpyFile(fname, mode, cfg);
+        // file_impl = new NumpyFile(fname, mode, cfg);
+        file_impl = std::make_unique<NumpyFile>(fname, mode, cfg);
     } else {
         throw std::runtime_error("Unsupported file type");
     }
 }
 
-void File::write(Frame &frame, sls_detector_header header) {
-    if (is_npy) {
-        // aare::logger::info("ignoring header for npy file");
-        dynamic_cast<NumpyFile *>(file_impl)->write(frame);
-    } else {
-        dynamic_cast<RawFile *>(file_impl)->write(frame, header);
-    }
+
+File::File(File &&other) noexcept{
+    std::swap(file_impl, other.file_impl);
+}
+
+File& File::operator=(File &&other) noexcept {
+    if (this != &other) {
+        File tmp(std::move(other));
+        std::swap(file_impl, tmp.file_impl);
+    }
+    return *this;
+}
+
+Frame File::read_frame() { return file_impl->read_frame(); }
+Frame File::read_frame(size_t frame_index) {
+    return file_impl->read_frame(frame_index);
 }
-Frame File::read() { return file_impl->read(); }
 size_t File::total_frames() const { return file_impl->total_frames(); }
-std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
+std::vector<Frame> File::read_n(size_t n_frames) {
+    return file_impl->read_n(n_frames);
+}
+
 void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
-void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
-size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
-size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
-size_t File::pixels_per_frame() { return file_impl->pixels_per_frame(); }
-void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
+void File::read_into(std::byte *image_buf, size_t n_frames) {
+    file_impl->read_into(image_buf, n_frames);
+}
+size_t File::frame_number(size_t frame_index) {
+    return file_impl->frame_number(frame_index);
+}
+
+size_t File::bytes_per_frame() const { return file_impl->bytes_per_frame(); }
+size_t File::pixels_per_frame() const{ return file_impl->pixels_per_frame(); }
+void File::seek(size_t frame_index) { file_impl->seek(frame_index); }
 size_t File::tell() const { return file_impl->tell(); }
 size_t File::rows() const { return file_impl->rows(); }
 size_t File::cols() const { return file_impl->cols(); }
 size_t File::bitdepth() const { return file_impl->bitdepth(); }
-size_t File::bytes_per_pixel() const { return file_impl->bitdepth()/8; }
-void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); }
-File::~File() { delete file_impl; }
+size_t File::bytes_per_pixel() const { return file_impl->bitdepth() / 8; }
+
 DetectorType File::detector_type() const { return file_impl->detector_type(); }
-xy File::geometry() const {
-    if (is_npy) {
-        return {1, 1};
-    }
-    return reinterpret_cast<RawFile *>(file_impl)->geometry();
-}
 
-Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
-
-File::File(File &&other) noexcept : file_impl(other.file_impl), is_npy(other.is_npy) { other.file_impl = nullptr; }
-
-// write move assignment operator
 
 } // namespace aare

src/Frame.cpp

@@ -7,28 +7,16 @@
 
 namespace aare {
 
-/**
- * @brief Construct a new Frame
- * @param bytes pointer to the data to be copied into the frame
- * @param rows number of rows
- * @param cols number of columns
- * @param bitdepth bitdepth of the pixels
- */
 Frame::Frame(const std::byte *bytes, uint32_t rows, uint32_t cols, Dtype dtype)
-    : m_rows(rows), m_cols(cols), m_dtype(dtype), m_data(new std::byte[rows * cols * m_dtype.bytes()]) {
+    : m_rows(rows), m_cols(cols), m_dtype(dtype),
+      m_data(new std::byte[rows * cols * m_dtype.bytes()]) {
 
     std::memcpy(m_data, bytes, rows * cols * m_dtype.bytes());
 }
 
-/**
- * @brief Construct a new Frame
- * @param rows number of rows
- * @param cols number of columns
- * @param bitdepth bitdepth of the pixels
- * @note the data is initialized to zero
- */
 Frame::Frame(uint32_t rows, uint32_t cols, Dtype dtype)
-    : m_rows(rows), m_cols(cols), m_dtype(dtype), m_data(new std::byte[rows * cols * dtype.bytes()]) {
+    : m_rows(rows), m_cols(cols), m_dtype(dtype),
+      m_data(new std::byte[rows * cols * dtype.bytes()]) {
 
     std::memset(m_data, 0, rows * cols * dtype.bytes());
 }
@@ -41,14 +29,8 @@ uint64_t Frame::size() const { return m_rows * m_cols; }
 size_t Frame::bytes() const { return m_rows * m_cols * m_dtype.bytes(); }
 std::byte *Frame::data() const { return m_data; }
 
-/**
- * @brief Get the pointer to the pixel at the given row and column
- * @param row row index
- * @param col column index
- * @return pointer to the pixel
- * @note the user should cast the pointer to the appropriate type
- */
-std::byte *Frame::get(uint32_t row, uint32_t col) {
+
+std::byte *Frame::pixel_ptr(uint32_t row, uint32_t col) const{
     if ((row >= m_rows) || (col >= m_cols)) {
         std::cerr << "Invalid row or column index" << '\n';
         return nullptr;
@@ -56,20 +38,7 @@ std::byte *Frame::get(uint32_t row, uint32_t col) {
     return m_data + (row * m_cols + col) * (m_dtype.bytes());
 }
 
-// Frame &Frame::operator=(const Frame &other) {
-//     if (this == &other) {
-//         return *this;
-//     }
-//     m_rows = other.rows();
-//     m_cols = other.cols();
-//     m_dtype = other.dtype();
-//     m_data = new std::byte[m_rows * m_cols * m_dtype.bytes()];
-//     if (m_data == nullptr) {
-//         throw std::bad_alloc();
-//     }
-//     std::memcpy(m_data, other.m_data, m_rows * m_cols * m_dtype.bytes());
-//     return *this;
-// }
+
 Frame &Frame::operator=(Frame &&other) noexcept {
     if (this == &other) {
         return *this;
@@ -87,24 +56,19 @@ Frame &Frame::operator=(Frame &&other) noexcept {
     return *this;
 }
 Frame::Frame(Frame &&other) noexcept
-    : m_rows(other.rows()), m_cols(other.cols()), m_dtype(other.dtype()), m_data(other.m_data) {
+    : m_rows(other.rows()), m_cols(other.cols()), m_dtype(other.dtype()),
+      m_data(other.m_data) {
 
     other.m_data = nullptr;
     other.m_rows = other.m_cols = 0;
     other.m_dtype = Dtype(Dtype::TypeIndex::ERROR);
 }
-// Frame::Frame(const Frame &other)
-//     : m_rows(other.rows()), m_cols(other.cols()), m_dtype(other.dtype()),
-//       m_data(new std::byte[m_rows * m_cols * m_dtype.bytes()]) {
 
-//     std::memcpy(m_data, other.m_data, m_rows * m_cols * m_dtype.bytes());
-// }
-
-Frame Frame::copy() const {
+Frame Frame::clone() const {
     Frame frame(m_rows, m_cols, m_dtype);
     std::memcpy(frame.m_data, m_data, m_rows * m_cols * m_dtype.bytes());
     return frame;
 }
 
-Frame::~Frame() noexcept { delete[] m_data; }
+
 } // namespace aare

src/Frame.test.cpp

@@ -19,7 +19,7 @@ TEST_CASE("Construct a frame") {
     // data should be initialized to 0
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint8_t *data = (uint8_t *)frame.get(i, j);
+            uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
             REQUIRE(data != nullptr);
             REQUIRE(*data == 0);
         }
@@ -40,7 +40,7 @@ TEST_CASE("Set a value in a 8 bit frame") {
     // only the value we did set should be non-zero
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint8_t *data = (uint8_t *)frame.get(i, j);
+            uint8_t *data = (uint8_t *)frame.pixel_ptr(i, j);
             REQUIRE(data != nullptr);
             if (i == 5 && j == 7) {
                 REQUIRE(*data == value);
@@ -65,7 +65,7 @@ TEST_CASE("Set a value in a 64 bit frame") {
     // only the value we did set should be non-zero
     for (size_t i = 0; i < rows; i++) {
         for (size_t j = 0; j < cols; j++) {
-            uint64_t *data = (uint64_t *)frame.get(i, j);
+            uint64_t *data = (uint64_t *)frame.pixel_ptr(i, j);
             REQUIRE(data != nullptr);
             if (i == 5 && j == 7) {
                 REQUIRE(*data == value);
@@ -134,7 +134,7 @@ TEST_CASE("test explicit copy constructor") {
     Frame frame(rows, cols, Dtype::from_bitdepth(bitdepth));
     std::byte *data = frame.data();
 
-    Frame frame2 = frame.copy();
+    Frame frame2 = frame.clone();
 
     // state of the original object
     REQUIRE(frame.rows() == rows);

src/NDArray.test.cpp

@@ -1,5 +1,6 @@
 #include "aare/NDArray.hpp"
 #include <array>
+#include <catch2/benchmark/catch_benchmark.hpp>
 #include <catch2/catch_test_macros.hpp>
 
 using aare::NDArray;
@@ -12,7 +13,7 @@ TEST_CASE("Initial size is zero if no size is specified") {
     REQUIRE(a.shape() == Shape<2>{0, 0});
 }
 
-TEST_CASE("Construct from a DataSpan") {
+TEST_CASE("Construct from an NDView") {
     std::vector<int> some_data(9, 42);
     NDView<int, 2> view(some_data.data(), Shape<2>{3, 3});
 
@@ -101,7 +102,8 @@ TEST_CASE("Elementwise multiplication of 3D image") {
         a(i) = i;
         b(i) = i;
     }
-    auto c = a * b;
+    // auto c = a * b; // This works but the result is a lazy ArrayMul object
+    NDArray<double, 3> c = a * b;
     REQUIRE(c(0, 0, 0) == 0 * 0);
     REQUIRE(c(0, 0, 1) == 1 * 1);
     REQUIRE(c(0, 1, 1) == 3 * 3);
@@ -109,6 +111,39 @@ TEST_CASE("Elementwise multiplication of 3D image") {
     REQUIRE(c(2, 3, 1) == 23 * 23);
 }
 
+NDArray<int> MultiplyNDArrayUsingOperator(NDArray<int> &a, NDArray<int> &b) {
+    // return a * a * b * b;
+    NDArray<int>c =  a*b;
+    return c;
+}
+
+NDArray<int> MultiplyNDArrayUsingIndex(NDArray<int> &a, NDArray<int> &b) {
+    NDArray<int> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        // res(i) = a(i) * a(i) * b(i) * b(i);
+        res(i) = a(i) * b(i);
+    }
+    return res;
+}
+
+NDArray<int> AddNDArrayUsingOperator(NDArray<int> &a, NDArray<int> &b) {
+    // return a * a * b * b;
+    // NDArray<int>c = a+b;
+    NDArray<int> c(a.shape());
+    c = a + b;
+    return c;
+}
+
+NDArray<int> AddNDArrayUsingIndex(NDArray<int> &a, NDArray<int> &b) {
+    NDArray<int> res(a.shape());
+    for (uint32_t i = 0; i < a.size(); i++) {
+        // res(i) = a(i) * a(i) * b(i) * b(i);
+        res(i) = a(i) + b(i);
+    }
+    return res;
+}
+
+
 TEST_CASE("Compare two images") {
     NDArray<int> a;
     NDArray<int> b;
@@ -168,42 +203,8 @@ TEST_CASE("Bitwise and on data") {
     REQUIRE(a(2) == 384);
 }
 
-// TEST_CASE("Benchmarks")
-// {
-//     NDArray<double> img;
-//     std::array<int64_t, 2> shape{ 512, 1024 };
-//     BENCHMARK("Allocate 500k double image")
-//     {
-//         NDArray<double>im{ shape };
-//     }
-//     BENCHMARK("Allocate 500k double image with initial value")
-//     {
-//         NDArray<double>im{ shape, 3.14 };
-//     }
 
-//     NDArray<double> a{ shape, 1.2 };
-//     NDArray<double> b{ shape, 53. };
-//     auto c = a + b;
-//     c      = a * b;
-//     BENCHMARK("Multiply two images")
-//     {
-//         c = a * b;
-//     }
-//     BENCHMARK("Divide two images")
-//     {
-//         c = a / b;
-//     }
-//     BENCHMARK("Add two images")
-//     {
-//         c = a + b;
-//     }
-//     BENCHMARK("Subtract two images")
-//     {
-//         c = a - b;
-//     }
-// }
-
-TEST_CASE("Elementwise operatios on images") {
+TEST_CASE("Elementwise operations on images") {
     std::array<int64_t, 2> shape{5, 5};
     double a_val = 3.0;
     double b_val = 8.0;
@@ -212,7 +213,8 @@ TEST_CASE("Elementwise operatios on images") {
         NDArray<double> A(shape, a_val);
         NDArray<double> B(shape, b_val);
 
-        auto C = A + B;
+        NDArray<double> C = A + B;
+        // auto C = A+B; // This works but the result is a lazy ArraySum object
 
         // Value of C matches
         for (uint32_t i = 0; i < C.size(); ++i) {
@@ -236,7 +238,8 @@ TEST_CASE("Elementwise operatios on images") {
     SECTION("Subtract two images") {
         NDArray<double> A(shape, a_val);
         NDArray<double> B(shape, b_val);
-        auto C = A - B;
+        NDArray<double> C = A - B;
+        // auto C = A - B; // This works but the result is a lazy ArraySub object
 
         // Value of C matches
         for (uint32_t i = 0; i < C.size(); ++i) {
@@ -260,7 +263,8 @@ TEST_CASE("Elementwise operatios on images") {
     SECTION("Multiply two images") {
         NDArray<double> A(shape, a_val);
         NDArray<double> B(shape, b_val);
-        auto C = A * B;
+        // auto C = A * B; // This works but the result is a lazy ArrayMul object
+        NDArray<double> C = A * B;
 
         // Value of C matches
         for (uint32_t i = 0; i < C.size(); ++i) {
@@ -284,7 +288,8 @@ TEST_CASE("Elementwise operatios on images") {
     SECTION("Divide two images") {
         NDArray<double> A(shape, a_val);
         NDArray<double> B(shape, b_val);
-        auto C = A / B;
+        // auto C = A / B; // This works but the result is a lazy ArrayDiv object
+        NDArray<double> C = A / B;
 
         // Value of C matches
         for (uint32_t i = 0; i < C.size(); ++i) {

src/NumpyFile.cpp

@@ -8,12 +8,12 @@ namespace aare {
 
 NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
     // TODO! add opts to constructor
-    m_fname = fname;
+
     m_mode = mode;
     if (mode == "r") {
-        fp = fopen(m_fname.string().c_str(), "rb");
+        fp = fopen(fname.string().c_str(), "rb");
         if (!fp) {
-            throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
+            throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
         }
         load_metadata();
     } else if (mode == "w") {
@@ -22,11 +22,11 @@ NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode
         m_cols = cfg.cols;
         m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}};
         m_header.shape = {0, cfg.rows, cfg.cols};
-        fp = fopen(m_fname.string().c_str(), "wb");
+        fp = fopen(fname.string().c_str(), "wb");
         if (!fp) {
-            throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
+            throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
         }
-        initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header);
+        initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(fname.c_str()), m_header);
     }
     m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>());
 
@@ -75,7 +75,7 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
 size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; };
 size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; };
 
-std::vector<Frame> NumpyFile::read(size_t n_frames) {
+std::vector<Frame> NumpyFile::read_n(size_t n_frames) {
     // TODO: implement this in a more efficient way
     std::vector<Frame> frames;
     for (size_t i = 0; i < n_frames; i++) {

src/NumpyFile.test.cpp

@@ -6,7 +6,7 @@
 
 using aare::Dtype;
 using aare::NumpyFile;
-TEST_CASE("Read a 1D numpy file with int32 data type") {
+TEST_CASE("Read a 1D numpy file with int32 data type", "[.integration]") {
 
     auto fpath = test_data_path() / "numpy" / "test_1d_int32.npy";
     REQUIRE(std::filesystem::exists(fpath));
@@ -23,8 +23,8 @@ TEST_CASE("Read a 1D numpy file with int32 data type") {
         REQUIRE(data(i) == i);
     }
 }
-
-TEST_CASE("Read a 3D numpy file with np.double data type") {
+ 
+TEST_CASE("Read a 3D numpy file with np.double data type", "[.integration]") {
 
     auto fpath = test_data_path() / "numpy" / "test_3d_double.npy";
     REQUIRE(std::filesystem::exists(fpath));

src/Pedestal.test.cpp

@@ -66,7 +66,7 @@ TEST_CASE("test pedestal push") {
                 }
                 REQUIRE(pedestal.mean(i, j) == (i + j));
                 REQUIRE(pedestal.variance(i, j) == 0);
-                REQUIRE(pedestal.standard_deviation(i, j) == 0);
+                REQUIRE(pedestal.std(i, j) == 0);
             }
         }
     }
@@ -91,7 +91,7 @@ TEST_CASE("test pedestal with normal distribution") {
     }
     auto mean = pedestal.mean();
     auto variance = pedestal.variance();
-    auto standard_deviation = pedestal.standard_deviation();
+    auto standard_deviation = pedestal.std();
 
     for (int i = 0; i < 3; i++) {
         for (int j = 0; j < 5; j++) {

src/PixelMap.cpp

@@ -0,0 +1,87 @@
+#include "aare/PixelMap.hpp"
+
+#include <array>
+
+namespace aare {
+NDArray<ssize_t, 2> GenerateMoench03PixelMap() {
+    std::array<int, 32> const adc_nr = {300, 325, 350, 375, 300, 325, 350, 375,
+                                        200, 225, 250, 275, 200, 225, 250, 275,
+                                        100, 125, 150, 175, 100, 125, 150, 175,
+                                        0,   25,  50,  75,  0,   25,  50,  75};
+    int const sc_width = 25;
+    int const nadc = 32;
+    int const pixels_per_sc = 5000;
+    NDArray<ssize_t, 2> order_map({400, 400});
+
+    int pixel = 0;
+    for (int i = 0; i != pixels_per_sc; ++i) {
+        for (int i_adc = 0; i_adc != nadc; ++i_adc) {
+            int const col = adc_nr[i_adc] + (i % sc_width);
+            int row = 0;
+            if ((i_adc / 4) % 2 == 0)
+                row = 199 - (i / sc_width);
+            else
+                row = 200 + (i / sc_width);
+
+            order_map(row, col) = pixel;
+            pixel++;
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 2> GenerateMoench05PixelMap() {
+    std::array<int, 3> adc_numbers = {9, 13, 1};
+    NDArray<ssize_t, 2> order_map({160, 150});
+    int n_pixel = 0;
+    for (int row = 0; row < 160; row++) {
+        for (int i_col = 0; i_col < 50; i_col++) {
+            n_pixel = row * 50 + i_col;
+            for (int i_sc = 0; i_sc < 3; i_sc++) {
+                int col = 50 * i_sc + i_col;
+                int adc_nr = adc_numbers[i_sc];
+                int i_analog = n_pixel * 32 + adc_nr;
+
+
+                // analog_frame[row * 150 + col] = analog_data[i_analog] & 0x3FFF;
+                order_map(row, col) = i_analog;
+                
+            }
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 2>GenerateEigerFlipRowsPixelMap(){
+    NDArray<ssize_t, 2> order_map({256, 512});
+    for(int row = 0; row < 256; row++){
+        for(int col = 0; col < 512; col++){
+            order_map(row, col) = 255*512-row*512 + col;
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 2>GenerateMH02SingleCounterPixelMap(){
+    NDArray<ssize_t, 2> order_map({48, 48});
+    for(int row = 0; row < 48; row++){
+        for(int col = 0; col < 48; col++){
+            order_map(row, col) = row*48 + col;
+        }
+    }
+    return order_map;
+}
+
+NDArray<ssize_t, 3> GenerateMH02FourCounterPixelMap(){
+    NDArray<ssize_t, 3> order_map({4, 48, 48});
+    for (int counter=0; counter<4; counter++){
+        for(int row = 0; row < 48; row++){
+            for(int col = 0; col < 48; col++){
+                order_map(counter, row, col) = counter*48*48 + row*48 + col;
+            }
+        }
+    }
+    return order_map;
+}
+
+} // namespace aare

src/RawFile.cpp

@@ -1,6 +1,6 @@
 #include "aare/RawFile.hpp"
+#include "aare/PixelMap.hpp"
 #include "aare/defs.hpp"
-#include "aare/json.hpp"
 
 #include <fmt/format.h>
 #include <nlohmann/json.hpp>
@@ -9,132 +9,120 @@ using json = nlohmann::json;
 
 namespace aare {
 
-RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) {
+RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
+    : m_master(fname) {
     m_mode = mode;
-    m_fname = fname;
-    if (mode == "r" || mode == "r+") {
-        if (config != FileConfig()) {
-            // aare::logger::warn(
-            //     "In read mode it is not necessary to provide a config, the provided config will be ignored");
-        }
-        parse_fname();
-        parse_metadata();
-        find_number_of_subfiles();
+    if (mode == "r") {
+
+        n_subfiles = find_number_of_subfiles(); // f0,f1...fn
+        n_subfile_parts =
+            m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
         find_geometry();
+        update_geometry_with_roi();
+
         open_subfiles();
-
-    } else if (mode == "w" || mode == "w+") {
-
-        if (std::filesystem::exists(fname)) {
-            // handle mode w as w+ (no overrwriting)
-            throw std::runtime_error(LOCATION + "File already exists");
-        }
-
-        parse_config(config);
-        parse_fname();
-        write_master_file();
-        n_subfiles = 1;
-        n_subfile_parts = 1;
-        subfile_cols = m_cols;
-        subfile_rows = m_rows;
-        open_subfiles();
-
     } else {
-        throw std::runtime_error(LOCATION + "Unsupported mode");
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported mode. Can only read RawFiles.");
     }
 }
 
-void RawFile::parse_config(const FileConfig &config) {
-    m_bitdepth = config.dtype.bitdepth();
-    m_total_frames = config.total_frames;
-    m_rows = config.rows;
-    m_cols = config.cols;
-    m_type = config.detector_type;
-    max_frames_per_file = config.max_frames_per_file;
-    m_geometry = config.geometry;
-    version = config.version;
-    subfile_rows = config.geometry.row;
-    subfile_cols = config.geometry.col;
+Frame RawFile::read_frame() { return get_frame(m_current_frame++); };
 
-    if (m_geometry != aare::xy{1, 1}) {
-        throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing");
+Frame RawFile::read_frame(size_t frame_number) {
+    seek(frame_number);
+    return read_frame();
+}
+
+void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
+    // TODO: implement this in a more efficient way
+
+    for (size_t i = 0; i < n_frames; i++) {
+        this->get_frame_into(m_current_frame++, image_buf);
+        image_buf += bytes_per_frame();
     }
 }
-void RawFile::write_master_file() {
-    if (m_ext != ".json") {
-        throw std::runtime_error(LOCATION + "only json master files are supported for writing");
+
+void RawFile::read_into(std::byte *image_buf) {
+    return get_frame_into(m_current_frame++, image_buf);
+};
+
+
+void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
+
+    return get_frame_into(m_current_frame++, image_buf, header);
+};
+
+void RawFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) {
+    // return get_frame_into(m_current_frame++, image_buf, header);
+
+    for (size_t i = 0; i < n_frames; i++) {
+        this->get_frame_into(m_current_frame++, image_buf, header);
+        image_buf += bytes_per_frame();
+        if(header) 
+            header+=n_mod();
     }
-    std::ofstream ofs(master_fname(), std::ios::binary);
-    std::string ss;
-    ss.reserve(1024);
-    ss += "{\n\t";
-    aare::write_str(ss, "Version", version);
-    ss += "\n\t";
-    aare::write_digit(ss, "Total Frames", m_total_frames);
-    ss += "\n\t";
-    aare::write_str(ss, "Detector Type", toString(m_type));
-    ss += "\n\t";
-    aare::write_str(ss, "Geometry", m_geometry.to_string());
-    ss += "\n\t";
 
-    uint64_t img_size = (m_cols * m_rows) / (static_cast<size_t>(m_geometry.col * m_geometry.row));
-    img_size *= m_bitdepth;
-    aare::write_digit(ss, "Image Size in bytes", img_size);
-    ss += "\n\t";
-    aare::write_digit(ss, "Max Frames Per File", max_frames_per_file);
-    ss += "\n\t";
-    aare::write_digit(ss, "Dynamic Range", m_bitdepth);
-    ss += "\n\t";
-    const aare::xy pixels = {static_cast<uint32_t>(m_rows / m_geometry.row),
-                             static_cast<uint32_t>(m_cols / m_geometry.col)};
-    aare::write_str(ss, "Pixels", pixels.to_string());
-    ss += "\n\t";
-    aare::write_digit(ss, "Number of rows", m_rows);
-    ss += "\n\t";
-    const std::string tmp = "{\n"
-                            "        \"Frame Number\": \"8 bytes\",\n"
-                            "        \"Exposure Length\": \"4 bytes\",\n"
-                            "        \"Packet Number\": \"4 bytes\",\n"
-                            "        \"Bunch Id\": \"8 bytes\",\n"
-                            "        \"Timestamp\": \"8 bytes\",\n"
-                            "        \"Module Id\": \"2 bytes\",\n"
-                            "        \"Row\": \"2 bytes\",\n"
-                            "        \"Column\": \"2 bytes\",\n"
-                            "        \"Reserved\": \"2 bytes\",\n"
-                            "        \"Debug\": \"4 bytes\",\n"
-                            "        \"RoundRNumber\": \"2 bytes\",\n"
-                            "        \"DetType\": \"1 byte\",\n"
-                            "        \"Version\": \"1 byte\",\n"
-                            "        \"Packet Mask\": \"64 bytes\"\n"
-                            "    }";
+};
 
-    ss += "\"Frame Header Format\":" + tmp + "\n";
-    ss += "}";
-    ofs << ss;
-    ofs.close();
+size_t RawFile::n_mod() const { return n_subfile_parts; }
+
+
+size_t RawFile::bytes_per_frame() {
+    return m_rows * m_cols * m_master.bitdepth() / 8;
 }
+size_t RawFile::pixels_per_frame() { return m_rows * m_cols; }
+
+DetectorType RawFile::detector_type() const { return m_master.detector_type(); }
+
+void RawFile::seek(size_t frame_index) {
+    // check if the frame number is greater than the total frames
+    // if frame_number == total_frames, then the next read will throw an error
+    if (frame_index > total_frames()) {
+        throw std::runtime_error(
+            fmt::format("frame number {} is greater than total frames {}",
+                        frame_index, total_frames()));
+    }
+    m_current_frame = frame_index;
+};
+
+size_t RawFile::tell() { return m_current_frame; };
+
+size_t RawFile::total_frames() const { return m_master.frames_in_file(); }
+size_t RawFile::rows() const { return m_rows; }
+size_t RawFile::cols() const { return m_cols; }
+size_t RawFile::bitdepth() const { return m_master.bitdepth(); }
+xy RawFile::geometry() { return m_master.geometry(); }
 
 void RawFile::open_subfiles() {
     if (m_mode == "r")
         for (size_t i = 0; i != n_subfiles; ++i) {
-            auto v = std::vector<SubFile *>(n_subfile_parts);
+            auto v = std::vector<RawSubFile *>(n_subfile_parts);
             for (size_t j = 0; j != n_subfile_parts; ++j) {
-                v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
+                fmt::print("{} pos: {},{}\n", j,positions[j].row, positions[j].col);
+                
+                auto pos = m_module_pixel_0[j];
+                fmt::print("{} pos: {},{}\n", j,pos.y, pos.x);
+                v[j] = new RawSubFile(m_master.data_fname(j, i),
+                                      m_master.detector_type(), pos.height,
+                                      pos.width, m_master.bitdepth(),
+                                      positions[j].row, positions[j].col);
+
             }
             subfiles.push_back(v);
         }
     else {
-        auto v = std::vector<SubFile *>(n_subfile_parts); // only one subfile is implemented
-        v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w");
-        subfiles.push_back(v);
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported mode. Can only read RawFiles.");
     }
 }
 
-sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
-    sls_detector_header h{};
+DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
+    DetectorHeader h{};
     FILE *fp = fopen(fname.string().c_str(), "r");
     if (!fp)
-        throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
+        throw std::runtime_error(
+            fmt::format("Could not open: {} for reading", fname.string()));
 
     size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
     if (rc != 1)
@@ -145,279 +133,266 @@ sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
 
     return h;
 }
-bool RawFile::is_master_file(const std::filesystem::path &fpath) {
-    std::string const stem = fpath.stem().string();
-    return stem.find("_master_") != std::string::npos;
+
+int RawFile::find_number_of_subfiles() {
+    int n_files = 0;
+    // f0,f1...fn How many files is the data split into?
+    while (std::filesystem::exists(m_master.data_fname(0, n_files)))
+        n_files++; // increment after test
+
+#ifdef AARE_VERBOSE
+    fmt::print("Found: {} subfiles\n", n_files);
+#endif
+    return n_files;
+
 }
 
-void RawFile::find_number_of_subfiles() {
-    int n_mod = 0;
-    while (std::filesystem::exists(data_fname(++n_mod, 0)))
-        ;
-    n_subfiles = n_mod;
-}
-inline std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) {
-    return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
-}
-
-inline std::filesystem::path RawFile::master_fname() {
-    return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
-}
+RawMasterFile RawFile::master() const { return m_master; }
 
 void RawFile::find_geometry() {
     uint16_t r{};
     uint16_t c{};
-    for (size_t i = 0; i < n_subfile_parts; i++) {
-        for (size_t j = 0; j != n_subfiles; ++j) {
-            auto h = this->read_header(data_fname(j, i));
-            r = std::max(r, h.row);
-            c = std::max(c, h.column);
 
-            positions.push_back({h.row, h.column});
-        }
+
+    for (size_t i = 0; i < n_subfile_parts; i++) {
+        auto h = this->read_header(m_master.data_fname(i, 0));
+        r = std::max(r, h.row);
+        c = std::max(c, h.column);
+        positions.push_back({h.row, h.column});
+        ModuleGeometry g;
+        g.x = h.column * m_master.pixels_x();
+        g.y = h.row * m_master.pixels_y();
+        g.width = m_master.pixels_x();
+        g.height = m_master.pixels_y();
+        m_module_pixel_0.push_back(g);
+
     }
 
     r++;
     c++;
 
-    m_rows = (r * subfile_rows);
-    m_cols = (c * subfile_cols);
+    m_rows = (r * m_master.pixels_y());
+    m_cols = (c * m_master.pixels_x());
 
     m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
+
+#ifdef AARE_VERBOSE
+    fmt::print("\nRawFile::find_geometry()\n");
+    for (size_t i = 0; i < m_module_pixel_0.size(); i++) {
+        fmt::print("Module {} at position: (r:{},c:{})\n", i,
+                   m_module_pixel_0[i].y, m_module_pixel_0[i].x);
+    }
+    fmt::print("Image size: {}x{}\n\n", m_rows, m_cols);
+#endif
 }
 
-void RawFile::parse_metadata() {
-    if (m_ext == ".raw") {
-        parse_raw_metadata();
-        if (m_bitdepth == 0) {
-            switch (m_type) {
-            case DetectorType::Eiger:
-                m_bitdepth = 32;
-                break;
-            default:
-                m_bitdepth = 16;
+void RawFile::update_geometry_with_roi() {
+    // TODO! implement this
+    if (m_master.roi()) {
+        auto roi = m_master.roi().value();
+
+        // TODO! can we do this cleaner?
+        int pos_y = 0;
+        int pos_y_increment = 0;
+        for (size_t row = 0; row < m_master.geometry().row; row++) {
+            int pos_x = 0;
+            for (size_t col = 0; col < m_master.geometry().col; col++) {
+                auto &m = m_module_pixel_0[row * m_master.geometry().col + col];
+                auto original_height = m.height;
+                auto original_width = m.width;
+
+                // module is to the left of the roi
+                if (m.x + m.width < roi.xmin) {
+                    m.width = 0;
+
+                    // roi is in module
+                } else {
+                    // here we only arrive when the roi is in or to the left of
+                    // the module
+                    if (roi.xmin > m.x) {
+                        m.width -= roi.xmin - m.x;
+                    }
+                    if (roi.xmax < m.x + m.width) {
+                        m.width -= m.x + original_width - roi.xmax;
+                    }
+                    m.x = pos_x;
+                    pos_x += m.width;
+                }
+
+                if (m.y + m.height < roi.ymin) {
+                    m.height = 0;
+                } else {
+                    if ((roi.ymin > m.y) && (roi.ymin < m.y + m.height)) {
+                        m.height -= roi.ymin - m.y;
+
+                    }
+                    if (roi.ymax < m.y + m.height) {
+                        m.height -= m.y + original_height - roi.ymax;
+                    }
+                    m.y = pos_y;
+                    pos_y_increment = m.height;
+                }
             }
+            // increment pos_y
+            pos_y += pos_y_increment;
         }
-    } else if (m_ext == ".json") {
-        parse_json_metadata();
-    } else {
-        throw std::runtime_error(LOCATION + "Unsupported file type");
-    }
-    n_subfile_parts = static_cast<size_t>(m_geometry.row) * m_geometry.col;
-}
 
-void RawFile::parse_json_metadata() {
-    std::ifstream ifs(master_fname());
-    json j;
-    ifs >> j;
-    double v = j["Version"];
-    version = fmt::format("{:.1f}", v);
-    m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
-    timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
-    m_total_frames = j["Frames in File"];
-    subfile_rows = j["Pixels"]["y"];
-    subfile_cols = j["Pixels"]["x"];
-    max_frames_per_file = j["Max Frames Per File"];
-    try {
-        m_bitdepth = j.at("Dynamic Range");
-    } catch (const json::out_of_range &e) {
-        m_bitdepth = 16;
-    }
-    // only Eiger had quad
-    if (m_type == DetectorType::Eiger) {
-        quad = (j["Quad"] == 1);
+        m_rows = roi.height();
+        m_cols = roi.width();
     }
 
-    m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
-}
-void RawFile::parse_raw_metadata() {
-    std::ifstream ifs(master_fname());
-    for (std::string line; std::getline(ifs, line);) {
-        if (line == "#Frame Header")
-            break;
-        auto pos = line.find(':');
-        auto key_pos = pos;
-        while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
-            ;
-        if (key_pos != std::string::npos) {
-            auto key = line.substr(0, key_pos + 1);
-            auto value = line.substr(pos + 2);
-            // do the actual parsing
-            if (key == "Version") {
-                version = value;
-            } else if (key == "TimeStamp") {
+#ifdef AARE_VERBOSE
+    fmt::print("RawFile::update_geometry_with_roi()\n");
+    for (const auto &m : m_module_pixel_0) {
+        fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x,
+                   m.height, m.width);
+    }
+    fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols);
+    fmt::print("\n");
+#endif
 
-            } else if (key == "Detector Type") {
-                m_type = StringTo<DetectorType>(value);
-            } else if (key == "Timing Mode") {
-                timing_mode = StringTo<TimingMode>(value);
-            } else if (key == "Pixels") {
-                // Total number of pixels cannot be found yet looking at
-                // submodule
-                pos = value.find(',');
-                subfile_cols = std::stoi(value.substr(1, pos));
-                subfile_rows = std::stoi(value.substr(pos + 1));
-            } else if (key == "Total Frames") {
-                m_total_frames = std::stoi(value);
-            } else if (key == "Dynamic Range") {
-                m_bitdepth = std::stoi(value);
-            } else if (key == "Quad") {
-                quad = (value == "1");
-            } else if (key == "Max Frames Per File") {
-                max_frames_per_file = std::stoi(value);
-            } else if (key == "Geometry") {
-                pos = value.find(',');
-                m_geometry = {static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
-                              static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
-            }
-        }
-    }
-}
-
-void RawFile::parse_fname() {
-    bool wrong_format = false;
-    m_base_path = m_fname.parent_path().string();
-    m_base_name = m_fname.stem().string();
-    m_ext = m_fname.extension().string();
-    try {
-        auto pos = m_base_name.rfind('_');
-        m_findex = std::stoi(m_base_name.substr(pos + 1));
-    } catch (const std::invalid_argument &e) {
-        m_findex = 0;
-        wrong_format = true;
-    }
-    auto pos = m_base_name.find("_master_");
-    if (pos != std::string::npos) {
-        m_base_name.erase(pos);
-        wrong_format = true;
-    }
-    if (wrong_format && (m_mode == "w+" || m_mode == "w")) {
-        // aare::logger::warn("Master Filename", m_fname, "is not in the correct format");
-        // aare::logger::warn("using", master_fname(), "as the master file");
-    }
 }
 
 Frame RawFile::get_frame(size_t frame_index) {
-    auto f = Frame(this->m_rows, this->m_cols, Dtype::from_bitdepth(this->m_bitdepth));
+    auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
     std::byte *frame_buffer = f.data();
     get_frame_into(frame_index, frame_buffer);
     return f;
 }
 
-void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
-    if (frame_index > this->m_total_frames) {
+
+size_t RawFile::bytes_per_pixel() const {
+    return m_master.bitdepth() / 8;
+}
+
+void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
+
+    if (frame_index > total_frames()) {
         throw std::runtime_error(LOCATION + "Frame number out of range");
     }
-    std::vector<size_t> frame_numbers(this->n_subfile_parts);
-    std::vector<size_t> frame_indices(this->n_subfile_parts, frame_index);
+    std::vector<size_t> frame_numbers(n_subfile_parts);
+    std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
+
+
+    // sync the frame numbers
 
     if (n_subfile_parts != 1) {
-        for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
-            auto subfile_id = frame_index / this->max_frames_per_file;
+        for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
+            auto subfile_id = frame_index / m_master.max_frames_per_file();
             frame_numbers[part_idx] =
-                this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file);
+                subfiles[subfile_id][part_idx]->frame_number(
+                    frame_index % m_master.max_frames_per_file());
         }
         // 1. if frame number vector is the same break
-        while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) !=
+        while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(),
+                                  std::not_equal_to<>()) !=
                frame_numbers.end()) {
             // 2. find the index of the minimum frame number,
-            auto min_frame_idx =
-                std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end()));
+            auto min_frame_idx = std::distance(
+                frame_numbers.begin(),
+                std::min_element(frame_numbers.begin(), frame_numbers.end()));
             // 3. increase its index and update its respective frame number
             frame_indices[min_frame_idx]++;
             // 4. if we can't increase its index => throw error
-            if (frame_indices[min_frame_idx] >= this->m_total_frames) {
-                throw std::runtime_error(LOCATION + "Frame number out of range");
+            if (frame_indices[min_frame_idx] >= total_frames()) {
+                throw std::runtime_error(LOCATION +
+                                         "Frame number out of range");
             }
-            auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file;
-            frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number(
-                frame_indices[min_frame_idx] % this->max_frames_per_file);
+            auto subfile_id =
+                frame_indices[min_frame_idx] / m_master.max_frames_per_file();
+            frame_numbers[min_frame_idx] =
+                subfiles[subfile_id][min_frame_idx]->frame_number(
+                    frame_indices[min_frame_idx] %
+                    m_master.max_frames_per_file());
         }
     }
 
-    if (this->m_geometry.col == 1) {
+    if (m_master.geometry().col == 1) {
         // get the part from each subfile and copy it to the frame
-        for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
+        for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto corrected_idx = frame_indices[part_idx];
-            auto subfile_id = corrected_idx / this->max_frames_per_file;
-            auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
-            this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
-                                                           corrected_idx % this->max_frames_per_file);
+            auto subfile_id = corrected_idx / m_master.max_frames_per_file();
+
+            // This is where we start writing
+            auto offset = (m_module_pixel_0[part_idx].y * m_cols +
+                          m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8;
+
+            if (m_module_pixel_0[part_idx].x!=0)
+                throw std::runtime_error(LOCATION + "Implementation error. x pos not 0.");
+
+            subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
+            subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header);
+            if (header)
+                ++header;
         }
 
     } else {
+        //TODO! should we read row by row?
 
-        // create a buffer that will hold a the frame part
-        auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
+        // create a buffer large enough to hold a full module
+
+        auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
+                              m_master.bitdepth() /
+                              8; // TODO! replace with image_size_in_bytes
         auto *part_buffer = new std::byte[bytes_per_part];
 
-        for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
-            auto corrected_idx = frame_indices[part_idx];
-            auto subfile_id = corrected_idx / this->max_frames_per_file;
+        // TODO! if we have many submodules we should reorder them on the module
+        // level
 
-            this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file);
-            for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
-                auto irow = cur_row + (part_idx / this->m_geometry.col) * this->subfile_rows;
-                auto icol = (part_idx % this->m_geometry.col) * this->subfile_cols;
+        for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
+            auto pos = m_module_pixel_0[part_idx];
+            auto corrected_idx = frame_indices[part_idx];
+            auto subfile_id = corrected_idx / m_master.max_frames_per_file();
+
+            subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
+            subfiles[subfile_id][part_idx]->read_into(part_buffer, header);
+            if(header)
+                ++header;
+
+            for (size_t cur_row = 0; cur_row < (pos.height);
+                 cur_row++) {
+
+                auto irow = (pos.y + cur_row);
+                auto icol = pos.x;
                 auto dest = (irow * this->m_cols + icol);
-                dest = dest * this->m_bitdepth / 8;
-                memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
-                       this->subfile_cols * this->m_bitdepth / 8);
+                dest = dest * m_master.bitdepth() / 8;
+                memcpy(frame_buffer + dest,
+                       part_buffer + cur_row * pos.width *
+                                         m_master.bitdepth() / 8,
+                       pos.width * m_master.bitdepth() / 8);
+
             }
         }
         delete[] part_buffer;
     }
 }
 
-void RawFile::write(Frame &frame, sls_detector_header header) {
-    if (m_mode == "r") {
-        throw std::runtime_error(LOCATION + "File is open in read mode");
-    }
-    size_t const subfile_id = this->current_frame / this->max_frames_per_file;
-    for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
-
-        this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header,
-                                                         this->current_frame % this->max_frames_per_file);
-    }
-    this->current_frame++;
-}
-
-std::vector<Frame> RawFile::read(size_t n_frames) {
+std::vector<Frame> RawFile::read_n(size_t n_frames) {
     // TODO: implement this in a more efficient way
     std::vector<Frame> frames;
     for (size_t i = 0; i < n_frames; i++) {
-        frames.push_back(this->get_frame(this->current_frame));
-        this->current_frame++;
+        frames.push_back(this->get_frame(m_current_frame));
+        m_current_frame++;
     }
     return frames;
 }
-void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
-    // TODO: implement this in a more efficient way
-    for (size_t i = 0; i < n_frames; i++) {
-        this->get_frame_into(this->current_frame++, image_buf);
-        image_buf += this->bytes_per_frame();
-    }
-}
 
 size_t RawFile::frame_number(size_t frame_index) {
-    if (frame_index > this->m_total_frames) {
-        throw std::runtime_error(LOCATION + "Frame number out of range");
+    if (frame_index >= m_master.frames_in_file()) {
+        throw std::runtime_error(LOCATION + " Frame number out of range");
     }
-    size_t const subfile_id = frame_index / this->max_frames_per_file;
-    return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
+    size_t subfile_id = frame_index / m_master.max_frames_per_file();
+    if (subfile_id >= subfiles.size()) {
+        throw std::runtime_error(
+            LOCATION + " Subfile out of range. Possible missing data.");
+    }
+    return subfiles[subfile_id][0]->frame_number(
+        frame_index % m_master.max_frames_per_file());
 }
 
-RawFile::~RawFile() noexcept {
-
-    // update master file
-    if (m_mode == "w" || m_mode == "w+" || m_mode == "r+") {
-        try {
-            write_master_file();
-        } catch (...) {
-            // aare::logger::warn(LOCATION + "Could not update master file");
-        }
-    }
+RawFile::~RawFile() {
 
+    // TODO! Fix this, for file closing
     for (auto &vec : subfiles) {
         for (auto *subfile : vec) {
             delete subfile;

src/RawFile.test.cpp

@@ -7,7 +7,7 @@
 
 using aare::File;
 
-TEST_CASE("Read number of frames from a jungfrau raw file") {
+TEST_CASE("Read number of frames from a jungfrau raw file", "[.integration]") {
 
     auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
@@ -16,7 +16,7 @@ TEST_CASE("Read number of frames from a jungfrau raw file") {
     REQUIRE(f.total_frames() == 10);
 }
 
-TEST_CASE("Read frame numbers from a jungfrau raw file") {
+TEST_CASE("Read frame numbers from a jungfrau raw file", "[.integration]") {
     auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -32,7 +32,42 @@ TEST_CASE("Read frame numbers from a jungfrau raw file") {
     }
 }
 
-TEST_CASE("Read data from a jungfrau 500k single port raw file") {
+TEST_CASE("Read a frame number too high throws", "[.integration]") {
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    File f(fpath, "r");
+
+    // we know this file has 10 frames with frame numbers 1 to 10
+    // f0 1,2,3
+    // f1 4,5,6
+    // f2 7,8,9
+    // f3 10
+    REQUIRE_THROWS(f.frame_number(10));
+}
+
+TEST_CASE("Read a frame numbers where the subfile is missing throws", "[.integration]") {
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_missing_subfile_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    File f(fpath, "r");
+
+    // we know this file has 10 frames with frame numbers 1 to 10
+    // f0 1,2,3
+    // f1 4,5,6 - but files f1-f3 are missing
+    // f2 7,8,9 - gone  
+    // f3 10    - gone
+    REQUIRE(f.frame_number(0) == 1);
+    REQUIRE(f.frame_number(1) == 2);
+    REQUIRE(f.frame_number(2) == 3);
+    REQUIRE_THROWS(f.frame_number(4));
+    REQUIRE_THROWS(f.frame_number(7));
+    REQUIRE_THROWS(f.frame_number(937));
+    REQUIRE_THROWS(f.frame_number(10));
+}
+
+
+TEST_CASE("Read data from a jungfrau 500k single port raw file", "[.integration]") {
     auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -41,14 +76,14 @@ TEST_CASE("Read data from a jungfrau 500k single port raw file") {
     // we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102
     std::vector<uint16_t> pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102};
     for (size_t i = 0; i < 10; i++) {
-        auto frame = f.read();
+        auto frame = f.read_frame();
         CHECK(frame.rows() == 512);
         CHECK(frame.cols() == 1024);
         CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
     }
 }
 
-TEST_CASE("Read frame numbers from a raw file") {
+TEST_CASE("Read frame numbers from a raw file", "[.integration]") {
     auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -61,7 +96,7 @@ TEST_CASE("Read frame numbers from a raw file") {
     }
 }
 
-TEST_CASE("Compare reading from a numpy file with a raw file") {
+TEST_CASE("Compare reading from a numpy file with a raw file", "[.integration]") {
     auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath_raw));
 
@@ -75,13 +110,13 @@ TEST_CASE("Compare reading from a numpy file with a raw file") {
     CHECK(npy.total_frames() == 10);
 
     for (size_t i = 0; i < 10; ++i) {
-        auto raw_frame = raw.read();
-        auto npy_frame = npy.read();
+        auto raw_frame = raw.read_frame();
+        auto npy_frame = npy.read_frame();
         CHECK((raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>()));
     }
 }
 
-TEST_CASE("Read multipart files") {
+TEST_CASE("Read multipart files", "[.integration]") {
     auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json";
     REQUIRE(std::filesystem::exists(fpath));
 
@@ -95,7 +130,7 @@ TEST_CASE("Read multipart files") {
     std::vector<uint16_t> pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598};
 
     for (size_t i = 0; i < 10; i++) {
-        auto frame = f.read();
+        auto frame = f.read_frame();
         CHECK(frame.rows() == 512);
         CHECK(frame.cols() == 1024);
         CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
@@ -106,9 +141,10 @@ TEST_CASE("Read multipart files") {
     }
 }
 
-TEST_CASE("Read file with unordered frames") {
+TEST_CASE("Read file with unordered frames", "[.integration]") {
+    //TODO! Better explanation and error message
     auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
     REQUIRE(std::filesystem::exists(fpath));
-    File f(fpath, "r");
-    REQUIRE_THROWS((f.read()));
+    File f(fpath);
+    REQUIRE_THROWS((f.read_frame()));
 }

src/RawMasterFile.cpp

@@ -0,0 +1,372 @@
+#include "aare/RawMasterFile.hpp"
+#include <sstream>  
+namespace aare {
+
+RawFileNameComponents::RawFileNameComponents(
+    const std::filesystem::path &fname) {
+    m_base_path = fname.parent_path();
+    m_base_name = fname.stem();
+    m_ext = fname.extension();
+
+    if (m_ext != ".json" && m_ext != ".raw") {
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported file type. (only .json or .raw)");
+    }
+
+    // parse file index
+    try {
+        auto pos = m_base_name.rfind('_');
+        m_file_index = std::stoi(m_base_name.substr(pos + 1));
+    } catch (const std::invalid_argument &e) {
+        throw std::runtime_error(LOCATION + "Could not parse file index");
+    }
+
+    // remove master from base name
+    auto pos = m_base_name.find("_master_");
+    if (pos != std::string::npos) {
+        m_base_name.erase(pos);
+    } else {
+        throw std::runtime_error(LOCATION +
+                                 "Could not find _master_ in file name");
+    }
+}
+
+std::filesystem::path RawFileNameComponents::master_fname() const {
+    return m_base_path /
+           fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
+}
+
+std::filesystem::path RawFileNameComponents::data_fname(size_t mod_id,
+                                                        size_t file_id) const {
+    return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name, mod_id,
+                                     file_id, m_file_index);
+}
+
+const std::filesystem::path &RawFileNameComponents::base_path() const {
+    return m_base_path;
+}
+const std::string &RawFileNameComponents::base_name() const {
+    return m_base_name;
+}
+const std::string &RawFileNameComponents::ext() const { return m_ext; }
+int RawFileNameComponents::file_index() const { return m_file_index; }
+
+// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
+ScanParameters::ScanParameters(const std::string& par){
+    std::istringstream iss(par.substr(1, par.size()-2));
+    std::string line;
+    while(std::getline(iss, line)){
+        if(line == "enabled"){
+            m_enabled = true;
+        }else if(line.find("dac") != std::string::npos){
+            m_dac = line.substr(4);
+        }else if(line.find("start") != std::string::npos){
+            m_start = std::stoi(line.substr(6));
+        }else if(line.find("stop") != std::string::npos){
+            m_stop = std::stoi(line.substr(5));
+        }else if(line.find("step") != std::string::npos){
+            m_step = std::stoi(line.substr(5));
+        }
+    }
+}
+
+int ScanParameters::start() const { return m_start; }
+int ScanParameters::stop() const { return m_stop; }
+void ScanParameters::increment_stop(){
+    m_stop += 1;
+};
+int ScanParameters::step() const { return m_step; }
+const std::string &ScanParameters::dac() const { return m_dac; }
+bool ScanParameters::enabled() const { return m_enabled; }
+
+
+RawMasterFile::RawMasterFile(const std::filesystem::path &fpath)
+    : m_fnc(fpath) {
+    if (!std::filesystem::exists(fpath)) {
+        throw std::runtime_error(LOCATION + " File does not exist");
+    }
+    if (m_fnc.ext() == ".json") {
+        parse_json(fpath);
+    } else if (m_fnc.ext() == ".raw") {
+        parse_raw(fpath);
+    } else {
+        throw std::runtime_error(LOCATION + "Unsupported file type");
+    }
+}
+
+std::filesystem::path RawMasterFile::data_fname(size_t mod_id,
+                                                size_t file_id) const {
+    return m_fnc.data_fname(mod_id, file_id);
+}
+
+const std::string &RawMasterFile::version() const { return m_version; }
+const DetectorType &RawMasterFile::detector_type() const { return m_type; }
+const TimingMode &RawMasterFile::timing_mode() const { return m_timing_mode; }
+size_t RawMasterFile::image_size_in_bytes() const {
+    return m_image_size_in_bytes;
+}
+size_t RawMasterFile::frames_in_file() const { return m_frames_in_file; }
+size_t RawMasterFile::pixels_y() const { return m_pixels_y; }
+size_t RawMasterFile::pixels_x() const { return m_pixels_x; }
+size_t RawMasterFile::max_frames_per_file() const {
+    return m_max_frames_per_file;
+}
+size_t RawMasterFile::bitdepth() const { return m_bitdepth; }
+size_t RawMasterFile::frame_padding() const { return m_frame_padding; }
+const FrameDiscardPolicy &RawMasterFile::frame_discard_policy() const {
+    return m_frame_discard_policy;
+}
+
+size_t RawMasterFile::total_frames_expected() const {
+    return m_total_frames_expected;
+}
+
+std::optional<size_t> RawMasterFile::number_of_rows() const {
+    return m_number_of_rows;
+}
+
+xy RawMasterFile::geometry() const { return m_geometry; }
+
+std::optional<uint8_t> RawMasterFile::quad() const { return m_quad; }
+
+// optional values, these may or may not be present in the master file
+// and are therefore modeled as std::optional
+std::optional<size_t> RawMasterFile::analog_samples() const {
+    return m_analog_samples;
+}
+std::optional<size_t> RawMasterFile::digital_samples() const {
+    return m_digital_samples;
+}
+
+std::optional<size_t> RawMasterFile::transceiver_samples() const {
+    return m_transceiver_samples;
+}
+
+ScanParameters RawMasterFile::scan_parameters() const {
+    return m_scan_parameters;
+}
+
+
+std::optional<ROI> RawMasterFile::roi() const { return m_roi; }
+
+
+void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
+    std::ifstream ifs(fpath);
+    json j;
+    ifs >> j;
+    double v = j["Version"];
+    m_version = fmt::format("{:.1f}", v);
+
+    m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
+    m_timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
+
+    m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
+
+    m_image_size_in_bytes = j["Image Size in bytes"];
+    m_frames_in_file = j["Frames in File"];
+    m_pixels_y = j["Pixels"]["y"];
+    m_pixels_x = j["Pixels"]["x"];
+
+    m_max_frames_per_file = j["Max Frames Per File"];
+
+    // Not all detectors write the bitdepth but in case
+    // its not there it is 16
+    try {
+        m_bitdepth = j.at("Dynamic Range");
+    } catch (const json::out_of_range &e) {
+        m_bitdepth = 16;
+    }
+    m_total_frames_expected = j["Total Frames"];
+
+    m_frame_padding = j["Frame Padding"];
+    m_frame_discard_policy = StringTo<FrameDiscardPolicy>(
+        j["Frame Discard Policy"].get<std::string>());
+
+    try {
+        m_number_of_rows = j.at("Number of rows");
+    } catch (const json::out_of_range &e) {
+        // keep the optional empty
+    }
+
+
+    // ----------------------------------------------------------------
+    // Special treatment of analog flag because of Moench03
+    try{
+        m_analog_flag = j.at("Analog Flag");
+    }catch (const json::out_of_range &e) {
+        // if it doesn't work still set it to one
+        // to try to decode analog samples (Old Moench03)
+        m_analog_flag = 1;
+    }
+    try {        
+        if (m_analog_flag) {
+            m_analog_samples = j.at("Analog Samples");
+        }
+
+    } catch (const json::out_of_range &e) {
+        // keep the optional empty
+        // and set analog flag to 0
+        m_analog_flag = 0;
+    }
+    //-----------------------------------------------------------------
+
+    try {
+        m_quad = j.at("Quad");
+    } catch (const json::out_of_range &e) {
+        // keep the optional empty
+    }
+    // try{
+    //     std::string adc_mask = j.at("ADC Mask");
+    //     m_adc_mask = std::stoul(adc_mask, nullptr, 16);
+    // }catch (const json::out_of_range &e) {
+    //     m_adc_mask = 0;
+    // }
+
+    try {
+        int digital_flag = j.at("Digital Flag");
+        if (digital_flag) {
+            m_digital_samples = j.at("Digital Samples");
+        }
+    } catch (const json::out_of_range &e) {
+        // keep the optional empty
+    }
+
+    try{
+        m_transceiver_flag = j.at("Transceiver Flag");
+        if(m_transceiver_flag){
+            m_transceiver_samples = j.at("Transceiver Samples");
+        }
+    }catch (const json::out_of_range &e) {
+        // keep the optional empty
+    }
+
+    try{
+        std::string scan_parameters = j.at("Scan Parameters");
+        m_scan_parameters = ScanParameters(scan_parameters);
+        if(v<7.21){
+                m_scan_parameters.increment_stop(); //adjust for endpoint being included
+            }
+    }catch (const json::out_of_range &e) {
+        // not a scan
+    }
+
+
+    try{
+        ROI tmp_roi;
+        auto obj = j.at("Receiver Roi");
+        tmp_roi.xmin = obj.at("xmin");
+        tmp_roi.xmax = obj.at("xmax");
+        tmp_roi.ymin = obj.at("ymin");
+        tmp_roi.ymax = obj.at("ymax");
+
+        //if any of the values are set update the roi
+        if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 ||
+            tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) {
+            
+            if(v<7.21){
+                tmp_roi.xmax++;
+                tmp_roi.ymax++;
+            }
+            
+            m_roi = tmp_roi;
+        }
+
+
+    }catch (const json::out_of_range &e) {
+        // leave the optional empty
+    }
+
+    //if we have an roi we need to update the geometry for the subfiles
+    if (m_roi){
+
+    }
+
+
+
+
+    // Update detector type for Moench
+    // TODO! How does this work with old .raw master files?
+    #ifdef AARE_VERBOSE
+    fmt::print("Detecting Moench03: m_pixels_y: {}, m_analog_samples: {}\n",
+               m_pixels_y, m_analog_samples.value_or(0));
+    #endif
+    if (m_type == DetectorType::Moench && !m_analog_samples &&
+        m_pixels_y == 400) {
+        m_type = DetectorType::Moench03;
+    } else if (m_type == DetectorType::Moench && m_pixels_y == 400 &&
+               m_analog_samples == 5000) {
+        m_type = DetectorType::Moench03_old;
+    }
+}
+void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
+
+    std::ifstream ifs(fpath);
+    for (std::string line; std::getline(ifs, line);) {
+        if (line == "#Frame Header")
+            break;
+        auto pos = line.find(':');
+        auto key_pos = pos;
+        while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
+            ;
+        if (key_pos != std::string::npos) {
+            auto key = line.substr(0, key_pos + 1);
+            auto value = line.substr(pos + 2);
+            // do the actual parsing
+            if (key == "Version") {
+                m_version = value;
+            } else if (key == "TimeStamp") {
+
+            } else if (key == "Detector Type") {
+                m_type = StringTo<DetectorType>(value);
+            } else if (key == "Timing Mode") {
+                m_timing_mode = StringTo<TimingMode>(value);
+            } else if (key == "Image Size") {
+                m_image_size_in_bytes = std::stoi(value);
+            } else if (key == "Frame Padding") {
+                m_frame_padding = std::stoi(value);
+                // } else if (key == "Frame Discard Policy"){
+                //     m_frame_discard_policy =
+                //     StringTo<FrameDiscardPolicy>(value);
+                // } else if (key == "Number of rows"){
+                //     m_number_of_rows = std::stoi(value);
+            } else if (key == "Analog Flag") {
+                m_analog_flag = std::stoi(value);
+            } else if (key == "Digital Flag") {
+                m_digital_flag = std::stoi(value);
+
+            } else if (key == "Analog Samples") {
+                if (m_analog_flag == 1) {
+                    m_analog_samples = std::stoi(value);
+                }
+            } else if (key == "Digital Samples") {
+                if (m_digital_flag == 1) {
+                    m_digital_samples = std::stoi(value);
+                }
+            } else if (key == "Frames in File") {
+                m_frames_in_file = std::stoi(value);
+                // } else if (key == "ADC Mask") {
+                //     m_adc_mask = std::stoi(value, nullptr, 16);
+            } else if (key == "Pixels") {
+                // Total number of pixels cannot be found yet looking at
+                // submodule
+                pos = value.find(',');
+                m_pixels_x = std::stoi(value.substr(1, pos));
+                m_pixels_y = std::stoi(value.substr(pos + 1));
+            } else if (key == "Total Frames") {
+                m_total_frames_expected = std::stoi(value);
+            } else if (key == "Dynamic Range") {
+                m_bitdepth = std::stoi(value);
+            } else if (key == "Quad") {
+                m_quad = std::stoi(value);
+            } else if (key == "Max Frames Per File") {
+                m_max_frames_per_file = std::stoi(value);
+            } else if (key == "Geometry") {
+                pos = value.find(',');
+                m_geometry = {
+                    static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
+                    static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
+            }
+        }
+    }
+}
+} // namespace aare

src/RawMasterFile.test.cpp

@@ -0,0 +1,278 @@
+#include "aare/RawMasterFile.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+#include "test_config.hpp"
+
+using namespace aare;
+
+
+TEST_CASE("Parse a master file fname"){
+    RawFileNameComponents m("test_master_1.json");
+    REQUIRE(m.base_name() == "test");
+    REQUIRE(m.ext() == ".json");
+    REQUIRE(m.file_index() == 1);
+    REQUIRE(m.base_path() == "");
+}
+
+TEST_CASE("Extraction of base path works"){
+    RawFileNameComponents m("some/path/test_master_73.json");
+    REQUIRE(m.base_name() == "test");
+    REQUIRE(m.ext() == ".json");
+    REQUIRE(m.file_index() == 73);
+    REQUIRE(m.base_path() == "some/path");
+}
+
+TEST_CASE("Construction of master file name and data files"){
+    RawFileNameComponents m("test_master_1.json");
+    REQUIRE(m.master_fname() == "test_master_1.json");
+    REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw");
+    REQUIRE(m.data_fname(1, 0) == "test_d1_f0_1.raw");
+    REQUIRE(m.data_fname(0, 1) == "test_d0_f1_1.raw");
+    REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
+}
+
+TEST_CASE("Master file name does not fit pattern"){
+    REQUIRE_THROWS(RawFileNameComponents("somefile.json"));
+    REQUIRE_THROWS(RawFileNameComponents("another_test_d0_f0_1.raw"));
+    REQUIRE_THROWS(RawFileNameComponents("test_master_1.txt"));
+}
+
+
+
+TEST_CASE("Parse scan parameters"){
+    ScanParameters s("[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]");
+    REQUIRE(s.enabled());
+    REQUIRE(s.dac() == "dac 4");
+    REQUIRE(s.start() == 500);
+    REQUIRE(s.stop() == 2200);
+    REQUIRE(s.step() == 5);
+}
+
+TEST_CASE("A disabled scan"){
+    ScanParameters s("[disabled]");
+    REQUIRE_FALSE(s.enabled());
+    REQUIRE(s.dac() == "");
+    REQUIRE(s.start() == 0);
+    REQUIRE(s.stop() == 0);
+    REQUIRE(s.step() == 0);
+}
+
+
+TEST_CASE("Parse a master file in .json format", "[.integration]"){
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawMasterFile f(fpath);
+
+    // "Version": 7.2,
+    REQUIRE(f.version() == "7.2");
+    // "Timestamp": "Tue Feb 20 08:28:24 2024",
+    // "Detector Type": "Jungfrau",
+    REQUIRE(f.detector_type() == DetectorType::Jungfrau);
+    // "Timing Mode": "auto",
+    REQUIRE(f.timing_mode() == TimingMode::Auto);
+    
+    // "Geometry": {
+    //     "x": 1,
+    //     "y": 1
+    // },
+    REQUIRE(f.geometry().col == 1);
+    REQUIRE(f.geometry().row == 1);
+
+    // "Image Size in bytes": 1048576,
+    REQUIRE(f.image_size_in_bytes() == 1048576);
+    // "Pixels": {
+    //     "x": 1024,
+    REQUIRE(f.pixels_x() == 1024);
+    //     "y": 512
+    REQUIRE(f.pixels_y() == 512);
+    // },
+
+    // "Max Frames Per File": 3,
+    REQUIRE(f.max_frames_per_file() == 3);
+
+    //Jungfrau doesn't write but it is 16
+    REQUIRE(f.bitdepth() == 16);
+
+
+    // "Frame Discard Policy": "nodiscard",
+
+    // "Frame Padding": 1,
+    REQUIRE(f.frame_padding() == 1);
+    // "Scan Parameters": "[disabled]",
+    // "Total Frames": 10,
+    REQUIRE(f.total_frames_expected() == 10);
+    // "Receiver Roi": {
+    //     "xmin": 4294967295,
+    //     "xmax": 4294967295,
+    //     "ymin": 4294967295,
+    //     "ymax": 4294967295
+    // },
+    // "Exptime": "10us",
+    // "Period": "1ms",
+    // "Number of UDP Interfaces": 1,
+    // "Number of rows": 512,
+    REQUIRE(f.number_of_rows() == 512);
+    // "Frames in File": 10,
+    REQUIRE(f.frames_in_file() == 10);
+
+    //TODO! Should we parse this? 
+    // "Frame Header Format": {
+    //     "Frame Number": "8 bytes",
+    //     "SubFrame Number/ExpLength": "4 bytes",
+    //     "Packet Number": "4 bytes",
+    //     "Bunch ID": "8 bytes",
+    //     "Timestamp": "8 bytes",
+    //     "Module Id": "2 bytes",
+    //     "Row": "2 bytes",
+    //     "Column": "2 bytes",
+    //     "Reserved": "2 bytes",
+    //     "Debug": "4 bytes",
+    //     "Round Robin Number": "2 bytes",
+    //     "Detector Type": "1 byte",
+    //     "Header Version": "1 byte",
+    //     "Packets Caught Mask": "64 bytes"
+    // }
+    // }         
+
+    REQUIRE_FALSE(f.analog_samples());
+    REQUIRE_FALSE(f.digital_samples());                
+
+}
+
+TEST_CASE("Parse a master file in .raw format", "[.integration]"){
+    
+    auto fpath = test_data_path() / "moench/moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawMasterFile f(fpath);
+
+    // Version                    : 6.4
+    REQUIRE(f.version() == "6.4");
+    // TimeStamp                  : Wed Aug 31 09:08:49 2022
+
+    // Detector Type              : ChipTestBoard
+    REQUIRE(f.detector_type() == DetectorType::ChipTestBoard);
+    // Timing Mode                : auto
+    REQUIRE(f.timing_mode() == TimingMode::Auto);
+    // Geometry                   : [1, 1]
+    REQUIRE(f.geometry().col == 1);
+    REQUIRE(f.geometry().row == 1);
+    // Image Size                 : 360000 bytes
+    REQUIRE(f.image_size_in_bytes() == 360000);
+    // Pixels                     : [96, 1]
+    REQUIRE(f.pixels_x() == 96);
+    REQUIRE(f.pixels_y() == 1);
+    // Max Frames Per File        : 20000
+    REQUIRE(f.max_frames_per_file() == 20000);
+    // Frame Discard Policy       : nodiscard
+    // Frame Padding              : 1
+    REQUIRE(f.frame_padding() == 1);
+    // Scan Parameters            : [disabled]
+    // Total Frames               : 100
+    REQUIRE(f.total_frames_expected() == 100);
+    // Exptime                    : 100us
+    // Period                     : 4ms
+    // Ten Giga                   : 1
+    // ADC Mask                   : 0xffffffff
+    // Analog Flag                : 1
+    // Analog Samples             : 5000
+    REQUIRE(f.analog_samples() == 5000);
+    // Digital Flag               : 1
+    // Digital Samples            : 5000
+    REQUIRE(f.digital_samples() == 5000);
+    // Dbit Offset                : 0
+    // Dbit Bitset                : 0
+    // Frames in File             : 100
+    REQUIRE(f.frames_in_file() == 100);
+
+    // #Frame Header
+    // Frame Number               : 8 bytes
+    // SubFrame Number/ExpLength  : 4 bytes
+    // Packet Number              : 4 bytes
+    // Bunch ID                   : 8 bytes
+    // Timestamp                  : 8 bytes
+    // Module Id                  : 2 bytes
+    // Row                        : 2 bytes
+    // Column                     : 2 bytes
+    // Reserved                   : 2 bytes
+    // Debug                      : 4 bytes
+    // Round Robin Number         : 2 bytes
+    // Detector Type              : 1 byte
+    // Header Version             : 1 byte
+    // Packets Caught Mask        : 64 bytes
+
+
+}
+
+
+TEST_CASE("Read eiger master file", "[.integration]"){
+auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawMasterFile f(fpath);
+    
+// {
+//     "Version": 7.2,
+REQUIRE(f.version() == "7.2");
+//     "Timestamp": "Tue Mar 26 17:24:34 2024",
+//     "Detector Type": "Eiger",
+REQUIRE(f.detector_type() == DetectorType::Eiger);
+//     "Timing Mode": "auto",
+REQUIRE(f.timing_mode() == TimingMode::Auto);
+//     "Geometry": {
+//         "x": 2,
+//         "y": 2
+//     },
+//     "Image Size in bytes": 524288,
+REQUIRE(f.image_size_in_bytes() == 524288);
+//     "Pixels": {
+//         "x": 512,
+REQUIRE(f.pixels_x() == 512);
+//         "y": 256
+REQUIRE(f.pixels_y() == 256);
+//     },
+//     "Max Frames Per File": 10000,
+REQUIRE(f.max_frames_per_file() == 10000);
+//     "Frame Discard Policy": "nodiscard",
+REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard);
+//     "Frame Padding": 1,
+REQUIRE(f.frame_padding() == 1);
+
+//     "Scan Parameters": "[disabled]",
+//     "Total Frames": 3,
+//     "Receiver Roi": {
+//         "xmin": 4294967295,
+//         "xmax": 4294967295,
+//         "ymin": 4294967295,
+//         "ymax": 4294967295
+//     },
+//     "Dynamic Range": 32,
+//     "Ten Giga": 0,
+//     "Exptime": "5s",
+//     "Period": "1s",
+//     "Threshold Energy": -1,
+//     "Sub Exptime": "2.62144ms",
+//     "Sub Period": "2.62144ms",
+//     "Quad": 0,
+//     "Number of rows": 256,
+//     "Rate Corrections": "[0, 0]",
+//     "Frames in File": 3,
+//     "Frame Header Format": {
+//         "Frame Number": "8 bytes",
+//         "SubFrame Number/ExpLength": "4 bytes",
+//         "Packet Number": "4 bytes",
+//         "Bunch ID": "8 bytes",
+//         "Timestamp": "8 bytes",
+//         "Module Id": "2 bytes",
+//         "Row": "2 bytes",
+//         "Column": "2 bytes",
+//         "Reserved": "2 bytes",
+//         "Debug": "4 bytes",
+//         "Round Robin Number": "2 bytes",
+//         "Detector Type": "1 byte",
+//         "Header Version": "1 byte",
+//         "Packets Caught Mask": "64 bytes"
+//     }
+// }            
+
+
+
+}

src/RawSubFile.cpp

@@ -0,0 +1,100 @@
+#include "aare/RawSubFile.hpp"
+#include "aare/PixelMap.hpp"
+#include <cstring> // memcpy
+#include <fmt/core.h>
+#include <iostream>
+
+namespace aare {
+
+RawSubFile::RawSubFile(const std::filesystem::path &fname,
+                       DetectorType detector, size_t rows, size_t cols,
+                       size_t bitdepth, uint32_t pos_row, uint32_t pos_col)
+    : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
+      m_detector_type(detector),
+      m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row), m_pos_col(pos_col) {
+    if (m_detector_type == DetectorType::Moench03_old) {
+        m_pixel_map = GenerateMoench03PixelMap();
+    }else if(m_detector_type == DetectorType::Eiger && m_pos_row % 2 == 0){
+        m_pixel_map = GenerateEigerFlipRowsPixelMap();
+        fmt::print("Flipping rows\n");
+    }
+
+    if (std::filesystem::exists(fname)) {
+        n_frames = std::filesystem::file_size(fname) /
+                   (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
+    } else {
+        throw std::runtime_error(
+            LOCATION + fmt::format("File {} does not exist", m_fname.string()));
+    }
+
+    // fp = fopen(m_fname.string().c_str(), "rb");
+    m_file.open(m_fname, std::ios::binary);
+    if (!m_file.is_open()) {
+        throw std::runtime_error(
+            LOCATION + fmt::format("Could not open file {}", m_fname.string()));
+    }
+
+#ifdef AARE_VERBOSE
+    fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
+    fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols,
+               m_bitdepth);
+    fmt::print("file size: {}\n", std::filesystem::file_size(fname));
+#endif
+}
+
+void RawSubFile::seek(size_t frame_index) {
+    if (frame_index >= n_frames) {
+        throw std::runtime_error("Frame number out of range");
+    }
+    m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index);
+}
+
+size_t RawSubFile::tell() {
+    return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame());
+}
+
+
+void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
+    if(header){
+         m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
+    } else {
+        m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
+    }
+
+    //TODO! expand support for different bitdepths
+    if(m_pixel_map){
+        // read into a temporary buffer and then copy the data to the buffer
+        // in the correct order
+        // currently this only supports 16 bit data!
+        auto part_buffer = new std::byte[bytes_per_frame()];
+        m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
+        auto *data = reinterpret_cast<uint16_t *>(image_buf);
+        auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
+        for (size_t i = 0; i < pixels_per_frame(); i++) {
+            data[i] = part_data[(*m_pixel_map)(i)];
+        }
+        delete[] part_buffer;
+    } else {
+        // read directly into the buffer
+        m_file.read(reinterpret_cast<char *>(image_buf), bytes_per_frame());
+    }
+}
+
+size_t RawSubFile::rows() const { return m_rows; }
+size_t RawSubFile::cols() const { return m_cols; }
+
+
+void RawSubFile::get_part(std::byte *buffer, size_t frame_index) {
+    seek(frame_index);
+    read_into(buffer, nullptr);
+}
+
+size_t RawSubFile::frame_number(size_t frame_index) {
+    seek(frame_index);
+    DetectorHeader h{};
+    m_file.read(reinterpret_cast<char *>(&h), sizeof(DetectorHeader));
+    return h.frameNumber;
+}
+
+
+} // namespace aare

src/SubFile.cpp

@@ -1,71 +0,0 @@
-#include "aare/SubFile.hpp"
-
-#include <cstring> // memcpy
-#include <fmt/core.h>
-#include <iostream>
-
-
-namespace aare {
-
-SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
-                 const std::string &mode)
-    : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) {
-
-    if (std::filesystem::exists(fname)) {
-        n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
-    } else {
-        n_frames = 0;
-    }
-
-    if (mode == "r") {
-        fp = fopen(m_fname.string().c_str(), "rb");
-    } else {
-        // if file exists, open in read/write mode (without truncating the file)
-        // if file does not exist, open in write mode
-        if (std::filesystem::exists(fname)) {
-            fp = fopen(m_fname.string().c_str(), "r+b");
-        } else {
-            fp = fopen(m_fname.string().c_str(), "wb");
-        }
-    }
-    if (fp == nullptr) {
-        throw std::runtime_error(LOCATION + "Could not open file for writing");
-    }
-}
-
-size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
-    if (frame_index >= n_frames) {
-        throw std::runtime_error("Frame number out of range");
-    }
-    fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT
-          SEEK_SET);
-    return fread(buffer, this->bytes_per_part(), 1, this->fp);
-}
-size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) {
-    if (frame_index > n_frames) {
-        throw std::runtime_error("Frame number out of range");
-    }
-    fseek(fp, static_cast<int64_t>((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET);
-    auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
-    wc += fwrite(buffer, bytes_per_part(), 1, fp);
-
-    return wc;
-}
-
-size_t SubFile::frame_number(size_t frame_index) {
-    sls_detector_header h{};
-    fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
-    size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
-    if (rc != 1)
-        throw std::runtime_error(LOCATION + "Could not read header from file");
-
-    return h.frameNumber;
-}
-
-SubFile::~SubFile() {
-    if (fp) {
-        fclose(fp);
-    }
-}
-
-} // namespace aare

src/defs.cpp

@@ -2,14 +2,24 @@
 #include <stdexcept>
 #include <string>
 
+#include <fmt/core.h>
 namespace aare {
 
+
+void assert_failed(const std::string &msg)
+ {
+    fmt::print(msg);
+    exit(1);
+}
+
+
+
 /**
  * @brief Convert a DetectorType to a string
  * @param type DetectorType
  * @return string representation of the DetectorType
  */
-template <> std::string toString(DetectorType arg) {
+template <> std::string ToString(DetectorType arg) {
     switch (arg) {
     case DetectorType::Jungfrau:
         return "Jungfrau";
@@ -19,6 +29,10 @@ template <> std::string toString(DetectorType arg) {
         return "Mythen3";
     case DetectorType::Moench:
         return "Moench";
+    case DetectorType::Moench03:
+        return "Moench03";
+    case DetectorType::Moench03_old:
+        return "Moench03_old";
     case DetectorType::ChipTestBoard:
         return "ChipTestBoard";
     default:
@@ -41,6 +55,10 @@ template <> DetectorType StringTo(const std::string &arg) {
         return DetectorType::Mythen3;
     if (arg == "Moench")
         return DetectorType::Moench;
+    if (arg == "Moench03")
+        return DetectorType::Moench03;
+    if (arg == "Moench03_old")
+        return DetectorType::Moench03_old;
     if (arg == "ChipTestBoard")
         return DetectorType::ChipTestBoard;
     throw std::runtime_error("Could not decode dector from: \"" + arg + "\"");
@@ -60,6 +78,16 @@ template <> TimingMode StringTo(const std::string &arg) {
     throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\"");
 }
 
+template <> FrameDiscardPolicy StringTo(const std::string &arg) {
+    if (arg == "nodiscard")
+        return FrameDiscardPolicy::NoDiscard;
+    if (arg == "discard")
+        return FrameDiscardPolicy::Discard;
+    if (arg == "discardpartial")
+        return FrameDiscardPolicy::DiscardPartial;
+    throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\"");
+}
+
 // template <> TimingMode StringTo<TimingMode>(std::string mode);
 
 } // namespace aare

src/defs.test.cpp

@@ -6,17 +6,17 @@
 TEST_CASE("Enum to string conversion") {
     // By the way I don't think the enum string conversions should be in the defs.hpp file
     // but let's use this to show a test
-    REQUIRE(toString(aare::DetectorType::Jungfrau) == "Jungfrau");
+    REQUIRE(ToString(aare::DetectorType::Jungfrau) == "Jungfrau");
 }
 
-TEST_CASE("Cluster creation") {
-    aare::Cluster c(13, 15);
+TEST_CASE("DynamicCluster creation") {
+    aare::DynamicCluster c(13, 15);
     REQUIRE(c.cluster_sizeX == 13);
     REQUIRE(c.cluster_sizeY == 15);
     REQUIRE(c.dt == aare::Dtype(typeid(int32_t)));
     REQUIRE(c.data() != nullptr);
 
-    aare::Cluster c2(c);
+    aare::DynamicCluster c2(c);
     REQUIRE(c2.cluster_sizeX == 13);
     REQUIRE(c2.cluster_sizeY == 15);
     REQUIRE(c2.dt == aare::Dtype(typeid(int32_t)));
@@ -25,7 +25,7 @@ TEST_CASE("Cluster creation") {
 
 // TEST_CASE("cluster set and get data") {
 
-//     aare::Cluster c2(33, 44, aare::Dtype(typeid(double)));
+//     aare::DynamicCluster c2(33, 44, aare::Dtype(typeid(double)));
 //     REQUIRE(c2.cluster_sizeX == 33);
 //     REQUIRE(c2.cluster_sizeY == 44);
 //     REQUIRE(c2.dt == aare::Dtype::DOUBLE);

tests/CMakeLists.txt

@@ -1,4 +1,5 @@
 
+# Download catch2 if configured to do so
 if (AARE_FETCH_CATCH)
     FetchContent_Declare(
         Catch2
@@ -8,6 +9,7 @@ if (AARE_FETCH_CATCH)
     )
     FetchContent_MakeAvailable(Catch2)
 else()
+    # Otherwise look for installed catch2
     find_package(Catch2 3 REQUIRED)
 endif()
 
@@ -32,11 +34,9 @@ set(TestSources
 target_sources(tests PRIVATE ${TestSources} )
 
 #Work around to remove, this is not the way to do it =) 
-# target_include_directories(tests PRIVATE ${CMAKE_SOURCE_DIR}/include/common)
 target_link_libraries(tests PRIVATE aare_core aare_compiler_flags)
 
 
-
 #configure a header to pass test file paths
 get_filename_component(TEST_FILE_PATH ${PROJECT_SOURCE_DIR}/data ABSOLUTE)
 configure_file(test_config.hpp.in test_config.hpp)

tests/test.cpp

@@ -4,12 +4,12 @@
 #include <filesystem>
 #include <fstream>
 
-TEST_CASE("Test suite can find data assets") {
+TEST_CASE("Test suite can find data assets", "[.integration]") {
     auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
     REQUIRE(std::filesystem::exists(fpath));
 }
 
-TEST_CASE("Test suite can open data assets") {
+TEST_CASE("Test suite can open data assets", "[.integration]") {
     auto fpath = test_data_path() / "numpy" / "test_numpy_file.npy";
     auto f = std::ifstream(fpath, std::ios::binary);
     REQUIRE(f.is_open());