Brining in changes (#93)

.clang-format

@@ -0,0 +1,7 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+
+UseTab: Never
+ColumnLimit: 80
+AlignConsecutiveAssignments: false 
+AlignConsecutiveMacros: true

.github/workflows/build_docs.yml

@@ -4,7 +4,8 @@ on:
   workflow_dispatch:
   push:
     branches:
-     - v1
+     - main
+
 
 permissions:
   contents: read

.github/workflows/build_pkg.yml

@@ -4,7 +4,9 @@ on:
   workflow_dispatch:
   push:
     branches:
-     - v1
+     - main
+     - developer
+
     #run on PRs as well?
 
 jobs:

.gitignore

@@ -0,0 +1,24 @@
+install/
+.cproject
+.project
+bin/
+.settings
+*.aux
+*.log
+*.out
+*.toc
+*.o
+*.so
+.*
+build/
+RELEASE.txt
+Testing/
+
+ctbDict.cpp
+ctbDict.h
+
+
+
+*.pyc
+*/__pycache__/*
+

CMakeLists.txt

@@ -33,6 +33,8 @@ option(AARE_EXAMPLES "Build examples" OFF)
 option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF)
 option(AARE_DOCS "Build documentation" OFF)
 option(AARE_VERBOSE "Verbose output" OFF)
+option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
+
 
 # Configure which of the dependencies to use FetchContent for
 option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
@@ -57,6 +59,10 @@ if(AARE_VERBOSE)
     add_compile_definitions(AARE_VERBOSE)
 endif()
 
+if(AARE_CUSTOM_ASSERT)
+    add_compile_definitions(AARE_CUSTOM_ASSERT)
+endif()
+
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -242,7 +248,6 @@ set(PUBLICHEADERS
     include/aare/Dtype.hpp
     include/aare/File.hpp
     include/aare/FileInterface.hpp
-    include/aare/RawMasterFile.hpp
     include/aare/Frame.hpp
     include/aare/NDArray.hpp
     include/aare/NDView.hpp
@@ -251,6 +256,8 @@ set(PUBLICHEADERS
     include/aare/Pedestal.hpp
     include/aare/PixelMap.hpp
     include/aare/RawFile.hpp
+    include/aare/RawSubFile.hpp
+    include/aare/RawMasterFile.hpp
     include/aare/SubFile.hpp
     include/aare/VarClusterFinder.hpp
 
@@ -263,11 +270,11 @@ set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
 )
 

conda-recipe/meta.yaml

@@ -1,6 +1,7 @@
 package:
   name: aare
-  version: 2024.11.07.dev0 #TODO! how to not duplicate this?
+  version: 2024.11.11.dev0 #TODO! how to not duplicate this?
+
 
 source:
   path: ..

docs/CMakeLists.txt

@@ -20,6 +20,7 @@ set(SPHINX_SOURCE_FILES
   src/ClusterFinder.rst
   src/Pedestal.rst
   src/RawFile.rst
+  src/RawSubFile.rst
   src/RawMasterFile.rst
   src/VarClusterFinder.rst
   src/pyVarClusterFinder.rst

docs/src/RawSubFile.rst

@@ -0,0 +1,8 @@
+RawSubFile
+===============
+
+
+.. doxygenclass:: aare::RawSubFile
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/index.rst

@@ -5,6 +5,16 @@ AARE
 
     Hello
 
+.. toctree::
+    :caption: Python API
+    :maxdepth: 1
+    
+    pyFile
+    pyCtbRawFile
+    pyRawMasterFile
+    pyVarClusterFinder
+
+
 .. toctree::
     :caption: C++ API
     :maxdepth: 1
@@ -17,15 +27,10 @@ AARE
     ClusterFinder
     Pedestal
     RawFile
+    RawSubFile
     RawMasterFile
     VarClusterFinder
 
-.. toctree::
-    :caption: Python API
-    :maxdepth: 1
-    
-    pyFile
-    pyCtbRawFile
-    pyRawMasterFile
-    pyVarClusterFinder
+
+
 

include/aare/NDArray.hpp

@@ -341,6 +341,21 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
     else
         Print_some();
 }
+
+
+template <typename T, int64_t Ndim>
+std::ostream& operator <<(std::ostream& os, const NDArray<T, Ndim>& arr){
+    for (auto row = 0; row < arr.shape(0); ++row) {
+        for (auto col = 0; col < arr.shape(1); ++col) {
+            os << std::setw(3);
+            os << arr(row, col) << " ";
+        }
+        os << "\n";
+    }
+    return os;
+}
+
+
 template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
     for (auto row = 0; row < shape_[0]; ++row) {
         for (auto col = 0; col < shape_[1]; ++col) {

include/aare/NDView.hpp

@@ -156,4 +156,18 @@ template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
     }
 }
 
+
+template <typename T, int64_t Ndim>
+std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
+    for (auto row = 0; row < arr.shape(0); ++row) {
+        for (auto col = 0; col < arr.shape(1); ++col) {
+            os << std::setw(3);
+            os << arr(row, col) << " ";
+        }
+        os << "\n";
+    }
+    return os;
+}
+
+
 } // namespace aare

include/aare/RawFile.hpp

@@ -3,7 +3,8 @@
 #include "aare/RawMasterFile.hpp"
 #include "aare/Frame.hpp"
 #include "aare/NDArray.hpp" //for pixel map
-#include "aare/SubFile.hpp"
+#include "aare/RawSubFile.hpp"
+
 
 #include <optional>
 
@@ -29,10 +30,12 @@ struct ModuleConfig {
  * Consider using that unless you need raw file specific functionality.
  */
 class RawFile : public FileInterface {
-    size_t n_subfiles{};
-    size_t n_subfile_parts{};
-    std::vector<std::vector<SubFile *>> subfiles;
+    size_t n_subfiles{}; //f0,f1...fn
+    size_t n_subfile_parts{}; // d0,d1...dn
+    //TODO! move to vector of SubFile instead of pointers
+    std::vector<std::vector<RawSubFile *>> subfiles; //subfiles[f0,f1...fn][d0,d1...dn]
     std::vector<xy> positions;
+    std::vector<ModuleGeometry> m_module_pixel_0;
     ModuleConfig cfg{0, 0};
 
     RawMasterFile m_master;
@@ -56,6 +59,11 @@ class RawFile : public FileInterface {
     std::vector<Frame> read_n(size_t n_frames) override;
     void read_into(std::byte *image_buf) override;
     void read_into(std::byte *image_buf, size_t n_frames) override;
+
+    //TODO! do we need to adapt the API? 
+    void read_into(std::byte *image_buf, DetectorHeader *header);
+
+
     size_t frame_number(size_t frame_index) override;
     size_t bytes_per_frame() override;
     size_t pixels_per_frame() override;
@@ -65,7 +73,10 @@ class RawFile : public FileInterface {
     size_t rows() const override;
     size_t cols() const override;
     size_t bitdepth() const override;
+    size_t bytes_per_pixel() const;
     xy geometry();
+    size_t n_mod() const;
+
 
     DetectorType detector_type() const override;
 
@@ -83,7 +94,9 @@ class RawFile : public FileInterface {
      * @param frame_number frame number to read
      * @param image_buf buffer to store the frame
      */
-    void get_frame_into(size_t frame_index, std::byte *frame_buffer);
+
+    void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
+
 
     /**
      * @brief get the frame at the given frame index
@@ -101,8 +114,9 @@ class RawFile : public FileInterface {
      */
     static DetectorHeader read_header(const std::filesystem::path &fname);
 
+    void update_geometry_with_roi();
+    int find_number_of_subfiles();
 
-    void find_number_of_subfiles();
     void open_subfiles();
     void find_geometry();
 };

include/aare/RawMasterFile.hpp

@@ -58,6 +58,17 @@ class ScanParameters {
     bool enabled() const;
 };
 
+
+struct ROI{
+  size_t xmin{};
+  size_t xmax{};
+  size_t ymin{};
+  size_t ymax{};
+
+  size_t height() const { return ymax - ymin; }
+  size_t width() const { return xmax - xmin; }
+};
+
 /**
  * @brief Class for parsing a master file either in our .json format or the old
  * .raw format
@@ -95,6 +106,9 @@ class RawMasterFile {
     std::optional<size_t> m_number_of_rows;
     std::optional<uint8_t> m_quad;
 
+    std::optional<ROI> m_roi;
+
+
   public:
     RawMasterFile(const std::filesystem::path &fpath);
 
@@ -121,6 +135,10 @@ class RawMasterFile {
     std::optional<size_t> number_of_rows() const;
     std::optional<uint8_t> quad() const;
 
+
+    std::optional<ROI> roi() const;
+
+
     ScanParameters scan_parameters() const;
 
   private:

include/aare/RawSubFile.hpp

@@ -0,0 +1,69 @@
+#pragma once
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <optional>
+
+namespace aare {
+
+/**
+ * @brief Class to read a singe subfile written in .raw format. Used from RawFile to read
+ * the entire detector. Can be used directly to read part of the image.
+ */
+class RawSubFile {
+  protected:
+    std::ifstream m_file;
+    size_t m_bitdepth;
+    std::filesystem::path m_fname;
+    size_t m_rows{};
+    size_t m_cols{};
+    size_t m_bytes_per_frame{};
+    size_t n_frames{};
+ 
+    DetectorType m_detector_type;
+    std::optional<NDArray<ssize_t, 2>> pixel_map;
+
+  public:
+    /**
+     * @brief SubFile constructor
+     * @param fname path to the subfile
+     * @param detector detector type
+     * @param rows number of rows in the subfile
+     * @param cols number of columns in the subfile
+     * @param bitdepth bitdepth of the subfile
+     * @throws std::invalid_argument if the detector,type pair is not supported
+     */
+    RawSubFile(const std::filesystem::path &fname, DetectorType detector,
+               size_t rows, size_t cols, size_t bitdepth);
+
+    ~RawSubFile() = default;
+    /**
+     * @brief Seek to the given frame number
+     * @note Puts the file pointer at the start of the header, not the start of the data
+     * @param frame_index frame position in file to seek to
+     * @throws std::runtime_error if the frame number is out of range
+     */
+    void seek(size_t frame_index);
+    size_t tell();
+
+    void read_into(std::byte *image_buf, DetectorHeader *header = nullptr);
+    void get_part(std::byte *buffer, size_t frame_index);
+    
+    void read_header(DetectorHeader *header);
+    
+    size_t rows() const;
+    size_t cols() const;
+    
+    size_t frame_number(size_t frame_index);
+
+    size_t bytes_per_frame() const { return m_bytes_per_frame; }
+    size_t pixels_per_frame() const { return m_rows * m_cols; }
+    size_t bytes_per_pixel() const { return m_bitdepth / 8; }
+
+
+};
+
+} // namespace aare

include/aare/defs.hpp

@@ -14,6 +14,7 @@
 #include <variant>
 #include <vector>
 
+
 /**
  * @brief LOCATION macro to get the current location in the code
  */
@@ -21,8 +22,25 @@
     std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) +      \
         ":" + std::string(__func__) + ":"
 
+
+
+#ifdef AARE_CUSTOM_ASSERT
+#define AARE_ASSERT(expr)\
+    if (expr)\
+        {}\
+    else\
+        aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n");
+#else
+#define AARE_ASSERT(cond)\
+    do { (void)sizeof(cond); } while(0)
+#endif
+
+
 namespace aare {
 
+void assert_failed(const std::string &msg);
+
+
 class Cluster {
   public:
     int cluster_sizeX;
@@ -164,6 +182,15 @@ template <typename T> struct t_xy {
 };
 using xy = t_xy<uint32_t>;
 
+
+struct ModuleGeometry{
+    int x{};
+    int y{};
+    int height{};
+    int width{};
+};
+
+
 using dynamic_shape = std::vector<int64_t>;
 
 //TODO! Can we uniform enums between the libraries?

pyproject.toml

@@ -4,7 +4,8 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "aare"
-version = "2024.11.07.dev0"
+version = "2024.11.11.dev0"
+
 
 [tool.scikit-build]
 cmake.verbose = true

python/aare/__init__.py

@@ -1,7 +1,10 @@
 # Make the compiled classes that live in _aare available from aare.
 from . import _aare
 
-from ._aare import VarClusterFinder, File, RawMasterFile
-from ._aare import Pedestal, ClusterFinder
+
+from ._aare import File, RawFile, RawMasterFile, RawSubFile
+from ._aare import Pedestal, ClusterFinder, VarClusterFinder
+from ._aare import DetectorType
+
 from .CtbRawFile import CtbRawFile
 from .ScanParameters import ScanParameters

python/examples/play.py

@@ -53,84 +53,36 @@ def get_Mh02_frames(fname):
 # plt.imshow(data[0,0])
 
 
-# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
+base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data')
+# p = Path(base / 'jungfrau/jungfrau_single_master_0.json')
 
 # f = aare.File(p)
-# frame = f.read_frame()
-
-# fig, ax = plt.subplots()
-# im = ax.imshow(frame, cmap='viridis')
-
-
-# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json')
-# # fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
-
-
-# # f = aare.CtbRawFile(fpath, transform = transform.moench05)
-# # with CtbRawFile(fpath, transform = transform.moench05) as f:
-# #     for header, image in f:
-# #         print(f'Frame number: {header["frameNumber"]}')
-
-# # m = aare.RawMasterFile(fpath)
-# f = aare.File(fpath)
+# for i in range(10):
+#     frame = f.read_frame()
 
 
 
-# cf = aare.ClusterFinder((400,400),(3,3))
-
-# for i in range(100):
-#     cf.push_pedestal_frame(f.read_frame())
-
-
-# f.seek(0)
-# pd = f.read_n(100).mean(axis=0)
-
-# clusters = cf.find_clusters_without_threshold(f.read_frame())
+# # f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
+# # header, data = f2.read()
+# # plt.plot(data[:,0,20,20])
+# from aare import RawMasterFile, File, RawSubFile, DetectorType, RawFile
+# base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/')
+# fpath = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json')
+# raw = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_d0_f0_0.raw')
 
 
 
-base = Path('/Users/erik/data/matterhorn/raw')
-fpath = Path(base / 'scan_15keV_vrf700_vrsh700_th0_master_0.json')
-f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
-f.seek(100)
-header1, image1 = f.read_frame()
+# m = RawMasterFile(base / fpath)
+# # roi = m.roi
+# # rows = roi.ymax-roi.ymin+1
+# # cols = 1024-roi.xmin
+# # sf = RawSubFile(base / raw, DetectorType.Jungfrau, rows, cols, 16)
 
-# fpath = Path(base / 'scan_all15keV_vrf500_vrsh700_th0_master_0.json')
-
-# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
-# f.seek(100)
-# header4, image4 = f.read_frame()
-
-# n_counters = image.shape[1] / 48**2 / 2
-
-# for i in range(100):
-#     header, image = f.read_frame()
-#     print(header['frameNumber'])
-
-
-
-
-
-#Data come in "blocks" of 4 pixels/receiver
-# data = get_Mh02_frames(fpath.as_posix())
-
-# rawi = np.zeros(48*48*4+56, dtype = np.uint16)
-# for i,v in enumerate(rawi[56:]):
-#     rawi[i+56] = i
-
-# raw = image.view(np.uint16)
-
-# pixel_map = decode(1, rawi)
-# # img = np.take(raw, pixel_map)
-
-# pm = np.zeros((4, 48,48), dtype = np.int64)
-# for counter in range(4):
-#     for row in range(48):
-#         for col in range(48):
-#             pm[counter, row, col] = row*48 + col+counter*48*48
-
-
-# f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02)
-# header, data = f2.read()
-# plt.plot(data[:,0,20,20])
+from aare import RawFile
 
+base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/')
+fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json')
+# fname = Path(base / 'jungfrau/jungfrau_single_master_0.json')
+f = RawFile(fname)
+h,img = f.read_frame()
+print(f'{h["frameNumber"]}')

python/src/file.hpp

@@ -1,7 +1,10 @@
 #include "aare/CtbRawFile.hpp"
-#include "aare/RawMasterFile.hpp"
 #include "aare/File.hpp"
 #include "aare/Frame.hpp"
+#include "aare/RawFile.hpp"
+#include "aare/RawMasterFile.hpp"
+#include "aare/RawSubFile.hpp"
+
 #include "aare/defs.hpp"
 // #include "aare/fClusterFileV2.hpp"
 
@@ -19,6 +22,18 @@ using namespace ::aare;
 
 void define_file_io_bindings(py::module &m) {
 
+
+    py::enum_<DetectorType>(m, "DetectorType")
+        .value("Jungfrau", DetectorType::Jungfrau)
+        .value("Eiger", DetectorType::Eiger)
+        .value("Mythen3", DetectorType::Mythen3)
+        .value("Moench", DetectorType::Moench)
+        .value("Moench03", DetectorType::Moench03)
+        .value("Moench03_old", DetectorType::Moench03_old)
+        .value("ChipTestBoard", DetectorType::ChipTestBoard)
+        .value("Unknown", DetectorType::Unknown);
+
+
     PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
                          bunchId, timestamp, modId, row, column, reserved,
                          debug, roundRNumber, detType, version, packetMask);
@@ -36,7 +51,6 @@ void define_file_io_bindings(py::module &m) {
 
                  py::array_t<DetectorHeader> header(1);
 
-
                  // always read bytes
                  image = py::array_t<uint8_t>(shape);
 
@@ -49,7 +63,10 @@ void define_file_io_bindings(py::module &m) {
         .def("seek", &CtbRawFile::seek)
         .def("tell", &CtbRawFile::tell)
         .def("master", &CtbRawFile::master)
-        .def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes)
+
+        .def_property_readonly("image_size_in_bytes",
+                               &CtbRawFile::image_size_in_bytes)
+
         .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
 
     py::class_<File>(m, "File")
@@ -169,26 +186,107 @@ void define_file_io_bindings(py::module &m) {
         .def_property_readonly("frame_padding", &RawMasterFile::frame_padding)
         .def_property_readonly("frame_discard_policy",
                                &RawMasterFile::frame_discard_policy)
-        .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected)
+
+        .def_property_readonly("total_frames_expected",
+                               &RawMasterFile::total_frames_expected)
         .def_property_readonly("geometry", &RawMasterFile::geometry)
         .def_property_readonly("analog_samples", &RawMasterFile::analog_samples)
         .def_property_readonly("digital_samples",
                                &RawMasterFile::digital_samples)
-        .def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples)
+
+        .def_property_readonly("transceiver_samples",
+                               &RawMasterFile::transceiver_samples)
         .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows)
         .def_property_readonly("quad", &RawMasterFile::quad)
-        .def_property_readonly("scan_parameters", &RawMasterFile::scan_parameters);
-
-
+        .def_property_readonly("scan_parameters",
+                               &RawMasterFile::scan_parameters)
+        .def_property_readonly("roi", &RawMasterFile::roi);
 
     py::class_<ScanParameters>(m, "ScanParameters")
         .def(py::init<const std::string &>())
-        .def(py::init<const ScanParameters&>())
+        .def(py::init<const ScanParameters &>())
+
         .def_property_readonly("enabled", &ScanParameters::enabled)
         .def_property_readonly("dac", &ScanParameters::dac)
         .def_property_readonly("start", &ScanParameters::start)
         .def_property_readonly("stop", &ScanParameters::stop)
         .def_property_readonly("step", &ScanParameters::step);
+
+
+    py::class_<ROI>(m, "ROI")
+        .def(py::init<>())
+        .def_readwrite("xmin", &ROI::xmin)
+        .def_readwrite("xmax", &ROI::xmax)
+        .def_readwrite("ymin", &ROI::ymin)
+        .def_readwrite("ymax", &ROI::ymax)
+        .def("__str__", [](const ROI& self){
+            return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax);
+        });
+
+    py::class_<RawFile>(m, "RawFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("read_frame",
+             [](RawFile &self) {
+                 size_t image_size = self.bytes_per_frame();
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+
+
+                //return headers from all subfiles
+                 py::array_t<DetectorHeader> header(self.n_mod());
+
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols());
+
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             });
+
+    py::class_<RawSubFile>(m, "RawSubFile")
+        .def(py::init<const std::filesystem::path &, DetectorType, size_t,
+                      size_t, size_t>())
+        .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame)
+        .def_property_readonly("pixels_per_frame",
+                               &RawSubFile::pixels_per_frame)
+        .def("seek", &RawSubFile::seek)
+        .def("tell", &RawSubFile::tell)
+        .def_property_readonly("rows", &RawSubFile::rows)
+        .def_property_readonly("cols", &RawSubFile::cols)
+        .def("read_frame",
+             [](RawSubFile &self) {
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols());
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+             });
+
+
     // py::class_<ClusterHeader>(m, "ClusterHeader")
     //     .def(py::init<>())
     //     .def_readwrite("frame_number", &ClusterHeader::frame_number)

src/RawFile.cpp

@@ -13,9 +13,13 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
     : m_master(fname) {
     m_mode = mode;
     if (mode == "r") {
-        find_number_of_subfiles();
-        n_subfile_parts = m_master.geometry().col * m_master.geometry().row;
+
+        n_subfiles = find_number_of_subfiles(); // f0,f1...fn
+        n_subfile_parts =
+            m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
         find_geometry();
+        update_geometry_with_roi();
+
         open_subfiles();
     } else {
         throw std::runtime_error(LOCATION +
@@ -38,10 +42,20 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
         image_buf += bytes_per_frame();
     }
 }
+
 void RawFile::read_into(std::byte *image_buf) {
     return get_frame_into(m_current_frame++, image_buf);
 };
 
+
+void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
+
+    return get_frame_into(m_current_frame++, image_buf, header);
+};
+
+size_t RawFile::n_mod() const { return n_subfile_parts; }
+
+
 size_t RawFile::bytes_per_frame() {
     return m_rows * m_cols * m_master.bitdepth() / 8;
 }
@@ -71,12 +85,13 @@ xy RawFile::geometry() { return m_master.geometry(); }
 void RawFile::open_subfiles() {
     if (m_mode == "r")
         for (size_t i = 0; i != n_subfiles; ++i) {
-            auto v = std::vector<SubFile *>(n_subfile_parts);
+            auto v = std::vector<RawSubFile *>(n_subfile_parts);
             for (size_t j = 0; j != n_subfile_parts; ++j) {
-                v[j] =
-                    new SubFile(m_master.data_fname(j, i),
-                                m_master.detector_type(), m_master.pixels_y(),
-                                m_master.pixels_x(), m_master.bitdepth());
+                auto pos = m_module_pixel_0[j];
+                v[j] = new RawSubFile(m_master.data_fname(j, i),
+                                      m_master.detector_type(), pos.height,
+                                      pos.width, m_master.bitdepth());
+
             }
             subfiles.push_back(v);
         }
@@ -107,27 +122,36 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) {
     return stem.find("_master_") != std::string::npos;
 }
 
-void RawFile::find_number_of_subfiles() {
-    int n_mod = 0;
-    while (std::filesystem::exists(m_master.data_fname(0, ++n_mod)))
-        ;
-    n_subfiles = n_mod;
-    #ifdef AARE_VERBOSE
-    fmt::print("Found: {} subfiles\n", n_subfiles);
-    #endif
+int RawFile::find_number_of_subfiles() {
+    int n_files = 0;
+    // f0,f1...fn How many files is the data split into?
+    while (std::filesystem::exists(m_master.data_fname(0, n_files)))
+        n_files++; // increment after test
+
+#ifdef AARE_VERBOSE
+    fmt::print("Found: {} subfiles\n", n_files);
+#endif
+    return n_files;
+
 }
 
 void RawFile::find_geometry() {
     uint16_t r{};
     uint16_t c{};
-    for (size_t i = 0; i < n_subfile_parts; i++) {
-        for (size_t j = 0; j != n_subfiles; ++j) {
-            auto h = this->read_header(m_master.data_fname(i, j));
-            r = std::max(r, h.row);
-            c = std::max(c, h.column);
 
-            positions.push_back({h.row, h.column});
-        }
+
+    for (size_t i = 0; i < n_subfile_parts; i++) {
+        auto h = this->read_header(m_master.data_fname(i, 0));
+        r = std::max(r, h.row);
+        c = std::max(c, h.column);
+        positions.push_back({h.row, h.column});
+        ModuleGeometry g;
+        g.x = h.column * m_master.pixels_x();
+        g.y = h.row * m_master.pixels_y();
+        g.width = m_master.pixels_x();
+        g.height = m_master.pixels_y();
+        m_module_pixel_0.push_back(g);
+
     }
 
     r++;
@@ -137,6 +161,81 @@ void RawFile::find_geometry() {
     m_cols = (c * m_master.pixels_x());
 
     m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
+
+#ifdef AARE_VERBOSE
+    fmt::print("\nRawFile::find_geometry()\n");
+    for (size_t i = 0; i < m_module_pixel_0.size(); i++) {
+        fmt::print("Module {} at position: (r:{},c:{})\n", i,
+                   m_module_pixel_0[i].y, m_module_pixel_0[i].x);
+    }
+    fmt::print("Image size: {}x{}\n\n", m_rows, m_cols);
+#endif
+}
+
+void RawFile::update_geometry_with_roi() {
+    // TODO! implement this
+    if (m_master.roi()) {
+        auto roi = m_master.roi().value();
+
+        // TODO! can we do this cleaner?
+        int pos_y = 0;
+        int pos_y_increment = 0;
+        for (size_t row = 0; row < m_master.geometry().row; row++) {
+            int pos_x = 0;
+            for (size_t col = 0; col < m_master.geometry().col; col++) {
+                auto &m = m_module_pixel_0[row * m_master.geometry().col + col];
+                auto original_height = m.height;
+                auto original_width = m.width;
+
+                // module is to the left of the roi
+                if (m.x + m.width < roi.xmin) {
+                    m.width = 0;
+
+                    // roi is in module
+                } else {
+                    // here we only arrive when the roi is in or to the left of
+                    // the module
+                    if (roi.xmin > m.x) {
+                        m.width -= roi.xmin - m.x;
+                    }
+                    if (roi.xmax < m.x + m.width) {
+                        m.width -= m.x + original_width - roi.xmax;
+                    }
+                    m.x = pos_x;
+                    pos_x += m.width;
+                }
+
+                if (m.y + m.height < roi.ymin) {
+                    m.height = 0;
+                } else {
+                    if (roi.ymin < m.y + m.height) {
+                        m.height -= roi.ymin;
+                    }
+                    if (roi.ymax < m.y + m.height) {
+                        m.height -= m.y + original_height - roi.ymax;
+                    }
+                    m.y = pos_y;
+                    pos_y_increment = m.height;
+                }
+            }
+            // increment pos_y
+            pos_y += pos_y_increment;
+        }
+
+        m_rows = roi.height();
+        m_cols = roi.width();
+    }
+
+#ifdef AARE_VERBOSE
+    fmt::print("RawFile::update_geometry_with_roi()\n");
+    for (const auto &m : m_module_pixel_0) {
+        fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x,
+                   m.height, m.width);
+    }
+    fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols);
+    fmt::print("\n");
+#endif
+
 }
 
 Frame RawFile::get_frame(size_t frame_index) {
@@ -146,13 +245,22 @@ Frame RawFile::get_frame(size_t frame_index) {
     return f;
 }
 
-void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
+
+size_t RawFile::bytes_per_pixel() const {
+    return m_master.bitdepth() / 8;
+}
+
+void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
+
     if (frame_index > total_frames()) {
         throw std::runtime_error(LOCATION + "Frame number out of range");
     }
     std::vector<size_t> frame_numbers(n_subfile_parts);
     std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
 
+
+    // sync the frame numbers
+
     if (n_subfile_parts != 1) {
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto subfile_id = frame_index / m_master.max_frames_per_file();
@@ -189,15 +297,25 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto corrected_idx = frame_indices[part_idx];
             auto subfile_id = corrected_idx / m_master.max_frames_per_file();
-            auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part();
-            subfiles[subfile_id][part_idx]->get_part(
-                frame_buffer + part_idx * part_offset,
-                corrected_idx % m_master.max_frames_per_file());
+
+            // This is where we start writing
+            auto offset = (m_module_pixel_0[part_idx].y * m_cols +
+                          m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8;
+
+            if (m_module_pixel_0[part_idx].x!=0)
+                throw std::runtime_error(LOCATION + "Implementation error. x pos not 0.");
+
+            subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
+            subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header);
+            if (header)
+                ++header;
         }
 
     } else {
+        //TODO! should we read row by row?
+
+        // create a buffer large enough to hold a full module
 
-        // create a buffer that will hold a the frame part
         auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
                               m_master.bitdepth() /
                               8; // TODO! replace with image_size_in_bytes
@@ -207,23 +325,27 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
         // level
 
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
+            auto pos = m_module_pixel_0[part_idx];
             auto corrected_idx = frame_indices[part_idx];
             auto subfile_id = corrected_idx / m_master.max_frames_per_file();
 
-            subfiles[subfile_id][part_idx]->get_part(
-                part_buffer, corrected_idx % m_master.max_frames_per_file());
-            for (size_t cur_row = 0; cur_row < (m_master.pixels_y());
+            subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file());
+            subfiles[subfile_id][part_idx]->read_into(part_buffer, header);
+            if(header)
+                ++header;
+
+            for (size_t cur_row = 0; cur_row < (pos.height);
                  cur_row++) {
-                auto irow = cur_row + (part_idx / m_master.geometry().col) *
-                                          m_master.pixels_y();
-                auto icol =
-                    (part_idx % m_master.geometry().col) * m_master.pixels_x();
+
+                auto irow = (pos.y + cur_row);
+                auto icol = pos.x;
                 auto dest = (irow * this->m_cols + icol);
                 dest = dest * m_master.bitdepth() / 8;
                 memcpy(frame_buffer + dest,
-                       part_buffer + cur_row * m_master.pixels_x() *
+                       part_buffer + cur_row * pos.width *
                                          m_master.bitdepth() / 8,
-                       m_master.pixels_x() * m_master.bitdepth() / 8);
+                       pos.width * m_master.bitdepth() / 8);
+
             }
         }
         delete[] part_buffer;

src/RawMasterFile.cpp

@@ -143,6 +143,10 @@ ScanParameters RawMasterFile::scan_parameters() const {
     return m_scan_parameters;
 }
 
+
+std::optional<ROI> RawMasterFile::roi() const { return m_roi; }
+
+
 void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
     std::ifstream ifs(fpath);
     json j;
@@ -240,6 +244,40 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
         // not a scan
     }
 
+
+    try{
+        ROI tmp_roi;
+        auto obj = j.at("Receiver Roi");
+        tmp_roi.xmin = obj.at("xmin");
+        tmp_roi.xmax = obj.at("xmax");
+        tmp_roi.ymin = obj.at("ymin");
+        tmp_roi.ymax = obj.at("ymax");
+
+        //if any of the values are set update the roi
+        if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 ||
+            tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) {
+            
+            if(v<7.21){
+                tmp_roi.xmax++;
+                tmp_roi.ymax++;
+            }
+            
+            m_roi = tmp_roi;
+        }
+
+
+    }catch (const json::out_of_range &e) {
+        // leave the optional empty
+    }
+
+    //if we have an roi we need to update the geometry for the subfiles
+    if (m_roi){
+
+    }
+
+
+
+
     // Update detector type for Moench
     // TODO! How does this work with old .raw master files?
     #ifdef AARE_VERBOSE

src/RawSubFile.cpp

@@ -0,0 +1,97 @@
+#include "aare/RawSubFile.hpp"
+#include "aare/PixelMap.hpp"
+#include <cstring> // memcpy
+#include <fmt/core.h>
+#include <iostream>
+
+namespace aare {
+
+RawSubFile::RawSubFile(const std::filesystem::path &fname,
+                       DetectorType detector, size_t rows, size_t cols,
+                       size_t bitdepth)
+    : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
+      m_detector_type(detector),
+      m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols) {
+    if (m_detector_type == DetectorType::Moench03_old) {
+        pixel_map = GenerateMoench03PixelMap();
+    }
+
+    if (std::filesystem::exists(fname)) {
+        n_frames = std::filesystem::file_size(fname) /
+                   (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
+    } else {
+        throw std::runtime_error(
+            LOCATION + fmt::format("File {} does not exist", m_fname.string()));
+    }
+
+    // fp = fopen(m_fname.string().c_str(), "rb");
+    m_file.open(m_fname, std::ios::binary);
+    if (!m_file.is_open()) {
+        throw std::runtime_error(
+            LOCATION + fmt::format("Could not open file {}", m_fname.string()));
+    }
+
+#ifdef AARE_VERBOSE
+    fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
+    fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols,
+               m_bitdepth);
+    fmt::print("file size: {}\n", std::filesystem::file_size(fname));
+#endif
+}
+
+void RawSubFile::seek(size_t frame_index) {
+    if (frame_index >= n_frames) {
+        throw std::runtime_error("Frame number out of range");
+    }
+    m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index);
+}
+
+size_t RawSubFile::tell() {
+    return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame());
+}
+
+
+void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
+    if(header){
+         m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
+    } else {
+        m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
+    }
+
+    //TODO! expand support for different bitdepths
+    if(pixel_map){
+        // read into a temporary buffer and then copy the data to the buffer
+        // in the correct order
+        // currently this only supports 16 bit data!
+        auto part_buffer = new std::byte[bytes_per_frame()];
+        m_file.read(reinterpret_cast<char *>(part_buffer), bytes_per_frame());
+        auto *data = reinterpret_cast<uint16_t *>(image_buf);
+        auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
+        for (size_t i = 0; i < pixels_per_frame(); i++) {
+            data[i] = part_data[(*pixel_map)(i)];
+        }
+        delete[] part_buffer;
+    } else {
+        // read directly into the buffer
+        m_file.read(reinterpret_cast<char *>(image_buf), bytes_per_frame());
+    }
+}
+
+size_t RawSubFile::rows() const { return m_rows; }
+size_t RawSubFile::cols() const { return m_cols; }
+
+
+void RawSubFile::get_part(std::byte *buffer, size_t frame_index) {
+    seek(frame_index);
+    read_into(buffer, nullptr);
+}
+
+size_t RawSubFile::frame_number(size_t frame_index) {
+    seek(frame_index);
+    DetectorHeader h{};
+    m_file.read(reinterpret_cast<char *>(&h), sizeof(DetectorHeader));
+    return h.frameNumber;
+}
+
+
+} // namespace aare

src/defs.cpp

@@ -2,8 +2,18 @@
 #include <stdexcept>
 #include <string>
 
+#include <fmt/core.h>
 namespace aare {
 
+
+void assert_failed(const std::string &msg)
+ {
+    fmt::print(msg);
+    exit(1);
+}
+
+
+
 /**
  * @brief Convert a DetectorType to a string
  * @param type DetectorType