added CtbRawFile

CMakeLists.txt

@@ -237,7 +237,7 @@ set(PUBLICHEADERS
     include/aare/Dtype.hpp
     include/aare/File.hpp
     include/aare/FileInterface.hpp
-    include/aare/file_utils.hpp
+    include/aare/RawMasterFile.hpp
     include/aare/Frame.hpp
     include/aare/json.hpp
     include/aare/NDArray.hpp
@@ -259,7 +259,7 @@ set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp
@@ -290,7 +290,7 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/file_utils.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDArray.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NDView.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinder.test.cpp

docs/CMakeLists.txt

@@ -22,6 +22,7 @@ set(SPHINX_SOURCE_FILES
   src/VarClusterFinder.rst
   src/pyVarClusterFinder.rst
   src/pyFile.rst
+  src/pyCtbRawFile.rst
 )
 
 foreach(filename ${SPHINX_SOURCE_FILES})

docs/src/index.rst

@@ -23,4 +23,6 @@ AARE
     :maxdepth: 1
     
     pyFile
+    pyCtbRawFile
     pyVarClusterFinder
+

docs/src/pyCtbRawFile.rst

@@ -0,0 +1,11 @@
+
+CtbRawFile
+============
+
+.. py:currentmodule:: aare
+
+.. autoclass:: CtbRawFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

include/aare/CtbRawFile.hpp

@@ -1,19 +1,37 @@
 #pragma once
 
 #include "aare/FileInterface.hpp"
-#include "aare/file_utils.hpp"
+#include "aare/RawMasterFile.hpp"
 #include "aare/Frame.hpp"
 
 #include <filesystem>
+#include <fstream>
 
 namespace aare{
 
 class CtbRawFile{
-    FileNameComponents m_fnc{};
+    RawMasterFile m_master;
+    std::ifstream m_file;
+    size_t m_current_frame{0};
+    size_t m_current_subfile{0};
+    size_t m_num_subfiles{0};
 public:
     CtbRawFile(const std::filesystem::path &fname);
 
+    void read_into(std::byte *image_buf, DetectorHeader* header = nullptr);
+    void seek(size_t frame_index);          //!< seek to the given frame index
+    size_t tell() const;                    //!< get the frame index of the file pointer
 
+    // in the specific class we can expose more functionality
+
+    size_t image_size_in_bytes() const { return m_master.image_size_in_bytes(); }
+
+private:
+    void find_subfiles();
+    size_t sub_file_index(size_t frame_index) const {
+        return frame_index / m_master.max_frames_per_file();
+    }
+    void open_data_file(size_t subfile_index);
 
 };
 

include/aare/RawFile.hpp

@@ -180,9 +180,9 @@ class RawFile : public FileInterface {
     /**
      * @brief read the header of the file
      * @param fname path to the data subfile
-     * @return sls_detector_header
+     * @return DetectorHeader
      */
-    static sls_detector_header read_header(const std::filesystem::path &fname);
+    static DetectorHeader read_header(const std::filesystem::path &fname);
 
     /**
      * @brief open the subfiles

include/aare/RawMasterFile.hpp

@@ -0,0 +1,96 @@
+#pragma once
+#include "aare/defs.hpp"
+#include <filesystem>
+#include <fmt/format.h>
+#include <fstream>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+namespace aare {
+
+class RawFileNameComponents {
+    std::filesystem::path m_base_path{};
+    std::string m_base_name{};
+    std::string m_ext{};
+    int m_file_index{}; // TODO! is this measurement_index?
+
+  public:
+    RawFileNameComponents(const std::filesystem::path &fname);
+    std::filesystem::path master_fname() const {
+        return m_base_path /
+               fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
+    }
+
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
+        return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name,
+                                         mod_id, file_id, m_file_index);
+    }
+
+    const std::filesystem::path &base_path() const { return m_base_path; }
+    const std::string &base_name() const { return m_base_name; }
+    const std::string &ext() const { return m_ext; }
+    int file_index() const { return m_file_index; }
+};
+
+class RawMasterFile {
+    RawFileNameComponents m_fnc;
+    std::string m_version;
+    DetectorType m_type;
+    TimingMode m_timing_mode;
+
+    size_t m_image_size_in_bytes;
+    size_t m_frames_in_file;
+    size_t m_pixels_y;
+    size_t m_pixels_x;
+    size_t m_bitdepth;
+
+    size_t m_max_frames_per_file;
+    uint32_t m_adc_mask;
+    FrameDiscardPolicy m_frame_discard_policy;
+    size_t m_frame_padding;
+
+    std::optional<size_t> m_analog_samples;
+    std::optional<size_t> m_digital_samples;
+
+  public:
+    RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) {
+        if (!std::filesystem::exists(fpath)) {
+            throw std::runtime_error(LOCATION + " File does not exist");
+        }
+        if (m_fnc.ext() == ".json") {
+            parse_json(fpath);
+        } else if (m_fnc.ext() == ".raw") {
+            parse_raw(fpath);
+        } else {
+            throw std::runtime_error(LOCATION + "Unsupported file type");
+        }
+    }
+
+    const std::string &version() const { return m_version; }
+    const DetectorType &detector_type() const { return m_type; }
+    const TimingMode &timing_mode() const { return m_timing_mode; }
+    size_t image_size_in_bytes() const { return m_image_size_in_bytes; }
+    size_t frames_in_file() const { return m_frames_in_file; }
+    size_t pixels_y() const { return m_pixels_y; }
+    size_t pixels_x() const { return m_pixels_x; }
+    size_t max_frames_per_file() const { return m_max_frames_per_file; }
+    size_t bitdepth() const { return m_bitdepth; }
+    size_t frame_padding() const { return m_frame_padding; }
+    const FrameDiscardPolicy &frame_discard_policy() const {
+        return m_frame_discard_policy;
+    }
+
+    std::optional<size_t> analog_samples() const { return m_analog_samples; }
+    std::optional<size_t> digital_samples() const { return m_digital_samples; }
+
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
+        return m_fnc.data_fname(mod_id, file_id);
+    }
+
+  private:
+    void parse_json(const std::filesystem::path &fpath);
+    void parse_raw(const std::filesystem::path &fpath);
+};
+
+} // namespace aare

include/aare/SubFile.hpp

@@ -15,7 +15,7 @@ namespace aare {
  */
 class SubFile {
   public:
-    size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
+    size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index);
     /**
      * @brief SubFile constructor
      * @param fname path to the subfile

include/aare/defs.hpp

@@ -111,7 +111,7 @@ class Cluster {
 /**
  * @brief header contained in parts of frames
  */
-struct sls_detector_header {
+struct DetectorHeader {
     uint64_t frameNumber;
     uint32_t expLength;
     uint32_t packetNumber;
@@ -166,6 +166,7 @@ using xy = t_xy<uint32_t>;
 
 using dynamic_shape = std::vector<int64_t>;
 
+//TODO! Can we uniform enums between the libraries?
 enum class DetectorType {
     Jungfrau,
     Eiger,
@@ -178,6 +179,7 @@ enum class DetectorType {
 };
 
 enum class TimingMode { Auto, Trigger };
+enum class FrameDiscardPolicy { NoDiscard, Discard };
 
 template <class T> T StringTo(const std::string &arg) { return T(arg); }
 
@@ -188,6 +190,8 @@ template <> std::string ToString(DetectorType arg);
 
 template <> TimingMode StringTo(const std::string & /*mode*/);
 
+template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
+
 using DataTypeVariants = std::variant<uint16_t, uint32_t>;
 
 } // namespace aare

include/aare/file_utils.hpp

@@ -1,119 +0,0 @@
-#pragma once
-#include "aare/defs.hpp"
-#include <filesystem>
-#include <fmt/format.h>
-#include <fstream>
-
-#include <nlohmann/json.hpp>
-using json = nlohmann::json;
-
-namespace aare {
-bool is_master_file(const std::filesystem::path &fpath);
-
-
-struct FileNameComponents {
-    std::filesystem::path base_path{};
-    std::string base_name{};
-    std::string ext{};
-    int findex{};
-    bool valid{false}; // TODO! how do we do error handling?
-
-    std::filesystem::path master_fname() const {
-        return base_path /
-               fmt::format("{}_master_{}{}", base_name, findex, ext);
-    }
-
-    std::filesystem::path data_fname(size_t mod_id, size_t file_id) {
-        return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, file_id,
-                                       mod_id, findex);
-    }
-};
-
-FileNameComponents parse_fname(const std::filesystem::path &fname);
-
-class MasterFile {
-    FileNameComponents m_fnc;
-    std::string m_version;
-    DetectorType m_type;
-    TimingMode m_timing_mode;
-    size_t m_total_frames;
-    size_t m_subfile_rows;
-    size_t m_subfile_cols;
-    size_t m_bitdepth;
-    size_t m_analog_samples;
-    size_t m_digital_samples;
-    size_t m_max_frames_per_file;
-    uint32_t m_adc_mask;
-public:
-    MasterFile(const std::filesystem::path &fpath) {
-        m_fnc = parse_fname(fpath);
-
-        
-
-        std::ifstream ifs(fpath);
-        json j;
-        ifs >> j;
-        double v = j["Version"];
-        m_version = fmt::format("{:.1f}", v);
-
-        m_type = StringTo<DetectorType>(j["Detector// Type"].get<std::string>()); 
-        m_timing_mode =
-        StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
-        m_total_frames = j["Frames in File"];
-        m_subfile_rows = j["Pixels"]["y"];
-        m_subfile_cols = j["Pixels"]["x"];
-        m_max_frames_per_file = j["Max Frames Per File"];
-        try {
-            m_bitdepth = j.at("Dynamic Range");
-        } catch (const json::out_of_range &e) {
-            m_bitdepth = 16;
-        }
-
-        try {
-            m_analog_samples = j.at("Analog Samples");
-        }catch (const json::out_of_range &e) {
-            m_analog_samples = 0;
-        }
-        try{
-            std::string adc_mask = j.at("ADC Mask");
-            m_adc_mask = std::stoul(adc_mask, nullptr, 16);
-        }catch (const json::out_of_range &e) {
-            m_adc_mask = 0;
-        }
-
-        try {
-            m_digital_samples = j.at("Digital Samples");
-            }catch (const json::out_of_range &e) {
-            m_digital_samples = 0;
-        }
-
-        //Update detector type for Moench
-        //TODO! How does this work with old .raw master files?
-        if (m_type == DetectorType::Moench && m_analog_samples == 0 &&
-        m_subfile_rows == 400) {
-            m_type = DetectorType::Moench03;
-        }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 &&
-        m_analog_samples == 5000) {
-            m_type = DetectorType::Moench03_old;
-        }
-
-        // //Here we know we have a ChipTestBoard file update the geometry?
-        // //TODO! Carry on information about digtial, and transceivers
-        // if (m_type == DetectorType::ChipTestBoard) {
-        //    subfile_rows = 1;
-        //    subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
-        // }
-
-        // // only Eiger had quad
-        // if (m_type == DetectorType::Eiger) {
-        //     quad = (j["Quad"] == 1);
-        // }
-
-        // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
-    }
-
-};
-
-
-
-} // namespace aare

python/CMakeLists.txt

@@ -27,6 +27,8 @@ target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
 # List of python files to be copied to the build directory
 set( PYTHON_FILES
     aare/__init__.py
+    aare/CtbRawFile.py
+    aare/transform.py
 )
 
 # Copy the python files to the build directory

python/aare/CtbRawFile.py

@@ -0,0 +1,69 @@
+
+from . import _aare
+import numpy as np
+
+class CtbRawFile(_aare.CtbRawFile):
+    def __init__(self, fname, transform = None):
+        super().__init__(fname)
+        self.transform = transform
+
+
+    def read_frame(self, frame_index = None):
+        """Read one frame from the file.
+
+        Args:
+            frame_index (int): If not None, seek to this frame before reading.
+
+        Returns:
+            tuple: header, data
+        """
+        if frame_index is not None:
+            self.seek(frame_index)
+
+        header, data = super().read_frame()
+        if header.shape == (1,):
+            header = header[0]
+
+
+        if self.transform:
+            res = self.transform(data)
+            if isinstance(res, tuple):
+                return header, *res
+            else:
+                return header, res
+            
+    def read_n(self, n_frames):
+        # Do the first read to figure out what we have
+        tmp_header, tmp_data = self.read_frame()
+        
+        # Allocate arrays for
+        header = np.zeros(n_frames, dtype = tmp_header.dtype)
+        data = np.zeros((n_frames, *tmp_data.shape), dtype = tmp_data.dtype)
+        
+        # Copy the first frame
+        header[0] = tmp_header
+        data[0] = tmp_data
+
+        # Do the rest of the reading
+        for i in range(1, n_frames):
+            header[i], data[i] = self.read_frame()
+
+        return header, data
+
+
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __iter__(self):
+        return self
+    
+    def __next__(self):
+        try:
+            return self.read_frame()
+        except RuntimeError:
+            # TODO! find a good way to check that we actually have the right exception
+            raise StopIteration
+

python/aare/__init__.py

@@ -1,4 +1,5 @@
 # Make the compiled classes that live in _aare available from aare.
-from ._aare import File
+from . import _aare
-from ._aare import VarClusterFinder
+
-from ._aare import GenerateMoench03PixelMap, GenerateMoench05PixelMap
+from ._aare import VarClusterFinder, File
+from .CtbRawFile import CtbRawFile

python/aare/transform.py

@@ -0,0 +1,15 @@
+import numpy as np
+from . import _aare
+
+
+class Moench05Transform:
+    #Could be moved to C++ without changing the interface
+    def __init__(self):
+        print('map created')
+        self.pixel_map = _aare.GenerateMoench05PixelMap()
+
+    def __call__(self, data):
+        return np.take(data.view(np.uint16), self.pixel_map)
+
+
+moench05 = Moench05Transform()

python/examples/play.py

@@ -3,6 +3,10 @@ import numpy as np
 plt.ion()
 
 import aare
+from aare import CtbRawFile
+print('aare imported')
+from aare import transform
+print('transform imported')
 from pathlib import Path
 
 # p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json')
@@ -15,11 +19,14 @@ from pathlib import Path
 
 
 # fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json')
-fpath = Path('/Users/erik/data/Moench05/moench05_master_0.json')
+fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json')
-f = aare.File(fpath)
-f.seek(437)
-frame = f.read_frame()
 
-m = aare.GenerateMoench05PixelMap()
+
-img = np.take(frame, m.astype(np.int64))
+# f = aare.CtbRawFile(fpath, transform = transform.moench05)
+
+
+
+with CtbRawFile(fpath, transform = transform.moench05) as f:
+    for header, image in f:
+        print(f'Frame number: {header["frameNumber"]}')
 

python/src/file.hpp

@@ -1,32 +1,63 @@
-#include "aare/Frame.hpp"
-#include "aare/File.hpp"
 #include "aare/CtbRawFile.hpp"
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
 #include "aare/defs.hpp"
 // #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
 #include <filesystem>
-#include <pybind11/numpy.h>
 #include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 #include <pybind11/stl/filesystem.h>
 #include <string>
 
 namespace py = pybind11;
-using namespace::aare;
+using namespace ::aare;
 
 void define_file_io_bindings(py::module &m) {
 
+    PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
+                         bunchId, timestamp, modId, row, column, reserved,
+                         debug, roundRNumber, detType, version, packetMask);
+
     py::class_<CtbRawFile>(m, "CtbRawFile")
-        .def(py::init<const std::filesystem::path &>());
+        .def(py::init<const std::filesystem::path &>())
-        
+        .def("read_frame",
+             [](CtbRawFile &self) {
+                 size_t image_size = self.image_size_in_bytes();
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(1);
+                 shape.push_back(image_size);
+
+                 py::array_t<DetectorHeader> header(1);
+
+
+                 // always read bytes
+                 image = py::array_t<uint8_t>(shape);
+
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def("seek", &CtbRawFile::seek)
+        .def("tell", &CtbRawFile::tell);
 
     py::class_<File>(m, "File")
-        .def(py::init([](const std::filesystem::path &fname) { return File(fname, "r", {}); }))
+        .def(py::init([](const std::filesystem::path &fname) {
-        .def(
+            return File(fname, "r", {});
-            py::init([](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); }))
+        }))
-        .def(py::init<const std::filesystem::path &, const std::string &, const FileConfig &>())
+        .def(py::init(
+            [](const std::filesystem::path &fname, const std::string &mode) {
+                return File(fname, mode, {});
+            }))
+        .def(py::init<const std::filesystem::path &, const std::string &,
+                      const FileConfig &>())
 
         .def("frame_number", &File::frame_number)
         .def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
@@ -38,44 +69,48 @@ void define_file_io_bindings(py::module &m) {
         .def_property_readonly("cols", &File::cols)
         .def_property_readonly("bitdepth", &File::bitdepth)
         .def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
-        .def_property_readonly("detector_type", [](File &self){
+        .def_property_readonly(
-            return ToString(self.detector_type());
+            "detector_type",
-        })
+            [](File &self) { return ToString(self.detector_type()); })
-        .def("read_frame", [](File &self) {
+        .def("read_frame",
-            const uint8_t item_size = self.bytes_per_pixel();
+             [](File &self) {
-            py::array image;
+                 const uint8_t item_size = self.bytes_per_pixel();
-            std::vector<ssize_t> shape;
+                 py::array image;
-            shape.reserve(2);
+                 std::vector<ssize_t> shape;
-            shape.push_back(self.rows());
+                 shape.reserve(2);
-            shape.push_back(self.cols());
+                 shape.push_back(self.rows());
-            if (item_size == 1) {
+                 shape.push_back(self.cols());
-                image = py::array_t<uint8_t>(shape);
+                 if (item_size == 1) {
-            } else if (item_size == 2) {
+                     image = py::array_t<uint8_t>(shape);
-                image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 2) {
-            } else if (item_size == 4) {
+                     image = py::array_t<uint16_t>(shape);
-                image = py::array_t<uint32_t>(shape);
+                 } else if (item_size == 4) {
-            }
+                     image = py::array_t<uint32_t>(shape);
-            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
+                 }
-            return image;
+                 self.read_into(
-        })
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
-        .def("read_frame", [](File &self, size_t frame_number) {
+                 return image;
-            self.seek(frame_number);
+             })
-            const uint8_t item_size = self.bytes_per_pixel();
+        .def("read_frame",
-            py::array image;
+             [](File &self, size_t frame_number) {
-            std::vector<ssize_t> shape;
+                 self.seek(frame_number);
-            shape.reserve(2);
+                 const uint8_t item_size = self.bytes_per_pixel();
-            shape.push_back(self.rows());
+                 py::array image;
-            shape.push_back(self.cols());
+                 std::vector<ssize_t> shape;
-            if (item_size == 1) {
+                 shape.reserve(2);
-                image = py::array_t<uint8_t>(shape);
+                 shape.push_back(self.rows());
-            } else if (item_size == 2) {
+                 shape.push_back(self.cols());
-                image = py::array_t<uint16_t>(shape);
+                 if (item_size == 1) {
-            } else if (item_size == 4) {
+                     image = py::array_t<uint8_t>(shape);
-                image = py::array_t<uint32_t>(shape);
+                 } else if (item_size == 2) {
-            }
+                     image = py::array_t<uint16_t>(shape);
-            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
+                 } else if (item_size == 4) {
-            return image;
+                     image = py::array_t<uint32_t>(shape);
-        })
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()));
+                 return image;
+             })
         .def("read_n", [](File &self, size_t n_frames) {
             const uint8_t item_size = self.bytes_per_pixel();
             py::array image;
@@ -91,7 +126,8 @@ void define_file_io_bindings(py::module &m) {
             } else if (item_size == 4) {
                 image = py::array_t<uint32_t>(shape);
             }
-            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()), n_frames);
+            self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()),
+                           n_frames);
             return image;
         });
 
@@ -107,37 +143,44 @@ void define_file_io_bindings(py::module &m) {
         .def_readwrite("dtype", &FileConfig::dtype)
         .def("__eq__", &FileConfig::operator==)
         .def("__ne__", &FileConfig::operator!=)
-        .def("__repr__", [](const FileConfig &a) { return "<FileConfig: " + a.to_string() + ">"; });
+        .def("__repr__", [](const FileConfig &a) {
+            return "<FileConfig: " + a.to_string() + ">";
+        });
 
     // py::class_<ClusterHeader>(m, "ClusterHeader")
     //     .def(py::init<>())
     //     .def_readwrite("frame_number", &ClusterHeader::frame_number)
     //     .def_readwrite("n_clusters", &ClusterHeader::n_clusters)
-    //     .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader: " + a.to_string() + ">"; });
+    //     .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader:
+    //     " + a.to_string() + ">"; });
 
     // py::class_<ClusterV2_>(m, "ClusterV2_")
     //     .def(py::init<>())
     //     .def_readwrite("x", &ClusterV2_::x)
     //     .def_readwrite("y", &ClusterV2_::y)
     //     .def_readwrite("data", &ClusterV2_::data)
-    //     .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " + a.to_string(false) + ">"; });
+    //     .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " +
+    //     a.to_string(false) + ">"; });
 
     // py::class_<ClusterV2>(m, "ClusterV2")
     //     .def(py::init<>())
     //     .def_readwrite("cluster", &ClusterV2::cluster)
     //     .def_readwrite("frame_number", &ClusterV2::frame_number)
-    //     .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " + a.to_string() + ">"; });
+    //     .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " +
+    //     a.to_string() + ">"; });
 
     // py::class_<ClusterFileV2>(m, "ClusterFileV2")
     //     .def(py::init<const std::filesystem::path &, const std::string &>())
     //     .def("read", py::overload_cast<>(&ClusterFileV2::read))
     //     .def("read", py::overload_cast<int>(&ClusterFileV2::read))
     //     .def("frame_number", &ClusterFileV2::frame_number)
-    //     .def("write", py::overload_cast<std::vector<ClusterV2> const &>(&ClusterFileV2::write))
+    //     .def("write", py::overload_cast<std::vector<ClusterV2> const
-        
+    //     &>(&ClusterFileV2::write))
+
     //     .def("close", &ClusterFileV2::close);
 
-    // m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int frame_number) {
+    // m.def("to_clustV2", [](std::vector<Cluster> &clusters, const int
+    // frame_number) {
     //     std::vector<ClusterV2> clusters_;
     //     for (auto &c : clusters) {
     //         ClusterV2 cluster;

src/CtbRawFile.cpp

@@ -1,22 +1,72 @@
 #include "aare/CtbRawFile.hpp"
 
+#include <fmt/format.h>
+namespace aare {
 
+CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) {
 
-namespace aare{
+    
 
-CtbRawFile::CtbRawFile(const std::filesystem::path &fname){
+    if (m_master.detector_type() != DetectorType::ChipTestBoard) {
+        throw std::runtime_error(LOCATION + "Not a Ctb file");
+    }
 
-    if(!std::filesystem::exists(fname)){
+    find_subfiles();
-        throw std::runtime_error(LOCATION + "File does not exist");
+
+    // open the first subfile
+    m_file.open(m_master.data_fname(0, 0), std::ios::binary);
+}
+
+size_t CtbRawFile::tell() const { return m_current_frame; }
+
+void CtbRawFile::seek(size_t frame_number) {
+    if (auto index = sub_file_index(frame_number); index != m_current_subfile) {
+        open_data_file(index);
+    }
+    size_t frame_number_in_file = frame_number % m_master.max_frames_per_file();
+    m_file.seekg((sizeof(DetectorHeader)+m_master.image_size_in_bytes()) * frame_number_in_file);
+    m_current_frame = frame_number;
+}
+
+void CtbRawFile::find_subfiles() {
+    // we can semi safely assume that there is only one module for CTB
+    while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles)))
+        m_num_subfiles++;
+
+    fmt::print("Found {} subfiles\n", m_num_subfiles);
+}
+
+void CtbRawFile::open_data_file(size_t subfile_index) {
+    if (subfile_index >= m_num_subfiles) {
+        throw std::runtime_error(LOCATION + "Subfile index out of range");
+    }
+    m_current_subfile = subfile_index;
+    m_file = std::ifstream(m_master.data_fname(0, subfile_index), std::ios::binary); // only one module for CTB
+    if (!m_file.is_open()) {
+        throw std::runtime_error(LOCATION + "Could not open data file");
+    }
+}
+
+void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) {
+    if(m_current_frame >= m_master.frames_in_file()){
+        throw std::runtime_error(LOCATION + "End of file reached");
+    }
+
+    if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){
+        open_data_file(m_current_subfile+1);
     }
     
-    m_fnc = parse_fname(fname);
+    if(header){
-    if(!m_fnc.valid){
+        m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
-        throw std::runtime_error(LOCATION + "Could not parse master file name");
+    }else{
+        m_file.seekg(sizeof(DetectorHeader), std::ios::cur);
     }
 
+    m_file.read(reinterpret_cast<char *>(image_buf), m_master.image_size_in_bytes());
+    m_current_frame++;
+
+    
 
 }
 
-
 } // namespace aare

src/RawFile.cpp

@@ -45,8 +45,8 @@ void RawFile::open_subfiles() {
     }
 }
 
-sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
+DetectorHeader RawFile::read_header(const std::filesystem::path &fname) {
-    sls_detector_header h{};
+    DetectorHeader h{};
     FILE *fp = fopen(fname.string().c_str(), "r");
     if (!fp)
         throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));

src/RawMasterFile.cpp

@@ -0,0 +1,103 @@
+#include "aare/RawMasterFile.hpp"
+
+namespace aare {
+
+
+RawFileNameComponents::RawFileNameComponents(const std::filesystem::path &fname) {
+    m_base_path = fname.parent_path();
+    m_base_name = fname.stem();
+    m_ext = fname.extension();
+
+    //parse file index
+    try {
+        auto pos = m_base_name.rfind('_');
+        m_file_index = std::stoi(m_base_name.substr(pos + 1));
+    } catch (const std::invalid_argument &e) {
+        throw std::runtime_error(LOCATION + "Could not parse file index");
+    }
+
+    //remove master from base name
+    auto pos = m_base_name.find("_master_");
+    if (pos != std::string::npos) {
+        m_base_name.erase(pos);
+    }else{
+        throw std::runtime_error(LOCATION + "Could not find _master_ in file name");
+    }
+}
+
+void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
+        std::ifstream ifs(fpath);
+        json j;
+        ifs >> j;
+        double v = j["Version"];
+        m_version = fmt::format("{:.1f}", v);
+
+        m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>()); 
+        m_timing_mode =
+        StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
+
+        m_image_size_in_bytes = j["Image Size in bytes"];
+        m_frames_in_file = j["Frames in File"];
+        m_pixels_y = j["Pixels"]["y"];
+        m_pixels_x = j["Pixels"]["x"];
+
+        m_max_frames_per_file = j["Max Frames Per File"];
+
+        //Not all detectors write the bitdepth but in case
+        //its not there it is 16
+        try {
+            m_bitdepth = j.at("Dynamic Range");
+        } catch (const json::out_of_range &e) {
+            m_bitdepth = 16;
+        }
+
+        m_frame_padding = j["Frame Padding"];
+        m_frame_discard_policy = StringTo<FrameDiscardPolicy>(
+            j["Frame Discard Policy"].get<std::string>());
+
+        try {
+            m_analog_samples = j.at("Analog Samples");
+        }catch (const json::out_of_range &e) {
+            // m_analog_samples = 0;
+        }
+        // try{
+        //     std::string adc_mask = j.at("ADC Mask");
+        //     m_adc_mask = std::stoul(adc_mask, nullptr, 16);
+        // }catch (const json::out_of_range &e) {
+        //     m_adc_mask = 0;
+        // }
+
+        try {
+            m_digital_samples = j.at("Digital Samples");
+        }catch (const json::out_of_range &e) {
+            // m_digital_samples = 0;
+        }
+
+        // //Update detector type for Moench
+        // //TODO! How does this work with old .raw master files?
+        // if (m_type == DetectorType::Moench && m_analog_samples == 0 &&
+        // m_subfile_rows == 400) {
+        //     m_type = DetectorType::Moench03;
+        // }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 &&
+        // m_analog_samples == 5000) {
+        //     m_type = DetectorType::Moench03_old;
+        // }
+
+        // //Here we know we have a ChipTestBoard file update the geometry?
+        // //TODO! Carry on information about digtial, and transceivers
+        // if (m_type == DetectorType::ChipTestBoard) {
+        //    subfile_rows = 1;
+        //    subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask);
+        // }
+
+        // // only Eiger had quad
+        // if (m_type == DetectorType::Eiger) {
+        //     quad = (j["Quad"] == 1);
+        // }
+
+        // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
+}
+void RawMasterFile::parse_raw(const std::filesystem::path &fpath) {
+    throw std::runtime_error("Not implemented");
+}
+} // namespace aare

src/RawMasterFile.test.cpp

@@ -0,0 +1,167 @@
+#include "aare/RawMasterFile.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+#include "test_config.hpp"
+
+using namespace aare;
+
+
+TEST_CASE("Parse a master file fname"){
+    RawFileNameComponents m("test_master_1.json");
+    REQUIRE(m.base_name() == "test");
+    REQUIRE(m.ext() == ".json");
+    REQUIRE(m.file_index() == 1);
+}
+
+TEST_CASE("Construction of master file name and data files"){
+    RawFileNameComponents m("test_master_1.json");
+    REQUIRE(m.master_fname() == "test_master_1.json");
+    REQUIRE(m.data_fname(0, 0) == "test_d0_f0_1.raw");
+    REQUIRE(m.data_fname(1, 0) == "test_d1_f0_1.raw");
+    REQUIRE(m.data_fname(0, 1) == "test_d0_f1_1.raw");
+    REQUIRE(m.data_fname(1, 1) == "test_d1_f1_1.raw");
+}
+
+
+TEST_CASE("Parse a master file"){
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawMasterFile f(fpath);
+
+    // "Version": 7.2,
+    REQUIRE(f.version() == "7.2");
+    // "Timestamp": "Tue Feb 20 08:28:24 2024",
+    // "Detector Type": "Jungfrau",
+    REQUIRE(f.detector_type() == DetectorType::Jungfrau);
+    // "Timing Mode": "auto",
+    REQUIRE(f.timing_mode() == TimingMode::Auto);
+    // "Geometry": {
+    //     "x": 1,
+    //     "y": 1
+    // },
+
+    // "Image Size in bytes": 1048576,
+    REQUIRE(f.image_size_in_bytes() == 1048576);
+    // "Pixels": {
+    //     "x": 1024,
+    REQUIRE(f.pixels_x() == 1024);
+    //     "y": 512
+    REQUIRE(f.pixels_y() == 512);
+    // },
+
+    // "Max Frames Per File": 3,
+    REQUIRE(f.max_frames_per_file() == 3);
+
+    //Jungfrau doesn't write but it is 16
+    REQUIRE(f.bitdepth() == 16);
+    // "Frame Discard Policy": "nodiscard",
+    // "Frame Padding": 1,
+    // "Scan Parameters": "[disabled]",
+    // "Total Frames": 10,
+    // "Receiver Roi": {
+    //     "xmin": 4294967295,
+    //     "xmax": 4294967295,
+    //     "ymin": 4294967295,
+    //     "ymax": 4294967295
+    // },
+    // "Exptime": "10us",
+    // "Period": "1ms",
+    // "Number of UDP Interfaces": 1,
+    // "Number of rows": 512,
+    // "Frames in File": 10,
+    // "Frame Header Format": {
+    //     "Frame Number": "8 bytes",
+    //     "SubFrame Number/ExpLength": "4 bytes",
+    //     "Packet Number": "4 bytes",
+    //     "Bunch ID": "8 bytes",
+    //     "Timestamp": "8 bytes",
+    //     "Module Id": "2 bytes",
+    //     "Row": "2 bytes",
+    //     "Column": "2 bytes",
+    //     "Reserved": "2 bytes",
+    //     "Debug": "4 bytes",
+    //     "Round Robin Number": "2 bytes",
+    //     "Detector Type": "1 byte",
+    //     "Header Version": "1 byte",
+    //     "Packets Caught Mask": "64 bytes"
+    // }
+    // }         
+
+    REQUIRE_FALSE(f.analog_samples());
+    REQUIRE_FALSE(f.digital_samples());                
+
+}
+
+
+TEST_CASE("Read eiger master file"){
+auto fpath = test_data_path() / "eiger" / "eiger_500k_32bit_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+    RawMasterFile f(fpath);
+    
+// {
+//     "Version": 7.2,
+REQUIRE(f.version() == "7.2");
+//     "Timestamp": "Tue Mar 26 17:24:34 2024",
+//     "Detector Type": "Eiger",
+REQUIRE(f.detector_type() == DetectorType::Eiger);
+//     "Timing Mode": "auto",
+REQUIRE(f.timing_mode() == TimingMode::Auto);
+//     "Geometry": {
+//         "x": 2,
+//         "y": 2
+//     },
+//     "Image Size in bytes": 524288,
+REQUIRE(f.image_size_in_bytes() == 524288);
+//     "Pixels": {
+//         "x": 512,
+REQUIRE(f.pixels_x() == 512);
+//         "y": 256
+REQUIRE(f.pixels_y() == 256);
+//     },
+//     "Max Frames Per File": 10000,
+REQUIRE(f.max_frames_per_file() == 10000);
+//     "Frame Discard Policy": "nodiscard",
+REQUIRE(f.frame_discard_policy() == FrameDiscardPolicy::NoDiscard);
+//     "Frame Padding": 1,
+REQUIRE(f.frame_padding() == 1);
+
+//     "Scan Parameters": "[disabled]",
+//     "Total Frames": 3,
+//     "Receiver Roi": {
+//         "xmin": 4294967295,
+//         "xmax": 4294967295,
+//         "ymin": 4294967295,
+//         "ymax": 4294967295
+//     },
+//     "Dynamic Range": 32,
+//     "Ten Giga": 0,
+//     "Exptime": "5s",
+//     "Period": "1s",
+//     "Threshold Energy": -1,
+//     "Sub Exptime": "2.62144ms",
+//     "Sub Period": "2.62144ms",
+//     "Quad": 0,
+//     "Number of rows": 256,
+//     "Rate Corrections": "[0, 0]",
+//     "Frames in File": 3,
+//     "Frame Header Format": {
+//         "Frame Number": "8 bytes",
+//         "SubFrame Number/ExpLength": "4 bytes",
+//         "Packet Number": "4 bytes",
+//         "Bunch ID": "8 bytes",
+//         "Timestamp": "8 bytes",
+//         "Module Id": "2 bytes",
+//         "Row": "2 bytes",
+//         "Column": "2 bytes",
+//         "Reserved": "2 bytes",
+//         "Debug": "4 bytes",
+//         "Round Robin Number": "2 bytes",
+//         "Detector Type": "1 byte",
+//         "Header Version": "1 byte",
+//         "Packets Caught Mask": "64 bytes"
+//     }
+// }            
+
+
+
+}

src/SubFile.cpp

@@ -17,7 +17,7 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size
     }
 
     if (std::filesystem::exists(fname)) {
-        n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
+        n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
     } else {
         n_frames = 0;
     }
@@ -42,7 +42,7 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
     if (frame_index >= n_frames) {
         throw std::runtime_error("Frame number out of range");
     }
-    fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT
+    fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT
           SEEK_SET);
 
     if (pixel_map){
@@ -63,11 +63,11 @@ size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
     }
     
 }
-size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) {
+size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) {
     if (frame_index > n_frames) {
         throw std::runtime_error("Frame number out of range");
     }
-    fseek(fp, static_cast<int64_t>((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET);
+    fseek(fp, static_cast<int64_t>((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET);
     auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
     wc += fwrite(buffer, bytes_per_part(), 1, fp);
 
@@ -75,8 +75,8 @@ size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t
 }
 
 size_t SubFile::frame_number(size_t frame_index) {
-    sls_detector_header h{};
+    DetectorHeader h{};
-    fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
+    fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
     size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
     if (rc != 1)
         throw std::runtime_error(LOCATION + "Could not read header from file");

src/defs.cpp

@@ -68,6 +68,14 @@ template <> TimingMode StringTo(const std::string &arg) {
     throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\"");
 }
 
+template <> FrameDiscardPolicy StringTo(const std::string &arg) {
+    if (arg == "nodiscard")
+        return FrameDiscardPolicy::NoDiscard;
+    if (arg == "discard")
+        return FrameDiscardPolicy::Discard;
+    throw std::runtime_error("Could not decode frame discard policy from: \"" + arg + "\"");
+}
+
 // template <> TimingMode StringTo<TimingMode>(std::string mode);
 
 } // namespace aare

src/file_utils.cpp

@@ -1,33 +0,0 @@
-#include "aare/file_utils.hpp"
-
-namespace aare {
-
-
-
-bool is_master_file(const std::filesystem::path &fpath) {
-    std::string const stem = fpath.stem().string();
-    return stem.find("_master_") != std::string::npos;
-}
-
-FileNameComponents parse_fname(const std::filesystem::path &fname) {
-    FileNameComponents fnc;
-    fnc.base_path = fname.parent_path();
-    fnc.base_name = fname.stem();
-    fnc.ext = fname.extension();
-    try {
-        auto pos = fnc.base_name.rfind('_');
-        fnc.findex = std::stoi(fnc.base_name.substr(pos + 1));
-    } catch (const std::invalid_argument &e) {
-        fnc.valid = false;
-    }
-    auto pos = fnc.base_name.find("_master_");
-    if (pos != std::string::npos) {
-        fnc.base_name.erase(pos);
-    }else{
-        fnc.valid = false;
-    }
-    fnc.valid = true;
-    return fnc;
-}
-
-} // namespace aare

src/file_utils.test.cpp

@@ -1,20 +0,0 @@
-#include "aare/file_utils.hpp"
-
-#include <catch2/catch_test_macros.hpp>
-
-using namespace aare;
-
-TEST_CASE("Use filename to determine if it is a master file") {
-
-    REQUIRE(is_master_file("test_master_1.json"));
-}
-
-TEST_CASE("Parse a master file fname"){
-    auto fnc = parse_fname("test_master_1.json");
-    REQUIRE(fnc.base_name == "test");
-    REQUIRE(fnc.ext == ".json");
-    REQUIRE(fnc.findex == 1);
-
-    REQUIRE(fnc.master_fname() == "test_master_1.json");
-    REQUIRE(fnc.data_fname(1, 2) == "test_d2_f1_1.raw");
-}