first draft of hdf5, reads master metadata, reads data dims, process of hyperslab

CMakeLists.txt

@@ -40,6 +40,7 @@ option(AARE_DOCS "Build documentation" OFF)
 option(AARE_VERBOSE "Verbose output" OFF)
 option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
 option(AARE_INSTALL_PYTHONEXT "Install the python extension in the install tree under CMAKE_INSTALL_PREFIX/aare/" OFF)
+option(AARE_HDF5 "Hdf5 File Format" OFF)
 
 
 # Configure which of the dependencies to use FetchContent for
@@ -309,6 +310,21 @@ set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp
 )
 
+# HDF5
+if (AARE_HDF5)
+    find_package(HDF5 1.10 COMPONENTS CXX REQUIRED)
+	    add_definitions( 
+	        ${HDF5_DEFINITIONS}
+	    )
+	    list (APPEND PUBLICHEADERS 
+            include/aare/Hdf5File.hpp
+            include/aare/Hdf5MasterFile.hpp
+	    )
+	    list (APPEND SourceFiles 
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5File.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5MasterFile.cpp
+	    )
+endif (AARE_HDF5)
 
 add_library(aare_core STATIC ${SourceFiles})
 target_include_directories(aare_core PUBLIC 
@@ -326,6 +342,16 @@ target_link_libraries(
     aare_compiler_flags 
 )
 
+if (AARE_HDF5 AND HDF5_FOUND)
+    add_definitions(-DHDF5_FOUND)
+    target_link_libraries(aare_core PUBLIC 
+        ${HDF5_LIBRARIES}
+    )
+    target_include_directories(aare_core PUBLIC
+        ${HDF5_INCLUDE_DIRS}
+    )
+endif()
+
 set_target_properties(aare_core PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
     PUBLIC_HEADER "${PUBLICHEADERS}"
@@ -348,13 +374,19 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
-
         )
+    if(HDF5_FOUND)
+        list (APPEND TestSources 
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5MasterFile.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5File.test.cpp
+        )
+    endif() 
     target_sources(tests PRIVATE ${TestSources} )
 endif()
 
 
 
+
 ###------------------------------------------------------------------------------------------
 ###------------------------------------------------------------------------------------------
 

cmake/Findh5py.cmake

@@ -0,0 +1,29 @@
+# Findh5py.cmake
+#
+# This module finds if h5py is installed and sets the H5PY_FOUND variable.
+# It also sets the H5PY_INCLUDE_DIRS and H5PY_LIBRARIES variables.
+
+find_package(PythonInterp REQUIRED)
+find_package(PythonLibs REQUIRED)
+
+execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} -c "import h5py"
+    RESULT_VARIABLE H5PY_IMPORT_RESULT
+    OUTPUT_QUIET
+    ERROR_QUIET
+)
+
+if(H5PY_IMPORT_RESULT EQUAL 0)
+    set(H5PY_FOUND TRUE)
+    execute_process(
+        COMMAND ${PYTHON_EXECUTABLE} -c "import h5py; print(h5py.get_include())"
+        OUTPUT_VARIABLE H5PY_INCLUDE_DIR
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    set(H5PY_INCLUDE_DIRS ${H5PY_INCLUDE_DIR})
+    set(H5PY_LIBRARIES ${PYTHON_LIBRARIES})
+else()
+    set(H5PY_FOUND FALSE)
+endif()
+
+mark_as_advanced(H5PY_INCLUDE_DIRS H5PY_LIBRARIES)

docs/CMakeLists.txt

@@ -27,12 +27,17 @@ file(GLOB SPHINX_SOURCE_FILES CONFIGURE_DEPENDS "src/*.rst")
 #   src/RawFile.rst
 #   src/RawSubFile.rst
 #   src/RawMasterFile.rst
+#   src/Hdf5File.rst
+#   src/Hdf5SubFile.rst
+#   src/Hdf5MasterFile.rst
 #   src/VarClusterFinder.rst
 #   src/pyVarClusterFinder.rst
 #   src/pyFile.rst
 #   src/pyCtbRawFile.rst
 #   src/pyRawFile.rst
 #   src/pyRawMasterFile.rst
+#   src/pyHdf5File.rst
+#   src/pyHdf5MasterFile.rst
 # )
 
 

docs/src/Hdf5File.rst

@@ -0,0 +1,8 @@
+Hdf5File
+===============
+
+
+.. doxygenclass:: aare::Hdf5File
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/Hdf5MasterFile.rst

@@ -0,0 +1,14 @@
+Hdf5MasterFile
+===============
+
+
+.. doxygenclass:: aare::Hdf5MasterFile
+   :members:
+   :undoc-members:
+   :private-members:
+
+
+.. doxygenclass:: aare::Hdf5FileNameComponents
+   :members:
+   :undoc-members:
+   :private-members:

docs/src/index.rst

@@ -32,6 +32,8 @@ AARE
     pyClusterFile
     pyRawFile
     pyRawMasterFile
+    pyHdf5File
+    pyHdf5MasterFile
     pyVarClusterFinder
 
 
@@ -50,6 +52,8 @@ AARE
     RawFile
     RawSubFile
     RawMasterFile
+    Hdf5File
+    Hdf5MasterFile
     VarClusterFinder
 
 

docs/src/pyHdf5File.rst

@@ -0,0 +1,10 @@
+Hdf5File
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: Hdf5File
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

docs/src/pyHdf5MasterFile.rst

@@ -0,0 +1,10 @@
+Hdf5MasterFile
+===================
+
+.. py:currentmodule:: aare
+
+.. autoclass:: Hdf5MasterFile
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:

include/aare/File.hpp

@@ -5,12 +5,12 @@
 namespace aare {
 
 /**
- * @brief RAII File class for reading, and in the future potentially writing 
- * image files in various formats. Minimal generic interface. For specail fuctions
- * plase use the RawFile or NumpyFile classes directly.
- * Wraps FileInterface to abstract the underlying file format
- * @note **frame_number** refers the the frame number sent by the detector while **frame_index**
- * is the position of the frame in the file
+ * @brief RAII File class for reading, and in the future potentially writing
+ * image files in various formats. Minimal generic interface. For specail
+ * fuctions plase use the RawFile, NumpyFile or Hdf5File classes directly. Wraps
+ * FileInterface to abstract the underlying file format
+ * @note **frame_number** refers the the frame number sent by the detector while
+ * **frame_index** is the position of the frame in the file
  */
 class File {
     std::unique_ptr<FileInterface> file_impl;

include/aare/FileInterface.hpp

@@ -41,8 +41,9 @@ struct FileConfig {
 
 /**
  * @brief FileInterface class to define the interface for file operations
- * @note parent class for NumpyFile and RawFile
- * @note all functions are pure virtual and must be implemented by the derived classes
+ * @note parent class for NumpyFile, RawFile and Hdf5File
+ * @note all functions are pure virtual and must be implemented by the derived
+ * classes
  */
 class FileInterface {
   public:

include/aare/Hdf5File.hpp

@@ -0,0 +1,96 @@
+#pragma once
+#include "aare/FileInterface.hpp"
+#include "aare/Frame.hpp"
+#include "aare/Hdf5MasterFile.hpp"
+#include "aare/NDArray.hpp" //for pixel map
+
+#include <optional>
+
+namespace aare {
+
+/**
+ * @brief Class to read .h5 files. The class will parse the master file
+ * to find the correct geometry for the frames.
+ * @note A more generic interface is available in the aare::File class.
+ * Consider using that unless you need hdf5 file specific functionality.
+ */
+class Hdf5File : public FileInterface {
+    Hdf5MasterFile m_master;
+
+    size_t m_current_frame{};
+    size_t m_total_frames{};
+    size_t m_rows{};
+    size_t m_cols{};
+    H5::DataType m_datatype{};
+
+    std::unique_ptr<H5::H5File> file{nullptr};
+    std::unique_ptr<H5::DataSet> dataset{nullptr};
+    std::unique_ptr<H5::DataSpace> dataspace{nullptr};
+
+  public:
+    /**
+     * @brief Hdf5File constructor
+     * @param fname path to the master file (.json)
+     * @param mode file mode (only "r" is supported at the moment)
+
+     */
+    Hdf5File(const std::filesystem::path &fname, const std::string &mode = "r");
+    virtual ~Hdf5File() override;
+
+    Frame read_frame() override;
+    Frame read_frame(size_t frame_number) override;
+    std::vector<Frame> read_n(size_t n_frames) override;
+    void read_into(std::byte *image_buf) override;
+    void read_into(std::byte *image_buf, size_t n_frames) override;
+
+    // TODO! do we need to adapt the API?
+    void read_into(std::byte *image_buf, DetectorHeader *header);
+    void read_into(std::byte *image_buf, size_t n_frames,
+                   DetectorHeader *header);
+
+    size_t frame_number(size_t frame_index) override;
+    size_t bytes_per_frame() override;
+    size_t pixels_per_frame() override;
+    size_t bytes_per_pixel() const;
+    void seek(size_t frame_index) override;
+    size_t tell() override;
+    size_t total_frames() const override;
+    size_t rows() const override;
+    size_t cols() const override;
+    size_t bitdepth() const override;
+    xy geometry();
+    size_t n_mod() const;
+
+    Hdf5MasterFile master() const;
+
+    DetectorType detector_type() const override;
+
+  private:
+    /**
+     * @brief read the frame at the given frame index into the image buffer
+     * @param frame_number frame number to read
+     * @param image_buf buffer to store the frame
+     */
+
+    void get_frame_into(size_t frame_index, std::byte *frame_buffer,
+                        DetectorHeader *header = nullptr);
+
+    /**
+     * @brief get the frame at the given frame index
+     * @param frame_number frame number to read
+     * @return Frame
+     */
+    Frame get_frame(size_t frame_index);
+
+    /**
+     * @brief read the header of the file
+     * @param fname path to the data subfile
+     * @return DetectorHeader
+     */
+    static DetectorHeader read_header(const std::filesystem::path &fname);
+
+    static const std::string metadata_group_name;
+    void open_file();
+};
+
+} // namespace aare

include/aare/Hdf5MasterFile.hpp

@@ -0,0 +1,173 @@
+#pragma once
+#include "H5Cpp.h"
+#include "aare/defs.hpp"
+#include <filesystem>
+#include <fmt/format.h>
+#include <fstream>
+#include <optional>
+
+namespace fmt {
+template <typename T> struct formatter<std::optional<T>> : formatter<T> {
+    template <typename FormatContext>
+    auto format(const std::optional<T> &opt, FormatContext &ctx) {
+        if (opt) {
+            return formatter<T>::format(*opt, ctx);
+        } else {
+            return format_to(ctx.out(), "nullopt");
+        }
+    }
+};
+} // namespace fmt
+
+namespace aare {
+
+/**
+ * @brief Implementation used in Hdf5MasterFile to parse the file name
+ */
+class Hdf5FileNameComponents {
+    bool m_old_scheme{false};
+    std::filesystem::path m_base_path{};
+    std::string m_base_name{};
+    std::string m_ext{};
+    int m_file_index{}; // TODO! is this measurement_index?
+
+  public:
+    Hdf5FileNameComponents(const std::filesystem::path &fname);
+
+    /// @brief Get the filename including path of the master file.
+    /// (i.e. what was passed in to the constructor))
+    std::filesystem::path master_fname() const;
+
+    /// @brief Get the filename including path of the data file.
+    /// @param mod_id module id run_d[module_id]_f0_0
+    /// @param file_id file id run_d0_f[file_id]_0
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
+
+    const std::filesystem::path &base_path() const;
+    const std::string &base_name() const;
+    const std::string &ext() const;
+    int file_index() const;
+    void set_old_scheme(bool old_scheme);
+};
+
+/*
+class ScanParameters {
+    bool m_enabled = false;
+    std::string m_dac;
+    int m_start = 0;
+    int m_stop = 0;
+    int m_step = 0;
+    // TODO! add settleTime, requires string to time conversion
+
+  public:
+    ScanParameters(const std::string &par);
+    ScanParameters() = default;
+    ScanParameters(const ScanParameters &) = default;
+    ScanParameters &operator=(const ScanParameters &) = default;
+    ScanParameters(ScanParameters &&) = default;
+    int start() const;
+    int stop() const;
+    int step() const;
+    const std::string &dac() const;
+    bool enabled() const;
+    void increment_stop();
+};
+
+struct ROI {
+    int64_t xmin{};
+    int64_t xmax{};
+    int64_t ymin{};
+    int64_t ymax{};
+
+    int64_t height() const { return ymax - ymin; }
+    int64_t width() const { return xmax - xmin; }
+};
+*/
+
+/**
+ * @brief Class for parsing a master file either in our .json format or the old
+ * .Hdf5 format
+ */
+class Hdf5MasterFile {
+    Hdf5FileNameComponents m_fnc;
+    std::string m_version;
+    DetectorType m_type;
+    TimingMode m_timing_mode;
+
+    size_t m_image_size_in_bytes{};
+    size_t m_frames_in_file{};
+    size_t m_total_frames_expected{};
+    size_t m_pixels_y{};
+    size_t m_pixels_x{};
+    size_t m_bitdepth{};
+
+    xy m_geometry{};
+
+    size_t m_max_frames_per_file{};
+    // uint32_t m_adc_mask{}; // TODO! implement reading
+    FrameDiscardPolicy m_frame_discard_policy{};
+    size_t m_frame_padding{};
+
+    // TODO! should these be bool?
+    uint8_t m_analog_flag{};
+    uint8_t m_digital_flag{};
+    uint8_t m_transceiver_flag{};
+
+    // ScanParameters m_scan_parameters;
+
+    std::optional<size_t> m_analog_samples;
+    std::optional<size_t> m_digital_samples;
+    std::optional<size_t> m_transceiver_samples;
+    std::optional<size_t> m_number_of_rows;
+    std::optional<uint8_t> m_quad;
+
+    // std::optional<ROI> m_roi;
+
+  public:
+    Hdf5MasterFile(const std::filesystem::path &fpath);
+
+    std::filesystem::path master_fname() const;
+    std::filesystem::path data_fname(size_t mod_id, size_t file_id) const;
+
+    const std::string &version() const; //!< For example "7.2"
+    const DetectorType &detector_type() const;
+    const TimingMode &timing_mode() const;
+    size_t image_size_in_bytes() const;
+    size_t frames_in_file() const;
+    size_t pixels_y() const;
+    size_t pixels_x() const;
+    size_t max_frames_per_file() const;
+    size_t bitdepth() const;
+    size_t frame_padding() const;
+    const FrameDiscardPolicy &frame_discard_policy() const;
+
+    size_t total_frames_expected() const;
+    xy geometry() const;
+
+    std::optional<size_t> analog_samples() const;
+    std::optional<size_t> digital_samples() const;
+    std::optional<size_t> transceiver_samples() const;
+    std::optional<size_t> number_of_rows() const;
+    std::optional<uint8_t> quad() const;
+
+    // std::optional<ROI> roi() const;
+
+    // ScanParameters scan_parameters() const;
+
+  private:
+    static const std::string metadata_group_name;
+    void parse_acquisition_metadata(const std::filesystem::path &fpath);
+
+    template <typename T>
+    T h5_read_scalar_dataset(const H5::DataSet &dataset,
+                             const H5::DataType &data_type);
+
+    template <typename T>
+    T h5_get_scalar_dataset(const H5::H5File &file,
+                            const std::string &dataset_name);
+};
+
+template <>
+std::string Hdf5MasterFile::h5_read_scalar_dataset<std::string>(
+    const H5::DataSet &dataset, const H5::DataType &data_type);
+} // namespace aare

include/aare/defs.hpp

@@ -225,8 +225,10 @@ template <> DetectorType StringTo(const std::string & /*name*/);
 template <> std::string ToString(DetectorType arg);
 
 template <> TimingMode StringTo(const std::string & /*mode*/);
+template <> std::string ToString(TimingMode arg);
 
 template <> FrameDiscardPolicy StringTo(const std::string & /*mode*/);
+template <> std::string ToString(FrameDiscardPolicy arg);
 
 using DataTypeVariants = std::variant<uint16_t, uint32_t>;
 

python/CMakeLists.txt

@@ -34,6 +34,31 @@ set( PYTHON_FILES
     aare/utils.py
 )
 
+# Conditionally add HDF5-related Python files
+# HDF5
+# if (AARE_HDF5)
+#     find_package(h5py REQUIRED)
+
+#     find_package(HDF5 1.10 COMPONENTS CXX REQUIRED)
+#     add_definitions( 
+#         ${HDF5_DEFINITIONS}
+#     )
+#     list(APPEND PYTHON_FILES
+#         aare/Hdf5File.py
+#     )
+#     if(HDF5_FOUND)
+#         target_sources(_aare PRIVATE
+#             ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5File.cpp
+#             ${CMAKE_CURRENT_SOURCE_DIR}/src/Hdf5MasterFile.cpp
+#         )
+#         target_link_libraries(_aare PUBLIC ${HDF5_LIBRARIES})
+#         target_include_directories(_aare PUBLIC ${HDF5_INCLUDE_DIRS})    
+#     endif()
+#     if(H5PY_FOUND)
+#         set(H5PY_INCLUDE_DIRS ${H5PY_INCLUDE_DIR})
+#         set(H5PY_LIBRARIES ${PYTHON_LIBRARIES})
+# endif()
+
 # Copy the python files to the build directory
 foreach(FILE ${PYTHON_FILES})
     configure_file(${FILE} ${CMAKE_BINARY_DIR}/${FILE}  )

python/aare/Hdf5File.py

@@ -0,0 +1,66 @@
+from . import _aare
+import numpy as np
+#from .ScanParameters import ScanParameters
+
+class Hdf5File(_aare.Hdf5File):
+    def __init__(self, fname, chunk_size = 1):
+        super().__init__(fname)
+        self._chunk_size = chunk_size
+
+
+    def read(self) -> tuple:
+        """Read the entire file.
+        Seeks to the beginning of the file before reading.
+
+        Returns:
+            tuple: header, data
+        """
+        self.seek(0)
+        return self.read_n(self.total_frames)
+
+    # @property
+    # def scan_parameters(self):
+    #     """Return the scan parameters.
+
+    #     Returns:
+    #         ScanParameters: Scan parameters.
+    #     """
+    #     return ScanParameters(self.master.scan_parameters)
+    
+    @property
+    def master(self):
+        """Return the master file.
+
+        Returns:
+            Hdf5MasterFile: Master file.
+        """
+        return super().master
+    
+    def __len__(self) -> int:
+        """Return the number of frames in the file.
+
+        Returns:
+            int: Number of frames in file.
+        """
+        return super().frames_in_file
+
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __iter__(self):
+        return self
+    
+    def __next__(self):
+        try:
+            if self._chunk_size == 1:
+                return self.read_frame()
+            else:
+                return self.read_n(self._chunk_size)
+            
+                
+        except RuntimeError:
+            # TODO! find a good way to check that we actually have the right exception
+            raise StopIteration

python/aare/__init__.py

@@ -11,4 +11,22 @@ from .CtbRawFile import CtbRawFile
 from .RawFile import RawFile
 from .ScanParameters import ScanParameters
 
-from .utils import random_pixels, random_pixel
+from .utils import random_pixels, random_pixel
+
+try:
+    import h5py
+    HDF5_FOUND = True
+except ImportError:
+    HDF5_FOUND = False
+
+if HDF5_FOUND:
+    from ._aare import Hdf5MasterFile
+    from .Hdf5File import Hdf5File
+else:
+    class Hdf5MasterFile:
+        def __init__(self, *args, **kwargs):
+            raise ImportError("h5py library not found. HDF5 Master File is not available.")
+        
+    class Hdf5File:
+        def __init__(self, *args, **kwargs):
+            raise ImportError("h5py library not found. HDF5 File is not available.")

python/src/file.hpp

@@ -5,6 +5,11 @@
 #include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
 
+#ifdef HDF5_FOUND
+#include "aare/Hdf5File.hpp"
+#include "aare/Hdf5MasterFile.hpp"
+#endif
+
 #include "aare/defs.hpp"
 // #include "aare/fClusterFileV2.hpp"
 

python/src/hdf5_file.hpp

@@ -0,0 +1,105 @@
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/Hdf5File.hpp"
+#include "aare/Hdf5MasterFile.hpp"
+
+#include "aare/defs.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_hdf5_file_io_bindings(py::module &m) {
+    py::class_<Hdf5File>(m, "Hdf5File")
+        .def(py::init<const std::filesystem::path &>())
+        .def("read_frame",
+             [](Hdf5File &self) {
+                 py::array image;
+                 std::vector<ssize_t> shape;
+                 shape.reserve(2);
+                 shape.push_back(self.rows());
+                 shape.push_back(self.cols());
+
+                 // return headers from all subfiles
+                 py::array_t<DetectorHeader> header(self.n_mod());
+
+                 const uint8_t item_size = self.bytes_per_pixel();
+                 if (item_size == 1) {
+                     image = py::array_t<uint8_t>(shape);
+                 } else if (item_size == 2) {
+                     image = py::array_t<uint16_t>(shape);
+                 } else if (item_size == 4) {
+                     image = py::array_t<uint32_t>(shape);
+                 }
+                 self.read_into(
+                     reinterpret_cast<std::byte *>(image.mutable_data()),
+                     header.mutable_data());
+
+                 return py::make_tuple(header, image);
+             })
+        .def(
+            "read_n",
+            [](Hdf5File &self, size_t n_frames) {
+                // adjust for actual frames left in the file
+                n_frames =
+                    std::min(n_frames, self.total_frames() - self.tell());
+                if (n_frames == 0) {
+                    throw std::runtime_error("No frames left in file");
+                }
+                std::vector<size_t> shape{n_frames, self.rows(), self.cols()};
+
+                // return headers from all subfiles
+                py::array_t<DetectorHeader> header;
+                if (self.n_mod() == 1) {
+                    header = py::array_t<DetectorHeader>(n_frames);
+                } else {
+                    header =
+                        py::array_t<DetectorHeader>({self.n_mod(), n_frames});
+                }
+                // py::array_t<DetectorHeader> header({self.n_mod(), n_frames});
+
+                py::array image;
+                const uint8_t item_size = self.bytes_per_pixel();
+                if (item_size == 1) {
+                    image = py::array_t<uint8_t>(shape);
+                } else if (item_size == 2) {
+                    image = py::array_t<uint16_t>(shape);
+                } else if (item_size == 4) {
+                    image = py::array_t<uint32_t>(shape);
+                }
+                self.read_into(
+                    reinterpret_cast<std::byte *>(image.mutable_data()),
+                    n_frames, header.mutable_data());
+
+                return py::make_tuple(header, image);
+            },
+            R"(
+             Read n frames from the file.
+             )")
+        .def("frame_number", &Hdf5File::frame_number)
+        .def_property_readonly("bytes_per_frame", &Hdf5File::bytes_per_frame)
+        .def_property_readonly("pixels_per_frame", &Hdf5File::pixels_per_frame)
+        .def_property_readonly("bytes_per_pixel", &Hdf5File::bytes_per_pixel)
+        .def("seek", &Hdf5File::seek, R"(
+            Seek to a frame index in file.
+            )")
+        .def("tell", &Hdf5File::tell, R"(
+            Return the current frame number.)")
+        .def_property_readonly("total_frames", &Hdf5File::total_frames)
+        .def_property_readonly("rows", &Hdf5File::rows)
+        .def_property_readonly("cols", &Hdf5File::cols)
+        .def_property_readonly("bitdepth", &Hdf5File::bitdepth)
+        .def_property_readonly("geometry", &Hdf5File::geometry)
+        .def_property_readonly("n_mod", &Hdf5File::n_mod)
+        .def_property_readonly("detector_type", &Hdf5File::detector_type)
+        .def_property_readonly("master", &Hdf5File::master);
+}

python/src/hdf5_master_file.hpp

@@ -0,0 +1,86 @@
+
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/Hdf5File.hpp"
+#include "aare/Hdf5MasterFile.hpp"
+
+#include "aare/defs.hpp"
+// #include "aare/fClusterFileV2.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <pybind11/iostream.h>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+#include <string>
+
+namespace py = pybind11;
+using namespace ::aare;
+
+void define_hdf5_master_file_bindings(py::module &m) {
+    py::class_<Hdf5MasterFile>(m, "Hdf5MasterFile")
+        .def(py::init<const std::filesystem::path &>())
+        .def("data_fname", &Hdf5MasterFile::data_fname, R"(
+
+            Parameters
+            ------------
+                module_index : int 
+                        module index (d0, d1 .. dN)
+                file_index : int 
+                        file index (f0, f1 .. fN)
+
+            Returns
+            ----------
+                os.PathLike
+                        The name of the data file.
+
+            )")
+        .def_property_readonly("version", &Hdf5MasterFile::version)
+        .def_property_readonly("detector_type", &Hdf5MasterFile::detector_type)
+        .def_property_readonly("timing_mode", &Hdf5MasterFile::timing_mode)
+        .def_property_readonly("image_size_in_bytes",
+                               &Hdf5MasterFile::image_size_in_bytes)
+        .def_property_readonly("frames_in_file",
+                               &Hdf5MasterFile::frames_in_file)
+        .def_property_readonly("pixels_y", &Hdf5MasterFile::pixels_y)
+        .def_property_readonly("pixels_x", &Hdf5MasterFile::pixels_x)
+        .def_property_readonly("max_frames_per_file",
+                               &Hdf5MasterFile::max_frames_per_file)
+        .def_property_readonly("bitdepth", &Hdf5MasterFile::bitdepth)
+        .def_property_readonly("frame_padding", &Hdf5MasterFile::frame_padding)
+        .def_property_readonly("frame_discard_policy",
+                               &Hdf5MasterFile::frame_discard_policy)
+
+        .def_property_readonly("total_frames_expected",
+                               &Hdf5MasterFile::total_frames_expected)
+        .def_property_readonly("geometry", &Hdf5MasterFile::geometry)
+        .def_property_readonly("analog_samples",
+                               &Hdf5MasterFile::analog_samples, R"(
+            Number of analog samples
+
+            Returns
+            ----------
+                int | None
+                    The number of analog samples in the file (or None if not enabled)
+            )")
+        .def_property_readonly("digital_samples",
+                               &Hdf5MasterFile::digital_samples, R"(
+            Number of digital samples
+
+            Returns
+            ----------
+                int | None
+                    The number of digital samples in the file (or None if not enabled)
+            )")
+
+        .def_property_readonly("transceiver_samples",
+                               &Hdf5MasterFile::transceiver_samples)
+        .def_property_readonly("number_of_rows",
+                               &Hdf5MasterFile::number_of_rows)
+        .def_property_readonly("quad", &Hdf5MasterFile::quad)
+        .def_property_readonly("scan_parameters",
+                               &Hdf5MasterFile::scan_parameters)
+        .def_property_readonly("roi", &Hdf5MasterFile::roi);
+}

python/src/module.cpp

@@ -3,6 +3,10 @@
 #include "raw_file.hpp"
 #include "ctb_raw_file.hpp"
 #include "raw_master_file.hpp"
+#ifdef HDF5_FOUND
+#include "hdf5_file.hpp"
+#include "hdf5_master_file.hpp"
+#endif
 #include "var_cluster.hpp"
 #include "pixel_map.hpp"
 #include "pedestal.hpp"
@@ -20,6 +24,10 @@ PYBIND11_MODULE(_aare, m) {
     define_raw_file_io_bindings(m);
     define_ctb_raw_file_io_bindings(m);
     define_raw_master_file_bindings(m);
+#ifdef HDF5_FOUND
+    define_hdf5_file_io_bindings(m);
+    define_hdf5_master_file_bindings(m);
+#endif
     define_var_cluster_finder_bindings(m);
     define_pixel_map_bindings(m);
     define_pedestal_bindings<double>(m, "Pedestal");

src/File.cpp

@@ -1,4 +1,7 @@
 #include "aare/File.hpp"
+#ifdef HDF5_FOUND
+#include "aare/Hdf5File.hpp"
+#endif
 #include "aare/NumpyFile.hpp"
 #include "aare/RawFile.hpp"
 
@@ -27,7 +30,17 @@ File::File(const std::filesystem::path &fname, const std::string &mode,
     else if (fname.extension() == ".npy") {
         // file_impl = new NumpyFile(fname, mode, cfg);
         file_impl = std::make_unique<NumpyFile>(fname, mode, cfg);
-    } else {
+    }
+#ifdef HDF5_FOUND
+    else if (fname.extension() == ".h5") {
+        file_impl = std::make_unique<Hdf5File>(fname, mode);
+    }
+#else
+    else if (fname.extension() == ".h5") {
+        throw std::runtime_error("Enable HDF5 compile option: AARE_HDF5=ON");
+    }
+#endif
+    else {
         throw std::runtime_error("Unsupported file type");
     }
 }

src/Hdf5File.cpp

@@ -0,0 +1,198 @@
+#include "aare/Hdf5File.hpp"
+#include "aare/PixelMap.hpp"
+#include "aare/defs.hpp"
+
+#include <fmt/format.h>
+
+namespace aare {
+
+Hdf5File::Hdf5File(const std::filesystem::path &fname, const std::string &mode)
+    : m_master(fname) {
+    m_mode = mode;
+    if (mode == "r") {
+        open_file();
+    } else {
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported mode. Can only read Hdf5Files.");
+    }
+}
+
+Frame Hdf5File::read_frame() { return get_frame(m_current_frame++); };
+
+Frame Hdf5File::read_frame(size_t frame_number) {
+    seek(frame_number);
+    return read_frame();
+}
+
+void Hdf5File::read_into(std::byte *image_buf, size_t n_frames) {
+    // TODO: implement this in a more efficient way
+
+    for (size_t i = 0; i < n_frames; i++) {
+        this->get_frame_into(m_current_frame++, image_buf);
+        image_buf += bytes_per_frame();
+    }
+}
+
+void Hdf5File::read_into(std::byte *image_buf) {
+    return get_frame_into(m_current_frame++, image_buf);
+};
+
+void Hdf5File::read_into(std::byte *image_buf, DetectorHeader *header) {
+
+    return get_frame_into(m_current_frame++, image_buf, header);
+};
+
+void Hdf5File::read_into(std::byte *image_buf, size_t n_frames,
+                         DetectorHeader *header) {
+    // return get_frame_into(m_current_frame++, image_buf, header);
+
+    for (size_t i = 0; i < n_frames; i++) {
+        this->get_frame_into(m_current_frame++, image_buf, header);
+        image_buf += bytes_per_frame();
+        if (header)
+            header += n_mod();
+    }
+};
+
+size_t Hdf5File::n_mod() const { return 1; } // n_subfile_parts; }
+
+size_t Hdf5File::bytes_per_frame() {
+    return m_rows * m_cols * m_master.bitdepth() / 8;
+}
+size_t Hdf5File::pixels_per_frame() { return m_rows * m_cols; }
+
+DetectorType Hdf5File::detector_type() const {
+    return m_master.detector_type();
+}
+
+void Hdf5File::seek(size_t frame_index) {
+    // check if the frame number is greater than the total frames
+    // if frame_number == total_frames, then the next read will throw an error
+    if (frame_index > total_frames()) {
+        throw std::runtime_error(
+            fmt::format("frame number {} is greater than total frames {}",
+                        frame_index, total_frames()));
+    }
+    m_current_frame = frame_index;
+};
+
+size_t Hdf5File::tell() { return m_current_frame; };
+
+size_t Hdf5File::total_frames() const { return m_total_frames; }
+size_t Hdf5File::rows() const { return m_rows; }
+size_t Hdf5File::cols() const { return m_cols; }
+size_t Hdf5File::bitdepth() const { return m_master.bitdepth(); }
+xy Hdf5File::geometry() { return m_master.geometry(); }
+
+DetectorHeader Hdf5File::read_header(const std::filesystem::path &fname) {
+    DetectorHeader h{};
+    FILE *fp = fopen(fname.string().c_str(), "r");
+    if (!fp)
+        throw std::runtime_error(
+            fmt::format("Could not open: {} for reading", fname.string()));
+
+    size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
+    if (rc != 1)
+        throw std::runtime_error(LOCATION + "Could not read header from file");
+    if (fclose(fp)) {
+        throw std::runtime_error(LOCATION + "Could not close file");
+    }
+
+    return h;
+}
+
+Hdf5MasterFile Hdf5File::master() const { return m_master; }
+
+Frame Hdf5File::get_frame(size_t frame_index) {
+    auto f = Frame(m_rows, m_cols, Dtype::from_bitdepth(m_master.bitdepth()));
+    std::byte *frame_buffer = f.data();
+    get_frame_into(frame_index, frame_buffer);
+    return f;
+}
+
+size_t Hdf5File::bytes_per_pixel() const { return m_master.bitdepth() / 8; }
+
+void Hdf5File::get_frame_into(size_t frame_index, std::byte *frame_buffer,
+                              DetectorHeader *header) {
+
+    // Check if the frame number is valid
+    if (frame_index < 0 || frame_index >= m_total_frames) {
+        throw std::runtime_error(LOCATION + "Invalid frame number");
+    }
+
+    // Define the hyperslab to select the 2D slice for the given frame number
+    hsize_t offset[3] = {static_cast<hsize_t>(frame_index), 0, 0};
+    hsize_t count[3] = {1, m_rows, m_cols};
+    dataspace->selectHyperslab(H5S_SELECT_SET, count, offset);
+
+    // Define the memory space for the 2D slice
+    hsize_t dimsm[2] = {m_rows, m_cols};
+    H5::DataSpace memspace(2, dimsm);
+
+    // Read the data into the provided 2D array
+    dataset->read(frame_buffer, m_datatype, memspace, dataspace);
+
+    fmt::print("Read 2D data for frame {}\n", frame_index);
+}
+
+std::vector<Frame> Hdf5File::read_n(size_t n_frames) {
+    // TODO: implement this in a more efficient way
+    std::vector<Frame> frames;
+    for (size_t i = 0; i < n_frames; i++) {
+        frames.push_back(this->get_frame(m_current_frame));
+        m_current_frame++;
+    }
+    return frames;
+}
+
+size_t Hdf5File::frame_number(size_t frame_index) {
+    if (frame_index >= m_master.frames_in_file()) {
+        throw std::runtime_error(LOCATION + " Frame number out of range");
+    }
+    size_t subfile_id = frame_index / m_master.max_frames_per_file();
+    /*if (subfile_id >= subfiles.size()) {
+        throw std::runtime_error(
+            LOCATION + " Subfile out of range. Possible missing data.");
+    }*/
+    return 1; // subfiles[subfile_id][0]->frame_number(
+              // frame_index % m_master.max_frames_per_file());
+}
+
+Hdf5File::~Hdf5File() {}
+
+const std::string Hdf5File::metadata_group_name = "/entry/data/";
+
+void Hdf5File::open_file() {
+    if (m_mode != "r")
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported mode. Can only read Hdf5 files.");
+    try {
+        file = std::make_unique<H5::H5File>(m_master.master_fname().string(),
+                                            H5F_ACC_RDONLY);
+        dataset = std::make_unique<H5::DataSet>(
+            file->openDataSet(metadata_group_name + "/data"));
+        dataspace = std::make_unique<H5::DataSpace>(dataset->getSpace());
+        int rank = dataspace->getSimpleExtentNdims();
+        if (rank != 3) {
+            throw std::runtime_error(
+                LOCATION + "Expected rank of '/data' dataset to be 3. Got " +
+                std::to_string(rank));
+        }
+        hsize_t dims[3];
+        dataspace->getSimpleExtentDims(dims, nullptr);
+        m_total_frames = dims[0];
+        m_rows = dims[1];
+        m_cols = dims[2];
+        m_datatype = dataset->getDataType();
+        fmt::print("Dataset dimensions: frames = {}, rows = {}, cols = {}\n",
+                   m_total_frames, m_rows, m_cols);
+    } catch (const H5::Exception &e) {
+        file->close();
+        fmt::print("Exception type: {}\n", typeid(e).name());
+        e.printErrorStack();
+        throw std::runtime_error(
+            LOCATION + "\nCould not to access 'data' dataset in master file.");
+    }
+}
+
+} // namespace aare

src/Hdf5MasterFile.cpp

@@ -0,0 +1,460 @@
+#include "aare/Hdf5MasterFile.hpp"
+#include <sstream>
+namespace aare {
+
+Hdf5FileNameComponents::Hdf5FileNameComponents(
+    const std::filesystem::path &fname) {
+    m_base_path = fname.parent_path();
+    m_base_name = fname.stem();
+    m_ext = fname.extension();
+
+    if (m_ext != ".h5") {
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported file type. (only .h5)");
+    }
+
+    // parse file index
+    try {
+        auto pos = m_base_name.rfind('_');
+        m_file_index = std::stoi(m_base_name.substr(pos + 1));
+    } catch (const std::invalid_argument &e) {
+        throw std::runtime_error(LOCATION + "Could not parse file index");
+    }
+
+    // remove master from base name
+    auto pos = m_base_name.find("_master_");
+    if (pos != std::string::npos) {
+        m_base_name.erase(pos);
+    } else {
+        throw std::runtime_error(LOCATION +
+                                 "Could not find _master_ in file name");
+    }
+}
+
+std::filesystem::path Hdf5FileNameComponents::master_fname() const {
+    return m_base_path /
+           fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext);
+}
+
+std::filesystem::path Hdf5FileNameComponents::data_fname(size_t mod_id,
+                                                         size_t file_id) const {
+
+    std::string fmt = "{}_d{}_f{}_{}.h5";
+    // Before version X we used to name the data files f000000000000
+    if (m_old_scheme) {
+        fmt = "{}_d{}_f{:012}_{}.h5";
+    }
+    return m_base_path /
+           fmt::format(fmt, m_base_name, mod_id, file_id, m_file_index);
+}
+
+void Hdf5FileNameComponents::set_old_scheme(bool old_scheme) {
+    m_old_scheme = old_scheme;
+}
+
+const std::filesystem::path &Hdf5FileNameComponents::base_path() const {
+    return m_base_path;
+}
+const std::string &Hdf5FileNameComponents::base_name() const {
+    return m_base_name;
+}
+const std::string &Hdf5FileNameComponents::ext() const { return m_ext; }
+int Hdf5FileNameComponents::file_index() const { return m_file_index; }
+
+// "[enabled\ndac dac 4\nstart 500\nstop 2200\nstep 5\nsettleTime 100us\n]"
+/*ScanParameters::ScanParameters(const std::string &par) {
+    std::istringstream iss(par.substr(1, par.size() - 2));
+    std::string line;
+    while (std::getline(iss, line)) {
+        if (line == "enabled") {
+            m_enabled = true;
+        } else if (line.find("dac") != std::string::npos) {
+            m_dac = line.substr(4);
+        } else if (line.find("start") != std::string::npos) {
+            m_start = std::stoi(line.substr(6));
+        } else if (line.find("stop") != std::string::npos) {
+            m_stop = std::stoi(line.substr(5));
+        } else if (line.find("step") != std::string::npos) {
+            m_step = std::stoi(line.substr(5));
+        }
+    }
+}
+
+int ScanParameters::start() const { return m_start; }
+int ScanParameters::stop() const { return m_stop; }
+void ScanParameters::increment_stop() { m_stop += 1; };
+//int ScanParameters::step() const { return m_step; }
+const std::string &ScanParameters::dac() const { return m_dac; }
+bool ScanParameters::enabled() const { return m_enabled; }
+*/
+Hdf5MasterFile::Hdf5MasterFile(const std::filesystem::path &fpath)
+    : m_fnc(fpath) {
+    if (!std::filesystem::exists(fpath)) {
+        throw std::runtime_error(LOCATION + " File does not exist");
+    }
+    if (m_fnc.ext() == ".h5") {
+        parse_acquisition_metadata(fpath);
+    } else {
+        throw std::runtime_error(LOCATION + "Unsupported file type");
+    }
+}
+
+std::filesystem::path Hdf5MasterFile::master_fname() const {
+    return m_fnc.master_fname();
+}
+
+std::filesystem::path Hdf5MasterFile::data_fname(size_t mod_id,
+                                                 size_t file_id) const {
+    return m_fnc.data_fname(mod_id, file_id);
+}
+
+const std::string &Hdf5MasterFile::version() const { return m_version; }
+const DetectorType &Hdf5MasterFile::detector_type() const { return m_type; }
+const TimingMode &Hdf5MasterFile::timing_mode() const { return m_timing_mode; }
+size_t Hdf5MasterFile::image_size_in_bytes() const {
+    return m_image_size_in_bytes;
+}
+size_t Hdf5MasterFile::frames_in_file() const { return m_frames_in_file; }
+size_t Hdf5MasterFile::pixels_y() const { return m_pixels_y; }
+size_t Hdf5MasterFile::pixels_x() const { return m_pixels_x; }
+size_t Hdf5MasterFile::max_frames_per_file() const {
+    return m_max_frames_per_file;
+}
+size_t Hdf5MasterFile::bitdepth() const { return m_bitdepth; }
+size_t Hdf5MasterFile::frame_padding() const { return m_frame_padding; }
+const FrameDiscardPolicy &Hdf5MasterFile::frame_discard_policy() const {
+    return m_frame_discard_policy;
+}
+
+size_t Hdf5MasterFile::total_frames_expected() const {
+    return m_total_frames_expected;
+}
+
+std::optional<size_t> Hdf5MasterFile::number_of_rows() const {
+    return m_number_of_rows;
+}
+
+xy Hdf5MasterFile::geometry() const { return m_geometry; }
+
+std::optional<uint8_t> Hdf5MasterFile::quad() const { return m_quad; }
+
+// optional values, these may or may not be present in the master file
+// and are therefore modeled as std::optional
+std::optional<size_t> Hdf5MasterFile::analog_samples() const {
+    return m_analog_samples;
+}
+std::optional<size_t> Hdf5MasterFile::digital_samples() const {
+    return m_digital_samples;
+}
+
+std::optional<size_t> Hdf5MasterFile::transceiver_samples() const {
+    return m_transceiver_samples;
+}
+
+/*
+ScanParameters Hdf5MasterFile::scan_parameters() const {
+    return m_scan_parameters;
+}
+*/
+
+// std::optional<ROI> Hdf5MasterFile::roi() const { return m_roi; }
+
+const std::string Hdf5MasterFile::metadata_group_name =
+    "/entry/instrument/detector/";
+
+template <typename T>
+T Hdf5MasterFile::h5_read_scalar_dataset(const H5::DataSet &dataset,
+                                         const H5::DataType &data_type) {
+    T value;
+    dataset.read(&value, data_type);
+    return value;
+}
+
+template <>
+std::string Hdf5MasterFile::h5_read_scalar_dataset<std::string>(
+    const H5::DataSet &dataset, const H5::DataType &data_type) {
+    char buffer[257]{0};
+    dataset.read(buffer, data_type);
+    return std::string(buffer);
+}
+
+template <typename T>
+T Hdf5MasterFile::h5_get_scalar_dataset(const H5::H5File &file,
+                                        const std::string &dataset_name) {
+    H5::DataSet dataset = file.openDataSet(dataset_name);
+    H5::DataSpace dataspace = dataset.getSpace();
+    if (dataspace.getSimpleExtentNdims() != 0) {
+        throw std::runtime_error(LOCATION + "Expected " + dataset_name +
+                                 " to be a scalar dataset");
+    }
+    H5::DataType data_type = dataset.getDataType();
+    return h5_read_scalar_dataset<T>(dataset, data_type);
+}
+
+void Hdf5MasterFile::parse_acquisition_metadata(
+    const std::filesystem::path &fpath) {
+    try {
+        H5::H5File file(fpath, H5F_ACC_RDONLY);
+
+        // Attribute - version
+        {
+            H5::Attribute attr = file.openAttribute("version");
+            H5::DataType attr_type = attr.getDataType();
+            double value{0.0};
+            attr.read(attr_type, &value);
+            std::ostringstream oss;
+            oss << std::fixed << std::setprecision(1) << value;
+            m_version = oss.str();
+            // fmt::print("Version: {}\n", m_version);
+        }
+
+        // Scalar Dataset
+        // Detector Type
+        m_type = StringTo<DetectorType>(h5_get_scalar_dataset<std::string>(
+            file, std::string(metadata_group_name + "Detector Type")));
+        // fmt::print("Detector Type: {}\n", (ToString(m_type)));
+
+        // Timing Mode
+        m_timing_mode = StringTo<TimingMode>(h5_get_scalar_dataset<std::string>(
+            file, std::string(metadata_group_name + "Timing Mode")));
+        // fmt::print("Timing Mode: {}\n", (ToString(m_timing_mode)));
+
+        // Geometry
+        m_geometry.row = h5_get_scalar_dataset<int>(
+            file, std::string(metadata_group_name + "Geometry in y axis"));
+        m_geometry.col = h5_get_scalar_dataset<int>(
+            file, std::string(metadata_group_name + "Geometry in x axis"));
+        // fmt::print("Geometry: {}\n", m_geometry.to_string());
+
+        // Image Size
+        m_image_size_in_bytes = h5_get_scalar_dataset<int>(
+            file, std::string(metadata_group_name + "Image Size"));
+        // fmt::print("Image size: {}\n", m_image_size_in_bytes);
+
+        // Frames in File
+        m_frames_in_file = h5_get_scalar_dataset<uint64_t>(
+            file, std::string(metadata_group_name + "Frames in File"));
+        // fmt::print("Frames in File: {}\n", m_frames_in_file);
+
+        // Pixels
+        m_pixels_y = h5_get_scalar_dataset<int>(
+            file,
+            std::string(metadata_group_name + "Number of pixels in y axis"));
+        // fmt::print("Pixels in y: {}\n", m_pixels_y);
+        m_pixels_x = h5_get_scalar_dataset<int>(
+            file,
+            std::string(metadata_group_name + "Number of pixels in x axis"));
+        // fmt::print("Pixels in x: {}\n", m_pixels_x);
+
+        // Max Frames per File
+        m_max_frames_per_file = h5_get_scalar_dataset<int>(
+            file, std::string(metadata_group_name + "Maximum frames per file"));
+        // fmt::print("Max frames per File: {}\n", m_max_frames_per_file);
+
+        // Bit Depth
+        // Not all detectors write the bitdepth but in case
+        // its not there it is 16
+        H5::Exception::dontPrint();
+        try {
+            m_bitdepth = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Dynamic Range"));
+        } catch (H5::FileIException &e) {
+            m_bitdepth = 16;
+        }
+        // fmt::print("Bit Depth: {}\n", m_bitdepth);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // Total Frames
+        m_total_frames_expected = h5_get_scalar_dataset<uint64_t>(
+            file, std::string(metadata_group_name + "Total Frames"));
+        // fmt::print("Total Frames: {}\n", m_total_frames_expected);
+
+        // Frame Padding
+        m_frame_padding = h5_get_scalar_dataset<int>(
+            file, std::string(metadata_group_name + "Frame Padding"));
+        // fmt::print("Frame Padding: {}\n", m_frame_padding);
+
+        // Frame Discard Policy
+        m_frame_discard_policy =
+            StringTo<FrameDiscardPolicy>(h5_get_scalar_dataset<std::string>(
+                file,
+                std::string(metadata_group_name + "Frame Discard Policy")));
+        // fmt::print("Frame Discard Policy: {}\n",
+        // (ToString(m_frame_discard_policy)));
+
+        // Number of rows
+        // Not all detectors write the Number of rows but in case
+        H5::Exception::dontPrint();
+        try {
+            m_number_of_rows = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Number of rows"));
+        } catch (H5::FileIException &e) {
+            // keep the optional empty
+        }
+        // fmt::print("Number of rows: {}\n", m_number_of_rows);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // Analog Flag
+        // ----------------------------------------------------------------
+        // Special treatment of analog flag because of Moench03
+        H5::Exception::dontPrint();
+        try {
+            m_analog_flag = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Analog Flag"));
+        } catch (H5::FileIException &e) {
+            // if it doesn't work still set it to one
+            // to try to decode analog samples (Old Moench03)
+            m_analog_flag = 1;
+        }
+        // fmt::print("Analog Flag: {}\n", m_analog_flag);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // Analog Samples
+        H5::Exception::dontPrint();
+        try {
+            if (m_analog_flag) {
+                m_analog_samples = h5_get_scalar_dataset<int>(
+                    file, std::string(metadata_group_name + "Analog Samples"));
+            }
+        } catch (H5::FileIException &e) {
+            // keep the optional empty
+            // and set analog flag to 0
+            m_analog_flag = 0;
+        }
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+        // fmt::print("Analog Samples: {}\n", m_analog_samples);
+        //-----------------------------------------------------------------
+
+        // Quad
+        H5::Exception::dontPrint();
+        try {
+            m_quad = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Quad"));
+        } catch (H5::FileIException &e) {
+            // keep the optional empty
+        }
+        // fmt::print("Quad: {}\n", m_quad);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // ADC Mask
+        // H5::Exception::dontPrint();
+        // try {
+        //     m_adc_mask = h5_get_scalar_dataset<int>(
+        //         file, std::string(metadata_group_name + "ADC
+        //         Mask"));
+        // } catch (H5::FileIException &e) {
+        //     m_adc_mask = 0;
+        // }
+        // fmt::print("ADC Mask: {}\n", m_adc_mask);
+        // H5Eset_auto(H5E_DEFAULT,
+        // reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+        //             stderr);
+
+        // Digital Flag, Digital Samples
+        H5::Exception::dontPrint();
+        try {
+            m_digital_flag = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Digital Flag"));
+            if (m_digital_flag) {
+                m_digital_samples = h5_get_scalar_dataset<int>(
+                    file, std::string(metadata_group_name + "Digital Samples"));
+            }
+        } catch (H5::FileIException &e) {
+            // keep the optional empty
+        }
+        // fmt::print("Digital Flag: {}\n", m_digital_flag);
+        // fmt::print("Digital Samples: {}\n", m_digital_samples);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // Transceiver Flag, Transceiver Samples
+        H5::Exception::dontPrint();
+        try {
+            m_transceiver_flag = h5_get_scalar_dataset<int>(
+                file, std::string(metadata_group_name + "Transceiver Flag"));
+            if (m_transceiver_flag) {
+                m_transceiver_samples = h5_get_scalar_dataset<int>(
+                    file,
+                    std::string(metadata_group_name + "Transceiver Samples"));
+            }
+        } catch (H5::FileIException &e) {
+            // keep the optional empty
+        }
+        // fmt::print("Transceiver Flag: {}\n", m_transceiver_flag);
+        // fmt::print("Transceiver Samples: {}\n",
+        // m_transceiver_samples);
+        H5Eset_auto(H5E_DEFAULT, reinterpret_cast<H5E_auto2_t>(H5Eprint2),
+                    stderr);
+
+        // scan parameters
+        /*try{
+            std::string scan_parameters = j.at("Scan Parameters");
+            m_scan_parameters = ScanParameters(scan_parameters);
+            if(v<7.21){
+                    m_scan_parameters.increment_stop(); //adjust for
+    endpoint being included
+                }
+        }catch (const json::out_of_range &e) {
+            // not a scan
+        }
+
+    try{
+        ROI tmp_roi;
+        auto obj = j.at("Receiver Roi");
+        tmp_roi.xmin = obj.at("xmin");
+        tmp_roi.xmax = obj.at("xmax");
+        tmp_roi.ymin = obj.at("ymin");
+        tmp_roi.ymax = obj.at("ymax");
+
+        //if any of the values are set update the roi
+        if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295
+    || tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) {
+
+            if(v<7.21){
+                tmp_roi.xmax++;
+                tmp_roi.ymax++;
+            }
+
+            m_roi = tmp_roi;
+        }
+
+
+    }catch (const json::out_of_range &e) {
+        // leave the optional empty
+    }
+           //if we have an roi we need to update the geometry for the
+    subfiles if (m_roi){
+
+    }
+        */
+
+        // Update detector type for Moench
+        // TODO! How does this work with old .h5 master files?
+#ifdef AARE_VERBOSE
+        fmt::print("Detecting Moench03: m_pixels_y: {}, "
+                   "m_analog_samples: {}\n",
+                   m_pixels_y, m_analog_samples.value_or(0));
+#endif
+        if (m_type == DetectorType::Moench && !m_analog_samples &&
+            m_pixels_y == 400) {
+            m_type = DetectorType::Moench03;
+        } else if (m_type == DetectorType::Moench && m_pixels_y == 400 &&
+                   m_analog_samples == 5000) {
+            m_type = DetectorType::Moench03_old;
+        }
+
+        file.close();
+
+    } catch (const H5::Exception &e) {
+        fmt::print("Exception type: {}\n", typeid(e).name());
+        e.printErrorStack();
+        throw std::runtime_error(LOCATION + "\nCould not parse master file");
+    }
+}
+
+} // namespace aare

src/defs.cpp

@@ -91,6 +91,24 @@ template <> DetectorType StringTo(const std::string &arg) {
     throw std::runtime_error("Could not decode detector from: \"" + arg + "\"");
 }
 
+/**
+ * @brief Convert a TimingMode to a string
+ * @param type TimingMode
+ * @return string representation of the TimingMode
+ */
+template <> std::string ToString(TimingMode arg) {
+    switch (arg) {
+    case TimingMode::Auto:
+        return "Auto";
+    case TimingMode::Trigger:
+        return "Trigger";
+
+        // no default case to trigger compiler warning if not all
+        // enum values are handled
+    }
+    throw std::runtime_error("Could not decode timing mode to string");
+}
+
 /**
  * @brief Convert a string to a TimingMode
  * @param mode string representation of the TimingMode
@@ -105,6 +123,26 @@ template <> TimingMode StringTo(const std::string &arg) {
     throw std::runtime_error("Could not decode timing mode from: \"" + arg + "\"");
 }
 
+/**
+ * @brief Convert a FrameDiscardPolicy to a string
+ * @param type FrameDiscardPolicy
+ * @return string representation of the FrameDiscardPolicy
+ */
+template <> std::string ToString(FrameDiscardPolicy arg) {
+    switch (arg) {
+    case FrameDiscardPolicy::NoDiscard:
+        return "nodiscard";
+    case FrameDiscardPolicy::Discard:
+        return "discard";
+    case FrameDiscardPolicy::DiscardPartial:
+        return "discardpartial";
+
+        // no default case to trigger compiler warning if not all
+        // enum values are handled
+    }
+    throw std::runtime_error("Could not decode frame discard policy to string");
+}
+
 template <> FrameDiscardPolicy StringTo(const std::string &arg) {
     if (arg == "nodiscard")
         return FrameDiscardPolicy::NoDiscard;