Files and structure for python interface

include/aare/ClusterFileV2.hpp

@@ -0,0 +1,148 @@
+#pragma once
+#include "aare/core/defs.hpp"
+#include <filesystem>
+#include <string>
+#include <fmt/format.h>
+
+namespace aare {
+struct ClusterHeader {
+    int32_t frame_number;
+    int32_t n_clusters;
+    std::string to_string() const {
+        return "frame_number: " + std::to_string(frame_number) + ", n_clusters: " + std::to_string(n_clusters);
+    }
+};
+
+struct ClusterV2_ {
+    int16_t x;
+    int16_t y;
+    std::array<int32_t, 9> data;
+    std::string to_string(bool detailed = false) const {
+        if (detailed) {
+            std::string data_str = "[";
+            for (auto &d : data) {
+                data_str += std::to_string(d) + ", ";
+            }
+            data_str += "]";
+            return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", data: " + data_str;
+        }
+        return "x: " + std::to_string(x) + ", y: " + std::to_string(y);
+    }
+};
+
+struct ClusterV2 {
+    ClusterV2_ cluster;
+    int32_t frame_number;
+    std::string to_string() const {
+        return "frame_number: " + std::to_string(frame_number) + ", " + cluster.to_string();
+    }
+};
+
+/**
+ * @brief
+ * important not: fp always points to the clusters header and does not point to individual clusters
+ *
+ */
+class ClusterFileV2 {
+    std::filesystem::path m_fpath; 
+    std::string m_mode;
+    FILE *fp{nullptr};
+
+    void check_open(){
+        if (!fp)
+            throw std::runtime_error(fmt::format("File: {} not open", m_fpath.string()));
+    }
+
+  public:
+    ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode): m_fpath(fpath), m_mode(mode) {
+        if (m_mode != "r" && m_mode != "w")
+            throw std::invalid_argument("mode must be 'r' or 'w'");
+        if (m_mode == "r" && !std::filesystem::exists(m_fpath))
+            throw std::invalid_argument("File does not exist");
+        if (mode == "r") {
+            fp = fopen(fpath.string().c_str(), "rb");
+        } else if (mode == "w") {
+            if (std::filesystem::exists(fpath)) {
+                fp = fopen(fpath.string().c_str(), "r+b");
+            } else {
+                fp = fopen(fpath.string().c_str(), "wb");
+            }
+        }
+        if (fp == nullptr) {
+            throw std::runtime_error("Failed to open file");
+        }
+    }
+    ~ClusterFileV2() { close(); }
+    std::vector<ClusterV2> read() {
+        check_open();
+
+        ClusterHeader header;
+        fread(&header, sizeof(ClusterHeader), 1, fp); 
+        std::vector<ClusterV2_> clusters_(header.n_clusters);
+        fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp);
+        std::vector<ClusterV2> clusters;
+        for (auto &c : clusters_) {
+            ClusterV2 cluster;
+            cluster.cluster = std::move(c);
+            cluster.frame_number = header.frame_number;
+            clusters.push_back(cluster);
+        }
+
+        return clusters;
+    }
+    std::vector<std::vector<ClusterV2>> read(int n_frames) {
+        std::vector<std::vector<ClusterV2>> clusters;
+        for (int i = 0; i < n_frames; i++) {
+            clusters.push_back(read());
+        }
+        return clusters;
+    }
+
+    size_t write(std::vector<ClusterV2> const &clusters) {
+        check_open();
+        if (m_mode != "w")
+            throw std::runtime_error("File not opened in write mode");
+        if (clusters.empty())
+            return 0;
+
+        ClusterHeader header;
+        header.frame_number = clusters[0].frame_number;
+        header.n_clusters = clusters.size();
+        fwrite(&header, sizeof(ClusterHeader), 1, fp);
+        for (auto &c : clusters) {
+            fwrite(&c.cluster, sizeof(ClusterV2_), 1, fp);
+        }
+        return clusters.size();
+    }
+
+    size_t write(std::vector<std::vector<ClusterV2>> const &clusters) {
+        check_open();
+        if (m_mode != "w") 
+            throw std::runtime_error("File not opened in write mode");
+
+        size_t n_clusters = 0;
+        for (auto &c : clusters) {
+            n_clusters += write(c);
+        }
+        return n_clusters;
+    }
+
+    int seek_to_begin() { return fseek(fp, 0, SEEK_SET); }
+    int seek_to_end() { return fseek(fp, 0, SEEK_END); }
+
+    int32_t frame_number() {
+        auto pos = ftell(fp);
+        ClusterHeader header;
+        fread(&header, sizeof(ClusterHeader), 1, fp);
+        fseek(fp, pos, SEEK_SET);
+        return header.frame_number;
+    }
+
+    void close() {
+        if (fp) {
+            fclose(fp);
+            fp = nullptr;
+        }
+    }
+};
+} // namespace aare

include/aare/File.hpp

@@ -0,0 +1,59 @@
+#pragma once
+#include "aare/FileInterface.hpp"
+
+
+namespace aare {
+
+/**
+ * @brief RAII File class for reading and writing image files in various formats
+ * wrapper on a FileInterface to abstract the underlying file format
+ * @note documentation for each function is in the FileInterface class
+ */
+class File {
+  private:
+    FileInterface *file_impl;
+    bool is_npy;
+
+  public:
+    /**
+     * @brief Construct a new File object
+     * @param fname path to the file
+     * @param mode file mode (r, w, a)
+     * @param cfg file configuration
+     * @throws std::runtime_error if the file cannot be opened
+     * @throws std::invalid_argument if the file mode is not supported
+     *
+     */
+    File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {});
+    void write(Frame &frame, sls_detector_header header = {});
+    Frame read();
+    Frame iread(size_t frame_number);
+    std::vector<Frame> read(size_t n_frames);
+    void read_into(std::byte *image_buf);
+    void read_into(std::byte *image_buf, size_t n_frames);
+    size_t frame_number(size_t frame_index);
+    size_t bytes_per_frame();
+    size_t pixels_per_frame();
+    void seek(size_t frame_number);
+    size_t tell() const;
+    size_t total_frames() const;
+    size_t rows() const;
+    size_t cols() const;
+    size_t bitdepth() const;
+    void set_total_frames(size_t total_frames);
+    DetectorType detector_type() const;
+    xy geometry() const;
+
+    /**
+     * @brief Move constructor
+     * @param other File object to move from
+     */
+    File(File &&other) noexcept;
+
+    /**
+     * @brief destructor: will only delete the FileInterface object
+     */
+    ~File();
+};
+
+} // namespace aare

include/aare/FileInterface.hpp

@@ -0,0 +1,196 @@
+#pragma once
+#include "aare/Dtype.hpp"
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+
+#include <filesystem>
+#include <vector>
+
+namespace aare {
+
+/**
+ * @brief FileConfig structure to store the configuration of a file
+ * dtype: data type of the file
+ * rows: number of rows in the file
+ * cols: number of columns in the file
+ * geometry: geometry of the file
+ */
+struct FileConfig {
+    aare::Dtype dtype{typeid(uint16_t)};
+    uint64_t rows{};
+    uint64_t cols{};
+    bool operator==(const FileConfig &other) const {
+        return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry &&
+               detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file;
+    }
+    bool operator!=(const FileConfig &other) const { return !(*this == other); }
+
+    // rawfile specific
+    std::string version{};
+    xy geometry{1, 1};
+    DetectorType detector_type{DetectorType::Unknown};
+    int max_frames_per_file{};
+    size_t total_frames{};
+    std::string to_string() const {
+        return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) +
+               ", geometry: " + geometry.to_string() + ", detector_type: " + toString(detector_type) +
+               ", max_frames_per_file: " + std::to_string(max_frames_per_file) +
+               ", total_frames: " + std::to_string(total_frames) + " }";
+    }
+};
+
+/**
+ * @brief FileInterface class to define the interface for file operations
+ * @note parent class for NumpyFile and RawFile
+ * @note all functions are pure virtual and must be implemented by the derived classes
+ */
+class FileInterface {
+  public:
+    /**
+     * @brief write a frame to the file
+     * @param frame frame to write
+     * @return void
+     * @throws std::runtime_error if the function is not implemented
+     */
+    // virtual void write(Frame &frame) = 0;
+
+    /**
+     * @brief write a vector of frames to the file
+     * @param frames vector of frames to write
+     * @return void
+     */
+    // virtual void write(std::vector<Frame> &frames) = 0;
+
+    /**
+     * @brief read one frame from the file at the current position
+     * @return Frame
+     */
+    virtual Frame read() = 0;
+
+    /**
+     * @brief read n_frames from the file at the current position
+     * @param n_frames number of frames to read
+     * @return vector of frames
+     */
+    virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
+
+    /**
+     * @brief read one frame from the file at the current position and store it in the provided buffer
+     * @param image_buf buffer to store the frame
+     * @return void
+     */
+    virtual void read_into(std::byte *image_buf) = 0;
+
+    /**
+     * @brief read n_frames from the file at the current position and store them in the provided buffer
+     * @param image_buf buffer to store the frames
+     * @param n_frames number of frames to read
+     * @return void
+     */
+    virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
+
+    /**
+     * @brief get the frame number at the given frame index
+     * @param frame_index index of the frame
+     * @return frame number
+     */
+    virtual size_t frame_number(size_t frame_index) = 0;
+
+    /**
+     * @brief get the size of one frame in bytes
+     * @return size of one frame
+     */
+    virtual size_t bytes_per_frame() = 0;
+
+    /**
+     * @brief get the number of pixels in one frame
+     * @return number of pixels in one frame
+     */
+    virtual size_t pixels_per_frame() = 0;
+
+    /**
+     * @brief seek to the given frame number
+     * @param frame_number frame number to seek to
+     * @return void
+     */
+    virtual void seek(size_t frame_number) = 0;
+
+    /**
+     * @brief get the current position of the file pointer
+     * @return current position of the file pointer
+     */
+    virtual size_t tell() = 0;
+
+    /**
+     * @brief get the total number of frames in the file
+     * @return total number of frames in the file
+     */
+    virtual size_t total_frames() const = 0;
+    /**
+     * @brief get the number of rows in the file
+     * @return number of rows in the file
+     */
+    virtual size_t rows() const = 0;
+    /**
+     * @brief get the number of columns in the file
+     * @return number of columns in the file
+     */
+    virtual size_t cols() const = 0;
+    /**
+     * @brief get the bitdepth of the file
+     * @return bitdepth of the file
+     */
+    virtual size_t bitdepth() const = 0;
+
+    /**
+     * @brief read one frame from the file at the given frame number
+     * @param frame_number frame number to read
+     * @return frame
+     */
+    Frame iread(size_t frame_number) {
+        auto old_pos = tell();
+        seek(frame_number);
+        Frame tmp = read();
+        seek(old_pos);
+        return tmp;
+    };
+
+    /**
+     * @brief read n_frames from the file starting at the given frame number
+     * @param frame_number frame number to start reading from
+     * @param n_frames number of frames to read
+     * @return vector of frames
+     */
+    std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
+        auto old_pos = tell();
+        seek(frame_number);
+        std::vector<Frame> tmp = read(n_frames);
+        seek(old_pos);
+        return tmp;
+    }
+    DetectorType detector_type() const { return m_type; }
+
+    // function to query the data type of the file
+    /*virtual DataType dtype = 0; */
+
+    virtual ~FileInterface() = default;
+
+    void set_total_frames(size_t total_frames) { m_total_frames = total_frames; }
+
+  protected:
+    std::string m_mode{};
+    std::filesystem::path m_fname{};
+    std::filesystem::path m_base_path{};
+    std::string m_base_name{}, m_ext{};
+    int m_findex{};
+    size_t m_total_frames{};
+    size_t max_frames_per_file{};
+    std::string version{};
+    DetectorType m_type{DetectorType::Unknown};
+    size_t m_rows{};
+    size_t m_cols{};
+    size_t m_bitdepth{};
+    size_t current_frame{};
+};
+
+} // namespace aare

include/aare/NumpyFile.hpp

@@ -0,0 +1,112 @@
+#pragma once
+#include "aare/Dtype.hpp"
+#include "aare/defs.hpp"
+#include "aare/FileInterface.hpp"
+#include "aare/NumpyHelpers.hpp"
+
+
+#include <filesystem>
+#include <iostream>
+#include <numeric>
+
+namespace aare {
+
+
+
+/**
+ * @brief NumpyFile class to read and write numpy files
+ * @note derived from FileInterface
+ * @note implements all the pure virtual functions from FileInterface
+ * @note documentation for the functions can also be found in the FileInterface class
+ */
+class NumpyFile : public FileInterface {
+
+  public:
+    /**
+     * @brief NumpyFile constructor
+     * @param fname path to the numpy file
+     * @param mode file mode (r, w)
+     * @param cfg file configuration
+     */
+    explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
+
+    void write(Frame &frame);
+    Frame read() override { return get_frame(this->current_frame++); }
+
+    std::vector<Frame> read(size_t n_frames) override;
+    void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
+    void read_into(std::byte *image_buf, size_t n_frames) override;
+    size_t frame_number(size_t frame_index) override { return frame_index; };
+    size_t bytes_per_frame() override;
+    size_t pixels_per_frame() override;
+    void seek(size_t frame_number) override { this->current_frame = frame_number; }
+    size_t tell() override { return this->current_frame; }
+    size_t total_frames() const override { return m_header.shape[0]; }
+    size_t rows() const override { return m_header.shape[1]; }
+    size_t cols() const override { return m_header.shape[2]; }
+    size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
+
+    /**
+     * @brief get the data type of the numpy file
+     * @return DType
+     */
+    Dtype dtype() const { return m_header.dtype; }
+
+    /**
+     * @brief get the shape of the numpy file
+     * @return vector of type size_t
+     */
+    std::vector<size_t> shape() const { return m_header.shape; }
+
+    /**
+     * @brief load the numpy file into an NDArray
+     * @tparam T data type of the NDArray
+     * @tparam NDim number of dimensions of the NDArray
+     * @return NDArray<T, NDim>
+     */
+    template <typename T, size_t NDim> NDArray<T, NDim> load() {
+        NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
+        if (fseek(fp, static_cast<int64_t>(header_size), SEEK_SET)) {
+            throw std::runtime_error(LOCATION + "Error seeking to the start of the data");
+        }
+        size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp);
+        if (rc != static_cast<size_t>(arr.size())) {
+            throw std::runtime_error(LOCATION + "Error reading data from file");
+        }
+        return arr;
+    }
+    template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &frame) {
+        write_impl(frame.data(), frame.total_bytes());
+    }
+    template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &frame) {
+        write_impl(frame.data(), frame.total_bytes());
+    }
+    template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &&frame) {
+        write_impl(frame.data(), frame.total_bytes());
+    }
+    template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &&frame) {
+        write_impl(frame.data(), frame.total_bytes());
+    }
+
+    ~NumpyFile() noexcept override;
+
+  private:
+    FILE *fp = nullptr;
+    size_t initial_header_len = 0;
+    size_t current_frame{};
+    uint32_t header_len{};
+    uint8_t header_len_size{};
+    size_t header_size{};
+    NumpyHeader m_header;
+    uint8_t major_ver_{};
+    uint8_t minor_ver_{};
+    size_t m_bytes_per_frame{};
+    size_t m_pixels_per_frame{};
+
+    void load_metadata();
+    void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
+    Frame get_frame(size_t frame_number);
+    void write_impl(void *data, uint64_t size);
+};
+
+} // namespace aare

include/aare/NumpyHelpers.hpp

@@ -0,0 +1,55 @@
+
+#pragma once
+#include <algorithm>
+#include <array>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "aare/Dtype.hpp"
+#include "aare/defs.hpp"
+
+namespace aare {
+
+struct NumpyHeader {
+    Dtype dtype{aare::Dtype::ERROR};
+    bool fortran_order{false};
+    std::vector<size_t> shape{};
+    std::string to_string() const;
+};
+
+namespace NumpyHelpers {
+
+const constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
+const uint8_t magic_string_length{6};
+
+std::string parse_str(const std::string &in);
+/**
+  Removes leading and trailing whitespaces
+  */
+std::string trim(const std::string &str);
+
+std::vector<std::string> parse_tuple(std::string in);
+
+bool parse_bool(const std::string &in);
+
+std::string get_value_from_map(const std::string &mapstr);
+
+std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys);
+
+template <typename T, size_t N> bool in_array(T val, const std::array<T, N> &arr) {
+    return std::find(std::begin(arr), std::end(arr), val) != std::end(arr);
+}
+bool is_digits(const std::string &str);
+
+aare::Dtype parse_descr(std::string typestring);
+size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header);
+size_t write_header(std::ostream &out, const NumpyHeader &header);
+
+} // namespace NumpyHelpers
+} // namespace aare

include/aare/RawFile.hpp

@@ -0,0 +1,186 @@
+#pragma once
+#include "aare/Frame.hpp"
+#include "aare/FileInterface.hpp"
+#include "aare/SubFile.hpp"
+
+namespace aare {
+
+struct ModuleConfig {
+    int module_gap_row{};
+    int module_gap_col{};
+
+    bool operator==(const ModuleConfig &other) const {
+        if (module_gap_col != other.module_gap_col)
+            return false;
+        if (module_gap_row != other.module_gap_row)
+            return false;
+        return true;
+    }
+};
+
+/**
+ * @brief RawFile class to read .raw and .json files
+ * @note derived from FileInterface
+ * @note documentation can also be found in the FileInterface class
+ */
+class RawFile : public FileInterface {
+  public:
+    /**
+     * @brief RawFile constructor
+     * @param fname path to the file
+     * @param mode file mode (r, w)
+     * @param cfg file configuration
+     */
+    explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r",
+                     const FileConfig &config = FileConfig{});
+
+    /**
+     * @brief write function is not implemented for RawFile
+     * @param frame frame to write
+     */
+    void write(Frame &frame, sls_detector_header header);
+    Frame read() override { return get_frame(this->current_frame++); };
+    std::vector<Frame> read(size_t n_frames) override;
+    void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
+    void read_into(std::byte *image_buf, size_t n_frames) override;
+    size_t frame_number(size_t frame_index) override;
+
+    /**
+     * @brief get the number of bytess per frame
+     * @return size of one frame in bytes
+     */
+    size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
+
+    /**
+     * @brief get the number of pixels in the frame
+     * @return number of pixels
+     */
+    size_t pixels_per_frame() override { return m_rows * m_cols; }
+
+    // goto frame index
+    void seek(size_t frame_index) override {
+        // check if the frame number is greater than the total frames
+        // if frame_number == total_frames, then the next read will throw an error
+        if (frame_index > this->total_frames()) {
+            throw std::runtime_error(
+                fmt::format("frame number {} is greater than total frames {}", frame_index, m_total_frames));
+        }
+        this->current_frame = frame_index;
+    };
+
+    // return the position of the file pointer (in number of frames)
+    size_t tell() override { return this->current_frame; };
+
+    /**
+     * @brief check if the file is a master file
+     * @param fpath path to the file
+     */
+    static bool is_master_file(const std::filesystem::path &fpath);
+
+    /**
+     * @brief set the module gap row and column
+     * @param row gap between rows
+     * @param col gap between columns
+     */
+    inline void set_config(int row, int col) {
+        cfg.module_gap_row = row;
+        cfg.module_gap_col = col;
+    }
+    // TODO! Deal with fast quad and missing files
+
+    /**
+     * @brief get the number of subfiles for the RawFile
+     * @return number of subfiles
+     */
+    void find_number_of_subfiles();
+
+    /**
+     * @brief get the master file name path for the RawFile
+     * @return path to the master file
+     */
+    inline std::filesystem::path master_fname();
+    /**
+     * @brief get the data file name path for the RawFile with the given module id and file id
+     * @param mod_id module id
+     * @param file_id file id
+     * @return path to the data file
+     */
+    inline std::filesystem::path data_fname(size_t mod_id, size_t file_id);
+
+    /**
+     * @brief destructor: will delete the subfiles
+     */
+    ~RawFile() noexcept override;
+
+    size_t total_frames() const override { return m_total_frames; }
+    size_t rows() const override { return m_rows; }
+    size_t cols() const override { return m_cols; }
+    size_t bitdepth() const override { return m_bitdepth; }
+    xy geometry() { return m_geometry; }
+
+  private:
+    void write_master_file();
+    /**
+     * @brief read the frame at the given frame index into the image buffer
+     * @param frame_number frame number to read
+     * @param image_buf buffer to store the frame
+     */
+    void get_frame_into(size_t frame_index, std::byte *frame_buffer);
+
+    /**
+     * @brief get the frame at the given frame index
+     * @param frame_number frame number to read
+     * @return Frame
+     */
+    Frame get_frame(size_t frame_index);
+
+    /**
+     * @brief parse the file name to get the extension, base name and index
+     */
+    void parse_fname();
+
+    /**
+     * @brief parse the metadata from the file
+     */
+    void parse_metadata();
+
+    /**
+     * @brief parse the metadata of a .raw file
+     */
+    void parse_raw_metadata();
+
+    /**
+     * @brief parse the metadata of a .json file
+     */
+    void parse_json_metadata();
+
+    /**
+     * @brief finds the geometry of the file
+     */
+    void find_geometry();
+
+    /**
+     * @brief read the header of the file
+     * @param fname path to the data subfile
+     * @return sls_detector_header
+     */
+    static sls_detector_header read_header(const std::filesystem::path &fname);
+
+    /**
+     * @brief open the subfiles
+     */
+    void open_subfiles();
+    void parse_config(const FileConfig &config);
+
+    size_t n_subfiles{};
+    size_t n_subfile_parts{};
+    std::vector<std::vector<SubFile *>> subfiles;
+    size_t subfile_rows{}, subfile_cols{};
+    xy m_geometry{};
+    std::vector<xy> positions;
+    ModuleConfig cfg{0, 0};
+    TimingMode timing_mode{};
+    bool quad{false};
+};
+
+} // namespace aare

include/aare/SubFile.hpp

@@ -0,0 +1,77 @@
+#pragma once
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <variant>
+
+namespace aare {
+
+/**
+ * @brief Class to read a subfile from a RawFile
+ */
+class SubFile {
+  public:
+    size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
+    /**
+     * @brief SubFile constructor
+     * @param fname path to the subfile
+     * @param detector detector type
+     * @param rows number of rows in the subfile
+     * @param cols number of columns in the subfile
+     * @param bitdepth bitdepth of the subfile
+     * @throws std::invalid_argument if the detector,type pair is not supported
+     */
+    SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
+            const std::string &mode = "r");
+
+    /**
+     * @brief read the subfile into a buffer
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    size_t read_impl_normal(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes flipped
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    template <typename DataType> size_t read_impl_flip(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes reordered
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes reordered and flipped
+     * @param buffer pointer to the buffer to read the data into
+     * @param frame_number frame number to read
+     * @return number of bytes read
+     */
+    size_t get_part(std::byte *buffer, size_t frame_index);
+    size_t frame_number(size_t frame_index);
+
+    // TODO: define the inlines as variables and assign them in constructor
+    inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
+    inline size_t pixels_per_part() const { return m_rows * m_cols; }
+
+    ~SubFile();
+
+  protected:
+    FILE *fp = nullptr;
+    size_t m_bitdepth;
+    std::filesystem::path m_fname;
+    size_t m_rows{};
+    size_t m_cols{};
+    std::string m_mode;
+    size_t n_frames{};
+    int m_sub_file_index_{};
+};
+
+} // namespace aare

include/aare/VarClusterFinder.hpp

@@ -10,7 +10,7 @@
 const int MAX_CLUSTER_SIZE = 200;
 namespace aare {
 
-template <typename T> class ClusterFinder {
+template <typename T> class VarClusterFinder {
   public:
     struct Hit {
         int16_t size{};
@@ -49,7 +49,7 @@ template <typename T> class ClusterFinder {
     int check_neighbours(int i, int j);
 
   public:
-    ClusterFinder(image_shape shape, T threshold)
+    VarClusterFinder(image_shape shape, T threshold)
         : shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) {
         hits.reserve(2000);
     }
@@ -112,7 +112,7 @@ template <typename T> class ClusterFinder {
         }
     }
 };
-template <typename T> int ClusterFinder<T>::check_neighbours(int i, int j) {
+template <typename T> int VarClusterFinder<T>::check_neighbours(int i, int j) {
     std::vector<int> neighbour_labels;
 
     for (int k = 0; k < 4; ++k) {
@@ -144,7 +144,7 @@ template <typename T> int ClusterFinder<T>::check_neighbours(int i, int j) {
     }
 }
 
-template <typename T> void ClusterFinder<T>::find_clusters(NDView<T, 2> img) {
+template <typename T> void VarClusterFinder<T>::find_clusters(NDView<T, 2> img) {
     original_ = img;
     labeled_ = 0;
     peripheral_labeled_ = 0;
@@ -156,7 +156,7 @@ template <typename T> void ClusterFinder<T>::find_clusters(NDView<T, 2> img) {
     store_clusters();
 }
 
-template <typename T> void ClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
+template <typename T> void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
     original_ = img;
     int clusterIndex = 0;
     for (int i = 0; i < shape_[0]; ++i) {
@@ -175,7 +175,7 @@ template <typename T> void ClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
     h_size.clear();
 }
 
-template <typename T> void ClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
+template <typename T> void VarClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
     // printf("original_(%d, %d)=%f\n", i, j, original_(i,j));
     // printf("h_size[%d].size=%d\n", clusterIndex, h_size[clusterIndex].size);
     if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE) {
@@ -213,7 +213,7 @@ template <typename T> void ClusterFinder<T>::rec_FillHit(int clusterIndex, int i
     }
 }
 
-template <typename T> void ClusterFinder<T>::single_pass(NDView<T, 2> img) {
+template <typename T> void VarClusterFinder<T>::single_pass(NDView<T, 2> img) {
     original_ = img;
     labeled_ = 0;
     current_label = 0;
@@ -224,7 +224,7 @@ template <typename T> void ClusterFinder<T>::single_pass(NDView<T, 2> img) {
     // store_clusters();
 }
 
-template <typename T> void ClusterFinder<T>::first_pass() {
+template <typename T> void VarClusterFinder<T>::first_pass() {
 
     for (int i = 0; i < original_.size(); ++i) {
         if (use_noise_map)
@@ -248,7 +248,7 @@ template <typename T> void ClusterFinder<T>::first_pass() {
     }
 }
 
-template <typename T> void ClusterFinder<T>::second_pass() {
+template <typename T> void VarClusterFinder<T>::second_pass() {
 
     for (int64_t i = 0; i != labeled_.size(); ++i) {
         auto current_label = labeled_(i);
@@ -265,7 +265,7 @@ template <typename T> void ClusterFinder<T>::second_pass() {
     }
 }
 
-template <typename T> void ClusterFinder<T>::store_clusters() {
+template <typename T> void VarClusterFinder<T>::store_clusters() {
 
     // Accumulate hit information in a map
     // Do we always have monotonic increasing

include/aare/json.hpp

@@ -0,0 +1,68 @@
+#pragma once
+#include <array>
+#include <map>
+#include <string>
+#include <vector>
+
+// helper functions to write json
+// append to string for better performance (not tested)
+
+namespace aare {
+
+/**
+ * @brief write a digit to a string
+ * takes key and value and outputs->"key": value,
+ * @tparam T type of value (int, uint32_t, ...)
+ * @param s string to append to
+ * @param key key to write
+ * @param value value to write
+ * @return void
+ * @note
+ * - can't use concepts here because we are using c++17
+ */
+template <typename T> inline void write_digit(std::string &s, const std::string &key, const T &value) {
+    s += "\"";
+    s += key;
+    s += "\": ";
+    s += std::to_string(value);
+    s += ", ";
+}
+inline void write_str(std::string &s, const std::string &key, const std::string &value) {
+    s += "\"";
+    s += key;
+    s += "\": \"";
+    s += value;
+    s += "\", ";
+}
+inline void write_map(std::string &s, const std::string &key, const std::map<std::string, std::string> &value) {
+    s += "\"";
+    s += key;
+    s += "\": {";
+    for (const auto &kv : value) {
+        write_str(s, kv.first, kv.second);
+    }
+    // remove last comma or trailing spaces
+    for (size_t i = s.size() - 1; i > 0; i--) {
+        if ((s[i] == ',') || (s[i] == ' ')) {
+            s.pop_back();
+        } else
+            break;
+    }
+    s += "}, ";
+}
+
+template <typename T, int N> void write_array(std::string &s, const std::string &key, const std::array<T, N> &value) {
+    s += "\"";
+    s += key;
+    s += "\": [";
+
+    for (size_t i = 0; i < N - 1; i++) {
+        s += std::to_string(value[i]);
+        s += ", ";
+    }
+    s += std::to_string(value[N - 1]);
+
+    s += "], ";
+}
+
+} // namespace aare