Merge branch 'main' into developer

.github/workflows/build_docs.yml

@@ -6,6 +6,7 @@ on:
     branches:
      - main
 
+
 permissions:
   contents: read
   pages: write

.github/workflows/build_pkg.yml

@@ -6,6 +6,7 @@ on:
     branches:
      - main
      - developer
+
     #run on PRs as well?
 
 jobs:

CMakeLists.txt

@@ -35,6 +35,7 @@ option(AARE_DOCS "Build documentation" OFF)
 option(AARE_VERBOSE "Verbose output" OFF)
 option(AARE_CUSTOM_ASSERT "Use custom assert" OFF)
 
+
 # Configure which of the dependencies to use FetchContent for
 option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON)
 option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON)
@@ -62,6 +63,7 @@ if(AARE_CUSTOM_ASSERT)
     add_compile_definitions(AARE_CUSTOM_ASSERT)
 endif()
 
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if(AARE_FETCH_ZMQ)
@@ -256,6 +258,7 @@ set(PUBLICHEADERS
     include/aare/RawFile.hpp
     include/aare/RawSubFile.hpp
     include/aare/RawMasterFile.hpp
+    include/aare/SubFile.hpp
     include/aare/VarClusterFinder.hpp
 
 )

conda-recipe/meta.yaml

@@ -2,6 +2,7 @@ package:
   name: aare
   version: 2024.11.11.dev0 #TODO! how to not duplicate this?
 
+
 source:
   path: ..
 

docs/src/index.rst

@@ -33,3 +33,4 @@ AARE
 
 
 
+

include/aare/NDArray.hpp

@@ -342,6 +342,7 @@ template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print() {
         Print_some();
 }
 
+
 template <typename T, int64_t Ndim>
 std::ostream& operator <<(std::ostream& os, const NDArray<T, Ndim>& arr){
     for (auto row = 0; row < arr.shape(0); ++row) {
@@ -354,6 +355,7 @@ std::ostream& operator <<(std::ostream& os, const NDArray<T, Ndim>& arr){
     return os;
 }
 
+
 template <typename T, int64_t Ndim> void NDArray<T, Ndim>::Print_all() {
     for (auto row = 0; row < shape_[0]; ++row) {
         for (auto col = 0; col < shape_[1]; ++col) {

include/aare/NDView.hpp

@@ -156,6 +156,7 @@ template <typename T, int64_t Ndim> void NDView<T, Ndim>::print_all() const {
     }
 }
 
+
 template <typename T, int64_t Ndim>
 std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
     for (auto row = 0; row < arr.shape(0); ++row) {
@@ -168,4 +169,5 @@ std::ostream& operator <<(std::ostream& os, const NDView<T, Ndim>& arr){
     return os;
 }
 
+
 } // namespace aare

include/aare/RawFile.hpp

@@ -5,6 +5,7 @@
 #include "aare/NDArray.hpp" //for pixel map
 #include "aare/RawSubFile.hpp"
 
+
 #include <optional>
 
 namespace aare {
@@ -62,6 +63,7 @@ class RawFile : public FileInterface {
     //TODO! do we need to adapt the API? 
     void read_into(std::byte *image_buf, DetectorHeader *header);
 
+
     size_t frame_number(size_t frame_index) override;
     size_t bytes_per_frame() override;
     size_t pixels_per_frame() override;
@@ -75,6 +77,7 @@ class RawFile : public FileInterface {
     xy geometry();
     size_t n_mod() const;
 
+
     DetectorType detector_type() const override;
 
   private:
@@ -91,8 +94,10 @@ class RawFile : public FileInterface {
      * @param frame_number frame number to read
      * @param image_buf buffer to store the frame
      */
+
     void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr);
 
+
     /**
      * @brief get the frame at the given frame index
      * @param frame_number frame number to read
@@ -111,6 +116,7 @@ class RawFile : public FileInterface {
 
     void update_geometry_with_roi();
     int find_number_of_subfiles();
+
     void open_subfiles();
     void find_geometry();
 };

include/aare/RawMasterFile.hpp

@@ -58,6 +58,7 @@ class ScanParameters {
     bool enabled() const;
 };
 
+
 struct ROI{
   size_t xmin{};
   size_t xmax{};
@@ -107,6 +108,7 @@ class RawMasterFile {
 
     std::optional<ROI> m_roi;
 
+
   public:
     RawMasterFile(const std::filesystem::path &fpath);
 
@@ -133,8 +135,10 @@ class RawMasterFile {
     std::optional<size_t> number_of_rows() const;
     std::optional<uint8_t> quad() const;
 
+
     std::optional<ROI> roi() const;
 
+
     ScanParameters scan_parameters() const;
 
   private:

include/aare/SubFile.hpp

@@ -0,0 +1,80 @@
+#pragma once
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+
+#include <cstdint>
+#include <filesystem>
+#include <map>
+#include <optional>
+
+
+namespace aare {
+
+/**
+ * @brief Class to read a subfile from a RawFile
+ */
+class SubFile {
+  public:
+    size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index);
+    /**
+     * @brief SubFile constructor
+     * @param fname path to the subfile
+     * @param detector detector type
+     * @param rows number of rows in the subfile
+     * @param cols number of columns in the subfile
+     * @param bitdepth bitdepth of the subfile
+     * @throws std::invalid_argument if the detector,type pair is not supported
+     */
+    SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
+            const std::string &mode = "r");
+
+    /**
+     * @brief read the subfile into a buffer
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    size_t read_impl_normal(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes flipped
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    template <typename DataType> size_t read_impl_flip(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes reordered
+     * @param buffer pointer to the buffer to read the data into
+     * @return number of bytes read
+     */
+    template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
+
+    /**
+     * @brief read the subfile into a buffer with the bytes reordered and flipped
+     * @param buffer pointer to the buffer to read the data into
+     * @param frame_number frame number to read
+     * @return number of bytes read
+     */
+    size_t get_part(std::byte *buffer, size_t frame_index);
+    size_t frame_number(size_t frame_index);
+
+    // TODO: define the inlines as variables and assign them in constructor
+    inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
+    inline size_t pixels_per_part() const { return m_rows * m_cols; }
+
+    ~SubFile();
+
+  protected:
+    FILE *fp = nullptr;
+    size_t m_bitdepth;
+    std::filesystem::path m_fname;
+    size_t m_rows{};
+    size_t m_cols{};
+    std::string m_mode;
+    size_t n_frames{};
+    int m_sub_file_index_{};
+    DetectorType m_detector_type;
+    std::optional<NDArray<ssize_t, 2>> pixel_map;
+};
+
+} // namespace aare

include/aare/defs.hpp

@@ -15,10 +15,6 @@
 #include <vector>
 
 
-
-
-
-
 /**
  * @brief LOCATION macro to get the current location in the code
  */
@@ -27,6 +23,7 @@
         ":" + std::string(__func__) + ":"
 
 
+
 #ifdef AARE_CUSTOM_ASSERT
 #define AARE_ASSERT(expr)\
     if (expr)\
@@ -43,6 +40,7 @@ namespace aare {
 
 void assert_failed(const std::string &msg);
 
+
 class Cluster {
   public:
     int cluster_sizeX;
@@ -192,6 +190,7 @@ struct ModuleGeometry{
     int width{};
 };
 
+
 using dynamic_shape = std::vector<int64_t>;
 
 //TODO! Can we uniform enums between the libraries?

pyproject.toml

@@ -6,6 +6,7 @@ build-backend = "scikit_build_core.build"
 name = "aare"
 version = "2024.11.11.dev0"
 
+
 [tool.scikit-build]
 cmake.verbose = true
 

python/aare/__init__.py

@@ -1,8 +1,10 @@
 # Make the compiled classes that live in _aare available from aare.
 from . import _aare
 
+
 from ._aare import File, RawFile, RawMasterFile, RawSubFile
 from ._aare import Pedestal, ClusterFinder, VarClusterFinder
 from ._aare import DetectorType
+
 from .CtbRawFile import CtbRawFile
 from .ScanParameters import ScanParameters

python/examples/play.py

@@ -52,6 +52,7 @@ def get_Mh02_frames(fname):
 # [frame, counter, row, col]
 # plt.imshow(data[0,0])
 
+
 base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data')
 # p = Path(base / 'jungfrau/jungfrau_single_master_0.json')
 
@@ -84,4 +85,4 @@ fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZP
 # fname = Path(base / 'jungfrau/jungfrau_single_master_0.json')
 f = RawFile(fname)
 h,img = f.read_frame()
-print(f'{h["frameNumber"]}')
+print(f'{h["frameNumber"]}')

python/src/file.hpp

@@ -4,6 +4,7 @@
 #include "aare/RawFile.hpp"
 #include "aare/RawMasterFile.hpp"
 #include "aare/RawSubFile.hpp"
+
 #include "aare/defs.hpp"
 // #include "aare/fClusterFileV2.hpp"
 
@@ -21,6 +22,7 @@ using namespace ::aare;
 
 void define_file_io_bindings(py::module &m) {
 
+
     py::enum_<DetectorType>(m, "DetectorType")
         .value("Jungfrau", DetectorType::Jungfrau)
         .value("Eiger", DetectorType::Eiger)
@@ -31,6 +33,7 @@ void define_file_io_bindings(py::module &m) {
         .value("ChipTestBoard", DetectorType::ChipTestBoard)
         .value("Unknown", DetectorType::Unknown);
 
+
     PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber,
                          bunchId, timestamp, modId, row, column, reserved,
                          debug, roundRNumber, detType, version, packetMask);
@@ -60,8 +63,10 @@ void define_file_io_bindings(py::module &m) {
         .def("seek", &CtbRawFile::seek)
         .def("tell", &CtbRawFile::tell)
         .def("master", &CtbRawFile::master)
+
         .def_property_readonly("image_size_in_bytes",
                                &CtbRawFile::image_size_in_bytes)
+
         .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file);
 
     py::class_<File>(m, "File")
@@ -163,6 +168,7 @@ void define_file_io_bindings(py::module &m) {
             return "<FileConfig: " + a.to_string() + ">";
         });
 
+
     py::class_<RawMasterFile>(m, "RawMasterFile")
         .def(py::init<const std::filesystem::path &>())
         .def("data_fname", &RawMasterFile::data_fname)
@@ -180,12 +186,14 @@ void define_file_io_bindings(py::module &m) {
         .def_property_readonly("frame_padding", &RawMasterFile::frame_padding)
         .def_property_readonly("frame_discard_policy",
                                &RawMasterFile::frame_discard_policy)
+
         .def_property_readonly("total_frames_expected",
                                &RawMasterFile::total_frames_expected)
         .def_property_readonly("geometry", &RawMasterFile::geometry)
         .def_property_readonly("analog_samples", &RawMasterFile::analog_samples)
         .def_property_readonly("digital_samples",
                                &RawMasterFile::digital_samples)
+
         .def_property_readonly("transceiver_samples",
                                &RawMasterFile::transceiver_samples)
         .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows)
@@ -197,12 +205,14 @@ void define_file_io_bindings(py::module &m) {
     py::class_<ScanParameters>(m, "ScanParameters")
         .def(py::init<const std::string &>())
         .def(py::init<const ScanParameters &>())
+
         .def_property_readonly("enabled", &ScanParameters::enabled)
         .def_property_readonly("dac", &ScanParameters::dac)
         .def_property_readonly("start", &ScanParameters::start)
         .def_property_readonly("stop", &ScanParameters::stop)
         .def_property_readonly("step", &ScanParameters::step);
 
+
     py::class_<ROI>(m, "ROI")
         .def(py::init<>())
         .def_readwrite("xmin", &ROI::xmin)
@@ -276,6 +286,7 @@ void define_file_io_bindings(py::module &m) {
                  return image;
              });
 
+
     // py::class_<ClusterHeader>(m, "ClusterHeader")
     //     .def(py::init<>())
     //     .def_readwrite("frame_number", &ClusterHeader::frame_number)

src/RawFile.cpp

@@ -13,11 +13,13 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode)
     : m_master(fname) {
     m_mode = mode;
     if (mode == "r") {
+
         n_subfiles = find_number_of_subfiles(); // f0,f1...fn
         n_subfile_parts =
             m_master.geometry().col * m_master.geometry().row; // d0,d1...dn
         find_geometry();
         update_geometry_with_roi();
+
         open_subfiles();
     } else {
         throw std::runtime_error(LOCATION +
@@ -41,11 +43,11 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
     }
 }
 
-
 void RawFile::read_into(std::byte *image_buf) {
     return get_frame_into(m_current_frame++, image_buf);
 };
 
+
 void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
 
     return get_frame_into(m_current_frame++, image_buf, header);
@@ -53,6 +55,7 @@ void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) {
 
 size_t RawFile::n_mod() const { return n_subfile_parts; }
 
+
 size_t RawFile::bytes_per_frame() {
     return m_rows * m_cols * m_master.bitdepth() / 8;
 }
@@ -88,6 +91,7 @@ void RawFile::open_subfiles() {
                 v[j] = new RawSubFile(m_master.data_fname(j, i),
                                       m_master.detector_type(), pos.height,
                                       pos.width, m_master.bitdepth());
+
             }
             subfiles.push_back(v);
         }
@@ -128,12 +132,14 @@ int RawFile::find_number_of_subfiles() {
     fmt::print("Found: {} subfiles\n", n_files);
 #endif
     return n_files;
+
 }
 
 void RawFile::find_geometry() {
     uint16_t r{};
     uint16_t c{};
 
+
     for (size_t i = 0; i < n_subfile_parts; i++) {
         auto h = this->read_header(m_master.data_fname(i, 0));
         r = std::max(r, h.row);
@@ -145,6 +151,7 @@ void RawFile::find_geometry() {
         g.width = m_master.pixels_x();
         g.height = m_master.pixels_y();
         m_module_pixel_0.push_back(g);
+
     }
 
     r++;
@@ -154,6 +161,7 @@ void RawFile::find_geometry() {
     m_cols = (c * m_master.pixels_x());
 
     m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
+
 #ifdef AARE_VERBOSE
     fmt::print("\nRawFile::find_geometry()\n");
     for (size_t i = 0; i < m_module_pixel_0.size(); i++) {
@@ -227,6 +235,7 @@ void RawFile::update_geometry_with_roi() {
     fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols);
     fmt::print("\n");
 #endif
+
 }
 
 Frame RawFile::get_frame(size_t frame_index) {
@@ -236,18 +245,22 @@ Frame RawFile::get_frame(size_t frame_index) {
     return f;
 }
 
+
 size_t RawFile::bytes_per_pixel() const {
     return m_master.bitdepth() / 8;
 }
 
 void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
+
     if (frame_index > total_frames()) {
         throw std::runtime_error(LOCATION + "Frame number out of range");
     }
     std::vector<size_t> frame_numbers(n_subfile_parts);
     std::vector<size_t> frame_indices(n_subfile_parts, frame_index);
 
+
     // sync the frame numbers
+
     if (n_subfile_parts != 1) {
         for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) {
             auto subfile_id = frame_index / m_master.max_frames_per_file();
@@ -302,6 +315,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
         //TODO! should we read row by row?
 
         // create a buffer large enough to hold a full module
+
         auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
                               m_master.bitdepth() /
                               8; // TODO! replace with image_size_in_bytes
@@ -331,6 +345,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
                        part_buffer + cur_row * pos.width *
                                          m_master.bitdepth() / 8,
                        pos.width * m_master.bitdepth() / 8);
+
             }
         }
         delete[] part_buffer;

src/RawMasterFile.cpp

@@ -8,8 +8,6 @@ RawFileNameComponents::RawFileNameComponents(
     m_base_name = fname.stem();
     m_ext = fname.extension();
 
-    AARE_ASSERT(false);
-
     if (m_ext != ".json" && m_ext != ".raw") {
         throw std::runtime_error(LOCATION +
                                  "Unsupported file type. (only .json or .raw)");
@@ -145,8 +143,10 @@ ScanParameters RawMasterFile::scan_parameters() const {
     return m_scan_parameters;
 }
 
+
 std::optional<ROI> RawMasterFile::roi() const { return m_roi; }
 
+
 void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
     std::ifstream ifs(fpath);
     json j;
@@ -244,6 +244,7 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
         // not a scan
     }
 
+
     try{
         ROI tmp_roi;
         auto obj = j.at("Receiver Roi");
@@ -276,6 +277,7 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) {
 
 
 
+
     // Update detector type for Moench
     // TODO! How does this work with old .raw master files?
     #ifdef AARE_VERBOSE

src/SubFile.cpp

@@ -0,0 +1,91 @@
+#include "aare/SubFile.hpp"
+#include "aare/PixelMap.hpp"
+#include <cstring> // memcpy
+#include <fmt/core.h>
+#include <iostream>
+
+
+namespace aare {
+
+SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
+                 const std::string &mode)
+    : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) {
+
+
+    if (m_detector_type == DetectorType::Moench03_old) {
+        pixel_map = GenerateMoench03PixelMap();
+    }
+
+    if (std::filesystem::exists(fname)) {
+        n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8);
+    } else {
+        n_frames = 0;
+    }
+
+    if (mode == "r") {
+        fp = fopen(m_fname.string().c_str(), "rb");
+    } else {
+        throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles.");
+    }
+    if (fp == nullptr) {
+        throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string()));
+    }
+#ifdef AARE_VERBOSE
+    fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames);
+    fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth);
+#endif
+}
+
+size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
+    if (frame_index >= n_frames) {
+        throw std::runtime_error("Frame number out of range");
+    }
+    fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT
+          SEEK_SET);
+
+    if (pixel_map){
+        // read into a temporary buffer and then copy the data to the buffer
+        // in the correct order
+        auto part_buffer = new std::byte[bytes_per_part()];
+        auto wc = fread(part_buffer, bytes_per_part(), 1, fp);
+        auto *data = reinterpret_cast<uint16_t *>(buffer);
+        auto *part_data = reinterpret_cast<uint16_t *>(part_buffer);
+        for (size_t i = 0; i < pixels_per_part(); i++) {
+            data[i] = part_data[(*pixel_map)(i)];
+        }
+        delete[] part_buffer;
+        return wc;
+    }else{
+        // read directly into the buffer
+        return fread(buffer, this->bytes_per_part(), 1, this->fp);
+    }
+    
+}
+size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) {
+    if (frame_index > n_frames) {
+        throw std::runtime_error("Frame number out of range");
+    }
+    fseek(fp, static_cast<int64_t>((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET);
+    auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
+    wc += fwrite(buffer, bytes_per_part(), 1, fp);
+
+    return wc;
+}
+
+size_t SubFile::frame_number(size_t frame_index) {
+    DetectorHeader h{};
+    fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
+    size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
+    if (rc != 1)
+        throw std::runtime_error(LOCATION + "Could not read header from file");
+
+    return h.frameNumber;
+}
+
+SubFile::~SubFile() {
+    if (fp) {
+        fclose(fp);
+    }
+}
+
+} // namespace aare

src/defs.cpp

@@ -1,6 +1,7 @@
 #include "aare/defs.hpp"
 #include <stdexcept>
 #include <string>
+
 #include <fmt/core.h>
 namespace aare {
 
@@ -12,6 +13,7 @@ void assert_failed(const std::string &msg)
 }
 
 
+
 /**
  * @brief Convert a DetectorType to a string
  * @param type DetectorType