diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index cfaa65a..57f15a3 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -6,6 +6,7 @@ on: branches: - main + permissions: contents: read pages: write diff --git a/.github/workflows/build_pkg.yml b/.github/workflows/build_pkg.yml index 11ccfae..2534a86 100644 --- a/.github/workflows/build_pkg.yml +++ b/.github/workflows/build_pkg.yml @@ -6,6 +6,7 @@ on: branches: - main - developer + #run on PRs as well? jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index 208e0be..d72d784 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ option(AARE_DOCS "Build documentation" OFF) option(AARE_VERBOSE "Verbose output" OFF) option(AARE_CUSTOM_ASSERT "Use custom assert" OFF) + # Configure which of the dependencies to use FetchContent for option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON) option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON) @@ -62,6 +63,7 @@ if(AARE_CUSTOM_ASSERT) add_compile_definitions(AARE_CUSTOM_ASSERT) endif() + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(AARE_FETCH_ZMQ) @@ -256,6 +258,7 @@ set(PUBLICHEADERS include/aare/RawFile.hpp include/aare/RawSubFile.hpp include/aare/RawMasterFile.hpp + include/aare/SubFile.hpp include/aare/VarClusterFinder.hpp ) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 3b77697..338aba8 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -2,6 +2,7 @@ package: name: aare version: 2024.11.11.dev0 #TODO! how to not duplicate this? + source: path: .. diff --git a/docs/src/index.rst b/docs/src/index.rst index 943e952..7bac72f 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -33,3 +33,4 @@ AARE + diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index fb5f446..e65b136 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -342,6 +342,7 @@ template void NDArray::Print() { Print_some(); } + template std::ostream& operator <<(std::ostream& os, const NDArray& arr){ for (auto row = 0; row < arr.shape(0); ++row) { @@ -354,6 +355,7 @@ std::ostream& operator <<(std::ostream& os, const NDArray& arr){ return os; } + template void NDArray::Print_all() { for (auto row = 0; row < shape_[0]; ++row) { for (auto col = 0; col < shape_[1]; ++col) { diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index 3e4fab8..906b215 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -156,6 +156,7 @@ template void NDView::print_all() const { } } + template std::ostream& operator <<(std::ostream& os, const NDView& arr){ for (auto row = 0; row < arr.shape(0); ++row) { @@ -168,4 +169,5 @@ std::ostream& operator <<(std::ostream& os, const NDView& arr){ return os; } + } // namespace aare \ No newline at end of file diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 8fefa34..5c28481 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -5,6 +5,7 @@ #include "aare/NDArray.hpp" //for pixel map #include "aare/RawSubFile.hpp" + #include namespace aare { @@ -62,6 +63,7 @@ class RawFile : public FileInterface { //TODO! do we need to adapt the API? void read_into(std::byte *image_buf, DetectorHeader *header); + size_t frame_number(size_t frame_index) override; size_t bytes_per_frame() override; size_t pixels_per_frame() override; @@ -75,6 +77,7 @@ class RawFile : public FileInterface { xy geometry(); size_t n_mod() const; + DetectorType detector_type() const override; private: @@ -91,8 +94,10 @@ class RawFile : public FileInterface { * @param frame_number frame number to read * @param image_buf buffer to store the frame */ + void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr); + /** * @brief get the frame at the given frame index * @param frame_number frame number to read @@ -111,6 +116,7 @@ class RawFile : public FileInterface { void update_geometry_with_roi(); int find_number_of_subfiles(); + void open_subfiles(); void find_geometry(); }; diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 52ce500..04eb1c9 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -58,6 +58,7 @@ class ScanParameters { bool enabled() const; }; + struct ROI{ size_t xmin{}; size_t xmax{}; @@ -107,6 +108,7 @@ class RawMasterFile { std::optional m_roi; + public: RawMasterFile(const std::filesystem::path &fpath); @@ -133,8 +135,10 @@ class RawMasterFile { std::optional number_of_rows() const; std::optional quad() const; + std::optional roi() const; + ScanParameters scan_parameters() const; private: diff --git a/include/aare/SubFile.hpp b/include/aare/SubFile.hpp new file mode 100644 index 0000000..47cc07c --- /dev/null +++ b/include/aare/SubFile.hpp @@ -0,0 +1,80 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/defs.hpp" + +#include +#include +#include +#include + + +namespace aare { + +/** + * @brief Class to read a subfile from a RawFile + */ +class SubFile { + public: + size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index); + /** + * @brief SubFile constructor + * @param fname path to the subfile + * @param detector detector type + * @param rows number of rows in the subfile + * @param cols number of columns in the subfile + * @param bitdepth bitdepth of the subfile + * @throws std::invalid_argument if the detector,type pair is not supported + */ + SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode = "r"); + + /** + * @brief read the subfile into a buffer + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + size_t read_impl_normal(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes flipped + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + template size_t read_impl_flip(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes reordered + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + template size_t read_impl_reorder(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes reordered and flipped + * @param buffer pointer to the buffer to read the data into + * @param frame_number frame number to read + * @return number of bytes read + */ + size_t get_part(std::byte *buffer, size_t frame_index); + size_t frame_number(size_t frame_index); + + // TODO: define the inlines as variables and assign them in constructor + inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } + inline size_t pixels_per_part() const { return m_rows * m_cols; } + + ~SubFile(); + + protected: + FILE *fp = nullptr; + size_t m_bitdepth; + std::filesystem::path m_fname; + size_t m_rows{}; + size_t m_cols{}; + std::string m_mode; + size_t n_frames{}; + int m_sub_file_index_{}; + DetectorType m_detector_type; + std::optional> pixel_map; +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index b32c6be..1fc369d 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -15,10 +15,6 @@ #include - - - - /** * @brief LOCATION macro to get the current location in the code */ @@ -27,6 +23,7 @@ ":" + std::string(__func__) + ":" + #ifdef AARE_CUSTOM_ASSERT #define AARE_ASSERT(expr)\ if (expr)\ @@ -43,6 +40,7 @@ namespace aare { void assert_failed(const std::string &msg); + class Cluster { public: int cluster_sizeX; @@ -192,6 +190,7 @@ struct ModuleGeometry{ int width{}; }; + using dynamic_shape = std::vector; //TODO! Can we uniform enums between the libraries? diff --git a/pyproject.toml b/pyproject.toml index 5cb6b71..fd3d35e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "scikit_build_core.build" name = "aare" version = "2024.11.11.dev0" + [tool.scikit-build] cmake.verbose = true diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 5892db0..f3ed63b 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -1,8 +1,10 @@ # Make the compiled classes that live in _aare available from aare. from . import _aare + from ._aare import File, RawFile, RawMasterFile, RawSubFile from ._aare import Pedestal, ClusterFinder, VarClusterFinder from ._aare import DetectorType + from .CtbRawFile import CtbRawFile from .ScanParameters import ScanParameters \ No newline at end of file diff --git a/python/examples/play.py b/python/examples/play.py index 0e03f9f..ceeb1d3 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -52,6 +52,7 @@ def get_Mh02_frames(fname): # [frame, counter, row, col] # plt.imshow(data[0,0]) + base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data') # p = Path(base / 'jungfrau/jungfrau_single_master_0.json') @@ -84,4 +85,4 @@ fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZP # fname = Path(base / 'jungfrau/jungfrau_single_master_0.json') f = RawFile(fname) h,img = f.read_frame() -print(f'{h["frameNumber"]}') \ No newline at end of file +print(f'{h["frameNumber"]}') diff --git a/python/src/file.hpp b/python/src/file.hpp index 54cb6a7..c7cb445 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -4,6 +4,7 @@ #include "aare/RawFile.hpp" #include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" + #include "aare/defs.hpp" // #include "aare/fClusterFileV2.hpp" @@ -21,6 +22,7 @@ using namespace ::aare; void define_file_io_bindings(py::module &m) { + py::enum_(m, "DetectorType") .value("Jungfrau", DetectorType::Jungfrau) .value("Eiger", DetectorType::Eiger) @@ -31,6 +33,7 @@ void define_file_io_bindings(py::module &m) { .value("ChipTestBoard", DetectorType::ChipTestBoard) .value("Unknown", DetectorType::Unknown); + PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber, bunchId, timestamp, modId, row, column, reserved, debug, roundRNumber, detType, version, packetMask); @@ -60,8 +63,10 @@ void define_file_io_bindings(py::module &m) { .def("seek", &CtbRawFile::seek) .def("tell", &CtbRawFile::tell) .def("master", &CtbRawFile::master) + .def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes) + .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file); py::class_(m, "File") @@ -163,6 +168,7 @@ void define_file_io_bindings(py::module &m) { return ""; }); + py::class_(m, "RawMasterFile") .def(py::init()) .def("data_fname", &RawMasterFile::data_fname) @@ -180,12 +186,14 @@ void define_file_io_bindings(py::module &m) { .def_property_readonly("frame_padding", &RawMasterFile::frame_padding) .def_property_readonly("frame_discard_policy", &RawMasterFile::frame_discard_policy) + .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected) .def_property_readonly("geometry", &RawMasterFile::geometry) .def_property_readonly("analog_samples", &RawMasterFile::analog_samples) .def_property_readonly("digital_samples", &RawMasterFile::digital_samples) + .def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples) .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows) @@ -197,12 +205,14 @@ void define_file_io_bindings(py::module &m) { py::class_(m, "ScanParameters") .def(py::init()) .def(py::init()) + .def_property_readonly("enabled", &ScanParameters::enabled) .def_property_readonly("dac", &ScanParameters::dac) .def_property_readonly("start", &ScanParameters::start) .def_property_readonly("stop", &ScanParameters::stop) .def_property_readonly("step", &ScanParameters::step); + py::class_(m, "ROI") .def(py::init<>()) .def_readwrite("xmin", &ROI::xmin) @@ -276,6 +286,7 @@ void define_file_io_bindings(py::module &m) { return image; }); + // py::class_(m, "ClusterHeader") // .def(py::init<>()) // .def_readwrite("frame_number", &ClusterHeader::frame_number) diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 35afdbd..a82b29b 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -13,11 +13,13 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) : m_master(fname) { m_mode = mode; if (mode == "r") { + n_subfiles = find_number_of_subfiles(); // f0,f1...fn n_subfile_parts = m_master.geometry().col * m_master.geometry().row; // d0,d1...dn find_geometry(); update_geometry_with_roi(); + open_subfiles(); } else { throw std::runtime_error(LOCATION + @@ -41,11 +43,11 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) { } } - void RawFile::read_into(std::byte *image_buf) { return get_frame_into(m_current_frame++, image_buf); }; + void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { return get_frame_into(m_current_frame++, image_buf, header); @@ -53,6 +55,7 @@ void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { size_t RawFile::n_mod() const { return n_subfile_parts; } + size_t RawFile::bytes_per_frame() { return m_rows * m_cols * m_master.bitdepth() / 8; } @@ -88,6 +91,7 @@ void RawFile::open_subfiles() { v[j] = new RawSubFile(m_master.data_fname(j, i), m_master.detector_type(), pos.height, pos.width, m_master.bitdepth()); + } subfiles.push_back(v); } @@ -128,12 +132,14 @@ int RawFile::find_number_of_subfiles() { fmt::print("Found: {} subfiles\n", n_files); #endif return n_files; + } void RawFile::find_geometry() { uint16_t r{}; uint16_t c{}; + for (size_t i = 0; i < n_subfile_parts; i++) { auto h = this->read_header(m_master.data_fname(i, 0)); r = std::max(r, h.row); @@ -145,6 +151,7 @@ void RawFile::find_geometry() { g.width = m_master.pixels_x(); g.height = m_master.pixels_y(); m_module_pixel_0.push_back(g); + } r++; @@ -154,6 +161,7 @@ void RawFile::find_geometry() { m_cols = (c * m_master.pixels_x()); m_rows += static_cast((r - 1) * cfg.module_gap_row); + #ifdef AARE_VERBOSE fmt::print("\nRawFile::find_geometry()\n"); for (size_t i = 0; i < m_module_pixel_0.size(); i++) { @@ -227,6 +235,7 @@ void RawFile::update_geometry_with_roi() { fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols); fmt::print("\n"); #endif + } Frame RawFile::get_frame(size_t frame_index) { @@ -236,18 +245,22 @@ Frame RawFile::get_frame(size_t frame_index) { return f; } + size_t RawFile::bytes_per_pixel() const { return m_master.bitdepth() / 8; } void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) { + if (frame_index > total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } std::vector frame_numbers(n_subfile_parts); std::vector frame_indices(n_subfile_parts, frame_index); + // sync the frame numbers + if (n_subfile_parts != 1) { for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { auto subfile_id = frame_index / m_master.max_frames_per_file(); @@ -302,6 +315,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect //TODO! should we read row by row? // create a buffer large enough to hold a full module + auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() * m_master.bitdepth() / 8; // TODO! replace with image_size_in_bytes @@ -331,6 +345,7 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect part_buffer + cur_row * pos.width * m_master.bitdepth() / 8, pos.width * m_master.bitdepth() / 8); + } } delete[] part_buffer; diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index ed16314..ae2caa1 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -8,8 +8,6 @@ RawFileNameComponents::RawFileNameComponents( m_base_name = fname.stem(); m_ext = fname.extension(); - AARE_ASSERT(false); - if (m_ext != ".json" && m_ext != ".raw") { throw std::runtime_error(LOCATION + "Unsupported file type. (only .json or .raw)"); @@ -145,8 +143,10 @@ ScanParameters RawMasterFile::scan_parameters() const { return m_scan_parameters; } + std::optional RawMasterFile::roi() const { return m_roi; } + void RawMasterFile::parse_json(const std::filesystem::path &fpath) { std::ifstream ifs(fpath); json j; @@ -244,6 +244,7 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { // not a scan } + try{ ROI tmp_roi; auto obj = j.at("Receiver Roi"); @@ -276,6 +277,7 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { + // Update detector type for Moench // TODO! How does this work with old .raw master files? #ifdef AARE_VERBOSE diff --git a/src/SubFile.cpp b/src/SubFile.cpp new file mode 100644 index 0000000..314fec5 --- /dev/null +++ b/src/SubFile.cpp @@ -0,0 +1,91 @@ +#include "aare/SubFile.hpp" +#include "aare/PixelMap.hpp" +#include // memcpy +#include +#include + + +namespace aare { + +SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) { + + + if (m_detector_type == DetectorType::Moench03_old) { + pixel_map = GenerateMoench03PixelMap(); + } + + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); + } else { + n_frames = 0; + } + + if (mode == "r") { + fp = fopen(m_fname.string().c_str(), "rb"); + } else { + throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles."); + } + if (fp == nullptr) { + throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string())); + } +#ifdef AARE_VERBOSE + fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); + fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth); +#endif +} + +size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { + if (frame_index >= n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT + SEEK_SET); + + if (pixel_map){ + // read into a temporary buffer and then copy the data to the buffer + // in the correct order + auto part_buffer = new std::byte[bytes_per_part()]; + auto wc = fread(part_buffer, bytes_per_part(), 1, fp); + auto *data = reinterpret_cast(buffer); + auto *part_data = reinterpret_cast(part_buffer); + for (size_t i = 0; i < pixels_per_part(); i++) { + data[i] = part_data[(*pixel_map)(i)]; + } + delete[] part_buffer; + return wc; + }else{ + // read directly into the buffer + return fread(buffer, this->bytes_per_part(), 1, this->fp); + } + +} +size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) { + if (frame_index > n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, static_cast((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET); + auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); + wc += fwrite(buffer, bytes_per_part(), 1, fp); + + return wc; +} + +size_t SubFile::frame_number(size_t frame_index) { + DetectorHeader h{}; + fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT + size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + if (rc != 1) + throw std::runtime_error(LOCATION + "Could not read header from file"); + + return h.frameNumber; +} + +SubFile::~SubFile() { + if (fp) { + fclose(fp); + } +} + +} // namespace aare \ No newline at end of file diff --git a/src/defs.cpp b/src/defs.cpp index b8171c1..16fa3d0 100644 --- a/src/defs.cpp +++ b/src/defs.cpp @@ -1,6 +1,7 @@ #include "aare/defs.hpp" #include #include + #include namespace aare { @@ -12,6 +13,7 @@ void assert_failed(const std::string &msg) } + /** * @brief Convert a DetectorType to a string * @param type DetectorType