From fbaf9dce890d7a106e023302dd9196cd1d25f0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Thu, 14 Nov 2024 08:03:18 +0100 Subject: [PATCH] Developer (#94) --- CMakeLists.txt | 7 +- docs/CMakeLists.txt | 3 + docs/src/Installation.rst | 101 +++++++++++++++++++ docs/src/Requirements.rst | 23 +++++ docs/src/index.rst | 20 ++++ docs/src/pyRawFile.rst | 10 ++ include/aare/NDArray.hpp | 173 ++++++++++++++++++++------------- include/aare/NDView.hpp | 4 +- include/aare/PixelMap.hpp | 3 + include/aare/RawFile.hpp | 13 +-- include/aare/RawMasterFile.hpp | 4 +- include/aare/RawSubFile.hpp | 6 +- include/aare/SubFile.hpp | 80 --------------- python/examples/play.py | 11 ++- python/src/file.hpp | 69 ++----------- python/src/module.cpp | 4 + python/src/raw_file.hpp | 95 ++++++++++++++++++ python/src/raw_master_file.hpp | 85 ++++++++++++++++ src/NDArray.test.cpp | 38 +------- src/PixelMap.cpp | 10 ++ src/RawFile.cpp | 29 ++++-- src/RawMasterFile.cpp | 6 ++ src/RawSubFile.cpp | 13 ++- src/SubFile.cpp | 91 ----------------- 24 files changed, 533 insertions(+), 365 deletions(-) create mode 100644 docs/src/Installation.rst create mode 100644 docs/src/Requirements.rst create mode 100644 docs/src/pyRawFile.rst delete mode 100644 include/aare/SubFile.hpp create mode 100644 python/src/raw_file.hpp create mode 100644 python/src/raw_master_file.hpp delete mode 100644 src/SubFile.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d72d784..98f1018 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -256,26 +256,25 @@ set(PUBLICHEADERS include/aare/Pedestal.hpp include/aare/PixelMap.hpp include/aare/RawFile.hpp - include/aare/RawSubFile.hpp include/aare/RawMasterFile.hpp - include/aare/SubFile.hpp + include/aare/RawSubFile.hpp include/aare/VarClusterFinder.hpp ) set(SourceFiles - ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp ) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index f4fb00f..b0c45ce 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -12,6 +12,8 @@ set(SPHINX_BUILD ${CMAKE_CURRENT_BINARY_DIR}) set(SPHINX_SOURCE_FILES src/index.rst + src/Installation.rst + src/Requirements.rst src/NDArray.rst src/NDView.rst src/File.rst @@ -26,6 +28,7 @@ set(SPHINX_SOURCE_FILES src/pyVarClusterFinder.rst src/pyFile.rst src/pyCtbRawFile.rst + src/pyRawFile.rst src/pyRawMasterFile.rst ) diff --git a/docs/src/Installation.rst b/docs/src/Installation.rst new file mode 100644 index 0000000..250310f --- /dev/null +++ b/docs/src/Installation.rst @@ -0,0 +1,101 @@ +Installation +=============== + +conda/mamaba +~~~~~~~~~~~~~~~~ + +.. note :: + + aare is developing rapidly. Check for the latest release by + using: **conda search aare -c slsdetectorgroup** + + +.. code-block:: bash + + # Install a specific version: + conda install aare=2024.11.11.dev0 -c slsdetectorgroup + + +cmake (development install) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http... + mkdir build + cd build + + #configure using cmake + cmake ../aare + + #build (replace 4 with the number of threads you want to use) + make -j4 + + + # add the build folder to your PYTHONPATH + +cmake install and use in your C++ project +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + #build and install aare + git clone git@github.com:slsdetectorgroup/aare.git --branch=v1 #or using http... + mkdir build + cd build + + #configure using cmake + cmake ../aare -DCMAKE_INSTALL_PREFIX=/where/to/put/aare + + #build (replace 4 with the number of threads you want to use) + make -j4 + + #install + make install + + + #Now configure your project + cmake .. -DCMAKE_PREFIX_PATH=SOME_PATH + + +cmake options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For detailed options see the CMakeLists.txt file in the root directory of the project. + +.. code-block:: bash + + # usage (or edit with ccmake .) + cmake ../aare -DOPTION1=ON -DOPTION2=OFF + + +**AARE_SYSTEM_LIBRARIES "Use system libraries" OFF** + +Use system libraries instead of using FetchContent to pull in dependencies. Default option is off. + + +**AARE_PYTHON_BINDINGS "Build python bindings" ON** + +Build the Python bindings. Default option is on. + +.. warning :: + + If you have a newer system Python compared to the one in your virtual environment, + you might have to pass -DPython_FIND_VIRTUALENV=ONLY to cmake. + +**AARE_TESTS "Build tests" OFF** + +Build unit tests. Default option is off. + +**AARE_EXAMPLES "Build examples" OFF** + +**AARE_DOCS "Build documentation" OFF** + +Build documentation. Needs doxygen, sphinx and breathe. Default option is off. +Requires a separate make docs. + +**AARE_VERBOSE "Verbose output" OFF** + +**AARE_CUSTOM_ASSERT "Use custom assert" OFF** + +Enable custom assert macro to check for errors. Default option is off. \ No newline at end of file diff --git a/docs/src/Requirements.rst b/docs/src/Requirements.rst new file mode 100644 index 0000000..c962f73 --- /dev/null +++ b/docs/src/Requirements.rst @@ -0,0 +1,23 @@ +Requirements +============================================== + +- C++17 compiler (gcc 8/clang 7) +- CMake 3.14+ + +**Internally used libraries** + +.. note :: + + These can also be picked up from the system/conda environment by specifying: + -DAARE_SYSTEM_LIBRARIES=ON during the cmake configuration. + +- pybind11 +- fmt +- nlohmann_json +- ZeroMQ + +**Extra dependencies for building documentation** + +- Sphinx +- Breathe +- Doxygen \ No newline at end of file diff --git a/docs/src/index.rst b/docs/src/index.rst index 7bac72f..588d651 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -5,6 +5,26 @@ AARE Hello + +.. toctree:: + :caption: Installation + :maxdepth: 1 + + Installation + Requirements + + +.. toctree:: + :caption: Python API + :maxdepth: 1 + + pyFile + pyCtbRawFile + pyRawFile + pyRawMasterFile + pyVarClusterFinder + + .. toctree:: :caption: Python API :maxdepth: 1 diff --git a/docs/src/pyRawFile.rst b/docs/src/pyRawFile.rst new file mode 100644 index 0000000..b81f413 --- /dev/null +++ b/docs/src/pyRawFile.rst @@ -0,0 +1,10 @@ +RawFile +=================== + +.. py:currentmodule:: aare + +.. autoclass:: RawFile + :members: + :undoc-members: + :show-inheritance: + :inherited-members: \ No newline at end of file diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index e65b136..75f4c1c 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -21,33 +21,62 @@ TODO! Add expression templates for operators namespace aare { template class NDArray { - public: - NDArray() : shape_(), strides_(c_strides(shape_)), data_(nullptr){}; + std::array shape_; + std::array strides_; + size_t size_{}; + T *data_; + public: + /** + * @brief Default constructor. Will construct an empty NDArray. + * + */ + NDArray() : shape_(), strides_(c_strides(shape_)), data_(nullptr) {}; + + /** + * @brief Construct a new NDArray object with a given shape. + * @note The data is uninitialized. + * + * @param shape shape of the new NDArray + */ explicit NDArray(std::array shape) : shape_(shape), strides_(c_strides(shape_)), - size_(std::accumulate(shape_.begin(), shape_.end(), 1, std::multiplies<>())), data_(new T[size_]){}; + size_(std::accumulate(shape_.begin(), shape_.end(), 1, + std::multiplies<>())), + data_(new T[size_]) {}; - NDArray(std::array shape, T value) : NDArray(shape) { this->operator=(value); } + /** + * @brief Construct a new NDArray object with a shape and value. + * + * @param shape shape of the new array + * @param value value to initialize the array with + */ + NDArray(std::array shape, T value) : NDArray(shape) { + this->operator=(value); + } - /* When constructing from a NDView we need to copy the data since - NDArray expect to own its data, and span is just a view*/ - explicit NDArray(NDView span) : NDArray(span.shape()) { - std::copy(span.begin(), span.end(), begin()); - // fmt::print("NDArray(NDView span)\n"); + /** + * @brief Construct a new NDArray object from a NDView. + * @note The data is copied from the view to the NDArray. + * + * @param v view of data to initialize the NDArray with + */ + explicit NDArray(const NDView v) : NDArray(v.shape()) { + std::copy(v.begin(), v.end(), begin()); } // Move constructor NDArray(NDArray &&other) noexcept - : shape_(other.shape_), strides_(c_strides(shape_)), size_(other.size_), data_(other.data_) { + : shape_(other.shape_), strides_(c_strides(shape_)), + size_(other.size_), data_(other.data_) { other.reset(); - // fmt::print("NDArray(NDArray &&other)\n"); } // Copy constructor NDArray(const NDArray &other) - : shape_(other.shape_), strides_(c_strides(shape_)), size_(other.size_), data_(new T[size_]) { + : shape_(other.shape_), strides_(c_strides(shape_)), + size_(other.size_), data_(new T[size_]) { std::copy(other.data_, other.data_ + size_, data_); // fmt::print("NDArray(const NDArray &other)\n"); } @@ -70,6 +99,7 @@ template class NDArray { NDArray &operator*=(const NDArray &other); NDArray operator/(const NDArray &other); // NDArray& operator/=(const NDArray& other); + template NDArray &operator/=(const NDArray &other) { // check shape if (shape_ == other.shape()) { @@ -106,15 +136,18 @@ template class NDArray { NDArray &operator++(); // pre inc - template std::enable_if_t operator()(Ix... index) { + template + std::enable_if_t operator()(Ix... index) { return data_[element_offset(strides_, index...)]; } - template std::enable_if_t operator()(Ix... index) const { + template + std::enable_if_t operator()(Ix... index) const { return data_[element_offset(strides_, index...)]; } - template std::enable_if_t value(Ix... index) { + template + std::enable_if_t value(Ix... index) { return data_[element_offset(strides_, index...)]; } @@ -129,15 +162,20 @@ template class NDArray { int64_t shape(int64_t i) const noexcept { return shape_[i]; } std::array strides() const noexcept { return strides_; } size_t bitdepth() const noexcept { return sizeof(T) * 8; } + std::array byte_strides() const noexcept { auto byte_strides = strides_; for (auto &val : byte_strides) val *= sizeof(T); return byte_strides; - // return strides_; } - NDView span() const { return NDView{data_, shape_}; } + /** + * @brief Create a view of the NDArray. + * + * @return NDView + */ + NDView view() const { return NDView{data_, shape_}; } void Print(); void Print_all(); @@ -149,16 +187,12 @@ template class NDArray { std::fill(shape_.begin(), shape_.end(), 0); std::fill(strides_.begin(), strides_.end(), 0); } - - private: - std::array shape_; - std::array strides_; - uint64_t size_{}; - T *data_; }; // Move assign -template NDArray &NDArray::operator=(NDArray &&other) noexcept { +template +NDArray & +NDArray::operator=(NDArray &&other) noexcept { if (this != &other) { delete[] data_; data_ = other.data_; @@ -170,12 +204,14 @@ template NDArray &NDArray::operator return *this; } -template NDArray NDArray::operator+(const NDArray &other) { +template +NDArray NDArray::operator+(const NDArray &other) { NDArray result(*this); result += other; return result; } -template NDArray &NDArray::operator+=(const NDArray &other) { +template +NDArray &NDArray::operator+=(const NDArray &other) { // check shape if (shape_ == other.shape_) { for (uint32_t i = 0; i < size_; ++i) { @@ -186,13 +222,15 @@ template NDArray &NDArray::operator throw(std::runtime_error("Shape of ImageDatas must match")); } -template NDArray NDArray::operator-(const NDArray &other) { +template +NDArray NDArray::operator-(const NDArray &other) { NDArray result{*this}; result -= other; return result; } -template NDArray &NDArray::operator-=(const NDArray &other) { +template +NDArray &NDArray::operator-=(const NDArray &other) { // check shape if (shape_ == other.shape_) { for (uint32_t i = 0; i < size_; ++i) { @@ -202,13 +240,15 @@ template NDArray &NDArray::operator } throw(std::runtime_error("Shape of ImageDatas must match")); } -template NDArray NDArray::operator*(const NDArray &other) { +template +NDArray NDArray::operator*(const NDArray &other) { NDArray result = *this; result *= other; return result; } -template NDArray &NDArray::operator*=(const NDArray &other) { +template +NDArray &NDArray::operator*=(const NDArray &other) { // check shape if (shape_ == other.shape_) { for (uint32_t i = 0; i < size_; ++i) { @@ -219,34 +259,22 @@ template NDArray &NDArray::operator throw(std::runtime_error("Shape of ImageDatas must match")); } -template NDArray NDArray::operator/(const NDArray &other) { +template +NDArray NDArray::operator/(const NDArray &other) { NDArray result = *this; result /= other; return result; } -template NDArray &NDArray::operator&=(const T &mask) { +template +NDArray &NDArray::operator&=(const T &mask) { for (auto it = begin(); it != end(); ++it) *it &= mask; return *this; } -// template -// NDArray& NDArray::operator/=(const NDArray& -// other) -// { -// //check shape -// if (shape_ == other.shape_) { -// for (int i = 0; i < size_; ++i) { -// data_[i] /= other.data_[i]; -// } -// return *this; -// } else { -// throw(std::runtime_error("Shape of ImageDatas must match")); -// } -// } - -template NDArray NDArray::operator>(const NDArray &other) { +template +NDArray NDArray::operator>(const NDArray &other) { if (shape_ == other.shape_) { NDArray result{shape_}; for (int i = 0; i < size_; ++i) { @@ -257,7 +285,8 @@ template NDArray NDArray::operat throw(std::runtime_error("Shape of ImageDatas must match")); } -template NDArray &NDArray::operator=(const NDArray &other) { +template +NDArray &NDArray::operator=(const NDArray &other) { if (this != &other) { delete[] data_; shape_ = other.shape_; @@ -269,7 +298,8 @@ template NDArray &NDArray::operator return *this; } -template bool NDArray::operator==(const NDArray &other) const { +template +bool NDArray::operator==(const NDArray &other) const { if (shape_ != other.shape_) return false; @@ -280,57 +310,68 @@ template bool NDArray::operator==(const NDAr return true; } -template bool NDArray::operator!=(const NDArray &other) const { +template +bool NDArray::operator!=(const NDArray &other) const { return !((*this) == other); } -template NDArray &NDArray::operator++() { +template +NDArray &NDArray::operator++() { for (uint32_t i = 0; i < size_; ++i) data_[i] += 1; return *this; } -template NDArray &NDArray::operator=(const T &value) { +template +NDArray &NDArray::operator=(const T &value) { std::fill_n(data_, size_, value); return *this; } -template NDArray &NDArray::operator+=(const T &value) { +template +NDArray &NDArray::operator+=(const T &value) { for (uint32_t i = 0; i < size_; ++i) data_[i] += value; return *this; } -template NDArray NDArray::operator+(const T &value) { +template +NDArray NDArray::operator+(const T &value) { NDArray result = *this; result += value; return result; } -template NDArray &NDArray::operator-=(const T &value) { +template +NDArray &NDArray::operator-=(const T &value) { for (uint32_t i = 0; i < size_; ++i) data_[i] -= value; return *this; } -template NDArray NDArray::operator-(const T &value) { +template +NDArray NDArray::operator-(const T &value) { NDArray result = *this; result -= value; return result; } -template NDArray &NDArray::operator/=(const T &value) { +template +NDArray &NDArray::operator/=(const T &value) { for (uint32_t i = 0; i < size_; ++i) data_[i] /= value; return *this; } -template NDArray NDArray::operator/(const T &value) { +template +NDArray NDArray::operator/(const T &value) { NDArray result = *this; result /= value; return result; } -template NDArray &NDArray::operator*=(const T &value) { +template +NDArray &NDArray::operator*=(const T &value) { for (uint32_t i = 0; i < size_; ++i) data_[i] *= value; return *this; } -template NDArray NDArray::operator*(const T &value) { +template +NDArray NDArray::operator*(const T &value) { NDArray result = *this; result *= value; return result; @@ -342,9 +383,8 @@ template void NDArray::Print() { Print_some(); } - template -std::ostream& operator <<(std::ostream& os, const NDArray& arr){ +std::ostream &operator<<(std::ostream &os, const NDArray &arr) { for (auto row = 0; row < arr.shape(0); ++row) { for (auto col = 0; col < arr.shape(1); ++col) { os << std::setw(3); @@ -355,7 +395,6 @@ std::ostream& operator <<(std::ostream& os, const NDArray& arr){ return os; } - template void NDArray::Print_all() { for (auto row = 0; row < shape_[0]; ++row) { for (auto col = 0; col < shape_[1]; ++col) { @@ -375,7 +414,8 @@ template void NDArray::Print_some() { } } -template void save(NDArray &img, std::string &pathname) { +template +void save(NDArray &img, std::string &pathname) { std::ofstream f; f.open(pathname, std::ios::binary); f.write(img.buffer(), img.size() * sizeof(T)); @@ -383,7 +423,8 @@ template void save(NDArray &img, std::string } template -NDArray load(const std::string &pathname, std::array shape) { +NDArray load(const std::string &pathname, + std::array shape) { NDArray img{shape}; std::ifstream f; f.open(pathname, std::ios::binary); diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index 906b215..e02b3b2 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -74,6 +74,8 @@ template class NDView { T *begin() { return buffer_; } T *end() { return buffer_ + size_; } + T const *begin() const { return buffer_; } + T const *end() const { return buffer_ + size_; } T &operator()(int64_t i) const { return buffer_[i]; } T &operator[](int64_t i) const { return buffer_[i]; } @@ -121,7 +123,7 @@ template class NDView { return *this; } - auto &shape() { return shape_; } + auto &shape() const { return shape_; } auto shape(int64_t i) const { return shape_[i]; } T *data() { return buffer_; } diff --git a/include/aare/PixelMap.hpp b/include/aare/PixelMap.hpp index 4838737..37fee95 100644 --- a/include/aare/PixelMap.hpp +++ b/include/aare/PixelMap.hpp @@ -8,8 +8,11 @@ namespace aare { NDArray GenerateMoench03PixelMap(); NDArray GenerateMoench05PixelMap(); +//Matterhorn02 NDArrayGenerateMH02SingleCounterPixelMap(); NDArray GenerateMH02FourCounterPixelMap(); +//Eiger +NDArrayGenerateEigerFlipRowsPixelMap(); } // namespace aare \ No newline at end of file diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 5c28481..eb044e3 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -62,32 +62,29 @@ class RawFile : public FileInterface { //TODO! do we need to adapt the API? void read_into(std::byte *image_buf, DetectorHeader *header); + void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header); size_t frame_number(size_t frame_index) override; size_t bytes_per_frame() override; size_t pixels_per_frame() override; + size_t bytes_per_pixel() const; void seek(size_t frame_index) override; size_t tell() override; size_t total_frames() const override; size_t rows() const override; size_t cols() const override; size_t bitdepth() const override; - size_t bytes_per_pixel() const; xy geometry(); size_t n_mod() const; + + RawMasterFile master() const; + DetectorType detector_type() const override; private: - /** - * @brief check if the file is a master file - * @param fpath path to the file - */ - static bool is_master_file(const std::filesystem::path &fpath); - - // TODO! Deal with fast quad and missing files /** * @brief read the frame at the given frame index into the image buffer diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 04eb1c9..f65d307 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -56,6 +56,7 @@ class ScanParameters { int step() const; const std::string &dac() const; bool enabled() const; + void increment_stop(); }; @@ -67,7 +68,8 @@ struct ROI{ size_t height() const { return ymax - ymin; } size_t width() const { return xmax - xmin; } -}; +}__attribute__((packed)); + /** * @brief Class for parsing a master file either in our .json format or the old diff --git a/include/aare/RawSubFile.hpp b/include/aare/RawSubFile.hpp index 5efa4b6..d5ff4f0 100644 --- a/include/aare/RawSubFile.hpp +++ b/include/aare/RawSubFile.hpp @@ -22,9 +22,11 @@ class RawSubFile { size_t m_cols{}; size_t m_bytes_per_frame{}; size_t n_frames{}; + uint32_t m_pos_row{}; + uint32_t m_pos_col{}; DetectorType m_detector_type; - std::optional> pixel_map; + std::optional> m_pixel_map; public: /** @@ -37,7 +39,7 @@ class RawSubFile { * @throws std::invalid_argument if the detector,type pair is not supported */ RawSubFile(const std::filesystem::path &fname, DetectorType detector, - size_t rows, size_t cols, size_t bitdepth); + size_t rows, size_t cols, size_t bitdepth, uint32_t pos_row = 0, uint32_t pos_col = 0); ~RawSubFile() = default; /** diff --git a/include/aare/SubFile.hpp b/include/aare/SubFile.hpp deleted file mode 100644 index 47cc07c..0000000 --- a/include/aare/SubFile.hpp +++ /dev/null @@ -1,80 +0,0 @@ -#pragma once -#include "aare/Frame.hpp" -#include "aare/defs.hpp" - -#include -#include -#include -#include - - -namespace aare { - -/** - * @brief Class to read a subfile from a RawFile - */ -class SubFile { - public: - size_t write_part(std::byte *buffer, DetectorHeader header, size_t frame_index); - /** - * @brief SubFile constructor - * @param fname path to the subfile - * @param detector detector type - * @param rows number of rows in the subfile - * @param cols number of columns in the subfile - * @param bitdepth bitdepth of the subfile - * @throws std::invalid_argument if the detector,type pair is not supported - */ - SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, - const std::string &mode = "r"); - - /** - * @brief read the subfile into a buffer - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - size_t read_impl_normal(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes flipped - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - template size_t read_impl_flip(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes reordered - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - template size_t read_impl_reorder(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes reordered and flipped - * @param buffer pointer to the buffer to read the data into - * @param frame_number frame number to read - * @return number of bytes read - */ - size_t get_part(std::byte *buffer, size_t frame_index); - size_t frame_number(size_t frame_index); - - // TODO: define the inlines as variables and assign them in constructor - inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } - inline size_t pixels_per_part() const { return m_rows * m_cols; } - - ~SubFile(); - - protected: - FILE *fp = nullptr; - size_t m_bitdepth; - std::filesystem::path m_fname; - size_t m_rows{}; - size_t m_cols{}; - std::string m_mode; - size_t n_frames{}; - int m_sub_file_index_{}; - DetectorType m_detector_type; - std::optional> pixel_map; -}; - -} // namespace aare \ No newline at end of file diff --git a/python/examples/play.py b/python/examples/play.py index ceeb1d3..43d585c 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -80,9 +80,16 @@ base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_da from aare import RawFile -base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/') -fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') + + +from aare import RawFile, File + +base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/') +fname = base / Path('SideBySide/241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') # fname = Path(base / 'jungfrau/jungfrau_single_master_0.json') +# fname = base / 'Stacked/241024_JF10_m450_m367_KnifeEdge_TestBesom_9keV_750umFilter_PedestalStart_ZPos_-6_master_0.json' + + f = RawFile(fname) h,img = f.read_frame() print(f'{h["frameNumber"]}') diff --git a/python/src/file.hpp b/python/src/file.hpp index c7cb445..79c0148 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -169,38 +169,6 @@ void define_file_io_bindings(py::module &m) { }); - py::class_(m, "RawMasterFile") - .def(py::init()) - .def("data_fname", &RawMasterFile::data_fname) - .def_property_readonly("version", &RawMasterFile::version) - .def_property_readonly("detector_type", &RawMasterFile::detector_type) - .def_property_readonly("timing_mode", &RawMasterFile::timing_mode) - .def_property_readonly("image_size_in_bytes", - &RawMasterFile::image_size_in_bytes) - .def_property_readonly("frames_in_file", &RawMasterFile::frames_in_file) - .def_property_readonly("pixels_y", &RawMasterFile::pixels_y) - .def_property_readonly("pixels_x", &RawMasterFile::pixels_x) - .def_property_readonly("max_frames_per_file", - &RawMasterFile::max_frames_per_file) - .def_property_readonly("bitdepth", &RawMasterFile::bitdepth) - .def_property_readonly("frame_padding", &RawMasterFile::frame_padding) - .def_property_readonly("frame_discard_policy", - &RawMasterFile::frame_discard_policy) - - .def_property_readonly("total_frames_expected", - &RawMasterFile::total_frames_expected) - .def_property_readonly("geometry", &RawMasterFile::geometry) - .def_property_readonly("analog_samples", &RawMasterFile::analog_samples) - .def_property_readonly("digital_samples", - &RawMasterFile::digital_samples) - - .def_property_readonly("transceiver_samples", - &RawMasterFile::transceiver_samples) - .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows) - .def_property_readonly("quad", &RawMasterFile::quad) - .def_property_readonly("scan_parameters", - &RawMasterFile::scan_parameters) - .def_property_readonly("roi", &RawMasterFile::roi); py::class_(m, "ScanParameters") .def(py::init()) @@ -221,40 +189,17 @@ void define_file_io_bindings(py::module &m) { .def_readwrite("ymax", &ROI::ymax) .def("__str__", [](const ROI& self){ return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax); + }) + .def("__repr__", [](const ROI& self){ + return fmt::format("", self.xmin, self.xmax, self.ymin, self.ymax); + }) + .def("__iter__", [](const ROI &self) { + return py::make_iterator(&self.xmin, &self.ymax+1); }); - py::class_(m, "RawFile") - .def(py::init()) - .def("read_frame", - [](RawFile &self) { - size_t image_size = self.bytes_per_frame(); - const uint8_t item_size = self.bytes_per_pixel(); - py::array image; - std::vector shape; - shape.reserve(2); - shape.push_back(self.rows()); - shape.push_back(self.cols()); + - //return headers from all subfiles - py::array_t header(self.n_mod()); - - if (item_size == 1) { - image = py::array_t(shape); - } else if (item_size == 2) { - image = py::array_t(shape); - } else if (item_size == 4) { - image = py::array_t(shape); - } - fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); - - self.read_into( - reinterpret_cast(image.mutable_data()), - header.mutable_data()); - - return py::make_tuple(header, image); - }); - py::class_(m, "RawSubFile") .def(py::init()) diff --git a/python/src/module.cpp b/python/src/module.cpp index ac4c4d2..9d68446 100644 --- a/python/src/module.cpp +++ b/python/src/module.cpp @@ -1,5 +1,7 @@ //Files with bindings to the different classes #include "file.hpp" +#include "raw_file.hpp" +#include "raw_master_file.hpp" #include "var_cluster.hpp" #include "pixel_map.hpp" #include "pedestal.hpp" @@ -13,6 +15,8 @@ namespace py = pybind11; PYBIND11_MODULE(_aare, m) { define_file_io_bindings(m); + define_raw_file_io_bindings(m); + define_raw_master_file_bindings(m); define_var_cluster_finder_bindings(m); define_pixel_map_bindings(m); define_pedestal_bindings(m, "Pedestal"); diff --git a/python/src/raw_file.hpp b/python/src/raw_file.hpp new file mode 100644 index 0000000..73442a4 --- /dev/null +++ b/python/src/raw_file.hpp @@ -0,0 +1,95 @@ +#include "aare/CtbRawFile.hpp" +#include "aare/File.hpp" +#include "aare/Frame.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" +#include "aare/RawSubFile.hpp" + +#include "aare/defs.hpp" +// #include "aare/fClusterFileV2.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace ::aare; + +void define_raw_file_io_bindings(py::module &m) { + py::class_(m, "RawFile") + .def(py::init()) + .def("read_frame", + [](RawFile &self) { + size_t image_size = self.bytes_per_frame(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + + // return headers from all subfiles + py::array_t header(self.n_mod()); + + const uint8_t item_size = self.bytes_per_pixel(); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into( + reinterpret_cast(image.mutable_data()), + header.mutable_data()); + + return py::make_tuple(header, image); + }) + .def("read_n", + [](RawFile &self, size_t n_frames) { + py::array image; + std::vector shape; + shape.reserve(3); + shape.push_back(n_frames); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + + // return headers from all subfiles + py::array_t header({self.n_mod(), n_frames}); + + const uint8_t item_size = self.bytes_per_pixel(); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into( + reinterpret_cast(image.mutable_data()), + n_frames, header.mutable_data()); + + return py::make_tuple(header, image); + }) + .def("frame_number", &RawFile::frame_number) + .def_property_readonly("bytes_per_frame", &RawFile::bytes_per_frame) + .def_property_readonly("pixels_per_frame", &RawFile::pixels_per_frame) + .def_property_readonly("bytes_per_pixel", &RawFile::bytes_per_pixel) + .def("seek", &RawFile::seek, R"( + Seek to a frame index in file. + )") + .def("tell", &RawFile::tell, R"( + Return the current frame number.)") + .def_property_readonly("total_frames", &RawFile::total_frames) + .def_property_readonly("rows", &RawFile::rows) + .def_property_readonly("cols", &RawFile::cols) + .def_property_readonly("bitdepth", &RawFile::bitdepth) + .def_property_readonly("geometry", &RawFile::geometry) + .def_property_readonly("n_mod", &RawFile::n_mod) + .def_property_readonly("detector_type", &RawFile::detector_type) + .def_property_readonly("master", &RawFile::master); +} \ No newline at end of file diff --git a/python/src/raw_master_file.hpp b/python/src/raw_master_file.hpp new file mode 100644 index 0000000..943437f --- /dev/null +++ b/python/src/raw_master_file.hpp @@ -0,0 +1,85 @@ + +#include "aare/CtbRawFile.hpp" +#include "aare/File.hpp" +#include "aare/Frame.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" +#include "aare/RawSubFile.hpp" + +#include "aare/defs.hpp" +// #include "aare/fClusterFileV2.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace ::aare; + +void define_raw_master_file_bindings(py::module &m) { + py::class_(m, "RawMasterFile") + .def(py::init()) + .def("data_fname", &RawMasterFile::data_fname, R"( + + Parameters + ------------ + module_index : int + module index (d0, d1 .. dN) + file_index : int + file index (f0, f1 .. fN) + + Returns + ---------- + os.PathLike + The name of the data file. + + )") + .def_property_readonly("version", &RawMasterFile::version) + .def_property_readonly("detector_type", &RawMasterFile::detector_type) + .def_property_readonly("timing_mode", &RawMasterFile::timing_mode) + .def_property_readonly("image_size_in_bytes", + &RawMasterFile::image_size_in_bytes) + .def_property_readonly("frames_in_file", &RawMasterFile::frames_in_file) + .def_property_readonly("pixels_y", &RawMasterFile::pixels_y) + .def_property_readonly("pixels_x", &RawMasterFile::pixels_x) + .def_property_readonly("max_frames_per_file", + &RawMasterFile::max_frames_per_file) + .def_property_readonly("bitdepth", &RawMasterFile::bitdepth) + .def_property_readonly("frame_padding", &RawMasterFile::frame_padding) + .def_property_readonly("frame_discard_policy", + &RawMasterFile::frame_discard_policy) + + .def_property_readonly("total_frames_expected", + &RawMasterFile::total_frames_expected) + .def_property_readonly("geometry", &RawMasterFile::geometry) + .def_property_readonly("analog_samples", &RawMasterFile::analog_samples, R"( + Number of analog samples + + Returns + ---------- + int | None + The number of analog samples in the file (or None if not enabled) + )") + .def_property_readonly("digital_samples", + &RawMasterFile::digital_samples, R"( + Number of digital samples + + Returns + ---------- + int | None + The number of digital samples in the file (or None if not enabled) + )") + + .def_property_readonly("transceiver_samples", + &RawMasterFile::transceiver_samples) + .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows) + .def_property_readonly("quad", &RawMasterFile::quad) + .def_property_readonly("scan_parameters", + &RawMasterFile::scan_parameters) + .def_property_readonly("roi", &RawMasterFile::roi); +} diff --git a/src/NDArray.test.cpp b/src/NDArray.test.cpp index 97aafa0..90a8d8e 100644 --- a/src/NDArray.test.cpp +++ b/src/NDArray.test.cpp @@ -12,7 +12,7 @@ TEST_CASE("Initial size is zero if no size is specified") { REQUIRE(a.shape() == Shape<2>{0, 0}); } -TEST_CASE("Construct from a DataSpan") { +TEST_CASE("Construct from an NDView") { std::vector some_data(9, 42); NDView view(some_data.data(), Shape<2>{3, 3}); @@ -168,42 +168,8 @@ TEST_CASE("Bitwise and on data") { REQUIRE(a(2) == 384); } -// TEST_CASE("Benchmarks") -// { -// NDArray img; -// std::array shape{ 512, 1024 }; -// BENCHMARK("Allocate 500k double image") -// { -// NDArrayim{ shape }; -// } -// BENCHMARK("Allocate 500k double image with initial value") -// { -// NDArrayim{ shape, 3.14 }; -// } -// NDArray a{ shape, 1.2 }; -// NDArray b{ shape, 53. }; -// auto c = a + b; -// c = a * b; -// BENCHMARK("Multiply two images") -// { -// c = a * b; -// } -// BENCHMARK("Divide two images") -// { -// c = a / b; -// } -// BENCHMARK("Add two images") -// { -// c = a + b; -// } -// BENCHMARK("Subtract two images") -// { -// c = a - b; -// } -// } - -TEST_CASE("Elementwise operatios on images") { +TEST_CASE("Elementwise operations on images") { std::array shape{5, 5}; double a_val = 3.0; double b_val = 8.0; diff --git a/src/PixelMap.cpp b/src/PixelMap.cpp index b8788c8..b1a6dc4 100644 --- a/src/PixelMap.cpp +++ b/src/PixelMap.cpp @@ -52,6 +52,16 @@ NDArray GenerateMoench05PixelMap() { return order_map; } +NDArrayGenerateEigerFlipRowsPixelMap(){ + NDArray order_map({256, 512}); + for(int row = 0; row < 256; row++){ + for(int col = 0; col < 512; col++){ + order_map(row, col) = 255*512-row*512 + col; + } + } + return order_map; +} + NDArrayGenerateMH02SingleCounterPixelMap(){ NDArray order_map({48, 48}); for(int row = 0; row < 48; row++){ diff --git a/src/RawFile.cpp b/src/RawFile.cpp index a82b29b..6ad4624 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -53,6 +53,18 @@ void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { return get_frame_into(m_current_frame++, image_buf, header); }; +void RawFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) { + // return get_frame_into(m_current_frame++, image_buf, header); + + for (size_t i = 0; i < n_frames; i++) { + this->get_frame_into(m_current_frame++, image_buf, header); + image_buf += bytes_per_frame(); + if(header) + header+=n_mod(); + } + +}; + size_t RawFile::n_mod() const { return n_subfile_parts; } @@ -87,10 +99,14 @@ void RawFile::open_subfiles() { for (size_t i = 0; i != n_subfiles; ++i) { auto v = std::vector(n_subfile_parts); for (size_t j = 0; j != n_subfile_parts; ++j) { + fmt::print("{} pos: {},{}\n", j,positions[j].row, positions[j].col); + auto pos = m_module_pixel_0[j]; + fmt::print("{} pos: {},{}\n", j,pos.y, pos.x); v[j] = new RawSubFile(m_master.data_fname(j, i), m_master.detector_type(), pos.height, - pos.width, m_master.bitdepth()); + pos.width, m_master.bitdepth(), + positions[j].row, positions[j].col); } subfiles.push_back(v); @@ -117,10 +133,6 @@ DetectorHeader RawFile::read_header(const std::filesystem::path &fname) { return h; } -bool RawFile::is_master_file(const std::filesystem::path &fpath) { - std::string const stem = fpath.stem().string(); - return stem.find("_master_") != std::string::npos; -} int RawFile::find_number_of_subfiles() { int n_files = 0; @@ -135,6 +147,8 @@ int RawFile::find_number_of_subfiles() { } +RawMasterFile RawFile::master() const { return m_master; } + void RawFile::find_geometry() { uint16_t r{}; uint16_t c{}; @@ -208,8 +222,9 @@ void RawFile::update_geometry_with_roi() { if (m.y + m.height < roi.ymin) { m.height = 0; } else { - if (roi.ymin < m.y + m.height) { - m.height -= roi.ymin; + if ((roi.ymin > m.y) && (roi.ymin < m.y + m.height)) { + m.height -= roi.ymin - m.y; + } if (roi.ymax < m.y + m.height) { m.height -= m.y + original_height - roi.ymax; diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index ae2caa1..0a5d146 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -72,6 +72,9 @@ ScanParameters::ScanParameters(const std::string& par){ int ScanParameters::start() const { return m_start; } int ScanParameters::stop() const { return m_stop; } +void ScanParameters::increment_stop(){ + m_stop += 1; +}; int ScanParameters::step() const { return m_step; } const std::string &ScanParameters::dac() const { return m_dac; } bool ScanParameters::enabled() const { return m_enabled; } @@ -240,6 +243,9 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { try{ std::string scan_parameters = j.at("Scan Parameters"); m_scan_parameters = ScanParameters(scan_parameters); + if(v<7.21){ + m_scan_parameters.increment_stop(); //adjust for endpoint being included + } }catch (const json::out_of_range &e) { // not a scan } diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp index c537007..4e322d2 100644 --- a/src/RawSubFile.cpp +++ b/src/RawSubFile.cpp @@ -8,12 +8,15 @@ namespace aare { RawSubFile::RawSubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, - size_t bitdepth) + size_t bitdepth, uint32_t pos_row, uint32_t pos_col) : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_detector_type(detector), - m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols) { + m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols), m_pos_row(pos_row), m_pos_col(pos_col) { if (m_detector_type == DetectorType::Moench03_old) { - pixel_map = GenerateMoench03PixelMap(); + m_pixel_map = GenerateMoench03PixelMap(); + }else if(m_detector_type == DetectorType::Eiger && m_pos_row % 2 == 0){ + m_pixel_map = GenerateEigerFlipRowsPixelMap(); + fmt::print("Flipping rows\n"); } if (std::filesystem::exists(fname)) { @@ -59,7 +62,7 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { } //TODO! expand support for different bitdepths - if(pixel_map){ + if(m_pixel_map){ // read into a temporary buffer and then copy the data to the buffer // in the correct order // currently this only supports 16 bit data! @@ -68,7 +71,7 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { auto *data = reinterpret_cast(image_buf); auto *part_data = reinterpret_cast(part_buffer); for (size_t i = 0; i < pixels_per_frame(); i++) { - data[i] = part_data[(*pixel_map)(i)]; + data[i] = part_data[(*m_pixel_map)(i)]; } delete[] part_buffer; } else { diff --git a/src/SubFile.cpp b/src/SubFile.cpp deleted file mode 100644 index 314fec5..0000000 --- a/src/SubFile.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "aare/SubFile.hpp" -#include "aare/PixelMap.hpp" -#include // memcpy -#include -#include - - -namespace aare { - -SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, - const std::string &mode) - : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) { - - - if (m_detector_type == DetectorType::Moench03_old) { - pixel_map = GenerateMoench03PixelMap(); - } - - if (std::filesystem::exists(fname)) { - n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); - } else { - n_frames = 0; - } - - if (mode == "r") { - fp = fopen(m_fname.string().c_str(), "rb"); - } else { - throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles."); - } - if (fp == nullptr) { - throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string())); - } -#ifdef AARE_VERBOSE - fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); - fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth); -#endif -} - -size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { - if (frame_index >= n_frames) { - throw std::runtime_error("Frame number out of range"); - } - fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT - SEEK_SET); - - if (pixel_map){ - // read into a temporary buffer and then copy the data to the buffer - // in the correct order - auto part_buffer = new std::byte[bytes_per_part()]; - auto wc = fread(part_buffer, bytes_per_part(), 1, fp); - auto *data = reinterpret_cast(buffer); - auto *part_data = reinterpret_cast(part_buffer); - for (size_t i = 0; i < pixels_per_part(); i++) { - data[i] = part_data[(*pixel_map)(i)]; - } - delete[] part_buffer; - return wc; - }else{ - // read directly into the buffer - return fread(buffer, this->bytes_per_part(), 1, this->fp); - } - -} -size_t SubFile::write_part(std::byte *buffer, DetectorHeader header, size_t frame_index) { - if (frame_index > n_frames) { - throw std::runtime_error("Frame number out of range"); - } - fseek(fp, static_cast((sizeof(DetectorHeader) + bytes_per_part()) * frame_index), SEEK_SET); - auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); - wc += fwrite(buffer, bytes_per_part(), 1, fp); - - return wc; -} - -size_t SubFile::frame_number(size_t frame_index) { - DetectorHeader h{}; - fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT - size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); - if (rc != 1) - throw std::runtime_error(LOCATION + "Could not read header from file"); - - return h.frameNumber; -} - -SubFile::~SubFile() { - if (fp) { - fclose(fp); - } -} - -} // namespace aare \ No newline at end of file