From ecf1b2a90b40b73d866f95a699aae77d465cb701 Mon Sep 17 00:00:00 2001 From: froejdh_e Date: Mon, 11 Nov 2024 17:13:48 +0100 Subject: [PATCH] WIP --- .clang-format | 7 ++ CMakeLists.txt | 12 ++- docs/CMakeLists.txt | 1 + docs/src/RawSubFile.rst | 8 ++ docs/src/index.rst | 20 ++-- include/aare/RawFile.hpp | 15 ++- include/aare/RawMasterFile.hpp | 3 + include/aare/RawSubFile.hpp | 69 ++++++++++++ include/aare/SubFile.hpp | 82 --------------- include/aare/defs.hpp | 28 +++++ python/aare/__init__.py | 5 +- python/examples/play.py | 98 ++++------------- python/src/file.hpp | 152 +++++++++++++++++++-------- src/RawFile.cpp | 186 ++++++++++++++++++++++++--------- src/RawMasterFile.cpp | 8 ++ src/RawSubFile.cpp | 99 ++++++++++++++++++ src/SubFile.cpp | 82 --------------- src/defs.cpp | 10 +- 18 files changed, 537 insertions(+), 348 deletions(-) create mode 100644 .clang-format create mode 100644 docs/src/RawSubFile.rst create mode 100644 include/aare/RawSubFile.hpp delete mode 100644 include/aare/SubFile.hpp create mode 100644 src/RawSubFile.cpp delete mode 100644 src/SubFile.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..2a4b795 --- /dev/null +++ b/.clang-format @@ -0,0 +1,7 @@ +BasedOnStyle: LLVM +IndentWidth: 4 + +UseTab: Never +ColumnLimit: 80 +AlignConsecutiveAssignments: false +AlignConsecutiveMacros: true \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e221cd..208e0be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ option(AARE_EXAMPLES "Build examples" OFF) option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF) option(AARE_DOCS "Build documentation" OFF) option(AARE_VERBOSE "Verbose output" OFF) +option(AARE_CUSTOM_ASSERT "Use custom assert" OFF) # Configure which of the dependencies to use FetchContent for option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON) @@ -57,6 +58,9 @@ if(AARE_VERBOSE) add_compile_definitions(AARE_VERBOSE) endif() +if(AARE_CUSTOM_ASSERT) + add_compile_definitions(AARE_CUSTOM_ASSERT) +endif() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -242,7 +246,6 @@ set(PUBLICHEADERS include/aare/Dtype.hpp include/aare/File.hpp include/aare/FileInterface.hpp - include/aare/RawMasterFile.hpp include/aare/Frame.hpp include/aare/NDArray.hpp include/aare/NDView.hpp @@ -251,7 +254,8 @@ set(PUBLICHEADERS include/aare/Pedestal.hpp include/aare/PixelMap.hpp include/aare/RawFile.hpp - include/aare/SubFile.hpp + include/aare/RawSubFile.hpp + include/aare/RawMasterFile.hpp include/aare/VarClusterFinder.hpp ) @@ -263,11 +267,11 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp ) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 51cb063..f4fb00f 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -20,6 +20,7 @@ set(SPHINX_SOURCE_FILES src/ClusterFinder.rst src/Pedestal.rst src/RawFile.rst + src/RawSubFile.rst src/RawMasterFile.rst src/VarClusterFinder.rst src/pyVarClusterFinder.rst diff --git a/docs/src/RawSubFile.rst b/docs/src/RawSubFile.rst new file mode 100644 index 0000000..3e4746a --- /dev/null +++ b/docs/src/RawSubFile.rst @@ -0,0 +1,8 @@ +RawSubFile +=============== + + +.. doxygenclass:: aare::RawSubFile + :members: + :undoc-members: + :private-members: \ No newline at end of file diff --git a/docs/src/index.rst b/docs/src/index.rst index 6e821a7..943e952 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -5,6 +5,16 @@ AARE Hello +.. toctree:: + :caption: Python API + :maxdepth: 1 + + pyFile + pyCtbRawFile + pyRawMasterFile + pyVarClusterFinder + + .. toctree:: :caption: C++ API :maxdepth: 1 @@ -17,15 +27,9 @@ AARE ClusterFinder Pedestal RawFile + RawSubFile RawMasterFile VarClusterFinder -.. toctree:: - :caption: Python API - :maxdepth: 1 - - pyFile - pyCtbRawFile - pyRawMasterFile - pyVarClusterFinder + diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 12acea3..8fefa34 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -3,7 +3,7 @@ #include "aare/RawMasterFile.hpp" #include "aare/Frame.hpp" #include "aare/NDArray.hpp" //for pixel map -#include "aare/SubFile.hpp" +#include "aare/RawSubFile.hpp" #include @@ -32,8 +32,9 @@ class RawFile : public FileInterface { size_t n_subfiles{}; //f0,f1...fn size_t n_subfile_parts{}; // d0,d1...dn //TODO! move to vector of SubFile instead of pointers - std::vector> subfiles; //subfiles[f0,f1...fn][d0,d1...dn] + std::vector> subfiles; //subfiles[f0,f1...fn][d0,d1...dn] std::vector positions; + std::vector m_module_pixel_0; ModuleConfig cfg{0, 0}; RawMasterFile m_master; @@ -57,6 +58,10 @@ class RawFile : public FileInterface { std::vector read_n(size_t n_frames) override; void read_into(std::byte *image_buf) override; void read_into(std::byte *image_buf, size_t n_frames) override; + + //TODO! do we need to adapt the API? + void read_into(std::byte *image_buf, DetectorHeader *header); + size_t frame_number(size_t frame_index) override; size_t bytes_per_frame() override; size_t pixels_per_frame() override; @@ -66,7 +71,9 @@ class RawFile : public FileInterface { size_t rows() const override; size_t cols() const override; size_t bitdepth() const override; + size_t bytes_per_pixel() const; xy geometry(); + size_t n_mod() const; DetectorType detector_type() const override; @@ -84,7 +91,7 @@ class RawFile : public FileInterface { * @param frame_number frame number to read * @param image_buf buffer to store the frame */ - void get_frame_into(size_t frame_index, std::byte *frame_buffer); + void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr); /** * @brief get the frame at the given frame index @@ -102,7 +109,7 @@ class RawFile : public FileInterface { */ static DetectorHeader read_header(const std::filesystem::path &fname); - + void update_geometry_with_roi(); int find_number_of_subfiles(); void open_subfiles(); void find_geometry(); diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 76292b7..52ce500 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -63,6 +63,9 @@ struct ROI{ size_t xmax{}; size_t ymin{}; size_t ymax{}; + + size_t height() const { return ymax - ymin; } + size_t width() const { return xmax - xmin; } }; /** diff --git a/include/aare/RawSubFile.hpp b/include/aare/RawSubFile.hpp new file mode 100644 index 0000000..5efa4b6 --- /dev/null +++ b/include/aare/RawSubFile.hpp @@ -0,0 +1,69 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/defs.hpp" + +#include +#include +#include +#include + +namespace aare { + +/** + * @brief Class to read a singe subfile written in .raw format. Used from RawFile to read + * the entire detector. Can be used directly to read part of the image. + */ +class RawSubFile { + protected: + std::ifstream m_file; + size_t m_bitdepth; + std::filesystem::path m_fname; + size_t m_rows{}; + size_t m_cols{}; + size_t m_bytes_per_frame{}; + size_t n_frames{}; + + DetectorType m_detector_type; + std::optional> pixel_map; + + public: + /** + * @brief SubFile constructor + * @param fname path to the subfile + * @param detector detector type + * @param rows number of rows in the subfile + * @param cols number of columns in the subfile + * @param bitdepth bitdepth of the subfile + * @throws std::invalid_argument if the detector,type pair is not supported + */ + RawSubFile(const std::filesystem::path &fname, DetectorType detector, + size_t rows, size_t cols, size_t bitdepth); + + ~RawSubFile() = default; + /** + * @brief Seek to the given frame number + * @note Puts the file pointer at the start of the header, not the start of the data + * @param frame_index frame position in file to seek to + * @throws std::runtime_error if the frame number is out of range + */ + void seek(size_t frame_index); + size_t tell(); + + void read_into(std::byte *image_buf, DetectorHeader *header = nullptr); + void get_part(std::byte *buffer, size_t frame_index); + + void read_header(DetectorHeader *header); + + size_t rows() const; + size_t cols() const; + + size_t frame_number(size_t frame_index); + + size_t bytes_per_frame() const { return m_bytes_per_frame; } + size_t pixels_per_frame() const { return m_rows * m_cols; } + size_t bytes_per_pixel() const { return m_bitdepth / 8; } + + +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/SubFile.hpp b/include/aare/SubFile.hpp deleted file mode 100644 index d9f7d0a..0000000 --- a/include/aare/SubFile.hpp +++ /dev/null @@ -1,82 +0,0 @@ -#pragma once -#include "aare/Frame.hpp" -#include "aare/defs.hpp" - -#include -#include -#include -#include - - -namespace aare { - -/** - * @brief Class to read a subfile from a RawFile - */ -class SubFile { - protected: - FILE *fp = nullptr; - size_t m_bitdepth; - std::filesystem::path m_fname; - size_t m_rows{}; - size_t m_cols{}; - std::string m_mode; - size_t n_frames{}; - int m_sub_file_index_{}; - DetectorType m_detector_type; - std::optional> pixel_map; - - - public: - /** - * @brief SubFile constructor - * @param fname path to the subfile - * @param detector detector type - * @param rows number of rows in the subfile - * @param cols number of columns in the subfile - * @param bitdepth bitdepth of the subfile - * @throws std::invalid_argument if the detector,type pair is not supported - */ - SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, - const std::string &mode = "r"); - - /** - * @brief read the subfile into a buffer - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - size_t read_impl_normal(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes flipped - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - template size_t read_impl_flip(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes reordered - * @param buffer pointer to the buffer to read the data into - * @return number of bytes read - */ - template size_t read_impl_reorder(std::byte *buffer); - - /** - * @brief read the subfile into a buffer with the bytes reordered and flipped - * @param buffer pointer to the buffer to read the data into - * @param frame_number frame number to read - * @return number of bytes read - */ - size_t get_part(std::byte *buffer, size_t frame_index); - size_t frame_number(size_t frame_index); - - - size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } - size_t pixels_per_part() const { return m_rows * m_cols; } - - ~SubFile(); - - -}; - -} // namespace aare \ No newline at end of file diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index f4a0bcf..b32c6be 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -14,6 +14,11 @@ #include #include + + + + + /** * @brief LOCATION macro to get the current location in the code */ @@ -21,8 +26,23 @@ std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + \ ":" + std::string(__func__) + ":" + +#ifdef AARE_CUSTOM_ASSERT +#define AARE_ASSERT(expr)\ + if (expr)\ + {}\ + else\ + aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n"); +#else +#define AARE_ASSERT(cond)\ + do { (void)sizeof(cond); } while(0) +#endif + + namespace aare { +void assert_failed(const std::string &msg); + class Cluster { public: int cluster_sizeX; @@ -164,6 +184,14 @@ template struct t_xy { }; using xy = t_xy; + +struct ModuleGeometry{ + int x{}; + int y{}; + int height{}; + int width{}; +}; + using dynamic_shape = std::vector; //TODO! Can we uniform enums between the libraries? diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 76af17a..5892db0 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -1,7 +1,8 @@ # Make the compiled classes that live in _aare available from aare. from . import _aare -from ._aare import VarClusterFinder, File, RawMasterFile -from ._aare import Pedestal, ClusterFinder +from ._aare import File, RawFile, RawMasterFile, RawSubFile +from ._aare import Pedestal, ClusterFinder, VarClusterFinder +from ._aare import DetectorType from .CtbRawFile import CtbRawFile from .ScanParameters import ScanParameters \ No newline at end of file diff --git a/python/examples/play.py b/python/examples/play.py index 42f3c33..0e03f9f 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -52,88 +52,36 @@ def get_Mh02_frames(fname): # [frame, counter, row, col] # plt.imshow(data[0,0]) - -# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json') +base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data') +# p = Path(base / 'jungfrau/jungfrau_single_master_0.json') # f = aare.File(p) -# frame = f.read_frame() - -# fig, ax = plt.subplots() -# im = ax.imshow(frame, cmap='viridis') - - -# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json') -# # fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json') - - -# # f = aare.CtbRawFile(fpath, transform = transform.moench05) -# # with CtbRawFile(fpath, transform = transform.moench05) as f: -# # for header, image in f: -# # print(f'Frame number: {header["frameNumber"]}') - -# # m = aare.RawMasterFile(fpath) -# f = aare.File(fpath) +# for i in range(10): +# frame = f.read_frame() -# cf = aare.ClusterFinder((400,400),(3,3)) - -# for i in range(100): -# cf.push_pedestal_frame(f.read_frame()) - - -# f.seek(0) -# pd = f.read_n(100).mean(axis=0) - -# clusters = cf.find_clusters_without_threshold(f.read_frame()) +# # f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02) +# # header, data = f2.read() +# # plt.plot(data[:,0,20,20]) +# from aare import RawMasterFile, File, RawSubFile, DetectorType, RawFile +# base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/') +# fpath = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') +# raw = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_d0_f0_0.raw') -# base = Path('/Users/erik/data/matterhorn/raw') -# fpath = Path(base / 'scan_15keV_vrf700_vrsh700_th0_master_0.json') -# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -# f.seek(100) -# header1, image1 = f.read_frame() +# m = RawMasterFile(base / fpath) +# # roi = m.roi +# # rows = roi.ymax-roi.ymin+1 +# # cols = 1024-roi.xmin +# # sf = RawSubFile(base / raw, DetectorType.Jungfrau, rows, cols, 16) -# fpath = Path(base / 'scan_all15keV_vrf500_vrsh700_th0_master_0.json') +from aare import RawFile -# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -# f.seek(100) -# header4, image4 = f.read_frame() - -# n_counters = image.shape[1] / 48**2 / 2 - -# for i in range(100): -# header, image = f.read_frame() -# print(header['frameNumber']) - - - - - -#Data come in "blocks" of 4 pixels/receiver -# data = get_Mh02_frames(fpath.as_posix()) - -# rawi = np.zeros(48*48*4+56, dtype = np.uint16) -# for i,v in enumerate(rawi[56:]): -# rawi[i+56] = i - -# raw = image.view(np.uint16) - -# pixel_map = decode(1, rawi) -# # img = np.take(raw, pixel_map) - -# pm = np.zeros((4, 48,48), dtype = np.int64) -# for counter in range(4): -# for row in range(48): -# for col in range(48): -# pm[counter, row, col] = row*48 + col+counter*48*48 - - -# f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -# header, data = f2.read() -# plt.plot(data[:,0,20,20]) -from aare import RawMasterFile, File -fpath = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') -m = RawMasterFile(fpath) -f = File(fpath) \ No newline at end of file +base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/') +fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') +# fname = Path(base / 'jungfrau/jungfrau_single_master_0.json') +f = RawFile(fname) +h,img = f.read_frame() +print(f'{h["frameNumber"]}') \ No newline at end of file diff --git a/python/src/file.hpp b/python/src/file.hpp index 2c3a36f..7c8a3c7 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -1,7 +1,9 @@ #include "aare/CtbRawFile.hpp" -#include "aare/RawMasterFile.hpp" #include "aare/File.hpp" #include "aare/Frame.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" +#include "aare/RawSubFile.hpp" #include "aare/defs.hpp" // #include "aare/fClusterFileV2.hpp" @@ -17,8 +19,17 @@ namespace py = pybind11; using namespace ::aare; -void define_file_io_bindings(py::module &m) -{ +void define_file_io_bindings(py::module &m) { + + py::enum_(m, "DetectorType") + .value("Jungfrau", DetectorType::Jungfrau) + .value("Eiger", DetectorType::Eiger) + .value("Mythen3", DetectorType::Mythen3) + .value("Moench", DetectorType::Moench) + .value("Moench03", DetectorType::Moench03) + .value("Moench03_old", DetectorType::Moench03_old) + .value("ChipTestBoard", DetectorType::ChipTestBoard) + .value("Unknown", DetectorType::Unknown); PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber, bunchId, timestamp, modId, row, column, reserved, @@ -27,8 +38,7 @@ void define_file_io_bindings(py::module &m) py::class_(m, "CtbRawFile") .def(py::init()) .def("read_frame", - [](CtbRawFile &self) - { + [](CtbRawFile &self) { size_t image_size = self.image_size_in_bytes(); py::array image; std::vector shape; @@ -50,15 +60,16 @@ void define_file_io_bindings(py::module &m) .def("seek", &CtbRawFile::seek) .def("tell", &CtbRawFile::tell) .def("master", &CtbRawFile::master) - .def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes) + .def_property_readonly("image_size_in_bytes", + &CtbRawFile::image_size_in_bytes) .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file); py::class_(m, "File") - .def(py::init([](const std::filesystem::path &fname) - { return File(fname, "r", {}); })) + .def(py::init([](const std::filesystem::path &fname) { + return File(fname, "r", {}); + })) .def(py::init( - [](const std::filesystem::path &fname, const std::string &mode) - { + [](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); })) .def(py::init shape; shape.reserve(2); shape.push_back(self.rows()); shape.push_back(self.cols()); - if (item_size == 1) - { + if (item_size == 1) { image = py::array_t(shape); - } - else if (item_size == 2) - { + } else if (item_size == 2) { image = py::array_t(shape); - } - else if (item_size == 4) - { + } else if (item_size == 4) { image = py::array_t(shape); } self.read_into( @@ -104,8 +108,7 @@ void define_file_io_bindings(py::module &m) return image; }) .def("read_frame", - [](File &self, size_t frame_number) - { + [](File &self, size_t frame_number) { self.seek(frame_number); const uint8_t item_size = self.bytes_per_pixel(); py::array image; @@ -113,24 +116,18 @@ void define_file_io_bindings(py::module &m) shape.reserve(2); shape.push_back(self.rows()); shape.push_back(self.cols()); - if (item_size == 1) - { + if (item_size == 1) { image = py::array_t(shape); - } - else if (item_size == 2) - { + } else if (item_size == 2) { image = py::array_t(shape); - } - else if (item_size == 4) - { + } else if (item_size == 4) { image = py::array_t(shape); } self.read_into( reinterpret_cast(image.mutable_data())); return image; }) - .def("read_n", [](File &self, size_t n_frames) - { + .def("read_n", [](File &self, size_t n_frames) { const uint8_t item_size = self.bytes_per_pixel(); py::array image; std::vector shape; @@ -147,7 +144,8 @@ void define_file_io_bindings(py::module &m) } self.read_into(reinterpret_cast(image.mutable_data()), n_frames); - return image; }); + return image; + }); py::class_(m, "FileConfig") .def(py::init<>()) @@ -161,8 +159,9 @@ void define_file_io_bindings(py::module &m) .def_readwrite("dtype", &FileConfig::dtype) .def("__eq__", &FileConfig::operator==) .def("__ne__", &FileConfig::operator!=) - .def("__repr__", [](const FileConfig &a) - { return ""; }); + .def("__repr__", [](const FileConfig &a) { + return ""; + }); py::class_(m, "RawMasterFile") .def(py::init()) @@ -181,15 +180,18 @@ void define_file_io_bindings(py::module &m) .def_property_readonly("frame_padding", &RawMasterFile::frame_padding) .def_property_readonly("frame_discard_policy", &RawMasterFile::frame_discard_policy) - .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected) + .def_property_readonly("total_frames_expected", + &RawMasterFile::total_frames_expected) .def_property_readonly("geometry", &RawMasterFile::geometry) .def_property_readonly("analog_samples", &RawMasterFile::analog_samples) .def_property_readonly("digital_samples", &RawMasterFile::digital_samples) - .def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples) + .def_property_readonly("transceiver_samples", + &RawMasterFile::transceiver_samples) .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows) .def_property_readonly("quad", &RawMasterFile::quad) - .def_property_readonly("scan_parameters", &RawMasterFile::scan_parameters) + .def_property_readonly("scan_parameters", + &RawMasterFile::scan_parameters) .def_property_readonly("roi", &RawMasterFile::roi); py::class_(m, "ScanParameters") @@ -206,7 +208,75 @@ void define_file_io_bindings(py::module &m) .def_readwrite("xmin", &ROI::xmin) .def_readwrite("xmax", &ROI::xmax) .def_readwrite("ymin", &ROI::ymin) - .def_readwrite("ymax", &ROI::ymax); + .def_readwrite("ymax", &ROI::ymax) + .def("__str__", [](const ROI& self){ + return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax); + }); + + py::class_(m, "RawFile") + .def(py::init()) + .def("read_frame", + [](RawFile &self) { + size_t image_size = self.bytes_per_frame(); + const uint8_t item_size = self.bytes_per_pixel(); + fmt::print("Image size in bytes: {}\n", image_size); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + + + //return headers from all subfiles + py::array_t header(self.n_mod()); + + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); + + self.read_into( + reinterpret_cast(image.mutable_data()), + header.mutable_data()); + + return py::make_tuple(header, image); + }); + + py::class_(m, "RawSubFile") + .def(py::init()) + .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame) + .def_property_readonly("pixels_per_frame", + &RawSubFile::pixels_per_frame) + .def("seek", &RawSubFile::seek) + .def("tell", &RawSubFile::tell) + .def_property_readonly("rows", &RawSubFile::rows) + .def_property_readonly("cols", &RawSubFile::cols) + .def("read_frame", + [](RawSubFile &self) { + const uint8_t item_size = self.bytes_per_pixel(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); + self.read_into( + reinterpret_cast(image.mutable_data())); + return image; + }); + // py::class_(m, "ClusterHeader") // .def(py::init<>()) // .def_readwrite("frame_number", &ClusterHeader::frame_number) diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 85f9736..35afdbd 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -13,10 +13,11 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) : m_master(fname) { m_mode = mode; if (mode == "r") { - n_subfiles = find_number_of_subfiles(); //f0,f1...fn - n_subfile_parts = m_master.geometry().col * m_master.geometry().row; // d0,d1...dn + n_subfiles = find_number_of_subfiles(); // f0,f1...fn + n_subfile_parts = + m_master.geometry().col * m_master.geometry().row; // d0,d1...dn find_geometry(); - //update_geometry_from_roi(); + update_geometry_with_roi(); open_subfiles(); } else { throw std::runtime_error(LOCATION + @@ -39,10 +40,19 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) { image_buf += bytes_per_frame(); } } + + void RawFile::read_into(std::byte *image_buf) { return get_frame_into(m_current_frame++, image_buf); }; +void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { + + return get_frame_into(m_current_frame++, image_buf, header); +}; + +size_t RawFile::n_mod() const { return n_subfile_parts; } + size_t RawFile::bytes_per_frame() { return m_rows * m_cols * m_master.bitdepth() / 8; } @@ -72,12 +82,12 @@ xy RawFile::geometry() { return m_master.geometry(); } void RawFile::open_subfiles() { if (m_mode == "r") for (size_t i = 0; i != n_subfiles; ++i) { - auto v = std::vector(n_subfile_parts); + auto v = std::vector(n_subfile_parts); for (size_t j = 0; j != n_subfile_parts; ++j) { - v[j] = - new SubFile(m_master.data_fname(j, i), - m_master.detector_type(), m_master.pixels_y(), - m_master.pixels_x(), m_master.bitdepth()); + auto pos = m_module_pixel_0[j]; + v[j] = new RawSubFile(m_master.data_fname(j, i), + m_master.detector_type(), pos.height, + pos.width, m_master.bitdepth()); } subfiles.push_back(v); } @@ -111,38 +121,30 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) { int RawFile::find_number_of_subfiles() { int n_files = 0; // f0,f1...fn How many files is the data split into? - while (std::filesystem::exists(m_master.data_fname(0, ++n_files))) - ; + while (std::filesystem::exists(m_master.data_fname(0, n_files))) + n_files++; // increment after test - #ifdef AARE_VERBOSE - fmt::print("Found: {} subfiles\n", n_subfiles); - #endif +#ifdef AARE_VERBOSE + fmt::print("Found: {} subfiles\n", n_files); +#endif return n_files; } void RawFile::find_geometry() { uint16_t r{}; uint16_t c{}; - // for (size_t i = 0; i < n_subfile_parts; i++) { - // for (size_t j = 0; j != n_subfiles; ++j) { - // auto h = this->read_header(m_master.data_fname(i, j)); - // r = std::max(r, h.row); - // c = std::max(c, h.column); - - // positions.push_back({h.row, h.column}); - // } - // } - - - std::vector module_position; for (size_t i = 0; i < n_subfile_parts; i++) { - auto h = this->read_header(m_master.data_fname(i, 0)); - r = std::max(r, h.row); - c = std::max(c, h.column); - positions.push_back({h.row, h.column}); - xy pos = {h.row * m_master.pixels_y(), h.column* m_master.pixels_x()}; - module_position.emplace_back(pos); + auto h = this->read_header(m_master.data_fname(i, 0)); + r = std::max(r, h.row); + c = std::max(c, h.column); + positions.push_back({h.row, h.column}); + ModuleGeometry g; + g.x = h.column * m_master.pixels_x(); + g.y = h.row * m_master.pixels_y(); + g.width = m_master.pixels_x(); + g.height = m_master.pixels_y(); + m_module_pixel_0.push_back(g); } r++; @@ -152,10 +154,79 @@ void RawFile::find_geometry() { m_cols = (c * m_master.pixels_x()); m_rows += static_cast((r - 1) * cfg.module_gap_row); - - for (size_t i=0; i < module_position.size(); i++){ - fmt::print("Module {} at position: ({},{})\n", i, module_position[i].row, module_position[i].col); +#ifdef AARE_VERBOSE + fmt::print("\nRawFile::find_geometry()\n"); + for (size_t i = 0; i < m_module_pixel_0.size(); i++) { + fmt::print("Module {} at position: (r:{},c:{})\n", i, + m_module_pixel_0[i].y, m_module_pixel_0[i].x); } + fmt::print("Image size: {}x{}\n\n", m_rows, m_cols); +#endif +} + +void RawFile::update_geometry_with_roi() { + // TODO! implement this + if (m_master.roi()) { + auto roi = m_master.roi().value(); + + // TODO! can we do this cleaner? + int pos_y = 0; + int pos_y_increment = 0; + for (size_t row = 0; row < m_master.geometry().row; row++) { + int pos_x = 0; + for (size_t col = 0; col < m_master.geometry().col; col++) { + auto &m = m_module_pixel_0[row * m_master.geometry().col + col]; + auto original_height = m.height; + auto original_width = m.width; + + // module is to the left of the roi + if (m.x + m.width < roi.xmin) { + m.width = 0; + + // roi is in module + } else { + // here we only arrive when the roi is in or to the left of + // the module + if (roi.xmin > m.x) { + m.width -= roi.xmin - m.x; + } + if (roi.xmax < m.x + m.width) { + m.width -= m.x + original_width - roi.xmax; + } + m.x = pos_x; + pos_x += m.width; + } + + if (m.y + m.height < roi.ymin) { + m.height = 0; + } else { + if (roi.ymin < m.y + m.height) { + m.height -= roi.ymin; + } + if (roi.ymax < m.y + m.height) { + m.height -= m.y + original_height - roi.ymax; + } + m.y = pos_y; + pos_y_increment = m.height; + } + } + // increment pos_y + pos_y += pos_y_increment; + } + + m_rows = roi.height(); + m_cols = roi.width(); + } + +#ifdef AARE_VERBOSE + fmt::print("RawFile::update_geometry_with_roi()\n"); + for (const auto &m : m_module_pixel_0) { + fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x, + m.height, m.width); + } + fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols); + fmt::print("\n"); +#endif } Frame RawFile::get_frame(size_t frame_index) { @@ -165,13 +236,18 @@ Frame RawFile::get_frame(size_t frame_index) { return f; } -void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { +size_t RawFile::bytes_per_pixel() const { + return m_master.bitdepth() / 8; +} + +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) { if (frame_index > total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } std::vector frame_numbers(n_subfile_parts); std::vector frame_indices(n_subfile_parts, frame_index); + // sync the frame numbers if (n_subfile_parts != 1) { for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { auto subfile_id = frame_index / m_master.max_frames_per_file(); @@ -208,15 +284,24 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { auto corrected_idx = frame_indices[part_idx]; auto subfile_id = corrected_idx / m_master.max_frames_per_file(); - auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part(); - subfiles[subfile_id][part_idx]->get_part( - frame_buffer + part_idx * part_offset, - corrected_idx % m_master.max_frames_per_file()); + + // This is where we start writing + auto offset = (m_module_pixel_0[part_idx].y * m_cols + + m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8; + + if (m_module_pixel_0[part_idx].x!=0) + throw std::runtime_error(LOCATION + "Implementation error. x pos not 0."); + + subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file()); + subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header); + if (header) + ++header; } } else { + //TODO! should we read row by row? - // create a buffer that will hold a the frame part + // create a buffer large enough to hold a full module auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() * m_master.bitdepth() / 8; // TODO! replace with image_size_in_bytes @@ -226,23 +311,26 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { // level for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { + auto pos = m_module_pixel_0[part_idx]; auto corrected_idx = frame_indices[part_idx]; auto subfile_id = corrected_idx / m_master.max_frames_per_file(); - subfiles[subfile_id][part_idx]->get_part( - part_buffer, corrected_idx % m_master.max_frames_per_file()); - for (size_t cur_row = 0; cur_row < (m_master.pixels_y()); + subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file()); + subfiles[subfile_id][part_idx]->read_into(part_buffer, header); + if(header) + ++header; + + for (size_t cur_row = 0; cur_row < (pos.height); cur_row++) { - auto irow = cur_row + (part_idx / m_master.geometry().col) * - m_master.pixels_y(); - auto icol = - (part_idx % m_master.geometry().col) * m_master.pixels_x(); + + auto irow = (pos.y + cur_row); + auto icol = pos.x; auto dest = (irow * this->m_cols + icol); dest = dest * m_master.bitdepth() / 8; memcpy(frame_buffer + dest, - part_buffer + cur_row * m_master.pixels_x() * + part_buffer + cur_row * pos.width * m_master.bitdepth() / 8, - m_master.pixels_x() * m_master.bitdepth() / 8); + pos.width * m_master.bitdepth() / 8); } } delete[] part_buffer; diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index 07de21e..8b3af85 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -8,6 +8,8 @@ RawFileNameComponents::RawFileNameComponents( m_base_name = fname.stem(); m_ext = fname.extension(); + AARE_ASSERT(false); + if (m_ext != ".json" && m_ext != ".raw") { throw std::runtime_error(LOCATION + "Unsupported file type. (only .json or .raw)"); @@ -254,6 +256,12 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { //if any of the values are set update the roi if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 || tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) { + + if(v<7.21){ + tmp_roi.xmax++; + tmp_roi.ymax++; + } + m_roi = tmp_roi; } diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp new file mode 100644 index 0000000..23c1201 --- /dev/null +++ b/src/RawSubFile.cpp @@ -0,0 +1,99 @@ +#include "aare/RawSubFile.hpp" +#include "aare/PixelMap.hpp" +#include // memcpy +#include +#include + +namespace aare { + +RawSubFile::RawSubFile(const std::filesystem::path &fname, + DetectorType detector, size_t rows, size_t cols, + size_t bitdepth) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), + m_detector_type(detector), + m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols) { + if (m_detector_type == DetectorType::Moench03_old) { + pixel_map = GenerateMoench03PixelMap(); + } + + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / + (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); + } else { + throw std::runtime_error( + LOCATION + fmt::format("File {} does not exist", m_fname.string())); + } + + // fp = fopen(m_fname.string().c_str(), "rb"); + m_file.open(m_fname, std::ios::binary); + if (!m_file.is_open()) { + throw std::runtime_error( + LOCATION + fmt::format("Could not open file {}", m_fname.string())); + } + +#ifdef AARE_VERBOSE + fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); + fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, + m_bitdepth); + fmt::print("file size: {}\n", std::filesystem::file_size(fname)); +#endif +} + +void RawSubFile::seek(size_t frame_index) { + if (frame_index >= n_frames) { + throw std::runtime_error("Frame number out of range"); + } + m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index); +} + +size_t RawSubFile::tell() { + return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame()); +} + + +void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { + if(header){ + fmt::print("Reading header\n"); + m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); + } else { + m_file.seekg(sizeof(DetectorHeader), std::ios::cur); + fmt::print("Skipping heading header\n"); + } + + //TODO! expand support for different bitdepths + if(pixel_map){ + // read into a temporary buffer and then copy the data to the buffer + // in the correct order + // currently this only supports 16 bit data! + auto part_buffer = new std::byte[bytes_per_frame()]; + m_file.read(reinterpret_cast(part_buffer), bytes_per_frame()); + auto *data = reinterpret_cast(image_buf); + auto *part_data = reinterpret_cast(part_buffer); + for (size_t i = 0; i < pixels_per_frame(); i++) { + data[i] = part_data[(*pixel_map)(i)]; + } + delete[] part_buffer; + } else { + // read directly into the buffer + m_file.read(reinterpret_cast(image_buf), bytes_per_frame()); + } +} + +size_t RawSubFile::rows() const { return m_rows; } +size_t RawSubFile::cols() const { return m_cols; } + + +void RawSubFile::get_part(std::byte *buffer, size_t frame_index) { + seek(frame_index); + read_into(buffer, nullptr); +} + +size_t RawSubFile::frame_number(size_t frame_index) { + seek(frame_index); + DetectorHeader h{}; + m_file.read(reinterpret_cast(&h), sizeof(DetectorHeader)); + return h.frameNumber; +} + + +} // namespace aare \ No newline at end of file diff --git a/src/SubFile.cpp b/src/SubFile.cpp deleted file mode 100644 index 04c686b..0000000 --- a/src/SubFile.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "aare/SubFile.hpp" -#include "aare/PixelMap.hpp" -#include // memcpy -#include -#include - - -namespace aare { - -SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, - const std::string &mode) - : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode), m_detector_type(detector) { - - - if (m_detector_type == DetectorType::Moench03_old) { - pixel_map = GenerateMoench03PixelMap(); - } - - if (std::filesystem::exists(fname)) { - n_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); - } else { - n_frames = 0; - } - - if (mode == "r") { - fp = fopen(m_fname.string().c_str(), "rb"); - } else { - throw std::runtime_error(LOCATION + "Unsupported mode. Can only read RawFiles."); - } - if (fp == nullptr) { - throw std::runtime_error(LOCATION + fmt::format("Could not open file {}", m_fname.string())); - } -#ifdef AARE_VERBOSE - fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); - fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth); -#endif -} - -size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { - if (frame_index >= n_frames) { - throw std::runtime_error("Frame number out of range"); - } - fseek(fp, sizeof(DetectorHeader) + (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, // NOLINT - SEEK_SET); - - if (pixel_map){ - // read into a temporary buffer and then copy the data to the buffer - // in the correct order - auto part_buffer = new std::byte[bytes_per_part()]; - auto wc = fread(part_buffer, bytes_per_part(), 1, fp); - auto *data = reinterpret_cast(buffer); - auto *part_data = reinterpret_cast(part_buffer); - for (size_t i = 0; i < pixels_per_part(); i++) { - data[i] = part_data[(*pixel_map)(i)]; - } - delete[] part_buffer; - return wc; - }else{ - // read directly into the buffer - return fread(buffer, this->bytes_per_part(), 1, this->fp); - } - -} - - -size_t SubFile::frame_number(size_t frame_index) { - DetectorHeader h{}; - fseek(fp, (sizeof(DetectorHeader) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT - size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); - if (rc != 1) - throw std::runtime_error(LOCATION + "Could not read header from file"); - - return h.frameNumber; -} - -SubFile::~SubFile() { - if (fp) { - fclose(fp); - } -} - -} // namespace aare \ No newline at end of file diff --git a/src/defs.cpp b/src/defs.cpp index 1b99045..b8171c1 100644 --- a/src/defs.cpp +++ b/src/defs.cpp @@ -1,9 +1,17 @@ #include "aare/defs.hpp" #include #include - +#include namespace aare { + +void assert_failed(const std::string &msg) + { + fmt::print(msg); + exit(1); +} + + /** * @brief Convert a DetectorType to a string * @param type DetectorType