From 349e3af8e1d0be38394a26c2104b78bee8f7e3b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Mon, 11 Nov 2024 19:59:55 +0100 Subject: [PATCH] Brining in changes (#93) --- .clang-format | 7 ++ .github/workflows/build_docs.yml | 3 +- .github/workflows/build_pkg.yml | 4 +- .gitignore | 24 ++++ CMakeLists.txt | 13 +- conda-recipe/meta.yaml | 3 +- docs/CMakeLists.txt | 1 + docs/src/RawSubFile.rst | 8 ++ docs/src/index.rst | 21 ++-- include/aare/NDArray.hpp | 15 +++ include/aare/NDView.hpp | 14 +++ include/aare/RawFile.hpp | 26 +++- include/aare/RawMasterFile.hpp | 18 +++ include/aare/RawSubFile.hpp | 69 +++++++++++ include/aare/defs.hpp | 27 +++++ pyproject.toml | 3 +- python/aare/__init__.py | 7 +- python/examples/play.py | 94 ++++----------- python/src/file.hpp | 116 ++++++++++++++++-- src/RawFile.cpp | 196 +++++++++++++++++++++++++------ src/RawMasterFile.cpp | 38 ++++++ src/RawSubFile.cpp | 97 +++++++++++++++ src/defs.cpp | 10 ++ 23 files changed, 674 insertions(+), 140 deletions(-) create mode 100644 .clang-format create mode 100644 .gitignore create mode 100644 docs/src/RawSubFile.rst create mode 100644 include/aare/RawSubFile.hpp create mode 100644 src/RawSubFile.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..2a4b795 --- /dev/null +++ b/.clang-format @@ -0,0 +1,7 @@ +BasedOnStyle: LLVM +IndentWidth: 4 + +UseTab: Never +ColumnLimit: 80 +AlignConsecutiveAssignments: false +AlignConsecutiveMacros: true \ No newline at end of file diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 355f934..57f15a3 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -4,7 +4,8 @@ on: workflow_dispatch: push: branches: - - v1 + - main + permissions: contents: read diff --git a/.github/workflows/build_pkg.yml b/.github/workflows/build_pkg.yml index 496945f..2534a86 100644 --- a/.github/workflows/build_pkg.yml +++ b/.github/workflows/build_pkg.yml @@ -4,7 +4,9 @@ on: workflow_dispatch: push: branches: - - v1 + - main + - developer + #run on PRs as well? jobs: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af3e3b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +install/ +.cproject +.project +bin/ +.settings +*.aux +*.log +*.out +*.toc +*.o +*.so +.* +build/ +RELEASE.txt +Testing/ + +ctbDict.cpp +ctbDict.h + + + +*.pyc +*/__pycache__/* + diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e221cd..d72d784 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,8 @@ option(AARE_EXAMPLES "Build examples" OFF) option(AARE_IN_GITHUB_ACTIONS "Running in Github Actions" OFF) option(AARE_DOCS "Build documentation" OFF) option(AARE_VERBOSE "Verbose output" OFF) +option(AARE_CUSTOM_ASSERT "Use custom assert" OFF) + # Configure which of the dependencies to use FetchContent for option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON) @@ -57,6 +59,10 @@ if(AARE_VERBOSE) add_compile_definitions(AARE_VERBOSE) endif() +if(AARE_CUSTOM_ASSERT) + add_compile_definitions(AARE_CUSTOM_ASSERT) +endif() + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -242,7 +248,6 @@ set(PUBLICHEADERS include/aare/Dtype.hpp include/aare/File.hpp include/aare/FileInterface.hpp - include/aare/RawMasterFile.hpp include/aare/Frame.hpp include/aare/NDArray.hpp include/aare/NDView.hpp @@ -251,6 +256,8 @@ set(PUBLICHEADERS include/aare/Pedestal.hpp include/aare/PixelMap.hpp include/aare/RawFile.hpp + include/aare/RawSubFile.hpp + include/aare/RawMasterFile.hpp include/aare/SubFile.hpp include/aare/VarClusterFinder.hpp @@ -263,11 +270,11 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp ) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 50b5968..338aba8 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,6 +1,7 @@ package: name: aare - version: 2024.11.07.dev0 #TODO! how to not duplicate this? + version: 2024.11.11.dev0 #TODO! how to not duplicate this? + source: path: .. diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 51cb063..f4fb00f 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -20,6 +20,7 @@ set(SPHINX_SOURCE_FILES src/ClusterFinder.rst src/Pedestal.rst src/RawFile.rst + src/RawSubFile.rst src/RawMasterFile.rst src/VarClusterFinder.rst src/pyVarClusterFinder.rst diff --git a/docs/src/RawSubFile.rst b/docs/src/RawSubFile.rst new file mode 100644 index 0000000..3e4746a --- /dev/null +++ b/docs/src/RawSubFile.rst @@ -0,0 +1,8 @@ +RawSubFile +=============== + + +.. doxygenclass:: aare::RawSubFile + :members: + :undoc-members: + :private-members: \ No newline at end of file diff --git a/docs/src/index.rst b/docs/src/index.rst index 6e821a7..7bac72f 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -5,6 +5,16 @@ AARE Hello +.. toctree:: + :caption: Python API + :maxdepth: 1 + + pyFile + pyCtbRawFile + pyRawMasterFile + pyVarClusterFinder + + .. toctree:: :caption: C++ API :maxdepth: 1 @@ -17,15 +27,10 @@ AARE ClusterFinder Pedestal RawFile + RawSubFile RawMasterFile VarClusterFinder -.. toctree:: - :caption: Python API - :maxdepth: 1 - - pyFile - pyCtbRawFile - pyRawMasterFile - pyVarClusterFinder + + diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index 0a54091..e65b136 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -341,6 +341,21 @@ template void NDArray::Print() { else Print_some(); } + + +template +std::ostream& operator <<(std::ostream& os, const NDArray& arr){ + for (auto row = 0; row < arr.shape(0); ++row) { + for (auto col = 0; col < arr.shape(1); ++col) { + os << std::setw(3); + os << arr(row, col) << " "; + } + os << "\n"; + } + return os; +} + + template void NDArray::Print_all() { for (auto row = 0; row < shape_[0]; ++row) { for (auto col = 0; col < shape_[1]; ++col) { diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index 8a73038..906b215 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -156,4 +156,18 @@ template void NDView::print_all() const { } } + +template +std::ostream& operator <<(std::ostream& os, const NDView& arr){ + for (auto row = 0; row < arr.shape(0); ++row) { + for (auto col = 0; col < arr.shape(1); ++col) { + os << std::setw(3); + os << arr(row, col) << " "; + } + os << "\n"; + } + return os; +} + + } // namespace aare \ No newline at end of file diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 78e68ef..5c28481 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -3,7 +3,8 @@ #include "aare/RawMasterFile.hpp" #include "aare/Frame.hpp" #include "aare/NDArray.hpp" //for pixel map -#include "aare/SubFile.hpp" +#include "aare/RawSubFile.hpp" + #include @@ -29,10 +30,12 @@ struct ModuleConfig { * Consider using that unless you need raw file specific functionality. */ class RawFile : public FileInterface { - size_t n_subfiles{}; - size_t n_subfile_parts{}; - std::vector> subfiles; + size_t n_subfiles{}; //f0,f1...fn + size_t n_subfile_parts{}; // d0,d1...dn + //TODO! move to vector of SubFile instead of pointers + std::vector> subfiles; //subfiles[f0,f1...fn][d0,d1...dn] std::vector positions; + std::vector m_module_pixel_0; ModuleConfig cfg{0, 0}; RawMasterFile m_master; @@ -56,6 +59,11 @@ class RawFile : public FileInterface { std::vector read_n(size_t n_frames) override; void read_into(std::byte *image_buf) override; void read_into(std::byte *image_buf, size_t n_frames) override; + + //TODO! do we need to adapt the API? + void read_into(std::byte *image_buf, DetectorHeader *header); + + size_t frame_number(size_t frame_index) override; size_t bytes_per_frame() override; size_t pixels_per_frame() override; @@ -65,7 +73,10 @@ class RawFile : public FileInterface { size_t rows() const override; size_t cols() const override; size_t bitdepth() const override; + size_t bytes_per_pixel() const; xy geometry(); + size_t n_mod() const; + DetectorType detector_type() const override; @@ -83,7 +94,9 @@ class RawFile : public FileInterface { * @param frame_number frame number to read * @param image_buf buffer to store the frame */ - void get_frame_into(size_t frame_index, std::byte *frame_buffer); + + void get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header = nullptr); + /** * @brief get the frame at the given frame index @@ -101,8 +114,9 @@ class RawFile : public FileInterface { */ static DetectorHeader read_header(const std::filesystem::path &fname); + void update_geometry_with_roi(); + int find_number_of_subfiles(); - void find_number_of_subfiles(); void open_subfiles(); void find_geometry(); }; diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index f86054a..04eb1c9 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -58,6 +58,17 @@ class ScanParameters { bool enabled() const; }; + +struct ROI{ + size_t xmin{}; + size_t xmax{}; + size_t ymin{}; + size_t ymax{}; + + size_t height() const { return ymax - ymin; } + size_t width() const { return xmax - xmin; } +}; + /** * @brief Class for parsing a master file either in our .json format or the old * .raw format @@ -95,6 +106,9 @@ class RawMasterFile { std::optional m_number_of_rows; std::optional m_quad; + std::optional m_roi; + + public: RawMasterFile(const std::filesystem::path &fpath); @@ -121,6 +135,10 @@ class RawMasterFile { std::optional number_of_rows() const; std::optional quad() const; + + std::optional roi() const; + + ScanParameters scan_parameters() const; private: diff --git a/include/aare/RawSubFile.hpp b/include/aare/RawSubFile.hpp new file mode 100644 index 0000000..5efa4b6 --- /dev/null +++ b/include/aare/RawSubFile.hpp @@ -0,0 +1,69 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/defs.hpp" + +#include +#include +#include +#include + +namespace aare { + +/** + * @brief Class to read a singe subfile written in .raw format. Used from RawFile to read + * the entire detector. Can be used directly to read part of the image. + */ +class RawSubFile { + protected: + std::ifstream m_file; + size_t m_bitdepth; + std::filesystem::path m_fname; + size_t m_rows{}; + size_t m_cols{}; + size_t m_bytes_per_frame{}; + size_t n_frames{}; + + DetectorType m_detector_type; + std::optional> pixel_map; + + public: + /** + * @brief SubFile constructor + * @param fname path to the subfile + * @param detector detector type + * @param rows number of rows in the subfile + * @param cols number of columns in the subfile + * @param bitdepth bitdepth of the subfile + * @throws std::invalid_argument if the detector,type pair is not supported + */ + RawSubFile(const std::filesystem::path &fname, DetectorType detector, + size_t rows, size_t cols, size_t bitdepth); + + ~RawSubFile() = default; + /** + * @brief Seek to the given frame number + * @note Puts the file pointer at the start of the header, not the start of the data + * @param frame_index frame position in file to seek to + * @throws std::runtime_error if the frame number is out of range + */ + void seek(size_t frame_index); + size_t tell(); + + void read_into(std::byte *image_buf, DetectorHeader *header = nullptr); + void get_part(std::byte *buffer, size_t frame_index); + + void read_header(DetectorHeader *header); + + size_t rows() const; + size_t cols() const; + + size_t frame_number(size_t frame_index); + + size_t bytes_per_frame() const { return m_bytes_per_frame; } + size_t pixels_per_frame() const { return m_rows * m_cols; } + size_t bytes_per_pixel() const { return m_bitdepth / 8; } + + +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index f4a0bcf..1fc369d 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -14,6 +14,7 @@ #include #include + /** * @brief LOCATION macro to get the current location in the code */ @@ -21,8 +22,25 @@ std::string(__FILE__) + std::string(":") + std::to_string(__LINE__) + \ ":" + std::string(__func__) + ":" + + +#ifdef AARE_CUSTOM_ASSERT +#define AARE_ASSERT(expr)\ + if (expr)\ + {}\ + else\ + aare::assert_failed(LOCATION + " Assertion failed: " + #expr + "\n"); +#else +#define AARE_ASSERT(cond)\ + do { (void)sizeof(cond); } while(0) +#endif + + namespace aare { +void assert_failed(const std::string &msg); + + class Cluster { public: int cluster_sizeX; @@ -164,6 +182,15 @@ template struct t_xy { }; using xy = t_xy; + +struct ModuleGeometry{ + int x{}; + int y{}; + int height{}; + int width{}; +}; + + using dynamic_shape = std::vector; //TODO! Can we uniform enums between the libraries? diff --git a/pyproject.toml b/pyproject.toml index baf6e38..fd3d35e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,8 @@ build-backend = "scikit_build_core.build" [project] name = "aare" -version = "2024.11.07.dev0" +version = "2024.11.11.dev0" + [tool.scikit-build] cmake.verbose = true diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 76af17a..f3ed63b 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -1,7 +1,10 @@ # Make the compiled classes that live in _aare available from aare. from . import _aare -from ._aare import VarClusterFinder, File, RawMasterFile -from ._aare import Pedestal, ClusterFinder + +from ._aare import File, RawFile, RawMasterFile, RawSubFile +from ._aare import Pedestal, ClusterFinder, VarClusterFinder +from ._aare import DetectorType + from .CtbRawFile import CtbRawFile from .ScanParameters import ScanParameters \ No newline at end of file diff --git a/python/examples/play.py b/python/examples/play.py index 3a12d28..ceeb1d3 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -53,84 +53,36 @@ def get_Mh02_frames(fname): # plt.imshow(data[0,0]) -# p = Path('/Users/erik/data/aare_test_data/jungfrau/jungfrau_single_master_0.json') +base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/ci/aare_test_data') +# p = Path(base / 'jungfrau/jungfrau_single_master_0.json') # f = aare.File(p) -# frame = f.read_frame() - -# fig, ax = plt.subplots() -# im = ax.imshow(frame, cmap='viridis') - - -# fpath = Path('/Users/erik/data/Moench03old/test_034_irradiated_noise_g4_hg_exptime_2000us_master_0.json') -# # fpath = Path('/Users/erik/data/Moench05/moench05_multifile_master_0.json') - - -# # f = aare.CtbRawFile(fpath, transform = transform.moench05) -# # with CtbRawFile(fpath, transform = transform.moench05) as f: -# # for header, image in f: -# # print(f'Frame number: {header["frameNumber"]}') - -# # m = aare.RawMasterFile(fpath) -# f = aare.File(fpath) +# for i in range(10): +# frame = f.read_frame() -# cf = aare.ClusterFinder((400,400),(3,3)) - -# for i in range(100): -# cf.push_pedestal_frame(f.read_frame()) - - -# f.seek(0) -# pd = f.read_n(100).mean(axis=0) - -# clusters = cf.find_clusters_without_threshold(f.read_frame()) +# # f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02) +# # header, data = f2.read() +# # plt.plot(data[:,0,20,20]) +# from aare import RawMasterFile, File, RawSubFile, DetectorType, RawFile +# base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/') +# fpath = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') +# raw = Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_d0_f0_0.raw') -base = Path('/Users/erik/data/matterhorn/raw') -fpath = Path(base / 'scan_15keV_vrf700_vrsh700_th0_master_0.json') -f = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -f.seek(100) -header1, image1 = f.read_frame() +# m = RawMasterFile(base / fpath) +# # roi = m.roi +# # rows = roi.ymax-roi.ymin+1 +# # cols = 1024-roi.xmin +# # sf = RawSubFile(base / raw, DetectorType.Jungfrau, rows, cols, 16) -# fpath = Path(base / 'scan_all15keV_vrf500_vrsh700_th0_master_0.json') - -# f = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -# f.seek(100) -# header4, image4 = f.read_frame() - -# n_counters = image.shape[1] / 48**2 / 2 - -# for i in range(100): -# header, image = f.read_frame() -# print(header['frameNumber']) - - - - - -#Data come in "blocks" of 4 pixels/receiver -# data = get_Mh02_frames(fpath.as_posix()) - -# rawi = np.zeros(48*48*4+56, dtype = np.uint16) -# for i,v in enumerate(rawi[56:]): -# rawi[i+56] = i - -# raw = image.view(np.uint16) - -# pixel_map = decode(1, rawi) -# # img = np.take(raw, pixel_map) - -# pm = np.zeros((4, 48,48), dtype = np.int64) -# for counter in range(4): -# for row in range(48): -# for col in range(48): -# pm[counter, row, col] = row*48 + col+counter*48*48 - - -# f2 = aare.CtbRawFile(fpath, transform=transform.matterhorn02) -# header, data = f2.read() -# plt.plot(data[:,0,20,20]) +from aare import RawFile +base = Path('/mnt/sls_det_storage/matterhorn_data/aare_test_data/Jungfrau10/Jungfrau_DoubleModule_1UDP_ROI/SideBySide/') +fname = base / Path('241019_JF_12keV_Si_FF_GaAs_FF_7p88mmFilter_PedestalStart_ZPos_5.5_master_0.json') +# fname = Path(base / 'jungfrau/jungfrau_single_master_0.json') +f = RawFile(fname) +h,img = f.read_frame() +print(f'{h["frameNumber"]}') diff --git a/python/src/file.hpp b/python/src/file.hpp index 6471b44..c7cb445 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -1,7 +1,10 @@ #include "aare/CtbRawFile.hpp" -#include "aare/RawMasterFile.hpp" #include "aare/File.hpp" #include "aare/Frame.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" +#include "aare/RawSubFile.hpp" + #include "aare/defs.hpp" // #include "aare/fClusterFileV2.hpp" @@ -19,6 +22,18 @@ using namespace ::aare; void define_file_io_bindings(py::module &m) { + + py::enum_(m, "DetectorType") + .value("Jungfrau", DetectorType::Jungfrau) + .value("Eiger", DetectorType::Eiger) + .value("Mythen3", DetectorType::Mythen3) + .value("Moench", DetectorType::Moench) + .value("Moench03", DetectorType::Moench03) + .value("Moench03_old", DetectorType::Moench03_old) + .value("ChipTestBoard", DetectorType::ChipTestBoard) + .value("Unknown", DetectorType::Unknown); + + PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber, bunchId, timestamp, modId, row, column, reserved, debug, roundRNumber, detType, version, packetMask); @@ -36,7 +51,6 @@ void define_file_io_bindings(py::module &m) { py::array_t header(1); - // always read bytes image = py::array_t(shape); @@ -49,7 +63,10 @@ void define_file_io_bindings(py::module &m) { .def("seek", &CtbRawFile::seek) .def("tell", &CtbRawFile::tell) .def("master", &CtbRawFile::master) - .def_property_readonly("image_size_in_bytes", &CtbRawFile::image_size_in_bytes) + + .def_property_readonly("image_size_in_bytes", + &CtbRawFile::image_size_in_bytes) + .def_property_readonly("frames_in_file", &CtbRawFile::frames_in_file); py::class_(m, "File") @@ -169,26 +186,107 @@ void define_file_io_bindings(py::module &m) { .def_property_readonly("frame_padding", &RawMasterFile::frame_padding) .def_property_readonly("frame_discard_policy", &RawMasterFile::frame_discard_policy) - .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected) + + .def_property_readonly("total_frames_expected", + &RawMasterFile::total_frames_expected) .def_property_readonly("geometry", &RawMasterFile::geometry) .def_property_readonly("analog_samples", &RawMasterFile::analog_samples) .def_property_readonly("digital_samples", &RawMasterFile::digital_samples) - .def_property_readonly("transceiver_samples", &RawMasterFile::transceiver_samples) + + .def_property_readonly("transceiver_samples", + &RawMasterFile::transceiver_samples) .def_property_readonly("number_of_rows", &RawMasterFile::number_of_rows) .def_property_readonly("quad", &RawMasterFile::quad) - .def_property_readonly("scan_parameters", &RawMasterFile::scan_parameters); - - + .def_property_readonly("scan_parameters", + &RawMasterFile::scan_parameters) + .def_property_readonly("roi", &RawMasterFile::roi); py::class_(m, "ScanParameters") .def(py::init()) - .def(py::init()) + .def(py::init()) + .def_property_readonly("enabled", &ScanParameters::enabled) .def_property_readonly("dac", &ScanParameters::dac) .def_property_readonly("start", &ScanParameters::start) .def_property_readonly("stop", &ScanParameters::stop) .def_property_readonly("step", &ScanParameters::step); + + + py::class_(m, "ROI") + .def(py::init<>()) + .def_readwrite("xmin", &ROI::xmin) + .def_readwrite("xmax", &ROI::xmax) + .def_readwrite("ymin", &ROI::ymin) + .def_readwrite("ymax", &ROI::ymax) + .def("__str__", [](const ROI& self){ + return fmt::format("ROI: xmin: {} xmax: {} ymin: {} ymax: {}", self.xmin, self.xmax, self.ymin, self.ymax); + }); + + py::class_(m, "RawFile") + .def(py::init()) + .def("read_frame", + [](RawFile &self) { + size_t image_size = self.bytes_per_frame(); + const uint8_t item_size = self.bytes_per_pixel(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + + + //return headers from all subfiles + py::array_t header(self.n_mod()); + + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); + + self.read_into( + reinterpret_cast(image.mutable_data()), + header.mutable_data()); + + return py::make_tuple(header, image); + }); + + py::class_(m, "RawSubFile") + .def(py::init()) + .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame) + .def_property_readonly("pixels_per_frame", + &RawSubFile::pixels_per_frame) + .def("seek", &RawSubFile::seek) + .def("tell", &RawSubFile::tell) + .def_property_readonly("rows", &RawSubFile::rows) + .def_property_readonly("cols", &RawSubFile::cols) + .def("read_frame", + [](RawSubFile &self) { + const uint8_t item_size = self.bytes_per_pixel(); + py::array image; + std::vector shape; + shape.reserve(2); + shape.push_back(self.rows()); + shape.push_back(self.cols()); + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); + self.read_into( + reinterpret_cast(image.mutable_data())); + return image; + }); + + // py::class_(m, "ClusterHeader") // .def(py::init<>()) // .def_readwrite("frame_number", &ClusterHeader::frame_number) diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 31b57c5..a82b29b 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -13,9 +13,13 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) : m_master(fname) { m_mode = mode; if (mode == "r") { - find_number_of_subfiles(); - n_subfile_parts = m_master.geometry().col * m_master.geometry().row; + + n_subfiles = find_number_of_subfiles(); // f0,f1...fn + n_subfile_parts = + m_master.geometry().col * m_master.geometry().row; // d0,d1...dn find_geometry(); + update_geometry_with_roi(); + open_subfiles(); } else { throw std::runtime_error(LOCATION + @@ -38,10 +42,20 @@ void RawFile::read_into(std::byte *image_buf, size_t n_frames) { image_buf += bytes_per_frame(); } } + void RawFile::read_into(std::byte *image_buf) { return get_frame_into(m_current_frame++, image_buf); }; + +void RawFile::read_into(std::byte *image_buf, DetectorHeader *header) { + + return get_frame_into(m_current_frame++, image_buf, header); +}; + +size_t RawFile::n_mod() const { return n_subfile_parts; } + + size_t RawFile::bytes_per_frame() { return m_rows * m_cols * m_master.bitdepth() / 8; } @@ -71,12 +85,13 @@ xy RawFile::geometry() { return m_master.geometry(); } void RawFile::open_subfiles() { if (m_mode == "r") for (size_t i = 0; i != n_subfiles; ++i) { - auto v = std::vector(n_subfile_parts); + auto v = std::vector(n_subfile_parts); for (size_t j = 0; j != n_subfile_parts; ++j) { - v[j] = - new SubFile(m_master.data_fname(j, i), - m_master.detector_type(), m_master.pixels_y(), - m_master.pixels_x(), m_master.bitdepth()); + auto pos = m_module_pixel_0[j]; + v[j] = new RawSubFile(m_master.data_fname(j, i), + m_master.detector_type(), pos.height, + pos.width, m_master.bitdepth()); + } subfiles.push_back(v); } @@ -107,27 +122,36 @@ bool RawFile::is_master_file(const std::filesystem::path &fpath) { return stem.find("_master_") != std::string::npos; } -void RawFile::find_number_of_subfiles() { - int n_mod = 0; - while (std::filesystem::exists(m_master.data_fname(0, ++n_mod))) - ; - n_subfiles = n_mod; - #ifdef AARE_VERBOSE - fmt::print("Found: {} subfiles\n", n_subfiles); - #endif +int RawFile::find_number_of_subfiles() { + int n_files = 0; + // f0,f1...fn How many files is the data split into? + while (std::filesystem::exists(m_master.data_fname(0, n_files))) + n_files++; // increment after test + +#ifdef AARE_VERBOSE + fmt::print("Found: {} subfiles\n", n_files); +#endif + return n_files; + } void RawFile::find_geometry() { uint16_t r{}; uint16_t c{}; - for (size_t i = 0; i < n_subfile_parts; i++) { - for (size_t j = 0; j != n_subfiles; ++j) { - auto h = this->read_header(m_master.data_fname(i, j)); - r = std::max(r, h.row); - c = std::max(c, h.column); - positions.push_back({h.row, h.column}); - } + + for (size_t i = 0; i < n_subfile_parts; i++) { + auto h = this->read_header(m_master.data_fname(i, 0)); + r = std::max(r, h.row); + c = std::max(c, h.column); + positions.push_back({h.row, h.column}); + ModuleGeometry g; + g.x = h.column * m_master.pixels_x(); + g.y = h.row * m_master.pixels_y(); + g.width = m_master.pixels_x(); + g.height = m_master.pixels_y(); + m_module_pixel_0.push_back(g); + } r++; @@ -137,6 +161,81 @@ void RawFile::find_geometry() { m_cols = (c * m_master.pixels_x()); m_rows += static_cast((r - 1) * cfg.module_gap_row); + +#ifdef AARE_VERBOSE + fmt::print("\nRawFile::find_geometry()\n"); + for (size_t i = 0; i < m_module_pixel_0.size(); i++) { + fmt::print("Module {} at position: (r:{},c:{})\n", i, + m_module_pixel_0[i].y, m_module_pixel_0[i].x); + } + fmt::print("Image size: {}x{}\n\n", m_rows, m_cols); +#endif +} + +void RawFile::update_geometry_with_roi() { + // TODO! implement this + if (m_master.roi()) { + auto roi = m_master.roi().value(); + + // TODO! can we do this cleaner? + int pos_y = 0; + int pos_y_increment = 0; + for (size_t row = 0; row < m_master.geometry().row; row++) { + int pos_x = 0; + for (size_t col = 0; col < m_master.geometry().col; col++) { + auto &m = m_module_pixel_0[row * m_master.geometry().col + col]; + auto original_height = m.height; + auto original_width = m.width; + + // module is to the left of the roi + if (m.x + m.width < roi.xmin) { + m.width = 0; + + // roi is in module + } else { + // here we only arrive when the roi is in or to the left of + // the module + if (roi.xmin > m.x) { + m.width -= roi.xmin - m.x; + } + if (roi.xmax < m.x + m.width) { + m.width -= m.x + original_width - roi.xmax; + } + m.x = pos_x; + pos_x += m.width; + } + + if (m.y + m.height < roi.ymin) { + m.height = 0; + } else { + if (roi.ymin < m.y + m.height) { + m.height -= roi.ymin; + } + if (roi.ymax < m.y + m.height) { + m.height -= m.y + original_height - roi.ymax; + } + m.y = pos_y; + pos_y_increment = m.height; + } + } + // increment pos_y + pos_y += pos_y_increment; + } + + m_rows = roi.height(); + m_cols = roi.width(); + } + +#ifdef AARE_VERBOSE + fmt::print("RawFile::update_geometry_with_roi()\n"); + for (const auto &m : m_module_pixel_0) { + fmt::print("Module at position: (r:{}, c:{}, h:{}, w:{})\n", m.y, m.x, + m.height, m.width); + } + fmt::print("Updated image size: {}x{}\n\n", m_rows, m_cols); + fmt::print("\n"); +#endif + } Frame RawFile::get_frame(size_t frame_index) { @@ -146,13 +245,22 @@ Frame RawFile::get_frame(size_t frame_index) { return f; } -void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { + +size_t RawFile::bytes_per_pixel() const { + return m_master.bitdepth() / 8; +} + +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) { + if (frame_index > total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } std::vector frame_numbers(n_subfile_parts); std::vector frame_indices(n_subfile_parts, frame_index); + + // sync the frame numbers + if (n_subfile_parts != 1) { for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { auto subfile_id = frame_index / m_master.max_frames_per_file(); @@ -189,15 +297,25 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { auto corrected_idx = frame_indices[part_idx]; auto subfile_id = corrected_idx / m_master.max_frames_per_file(); - auto part_offset = subfiles[subfile_id][part_idx]->bytes_per_part(); - subfiles[subfile_id][part_idx]->get_part( - frame_buffer + part_idx * part_offset, - corrected_idx % m_master.max_frames_per_file()); + + // This is where we start writing + auto offset = (m_module_pixel_0[part_idx].y * m_cols + + m_module_pixel_0[part_idx].x)*m_master.bitdepth()/8; + + if (m_module_pixel_0[part_idx].x!=0) + throw std::runtime_error(LOCATION + "Implementation error. x pos not 0."); + + subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file()); + subfiles[subfile_id][part_idx]->read_into(frame_buffer + offset, header); + if (header) + ++header; } } else { + //TODO! should we read row by row? + + // create a buffer large enough to hold a full module - // create a buffer that will hold a the frame part auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() * m_master.bitdepth() / 8; // TODO! replace with image_size_in_bytes @@ -207,23 +325,27 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { // level for (size_t part_idx = 0; part_idx != n_subfile_parts; ++part_idx) { + auto pos = m_module_pixel_0[part_idx]; auto corrected_idx = frame_indices[part_idx]; auto subfile_id = corrected_idx / m_master.max_frames_per_file(); - subfiles[subfile_id][part_idx]->get_part( - part_buffer, corrected_idx % m_master.max_frames_per_file()); - for (size_t cur_row = 0; cur_row < (m_master.pixels_y()); + subfiles[subfile_id][part_idx]->seek(corrected_idx % m_master.max_frames_per_file()); + subfiles[subfile_id][part_idx]->read_into(part_buffer, header); + if(header) + ++header; + + for (size_t cur_row = 0; cur_row < (pos.height); cur_row++) { - auto irow = cur_row + (part_idx / m_master.geometry().col) * - m_master.pixels_y(); - auto icol = - (part_idx % m_master.geometry().col) * m_master.pixels_x(); + + auto irow = (pos.y + cur_row); + auto icol = pos.x; auto dest = (irow * this->m_cols + icol); dest = dest * m_master.bitdepth() / 8; memcpy(frame_buffer + dest, - part_buffer + cur_row * m_master.pixels_x() * + part_buffer + cur_row * pos.width * m_master.bitdepth() / 8, - m_master.pixels_x() * m_master.bitdepth() / 8); + pos.width * m_master.bitdepth() / 8); + } } delete[] part_buffer; diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index 09a0006..ae2caa1 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -143,6 +143,10 @@ ScanParameters RawMasterFile::scan_parameters() const { return m_scan_parameters; } + +std::optional RawMasterFile::roi() const { return m_roi; } + + void RawMasterFile::parse_json(const std::filesystem::path &fpath) { std::ifstream ifs(fpath); json j; @@ -240,6 +244,40 @@ void RawMasterFile::parse_json(const std::filesystem::path &fpath) { // not a scan } + + try{ + ROI tmp_roi; + auto obj = j.at("Receiver Roi"); + tmp_roi.xmin = obj.at("xmin"); + tmp_roi.xmax = obj.at("xmax"); + tmp_roi.ymin = obj.at("ymin"); + tmp_roi.ymax = obj.at("ymax"); + + //if any of the values are set update the roi + if (tmp_roi.xmin != 4294967295 || tmp_roi.xmax != 4294967295 || + tmp_roi.ymin != 4294967295 || tmp_roi.ymax != 4294967295) { + + if(v<7.21){ + tmp_roi.xmax++; + tmp_roi.ymax++; + } + + m_roi = tmp_roi; + } + + + }catch (const json::out_of_range &e) { + // leave the optional empty + } + + //if we have an roi we need to update the geometry for the subfiles + if (m_roi){ + + } + + + + // Update detector type for Moench // TODO! How does this work with old .raw master files? #ifdef AARE_VERBOSE diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp new file mode 100644 index 0000000..c537007 --- /dev/null +++ b/src/RawSubFile.cpp @@ -0,0 +1,97 @@ +#include "aare/RawSubFile.hpp" +#include "aare/PixelMap.hpp" +#include // memcpy +#include +#include + +namespace aare { + +RawSubFile::RawSubFile(const std::filesystem::path &fname, + DetectorType detector, size_t rows, size_t cols, + size_t bitdepth) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), + m_detector_type(detector), + m_bytes_per_frame((m_bitdepth / 8) * m_rows * m_cols) { + if (m_detector_type == DetectorType::Moench03_old) { + pixel_map = GenerateMoench03PixelMap(); + } + + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / + (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); + } else { + throw std::runtime_error( + LOCATION + fmt::format("File {} does not exist", m_fname.string())); + } + + // fp = fopen(m_fname.string().c_str(), "rb"); + m_file.open(m_fname, std::ios::binary); + if (!m_file.is_open()) { + throw std::runtime_error( + LOCATION + fmt::format("Could not open file {}", m_fname.string())); + } + +#ifdef AARE_VERBOSE + fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); + fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, + m_bitdepth); + fmt::print("file size: {}\n", std::filesystem::file_size(fname)); +#endif +} + +void RawSubFile::seek(size_t frame_index) { + if (frame_index >= n_frames) { + throw std::runtime_error("Frame number out of range"); + } + m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index); +} + +size_t RawSubFile::tell() { + return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame()); +} + + +void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { + if(header){ + m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); + } else { + m_file.seekg(sizeof(DetectorHeader), std::ios::cur); + } + + //TODO! expand support for different bitdepths + if(pixel_map){ + // read into a temporary buffer and then copy the data to the buffer + // in the correct order + // currently this only supports 16 bit data! + auto part_buffer = new std::byte[bytes_per_frame()]; + m_file.read(reinterpret_cast(part_buffer), bytes_per_frame()); + auto *data = reinterpret_cast(image_buf); + auto *part_data = reinterpret_cast(part_buffer); + for (size_t i = 0; i < pixels_per_frame(); i++) { + data[i] = part_data[(*pixel_map)(i)]; + } + delete[] part_buffer; + } else { + // read directly into the buffer + m_file.read(reinterpret_cast(image_buf), bytes_per_frame()); + } +} + +size_t RawSubFile::rows() const { return m_rows; } +size_t RawSubFile::cols() const { return m_cols; } + + +void RawSubFile::get_part(std::byte *buffer, size_t frame_index) { + seek(frame_index); + read_into(buffer, nullptr); +} + +size_t RawSubFile::frame_number(size_t frame_index) { + seek(frame_index); + DetectorHeader h{}; + m_file.read(reinterpret_cast(&h), sizeof(DetectorHeader)); + return h.frameNumber; +} + + +} // namespace aare \ No newline at end of file diff --git a/src/defs.cpp b/src/defs.cpp index 1b99045..16fa3d0 100644 --- a/src/defs.cpp +++ b/src/defs.cpp @@ -2,8 +2,18 @@ #include #include +#include namespace aare { + +void assert_failed(const std::string &msg) + { + fmt::print(msg); + exit(1); +} + + + /** * @brief Convert a DetectorType to a string * @param type DetectorType