diff --git a/CMakeLists.txt b/CMakeLists.txt index 039545e..2f2a7b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -388,7 +388,9 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawMasterFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/ifstream_helpers.cpp ) diff --git a/include/aare/RawSubFile.hpp b/include/aare/RawSubFile.hpp index 1d554e8..350a475 100644 --- a/include/aare/RawSubFile.hpp +++ b/include/aare/RawSubFile.hpp @@ -22,7 +22,7 @@ class RawSubFile { size_t m_rows{}; size_t m_cols{}; size_t m_bytes_per_frame{}; - size_t n_frames{}; + size_t m_num_frames{}; uint32_t m_pos_row{}; uint32_t m_pos_col{}; @@ -53,6 +53,7 @@ class RawSubFile { size_t tell(); void read_into(std::byte *image_buf, DetectorHeader *header = nullptr); + void read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header= nullptr); void get_part(std::byte *buffer, size_t frame_index); void read_header(DetectorHeader *header); @@ -66,6 +67,8 @@ class RawSubFile { size_t pixels_per_frame() const { return m_rows * m_cols; } size_t bytes_per_pixel() const { return m_bitdepth / bits_per_byte; } + size_t frames_in_file() const { return m_num_frames; } + private: template void read_with_map(std::byte *image_buf); diff --git a/include/aare/utils/ifstream_helpers.hpp b/include/aare/utils/ifstream_helpers.hpp new file mode 100644 index 0000000..0a842ed --- /dev/null +++ b/include/aare/utils/ifstream_helpers.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include +#include +namespace aare { + +/** + * @brief Get the error message from an ifstream object +*/ +std::string ifstream_error_msg(std::ifstream &ifs); + +} // namespace aare \ No newline at end of file diff --git a/python/src/file.hpp b/python/src/file.hpp index 0d64e16..2d0f53e 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -20,6 +20,9 @@ namespace py = pybind11; using namespace ::aare; + + + //Disable warnings for unused parameters, as we ignore some //in the __exit__ method #pragma GCC diagnostic push @@ -214,36 +217,9 @@ void define_file_io_bindings(py::module &m) { - py::class_(m, "RawSubFile") - .def(py::init()) - .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame) - .def_property_readonly("pixels_per_frame", - &RawSubFile::pixels_per_frame) - .def("seek", &RawSubFile::seek) - .def("tell", &RawSubFile::tell) - .def_property_readonly("rows", &RawSubFile::rows) - .def_property_readonly("cols", &RawSubFile::cols) - .def("read_frame", - [](RawSubFile &self) { - const uint8_t item_size = self.bytes_per_pixel(); - py::array image; - std::vector shape; - shape.reserve(2); - shape.push_back(self.rows()); - shape.push_back(self.cols()); - if (item_size == 1) { - image = py::array_t(shape); - } else if (item_size == 2) { - image = py::array_t(shape); - } else if (item_size == 4) { - image = py::array_t(shape); - } - fmt::print("item_size: {} rows: {} cols: {}\n", item_size, self.rows(), self.cols()); - self.read_into( - reinterpret_cast(image.mutable_data())); - return image; - }); + + + #pragma GCC diagnostic pop // py::class_(m, "ClusterHeader") diff --git a/python/src/module.cpp b/python/src/module.cpp index 7a17e78..75fe237 100644 --- a/python/src/module.cpp +++ b/python/src/module.cpp @@ -10,6 +10,7 @@ #include "cluster_file.hpp" #include "fit.hpp" #include "interpolation.hpp" +#include "raw_sub_file.hpp" #include "jungfrau_data_file.hpp" @@ -22,6 +23,7 @@ namespace py = pybind11; PYBIND11_MODULE(_aare, m) { define_file_io_bindings(m); define_raw_file_io_bindings(m); + define_raw_sub_file_io_bindings(m); define_ctb_raw_file_io_bindings(m); define_raw_master_file_bindings(m); define_var_cluster_finder_bindings(m); diff --git a/python/src/raw_sub_file.hpp b/python/src/raw_sub_file.hpp new file mode 100644 index 0000000..2cb83fc --- /dev/null +++ b/python/src/raw_sub_file.hpp @@ -0,0 +1,110 @@ +#include "aare/CtbRawFile.hpp" +#include "aare/File.hpp" +#include "aare/Frame.hpp" +#include "aare/RawFile.hpp" +#include "aare/RawMasterFile.hpp" +#include "aare/RawSubFile.hpp" + +#include "aare/defs.hpp" +// #include "aare/fClusterFileV2.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace ::aare; + +auto read_frame_from_RawSubFile(RawSubFile &self) { + py::array_t header(1); + const uint8_t item_size = self.bytes_per_pixel(); + std::vector shape{static_cast(self.rows()), + static_cast(self.cols())}; + + py::array image; + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into(reinterpret_cast(image.mutable_data()), + header.mutable_data()); + + return py::make_tuple(header, image); +} + +auto read_n_frames_from_RawSubFile(RawSubFile &self, size_t n_frames) { + py::array_t header(n_frames); + const uint8_t item_size = self.bytes_per_pixel(); + std::vector shape{ + static_cast(n_frames), + static_cast(self.rows()), + static_cast(self.cols()) + }; + + py::array image; + if (item_size == 1) { + image = py::array_t(shape); + } else if (item_size == 2) { + image = py::array_t(shape); + } else if (item_size == 4) { + image = py::array_t(shape); + } + self.read_into(reinterpret_cast(image.mutable_data()), n_frames, + header.mutable_data()); + + return py::make_tuple(header, image); +} + + +//Disable warnings for unused parameters, as we ignore some +//in the __exit__ method +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" + +void define_raw_sub_file_io_bindings(py::module &m) { + py::class_(m, "RawSubFile") + .def(py::init()) + .def_property_readonly("bytes_per_frame", &RawSubFile::bytes_per_frame) + .def_property_readonly("pixels_per_frame", + &RawSubFile::pixels_per_frame) + .def_property_readonly("bytes_per_pixel", &RawSubFile::bytes_per_pixel) + .def("seek", &RawSubFile::seek) + .def("tell", &RawSubFile::tell) + .def_property_readonly("rows", &RawSubFile::rows) + .def_property_readonly("cols", &RawSubFile::cols) + .def_property_readonly("frames_in_file", &RawSubFile::frames_in_file) + .def("read_frame", &read_frame_from_RawSubFile) + .def("read_n", &read_n_frames_from_RawSubFile) + .def("read", [](RawSubFile &self){ + self.seek(0); + auto n_frames = self.frames_in_file(); + return read_n_frames_from_RawSubFile(self, n_frames); + }) + .def("__enter__", [](RawSubFile &self) { return &self; }) + .def("__exit__", + [](RawSubFile &self, + const std::optional &exc_type, + const std::optional &exc_value, + const std::optional &traceback) { + }) + .def("__iter__", [](RawSubFile &self) { return &self; }) + .def("__next__", [](RawSubFile &self) { + try { + return read_frame_from_RawSubFile(self); + } catch (std::runtime_error &e) { + throw py::stop_iteration(); + } + }); + +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/tests/test_RawSubFile.py b/python/tests/test_RawSubFile.py new file mode 100644 index 0000000..a5eea91 --- /dev/null +++ b/python/tests/test_RawSubFile.py @@ -0,0 +1,36 @@ +import pytest +import numpy as np +from aare import RawSubFile, DetectorType + + +@pytest.mark.files +def test_read_a_jungfrau_RawSubFile(test_data_path): + with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f1_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f: + assert f.frames_in_file == 3 + + headers, frames = f.read() + + assert headers.size == 3 + assert frames.shape == (3, 512, 1024) + + # Frame numbers in this file should be 4, 5, 6 + for i,h in zip(range(4,7,1), headers): + assert h["frameNumber"] == i + + # Compare to canned data using numpy + data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy") + assert np.all(data[3:6] == frames) + +@pytest.mark.files +def test_iterate_over_a_jungfrau_RawSubFile(test_data_path): + + data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy") + + with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f: + i = 0 + for header, frame in f: + assert header["frameNumber"] == i+1 + assert np.all(frame == data[i]) + i += 1 + assert i == 3 + assert header["frameNumber"] == 3 \ No newline at end of file diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp index a3bb79c..9e7a421 100644 --- a/src/RawSubFile.cpp +++ b/src/RawSubFile.cpp @@ -1,9 +1,12 @@ #include "aare/RawSubFile.hpp" #include "aare/PixelMap.hpp" +#include "aare/utils/ifstream_helpers.hpp" #include // memcpy #include #include + + namespace aare { RawSubFile::RawSubFile(const std::filesystem::path &fname, @@ -20,7 +23,7 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname, } if (std::filesystem::exists(fname)) { - n_frames = std::filesystem::file_size(fname) / + m_num_frames = std::filesystem::file_size(fname) / (sizeof(DetectorHeader) + rows * cols * bitdepth / 8); } else { throw std::runtime_error( @@ -35,7 +38,7 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname, } #ifdef AARE_VERBOSE - fmt::print("Opened file: {} with {} frames\n", m_fname.string(), n_frames); + fmt::print("Opened file: {} with {} frames\n", m_fname.string(), m_num_frames); fmt::print("m_rows: {}, m_cols: {}, m_bitdepth: {}\n", m_rows, m_cols, m_bitdepth); fmt::print("file size: {}\n", std::filesystem::file_size(fname)); @@ -43,8 +46,8 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname, } void RawSubFile::seek(size_t frame_index) { - if (frame_index >= n_frames) { - throw std::runtime_error(LOCATION + fmt::format("Frame index {} out of range in a file with {} frames", frame_index, n_frames)); + if (frame_index >= m_num_frames) { + throw std::runtime_error(LOCATION + fmt::format("Frame index {} out of range in a file with {} frames", frame_index, m_num_frames)); } m_file.seekg((sizeof(DetectorHeader) + bytes_per_frame()) * frame_index); } @@ -60,6 +63,10 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { m_file.seekg(sizeof(DetectorHeader), std::ios::cur); } + if (m_file.fail()){ + throw std::runtime_error(LOCATION + ifstream_error_msg(m_file)); + } + // TODO! expand support for different bitdepths if (m_pixel_map) { // read into a temporary buffer and then copy the data to the buffer @@ -79,8 +86,24 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { // read directly into the buffer m_file.read(reinterpret_cast(image_buf), bytes_per_frame()); } + + if (m_file.fail()){ + throw std::runtime_error(LOCATION + ifstream_error_msg(m_file)); + } } +void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) { + for (size_t i = 0; i < n_frames; i++) { + read_into(image_buf, header); + image_buf += bytes_per_frame(); + if (header) { + ++header; + } + } +} + + + template void RawSubFile::read_with_map(std::byte *image_buf) { auto part_buffer = new std::byte[bytes_per_frame()]; diff --git a/src/utils/ifstream_helpers.cpp b/src/utils/ifstream_helpers.cpp new file mode 100644 index 0000000..74c56f3 --- /dev/null +++ b/src/utils/ifstream_helpers.cpp @@ -0,0 +1,18 @@ +#include "aare/utils/ifstream_helpers.hpp" + +namespace aare { + +std::string ifstream_error_msg(std::ifstream &ifs) { + std::ios_base::iostate state = ifs.rdstate(); + if (state & std::ios_base::eofbit) { + return " End of file reached"; + } else if (state & std::ios_base::badbit) { + return " Bad file stream"; + } else if (state & std::ios_base::failbit) { + return " File read failed"; + }else{ + return " Unknown/no error"; + } +} + +} // namespace aare