diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index f3494f1..44e001c 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -19,6 +19,7 @@ set(SPHINX_SOURCE_FILES src/Dtype.rst src/ClusterFinder.rst src/Pedestal.rst + src/RawMasterFile.rst src/VarClusterFinder.rst src/pyVarClusterFinder.rst src/pyFile.rst diff --git a/docs/src/RawMasterFile.rst b/docs/src/RawMasterFile.rst new file mode 100644 index 0000000..95f772c --- /dev/null +++ b/docs/src/RawMasterFile.rst @@ -0,0 +1,8 @@ +RawMasterFile +=============== + + +.. doxygenclass:: aare::RawMasterFile + :members: + :undoc-members: + :private-members: \ No newline at end of file diff --git a/docs/src/index.rst b/docs/src/index.rst index f24170b..d2dde47 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -16,6 +16,7 @@ AARE Dtype ClusterFinder Pedestal + RawMasterFile VarClusterFinder .. toctree:: diff --git a/include/aare/CtbRawFile.hpp b/include/aare/CtbRawFile.hpp index 24d67f5..a615188 100644 --- a/include/aare/CtbRawFile.hpp +++ b/include/aare/CtbRawFile.hpp @@ -9,6 +9,7 @@ namespace aare{ + class CtbRawFile{ RawMasterFile m_master; std::ifstream m_file; @@ -24,8 +25,8 @@ public: // in the specific class we can expose more functionality - size_t image_size_in_bytes() const { return m_master.image_size_in_bytes(); } - + size_t image_size_in_bytes() const; + size_t frames_in_file() const; private: void find_subfiles(); size_t sub_file_index(size_t frame_index) const { diff --git a/include/aare/RawMasterFile.hpp b/include/aare/RawMasterFile.hpp index 3908090..5cad2d2 100644 --- a/include/aare/RawMasterFile.hpp +++ b/include/aare/RawMasterFile.hpp @@ -23,7 +23,7 @@ class RawFileNameComponents { fmt::format("{}_master_{}{}", m_base_name, m_file_index, m_ext); } - std::filesystem::path data_fname(size_t mod_id, size_t file_id) { + std::filesystem::path data_fname(size_t mod_id, size_t file_id) const{ return m_base_path / fmt::format("{}_d{}_f{}_{}.raw", m_base_name, mod_id, file_id, m_file_index); } @@ -34,6 +34,10 @@ class RawFileNameComponents { int file_index() const { return m_file_index; } }; + +/** + * @brief Class for parsing a master file either in our .json format or the old .raw format + */ class RawMasterFile { RawFileNameComponents m_fnc; std::string m_version; @@ -55,39 +59,26 @@ class RawMasterFile { std::optional m_digital_samples; public: - RawMasterFile(const std::filesystem::path &fpath) : m_fnc(fpath) { - if (!std::filesystem::exists(fpath)) { - throw std::runtime_error(LOCATION + " File does not exist"); - } - if (m_fnc.ext() == ".json") { - parse_json(fpath); - } else if (m_fnc.ext() == ".raw") { - parse_raw(fpath); - } else { - throw std::runtime_error(LOCATION + "Unsupported file type"); - } - } + RawMasterFile(const std::filesystem::path &fpath); - const std::string &version() const { return m_version; } - const DetectorType &detector_type() const { return m_type; } - const TimingMode &timing_mode() const { return m_timing_mode; } - size_t image_size_in_bytes() const { return m_image_size_in_bytes; } - size_t frames_in_file() const { return m_frames_in_file; } - size_t pixels_y() const { return m_pixels_y; } - size_t pixels_x() const { return m_pixels_x; } - size_t max_frames_per_file() const { return m_max_frames_per_file; } - size_t bitdepth() const { return m_bitdepth; } - size_t frame_padding() const { return m_frame_padding; } - const FrameDiscardPolicy &frame_discard_policy() const { - return m_frame_discard_policy; - } + std::filesystem::path data_fname(size_t mod_id, size_t file_id) const; - std::optional analog_samples() const { return m_analog_samples; } - std::optional digital_samples() const { return m_digital_samples; } + const std::string &version() const; //!< For example "7.2" + const DetectorType &detector_type() const; + const TimingMode &timing_mode() const; + size_t image_size_in_bytes() const; + size_t frames_in_file() const; + size_t pixels_y() const; + size_t pixels_x() const; + size_t max_frames_per_file() const; + size_t bitdepth() const; + size_t frame_padding() const; + const FrameDiscardPolicy &frame_discard_policy() const; - std::filesystem::path data_fname(size_t mod_id, size_t file_id) { - return m_fnc.data_fname(mod_id, file_id); - } + std::optional analog_samples() const; + std::optional digital_samples() const; + + private: void parse_json(const std::filesystem::path &fpath); diff --git a/python/aare/CtbRawFile.py b/python/aare/CtbRawFile.py index abd2566..435beba 100644 --- a/python/aare/CtbRawFile.py +++ b/python/aare/CtbRawFile.py @@ -3,19 +3,35 @@ from . import _aare import numpy as np class CtbRawFile(_aare.CtbRawFile): + """File reader for the CTB raw file format. + + Args: + fname (pathlib.Path | str): Path to the file to be read. + transform (function): Function to apply to the data after reading it. + The function should take a numpy array of type uint8 and return one + or several numpy arrays. + """ def __init__(self, fname, transform = None): super().__init__(fname) self.transform = transform - def read_frame(self, frame_index = None): - """Read one frame from the file. + def read_frame(self, frame_index: int | None = None ) -> tuple: + """Read one frame from the file and then advance the file pointer. + + .. note:: + + Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine + which frame to read unless frame_index is specified. Args: frame_index (int): If not None, seek to this frame before reading. Returns: tuple: header, data + + Raises: + RuntimeError: If the file is at the end. """ if frame_index is not None: self.seek(frame_index) @@ -32,7 +48,23 @@ class CtbRawFile(_aare.CtbRawFile): else: return header, res - def read_n(self, n_frames): + def read_n(self, n_frames:int) -> tuple: + """Read several frames from the file. + + .. note:: + + Uses the position of the file pointer :py:meth:`~CtbRawFile.tell` to determine + where to start reading from. + + Args: + n_frames (int): Number of frames to read. + + Returns: + tuple: header, data + + Raises: + RuntimeError: If EOF is reached. + """ # Do the first read to figure out what we have tmp_header, tmp_data = self.read_frame() @@ -51,6 +83,47 @@ class CtbRawFile(_aare.CtbRawFile): return header, data + def seek(self, frame_index:int) -> None: + """Seek to a specific frame in the file. + + Args: + frame_index (int): Frame position in file to seek to. + """ + super().seek(frame_index) + + def tell() -> int: + """Return the current frame position in the file. + + Returns: + int: Frame position in file. + """ + return super().tell() + + + def image_size_in_bytes(self) -> int: + """Return the size of the image in bytes. + + Returns: + int: Size of image in bytes. + """ + return super().image_size_in_bytes() + + def __len__(self) -> int: + """Return the number of frames in the file. + + Returns: + int: Number of frames in file. + """ + return super().frames_in_file() + + def frames_in_file(self) -> int: + """Return the number of frames in the file. + + Returns: + int: Number of frames in file. + """ + return super().frames_in_file() + def __enter__(self): return self diff --git a/python/src/file.hpp b/python/src/file.hpp index d4032d5..7d11c85 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -46,7 +46,9 @@ void define_file_io_bindings(py::module &m) { return py::make_tuple(header, image); }) .def("seek", &CtbRawFile::seek) - .def("tell", &CtbRawFile::tell); + .def("tell", &CtbRawFile::tell) + .def("image_size_in_bytes", &CtbRawFile::image_size_in_bytes) + .def("frames_in_file", &CtbRawFile::frames_in_file); py::class_(m, "File") .def(py::init([](const std::filesystem::path &fname) { diff --git a/src/CtbRawFile.cpp b/src/CtbRawFile.cpp index a5ae2c7..94cf77c 100644 --- a/src/CtbRawFile.cpp +++ b/src/CtbRawFile.cpp @@ -4,9 +4,6 @@ namespace aare { CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) { - - - if (m_master.detector_type() != DetectorType::ChipTestBoard) { throw std::runtime_error(LOCATION + "Not a Ctb file"); } @@ -17,7 +14,24 @@ CtbRawFile::CtbRawFile(const std::filesystem::path &fname) : m_master(fname) { m_file.open(m_master.data_fname(0, 0), std::ios::binary); } -size_t CtbRawFile::tell() const { return m_current_frame; } +void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) { + if(m_current_frame >= m_master.frames_in_file()){ + throw std::runtime_error(LOCATION + "End of file reached"); + } + + if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){ + open_data_file(m_current_subfile+1); + } + + if(header){ + m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); + }else{ + m_file.seekg(sizeof(DetectorHeader), std::ios::cur); + } + + m_file.read(reinterpret_cast(image_buf), m_master.image_size_in_bytes()); + m_current_frame++; +} void CtbRawFile::seek(size_t frame_number) { if (auto index = sub_file_index(frame_number); index != m_current_subfile) { @@ -28,6 +42,12 @@ void CtbRawFile::seek(size_t frame_number) { m_current_frame = frame_number; } +size_t CtbRawFile::tell() const { return m_current_frame; } + +size_t CtbRawFile::image_size_in_bytes() const { return m_master.image_size_in_bytes(); } + +size_t CtbRawFile::frames_in_file() const { return m_master.frames_in_file(); } + void CtbRawFile::find_subfiles() { // we can semi safely assume that there is only one module for CTB while (std::filesystem::exists(m_master.data_fname(0, m_num_subfiles))) @@ -47,26 +67,6 @@ void CtbRawFile::open_data_file(size_t subfile_index) { } } -void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader* header) { - if(m_current_frame >= m_master.frames_in_file()){ - throw std::runtime_error(LOCATION + "End of file reached"); - } - if(m_current_frame != 0 && m_current_frame % m_master.max_frames_per_file() == 0){ - open_data_file(m_current_subfile+1); - } - - if(header){ - m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); - }else{ - m_file.seekg(sizeof(DetectorHeader), std::ios::cur); - } - - m_file.read(reinterpret_cast(image_buf), m_master.image_size_in_bytes()); - m_current_frame++; - - - -} } // namespace aare \ No newline at end of file diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index 9ad7083..19f014f 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -2,13 +2,13 @@ namespace aare { - -RawFileNameComponents::RawFileNameComponents(const std::filesystem::path &fname) { +RawFileNameComponents::RawFileNameComponents( + const std::filesystem::path &fname) { m_base_path = fname.parent_path(); m_base_name = fname.stem(); m_ext = fname.extension(); - //parse file index + // parse file index try { auto pos = m_base_name.rfind('_'); m_file_index = std::stoi(m_base_name.substr(pos + 1)); @@ -16,86 +16,132 @@ RawFileNameComponents::RawFileNameComponents(const std::filesystem::path &fname) throw std::runtime_error(LOCATION + "Could not parse file index"); } - //remove master from base name + // remove master from base name auto pos = m_base_name.find("_master_"); if (pos != std::string::npos) { m_base_name.erase(pos); - }else{ - throw std::runtime_error(LOCATION + "Could not find _master_ in file name"); + } else { + throw std::runtime_error(LOCATION + + "Could not find _master_ in file name"); } } +RawMasterFile::RawMasterFile(const std::filesystem::path &fpath) + : m_fnc(fpath) { + if (!std::filesystem::exists(fpath)) { + throw std::runtime_error(LOCATION + " File does not exist"); + } + if (m_fnc.ext() == ".json") { + parse_json(fpath); + } else if (m_fnc.ext() == ".raw") { + parse_raw(fpath); + } else { + throw std::runtime_error(LOCATION + "Unsupported file type"); + } +} + +std::filesystem::path RawMasterFile::data_fname(size_t mod_id, + size_t file_id) const { + return m_fnc.data_fname(mod_id, file_id); +} + +const std::string &RawMasterFile::version() const { return m_version; } +const DetectorType &RawMasterFile::detector_type() const { return m_type; } +const TimingMode &RawMasterFile::timing_mode() const { return m_timing_mode; } +size_t RawMasterFile::image_size_in_bytes() const { + return m_image_size_in_bytes; +} +size_t RawMasterFile::frames_in_file() const { return m_frames_in_file; } +size_t RawMasterFile::pixels_y() const { return m_pixels_y; } +size_t RawMasterFile::pixels_x() const { return m_pixels_x; } +size_t RawMasterFile::max_frames_per_file() const { + return m_max_frames_per_file; +} +size_t RawMasterFile::bitdepth() const { return m_bitdepth; } +size_t RawMasterFile::frame_padding() const { return m_frame_padding; } +const FrameDiscardPolicy &RawMasterFile::frame_discard_policy() const { + return m_frame_discard_policy; +} + +// optional values, these may or may not be present in the master file +// and are therefore modeled as std::optional +std::optional RawMasterFile::analog_samples() const { + return m_analog_samples; +} +std::optional RawMasterFile::digital_samples() const { + return m_digital_samples; +} + void RawMasterFile::parse_json(const std::filesystem::path &fpath) { - std::ifstream ifs(fpath); - json j; - ifs >> j; - double v = j["Version"]; - m_version = fmt::format("{:.1f}", v); + std::ifstream ifs(fpath); + json j; + ifs >> j; + double v = j["Version"]; + m_version = fmt::format("{:.1f}", v); - m_type = StringTo(j["Detector Type"].get()); - m_timing_mode = - StringTo(j["Timing Mode"].get()); + m_type = StringTo(j["Detector Type"].get()); + m_timing_mode = StringTo(j["Timing Mode"].get()); - m_image_size_in_bytes = j["Image Size in bytes"]; - m_frames_in_file = j["Frames in File"]; - m_pixels_y = j["Pixels"]["y"]; - m_pixels_x = j["Pixels"]["x"]; + m_image_size_in_bytes = j["Image Size in bytes"]; + m_frames_in_file = j["Frames in File"]; + m_pixels_y = j["Pixels"]["y"]; + m_pixels_x = j["Pixels"]["x"]; - m_max_frames_per_file = j["Max Frames Per File"]; + m_max_frames_per_file = j["Max Frames Per File"]; - //Not all detectors write the bitdepth but in case - //its not there it is 16 - try { - m_bitdepth = j.at("Dynamic Range"); - } catch (const json::out_of_range &e) { - m_bitdepth = 16; - } + // Not all detectors write the bitdepth but in case + // its not there it is 16 + try { + m_bitdepth = j.at("Dynamic Range"); + } catch (const json::out_of_range &e) { + m_bitdepth = 16; + } - m_frame_padding = j["Frame Padding"]; - m_frame_discard_policy = StringTo( - j["Frame Discard Policy"].get()); + m_frame_padding = j["Frame Padding"]; + m_frame_discard_policy = StringTo( + j["Frame Discard Policy"].get()); - try { - m_analog_samples = j.at("Analog Samples"); - }catch (const json::out_of_range &e) { - // m_analog_samples = 0; - } - // try{ - // std::string adc_mask = j.at("ADC Mask"); - // m_adc_mask = std::stoul(adc_mask, nullptr, 16); - // }catch (const json::out_of_range &e) { - // m_adc_mask = 0; - // } + try { + m_analog_samples = j.at("Analog Samples"); + } catch (const json::out_of_range &e) { + // m_analog_samples = 0; + } + // try{ + // std::string adc_mask = j.at("ADC Mask"); + // m_adc_mask = std::stoul(adc_mask, nullptr, 16); + // }catch (const json::out_of_range &e) { + // m_adc_mask = 0; + // } - try { - m_digital_samples = j.at("Digital Samples"); - }catch (const json::out_of_range &e) { - // m_digital_samples = 0; - } + try { + m_digital_samples = j.at("Digital Samples"); + } catch (const json::out_of_range &e) { + // m_digital_samples = 0; + } - // //Update detector type for Moench - // //TODO! How does this work with old .raw master files? - // if (m_type == DetectorType::Moench && m_analog_samples == 0 && - // m_subfile_rows == 400) { - // m_type = DetectorType::Moench03; - // }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 && - // m_analog_samples == 5000) { - // m_type = DetectorType::Moench03_old; - // } + // //Update detector type for Moench + // //TODO! How does this work with old .raw master files? + // if (m_type == DetectorType::Moench && m_analog_samples == 0 && + // m_subfile_rows == 400) { + // m_type = DetectorType::Moench03; + // }else if (m_type == DetectorType::Moench && m_subfile_rows == 400 && + // m_analog_samples == 5000) { + // m_type = DetectorType::Moench03_old; + // } - // //Here we know we have a ChipTestBoard file update the geometry? - // //TODO! Carry on information about digtial, and transceivers - // if (m_type == DetectorType::ChipTestBoard) { - // subfile_rows = 1; - // subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask); - // } + // //Here we know we have a ChipTestBoard file update the geometry? + // //TODO! Carry on information about digtial, and transceivers + // if (m_type == DetectorType::ChipTestBoard) { + // subfile_rows = 1; + // subfile_cols = m_analog_samples*__builtin_popcount(m_adc_mask); + // } - // // only Eiger had quad - // if (m_type == DetectorType::Eiger) { - // quad = (j["Quad"] == 1); - // } + // // only Eiger had quad + // if (m_type == DetectorType::Eiger) { + // quad = (j["Quad"] == 1); + // } - // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; + // m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; } void RawMasterFile::parse_raw(const std::filesystem::path &fpath) { throw std::runtime_error("Not implemented");