From bfb59d650bb05a6f615fb6fddac7f52af330435f Mon Sep 17 00:00:00 2001 From: Bechir Braham Date: Wed, 10 Apr 2024 13:34:31 +0200 Subject: [PATCH] remove factories and change to size_t --- core/include/aare/core/defs.hpp | 2 + examples/numpy_write_example.cpp | 2 +- file_io/CMakeLists.txt | 3 - file_io/include/aare/file_io/File.hpp | 6 +- .../include/aare/file_io/FileInterface.hpp | 23 ++- file_io/include/aare/file_io/NumpyFile.hpp | 38 ++-- .../include/aare/file_io/NumpyFileFactory.hpp | 22 --- file_io/include/aare/file_io/RawFile.hpp | 23 ++- ...{RawFileFactory.hpp => RawFileHelpers.hpp} | 9 +- file_io/include/aare/file_io/SubFile.hpp | 10 +- file_io/src/File.cpp | 27 ++- file_io/src/FileFactory.cpp | 28 --- file_io/src/NumpyFile.cpp | 44 ++--- file_io/src/NumpyFileFactory.cpp | 19 -- file_io/src/RawFile.cpp | 167 ++++++++++++++++ file_io/src/RawFileFactory.cpp | 185 ------------------ file_io/src/RawFileHelpers.cpp | 32 +++ file_io/src/SubFile.cpp | 2 +- python/src/bindings.cpp | 2 +- 19 files changed, 302 insertions(+), 342 deletions(-) delete mode 100644 file_io/include/aare/file_io/NumpyFileFactory.hpp rename file_io/include/aare/file_io/{RawFileFactory.hpp => RawFileHelpers.hpp} (72%) delete mode 100644 file_io/src/FileFactory.cpp delete mode 100644 file_io/src/NumpyFileFactory.cpp delete mode 100644 file_io/src/RawFileFactory.cpp create mode 100644 file_io/src/RawFileHelpers.cpp diff --git a/core/include/aare/core/defs.hpp b/core/include/aare/core/defs.hpp index 9838be7..3e2fa50 100644 --- a/core/include/aare/core/defs.hpp +++ b/core/include/aare/core/defs.hpp @@ -31,6 +31,8 @@ typedef struct { struct xy { int row; int col; + bool operator==(const xy &other) const { return row == other.row && col == other.col; } + bool operator!=(const xy &other) const { return !(*this == other); } }; // using image_shape = std::array; diff --git a/examples/numpy_write_example.cpp b/examples/numpy_write_example.cpp index c9029ff..d02bc35 100644 --- a/examples/numpy_write_example.cpp +++ b/examples/numpy_write_example.cpp @@ -12,7 +12,7 @@ using aare::Frame; int main() { auto path = std::filesystem::path("/tmp/test.npy"); auto dtype = aare::DType(typeid(uint32_t)); - FileConfig cfg = {path, dtype, 100, 100}; + FileConfig cfg = {dtype, 100, 100}; File npy(path, "w", cfg); Frame f(100, 100, dtype.bitdepth()); for (int i = 0; i < 10000; i++) { diff --git a/file_io/CMakeLists.txt b/file_io/CMakeLists.txt index 845a5d7..eb13d2f 100644 --- a/file_io/CMakeLists.txt +++ b/file_io/CMakeLists.txt @@ -11,13 +11,10 @@ endif() set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/FileFactory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFileFactory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFileFactory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp ) diff --git a/file_io/include/aare/file_io/File.hpp b/file_io/include/aare/file_io/File.hpp index 09a2dd2..6410091 100644 --- a/file_io/include/aare/file_io/File.hpp +++ b/file_io/include/aare/file_io/File.hpp @@ -25,9 +25,9 @@ class File { void seek(size_t frame_number); size_t tell() const; size_t total_frames() const; - ssize_t rows() const; - ssize_t cols() const; - ssize_t bitdepth() const; + size_t rows() const; + size_t cols() const; + size_t bitdepth() const; File(File &&other); ~File(); diff --git a/file_io/include/aare/file_io/FileInterface.hpp b/file_io/include/aare/file_io/FileInterface.hpp index 67075e5..ad1f685 100644 --- a/file_io/include/aare/file_io/FileInterface.hpp +++ b/file_io/include/aare/file_io/FileInterface.hpp @@ -9,11 +9,14 @@ namespace aare { struct FileConfig { - std::filesystem::path fname; aare::DType dtype = aare::DType(typeid(uint16_t)); uint64_t rows; uint64_t cols; xy geometry{1, 1}; + bool operator==(const FileConfig &other) const { + return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry; + } + bool operator!=(const FileConfig &other) const { return !(*this == other); } }; class FileInterface { public: @@ -52,9 +55,9 @@ class FileInterface { // Getter functions virtual size_t total_frames() const = 0; - virtual ssize_t rows() const = 0; - virtual ssize_t cols() const = 0; - virtual ssize_t bitdepth() const = 0; + virtual size_t rows() const = 0; + virtual size_t cols() const = 0; + virtual size_t bitdepth() const = 0; // read one frame at position frame_number Frame iread(size_t frame_number) { @@ -82,8 +85,8 @@ class FileInterface { }; public: - std::string mode; - // std::filesystem::path m_fname; + std::string m_mode; + std::filesystem::path m_fname; std::filesystem::path m_base_path; std::string m_base_name, m_ext; int m_findex; @@ -91,9 +94,11 @@ class FileInterface { size_t max_frames_per_file{}; std::string version; DetectorType m_type; - ssize_t m_rows{}; - ssize_t m_cols{}; - ssize_t m_bitdepth{}; + size_t m_rows{}; + size_t m_cols{}; + size_t m_bitdepth{}; + size_t current_frame{}; + }; } // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/NumpyFile.hpp b/file_io/include/aare/file_io/NumpyFile.hpp index 8068745..93e09fd 100644 --- a/file_io/include/aare/file_io/NumpyFile.hpp +++ b/file_io/include/aare/file_io/NumpyFile.hpp @@ -10,24 +10,9 @@ namespace aare { class NumpyFile : public FileInterface { - FILE *fp = nullptr; - size_t initial_header_len = 0; - size_t current_frame{}; - std::filesystem::path m_fname; - uint32_t header_len{}; - uint8_t header_len_size{}; - ssize_t header_size{}; - NumpyHeader m_header; - uint8_t major_ver_{}; - uint8_t minor_ver_{}; - - void load_metadata(); - void get_frame_into(size_t, std::byte *); - Frame get_frame(size_t frame_number); public: - NumpyFile(const std::filesystem::path &fname); - NumpyFile(FileConfig, NumpyHeader); + NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {}); void write(Frame &frame) override; Frame read() override { return get_frame(this->current_frame++); } @@ -40,9 +25,9 @@ class NumpyFile : public FileInterface { void seek(size_t frame_number) override { this->current_frame = frame_number; } size_t tell() override { return this->current_frame; } size_t total_frames() const override { return m_header.shape[0]; } - ssize_t rows() const override { return m_header.shape[1]; } - ssize_t cols() const override { return m_header.shape[2]; } - ssize_t bitdepth() const override { return m_header.dtype.bitdepth(); } + size_t rows() const override { return m_header.shape[1]; } + size_t cols() const override { return m_header.shape[2]; } + size_t bitdepth() const override { return m_header.dtype.bitdepth(); } DType dtype() const { return m_header.dtype; } std::vector shape() const { return m_header.shape; } @@ -56,6 +41,21 @@ class NumpyFile : public FileInterface { } ~NumpyFile(); + + private: + FILE *fp = nullptr; + size_t initial_header_len = 0; + size_t current_frame{}; + uint32_t header_len{}; + uint8_t header_len_size{}; + size_t header_size{}; + NumpyHeader m_header; + uint8_t major_ver_{}; + uint8_t minor_ver_{}; + + void load_metadata(); + void get_frame_into(size_t, std::byte *); + Frame get_frame(size_t frame_number); }; } // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/NumpyFileFactory.hpp b/file_io/include/aare/file_io/NumpyFileFactory.hpp deleted file mode 100644 index f725c91..0000000 --- a/file_io/include/aare/file_io/NumpyFileFactory.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once -#include "aare/core/defs.hpp" -#include "aare/file_io/FileFactory.hpp" -#include "aare/file_io/NumpyFile.hpp" -#include - -namespace aare { - -class NumpyFileFactory : public FileFactory { - private: - std::ifstream f; - void read_data(FileInterface *_file); - - public: - NumpyFileFactory(std::filesystem::path fpath); - void parse_metadata(FileInterface *_file) override{/*TODO! remove after refactor*/}; - NumpyFile *load_file_read() override; - NumpyFile *load_file_write(FileConfig) override; - void parse_fname(FileInterface *) override{}; -}; - -} // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/RawFile.hpp b/file_io/include/aare/file_io/RawFile.hpp index 5e4b544..dcd0f46 100644 --- a/file_io/include/aare/file_io/RawFile.hpp +++ b/file_io/include/aare/file_io/RawFile.hpp @@ -1,19 +1,16 @@ #pragma once #include "aare/core/Frame.hpp" -#include "aare/core/defs.hpp" #include "aare/file_io/FileInterface.hpp" #include "aare/file_io/SubFile.hpp" namespace aare { class RawFile : public FileInterface { - - using config = RawFileConfig; - + private: public: std::filesystem::path m_fname; // TO be made private! - // pragma to ignore warnings + RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &cfg = {}); void write(Frame &frame) override { throw std::runtime_error("Not implemented"); }; Frame read() override { return get_frame(this->current_frame++); }; @@ -40,7 +37,7 @@ class RawFile : public FileInterface { int subfile_rows, subfile_cols; xy geometry; std::vector positions; - config cfg{0, 0}; + RawFileConfig cfg{0, 0}; TimingMode timing_mode; bool quad{false}; @@ -67,14 +64,20 @@ class RawFile : public FileInterface { ~RawFile(); size_t total_frames() const override { return m_total_frames; } - ssize_t rows() const override { return m_rows; } - ssize_t cols() const override { return m_cols; } - ssize_t bitdepth() const override { return m_bitdepth; } + size_t rows() const override { return m_rows; } + size_t cols() const override { return m_cols; } + size_t bitdepth() const override { return m_bitdepth; } private: - size_t current_frame{}; void get_frame_into(size_t frame_number, std::byte *image_buf); Frame get_frame(size_t frame_number); + void parse_fname(); + void parse_metadata(); + void parse_raw_metadata(); + void parse_json_metadata(); + void find_geometry(); + sls_detector_header read_header(const std::filesystem::path &fname); + void open_subfiles(); }; } // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/RawFileFactory.hpp b/file_io/include/aare/file_io/RawFileHelpers.hpp similarity index 72% rename from file_io/include/aare/file_io/RawFileFactory.hpp rename to file_io/include/aare/file_io/RawFileHelpers.hpp index 569ad06..46af6f2 100644 --- a/file_io/include/aare/file_io/RawFileFactory.hpp +++ b/file_io/include/aare/file_io/RawFileHelpers.hpp @@ -1,16 +1,10 @@ #pragma once -#include "aare/file_io/FileFactory.hpp" #include "aare/file_io/RawFile.hpp" -namespace aare { -class RawFileFactory : public FileFactory { - private: +namespace aare::RawFileHelpers { void parse_json_metadata(RawFile *file); void parse_raw_metadata(RawFile *file); - - public: - RawFileFactory(std::filesystem::path fpath); RawFile *load_file_read() override; RawFile *load_file_write(FileConfig) override { return new RawFile(); }; void parse_metadata(FileInterface *) override; @@ -20,4 +14,3 @@ class RawFileFactory : public FileFactory { void find_geometry(FileInterface *); }; -} // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/SubFile.hpp b/file_io/include/aare/file_io/SubFile.hpp index eea343d..bee7630 100644 --- a/file_io/include/aare/file_io/SubFile.hpp +++ b/file_io/include/aare/file_io/SubFile.hpp @@ -10,11 +10,11 @@ namespace aare { class SubFile { protected: FILE *fp = nullptr; - ssize_t m_bitdepth; + size_t m_bitdepth; std::filesystem::path m_fname; - ssize_t m_rows{}; - ssize_t m_cols{}; - ssize_t n_frames{}; + size_t m_rows{}; + size_t m_cols{}; + size_t n_frames{}; int m_sub_file_index_{}; // pointer to functions that will read frames using pfunc = size_t (SubFile::*)(std::byte *); @@ -35,7 +35,7 @@ class SubFile { template size_t read_impl_flip(std::byte *buffer); template size_t read_impl_reorder(std::byte *buffer); - SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth); + SubFile(std::filesystem::path fname, DetectorType detector, size_t rows, size_t cols, uint16_t bitdepth); size_t get_part(std::byte *buffer, int frame_number); size_t frame_number(int frame_index); diff --git a/file_io/src/File.cpp b/file_io/src/File.cpp index 462d2e4..ce2a5ad 100644 --- a/file_io/src/File.cpp +++ b/file_io/src/File.cpp @@ -1,11 +1,28 @@ #include "aare/file_io/File.hpp" -#include "aare/file_io/FileFactory.hpp" +#include "aare/file_io/NumpyFile.hpp" +#include "aare/file_io/RawFile.hpp" #include "aare/utils/logger.hpp" +#include namespace aare { File::File(std::filesystem::path fname, std::string mode, FileConfig cfg) { - file_impl = FileFactory::load_file(fname, mode, cfg); + + if ((mode == "r" or mode == "a") and not std::filesystem::exists(fname)) { + throw std::runtime_error(fmt::format("File does not exist: {}", fname.c_str())); + } + + if (fname.extension() == ".raw" || fname.extension() == ".json") { + aare::logger::debug("Loading raw file"); + file_impl = new RawFile(fname, mode, cfg); + } + // check if extension is numpy + else if (fname.extension() == ".npy") { + aare::logger::debug("Loading numpy file"); + file_impl = new NumpyFile(fname, mode, cfg); + } else { + throw std::runtime_error("Unsupported file type"); + } } void File::write(Frame &frame) { file_impl->write(frame); } @@ -19,9 +36,9 @@ size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); } size_t File::pixels() { return file_impl->pixels(); } void File::seek(size_t frame_number) { file_impl->seek(frame_number); } size_t File::tell() const { return file_impl->tell(); } -ssize_t File::rows() const { return file_impl->rows(); } -ssize_t File::cols() const { return file_impl->cols(); } -ssize_t File::bitdepth() const { return file_impl->bitdepth(); } +size_t File::rows() const { return file_impl->rows(); } +size_t File::cols() const { return file_impl->cols(); } +size_t File::bitdepth() const { return file_impl->bitdepth(); } File::~File() { delete file_impl; } Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); } diff --git a/file_io/src/FileFactory.cpp b/file_io/src/FileFactory.cpp deleted file mode 100644 index bbe8ed3..0000000 --- a/file_io/src/FileFactory.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "aare/file_io/FileFactory.hpp" -#include "aare/file_io/FileInterface.hpp" -#include "aare/file_io/NumpyFileFactory.hpp" -#include "aare/file_io/RawFileFactory.hpp" -#include "aare/utils/logger.hpp" -#include - -namespace aare { - -FileFactory *FileFactory::get_factory(std::filesystem::path fpath) { - if (fpath.extension() == ".raw" || fpath.extension() == ".json") { - aare::logger::debug("Loading", fpath.extension(), "file"); - return new RawFileFactory(fpath); - } - if (fpath.extension() == ".raw" || fpath.extension() == ".json") { - aare::logger::debug("Loading", fpath.extension(), "file"); - return new RawFileFactory(fpath); - } - // check if extension is numpy - else if (fpath.extension() == ".npy") { - aare::logger::debug("Loading numpy file"); - return new aare::NumpyFileFactory(fpath); - } - - throw std::runtime_error("Unsupported file type"); -} - -} // namespace aare diff --git a/file_io/src/NumpyFile.cpp b/file_io/src/NumpyFile.cpp index d18b5ac..68ccd7a 100644 --- a/file_io/src/NumpyFile.cpp +++ b/file_io/src/NumpyFile.cpp @@ -3,37 +3,35 @@ namespace aare { -NumpyFile::NumpyFile(const std::filesystem::path &fname) { +NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) { // TODO! add opts to constructor m_fname = fname; - fp = fopen(m_fname.c_str(), "rb"); - if (!fp) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); + m_mode = mode; + if (mode == "r") { + fp = fopen(m_fname.c_str(), "rb"); + if (!fp) { + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); + } + load_metadata(); + } else if (mode == "w") { + m_bitdepth = cfg.dtype.bitdepth(); + m_rows = cfg.rows; + m_cols = cfg.cols; + m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}}; + m_header.shape = {0, cfg.rows, cfg.cols}; + fp = fopen(m_fname.c_str(), "wb"); + if (!fp) { + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); + } + initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header); } - load_metadata(); -} -NumpyFile::NumpyFile(FileConfig config, NumpyHeader header) { - mode = "w"; - m_fname = config.fname; - m_bitdepth = config.dtype.bitdepth(); - m_rows = config.rows; - m_cols = config.cols; - m_header = header; - m_header.shape = {0, config.rows, config.cols}; - - fp = fopen(m_fname.c_str(), "wb"); - if (!fp) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); - } - - initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), header); } void NumpyFile::write(Frame &frame) { if (fp == nullptr) { throw std::runtime_error("File not open"); } - if (not(mode == "w" or mode == "a")) { + if (not(m_mode == "w" or m_mode == "a")) { throw std::runtime_error("File not open for writing"); } fseek(fp, 0, SEEK_END); @@ -79,7 +77,7 @@ void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) { } NumpyFile::~NumpyFile() { - if (mode == "w" or mode == "a") { + if (m_mode == "w" or m_mode == "a") { // determine number of frames fseek(fp, 0, SEEK_END); size_t file_size = ftell(fp); diff --git a/file_io/src/NumpyFileFactory.cpp b/file_io/src/NumpyFileFactory.cpp deleted file mode 100644 index d29c753..0000000 --- a/file_io/src/NumpyFileFactory.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "aare/file_io/NumpyFileFactory.hpp" -#include "aare/file_io/NumpyHelpers.hpp" - -namespace aare { - -NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; } - -NumpyFile *NumpyFileFactory::load_file_read() { - NumpyFile *file = new NumpyFile(this->m_fpath); - return file; -}; - -NumpyFile *NumpyFileFactory::load_file_write(FileConfig config) { - NumpyFile *file = new NumpyFile(config, {config.dtype, false, {config.rows, config.cols}}); - - return file; -}; - -} // namespace aare \ No newline at end of file diff --git a/file_io/src/RawFile.cpp b/file_io/src/RawFile.cpp index 7094614..c657167 100644 --- a/file_io/src/RawFile.cpp +++ b/file_io/src/RawFile.cpp @@ -1,8 +1,175 @@ #include "aare/file_io/RawFile.hpp" +#include "aare/core/defs.hpp" #include "aare/utils/logger.hpp" +#include +#include + +using json = nlohmann::json; namespace aare { +RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) { + m_fname = fname; + if (mode == "r") { + if (config != FileConfig()) { + aare::logger::warn( + "In read mode it is not necessary to provide a config, the provided config will be ignored"); + } + aare::logger::debug("XXXXXXXXLoading raw file"); + parse_fname(); + parse_metadata(); + find_number_of_subfiles(); + find_geometry(); + open_subfiles(); + + } else { + throw std::runtime_error(LOCATION + "Unsupported mode"); + } +} + +void RawFile::open_subfiles() { + for (size_t i = 0; i != n_subfiles; ++i) { + auto v = std::vector(n_subfile_parts); + for (size_t j = 0; j != n_subfile_parts; ++j) { + v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, bitdepth()); + } + subfiles.push_back(v); + } +} + +sls_detector_header RawFile::read_header(const std::filesystem::path &fname) { + sls_detector_header h{}; + FILE *fp = fopen(fname.c_str(), "r"); + if (!fp) + throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str())); + + size_t rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + fclose(fp); + if (rc != 1) + throw std::runtime_error("Could not read header from file"); + return h; +} + +void RawFile::find_geometry() { + uint16_t r{}; + uint16_t c{}; + for (size_t i = 0; i < n_subfile_parts; i++) { + for (size_t j = 0; j != n_subfiles; ++j) { + auto h = this->read_header(data_fname(j, i)); + r = std::max(r, h.row); + c = std::max(c, h.column); + + positions.push_back({h.row, h.column}); + } + } + + r++; + c++; + + m_rows = r * subfile_rows; + m_cols = c * subfile_cols; + + m_rows += (r - 1) * cfg.module_gap_row; +} + +void RawFile::parse_metadata() { + if (m_ext == ".raw") { + parse_raw_metadata(); + if (m_bitdepth == 0) { + switch (m_type) { + case DetectorType::Eiger: + m_bitdepth = 32; + break; + default: + m_bitdepth = 16; + } + } + } else if (m_ext == ".json") { + parse_json_metadata(); + } else { + throw std::runtime_error(LOCATION + "Unsupported file type"); + } + n_subfile_parts = geometry.row * geometry.col; +} + +void RawFile::parse_json_metadata() { + std::ifstream ifs(master_fname()); + json j; + ifs >> j; + double v = j["Version"]; + version = fmt::format("{:.1f}", v); + m_type = StringTo(j["Detector Type"].get()); + timing_mode = StringTo(j["Timing Mode"].get()); + m_total_frames = j["Frames in File"]; + subfile_rows = j["Pixels"]["y"]; + subfile_cols = j["Pixels"]["x"]; + max_frames_per_file = j["Max Frames Per File"]; + try { + m_bitdepth = j.at("Dynamic Range"); + } catch (const json::out_of_range &e) { + m_bitdepth = 16; + } + // only Eiger had quad + if (m_type == DetectorType::Eiger) { + quad = (j["Quad"] == 1); + } + + geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; +} +void RawFile::parse_raw_metadata() { + std::ifstream ifs(master_fname()); + for (std::string line; std::getline(ifs, line);) { + if (line == "#Frame Header") + break; + auto pos = line.find(":"); + auto key_pos = pos; + while (key_pos != std::string::npos && std::isspace(line[--key_pos])) + ; + if (key_pos != std::string::npos) { + auto key = line.substr(0, key_pos + 1); + auto value = line.substr(pos + 2); + // do the actual parsing + if (key == "Version") { + version = value; + } else if (key == "TimeStamp") { + + } else if (key == "Detector Type") { + m_type = StringTo(value); + } else if (key == "Timing Mode") { + timing_mode = StringTo(value); + } else if (key == "Pixels") { + // Total number of pixels cannot be found yet looking at + // submodule + pos = value.find(','); + subfile_cols = std::stoi(value.substr(1, pos)); + subfile_rows = std::stoi(value.substr(pos + 1)); + } else if (key == "Total Frames") { + m_total_frames = std::stoi(value); + } else if (key == "Dynamic Range") { + m_bitdepth = std::stoi(value); + } else if (key == "Quad") { + quad = (value == "1"); + } else if (key == "Max Frames Per File") { + max_frames_per_file = std::stoi(value); + } else if (key == "Geometry") { + pos = value.find(','); + geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))}; + } + } + } +} + +void RawFile::parse_fname() { + m_base_path = m_fname.parent_path(); + m_base_name = m_fname.stem(); + m_ext = m_fname.extension(); + aare::logger::debug("EXTEXT", m_ext); + auto pos = m_base_name.rfind("_"); + m_findex = std::stoi(m_base_name.substr(pos + 1)); + pos = m_base_name.find("_master_"); + m_base_name.erase(pos); +} + Frame RawFile::get_frame(size_t frame_number) { auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth); std::byte *frame_buffer = f.data(); diff --git a/file_io/src/RawFileFactory.cpp b/file_io/src/RawFileFactory.cpp deleted file mode 100644 index 80d6ffb..0000000 --- a/file_io/src/RawFileFactory.cpp +++ /dev/null @@ -1,185 +0,0 @@ -#include "aare/file_io/RawFileFactory.hpp" -#include "aare/core/defs.hpp" -#include "aare/file_io/RawFile.hpp" -#include "aare/file_io/SubFile.hpp" -#include "aare/file_io/helpers.hpp" -#include "aare/utils/logger.hpp" - -#include -#include -#include - -using json = nlohmann::json; - -namespace aare { - -RawFileFactory::RawFileFactory(std::filesystem::path fpath) { - if (not is_master_file(fpath)) - throw std::runtime_error("Json file is not a master file"); - this->m_fpath = fpath; -} - -void RawFileFactory::parse_metadata(FileInterface *_file) { - auto file = dynamic_cast(_file); - if (file->m_ext == ".raw") { - this->parse_raw_metadata(file); - if (file->m_bitdepth == 0) { - switch (file->m_type) { - case DetectorType::Eiger: - file->m_bitdepth = 32; - break; - - default: - file->m_bitdepth = 16; - } - } - } else if (file->m_ext == ".json") { - this->parse_json_metadata(file); - } else { - throw std::runtime_error("Unsupported file type"); - } - file->n_subfile_parts = file->geometry.row * file->geometry.col; -} - -void RawFileFactory::parse_raw_metadata(RawFile *file) { - std::ifstream ifs(file->master_fname()); - for (std::string line; std::getline(ifs, line);) { - if (line == "#Frame Header") - break; - auto pos = line.find(":"); - auto key_pos = pos; - while (key_pos != std::string::npos && std::isspace(line[--key_pos])) - ; - if (key_pos != std::string::npos) { - auto key = line.substr(0, key_pos + 1); - auto value = line.substr(pos + 2); - // do the actual parsing - if (key == "Version") { - file->version = value; - } else if (key == "TimeStamp") { - - } else if (key == "Detector Type") { - file->m_type = StringTo(value); - } else if (key == "Timing Mode") { - file->timing_mode = StringTo(value); - } else if (key == "Pixels") { - // Total number of pixels cannot be found yet looking at - // submodule - pos = value.find(','); - file->subfile_cols = std::stoi(value.substr(1, pos)); - file->subfile_rows = std::stoi(value.substr(pos + 1)); - } else if (key == "Total Frames") { - file->m_total_frames = std::stoi(value); - } else if (key == "Dynamic Range") { - file->m_bitdepth = std::stoi(value); - } else if (key == "Quad") { - file->quad = (value == "1"); - } else if (key == "Max Frames Per File") { - file->max_frames_per_file = std::stoi(value); - } else if (key == "Geometry") { - pos = value.find(','); - file->geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))}; - } - } - } -} - -void RawFileFactory::parse_json_metadata(RawFile *file) { - std::ifstream ifs(file->master_fname()); - json j; - ifs >> j; - double v = j["Version"]; - file->version = fmt::format("{:.1f}", v); - file->m_type = StringTo(j["Detector Type"].get()); - file->timing_mode = StringTo(j["Timing Mode"].get()); - file->m_total_frames = j["Frames in File"]; - file->subfile_rows = j["Pixels"]["y"]; - file->subfile_cols = j["Pixels"]["x"]; - file->max_frames_per_file = j["Max Frames Per File"]; - try { - file->m_bitdepth = j.at("Dynamic Range"); - } catch (const json::out_of_range &e) { - file->m_bitdepth = 16; - } - // only Eiger had quad - if (file->m_type == DetectorType::Eiger) { - file->quad = (j["Quad"] == 1); - } - - file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; -} - -void RawFileFactory::open_subfiles(FileInterface *_file) { - auto file = dynamic_cast(_file); - for (size_t i = 0; i != file->n_subfiles; ++i) { - auto v = std::vector(file->n_subfile_parts); - for (size_t j = 0; j != file->n_subfile_parts; ++j) { - v[j] = new SubFile(file->data_fname(i, j), file->m_type, file->subfile_rows, file->subfile_cols, - file->bitdepth()); - } - file->subfiles.push_back(v); - } -} - -RawFile *RawFileFactory::load_file_read() { - RawFile *file = new RawFile(); - file->m_fname = this->m_fpath; - this->parse_fname(file); - this->parse_metadata(file); - file->find_number_of_subfiles(); - - this->find_geometry(file); - this->open_subfiles(file); - - return file; -} - -sls_detector_header RawFileFactory::read_header(const std::filesystem::path &fname) { - sls_detector_header h{}; - FILE *fp = fopen(fname.c_str(), "r"); - if (!fp) - throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str())); - - size_t rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); - fclose(fp); - if (rc != 1) - throw std::runtime_error("Could not read header from file"); - return h; -} - -void RawFileFactory::find_geometry(FileInterface *_file) { - auto file = dynamic_cast(_file); - uint16_t r{}; - uint16_t c{}; - for (size_t i = 0; i < file->n_subfile_parts; i++) { - for (size_t j = 0; j != file->n_subfiles; ++j) { - auto h = this->read_header(file->data_fname(j, i)); - r = std::max(r, h.row); - c = std::max(c, h.column); - - file->positions.push_back({h.row, h.column}); - } - } - - r++; - c++; - - file->m_rows = r * file->subfile_rows; - file->m_cols = c * file->subfile_cols; - - file->m_rows += (r - 1) * file->cfg.module_gap_row; -} - -void RawFileFactory::parse_fname(FileInterface *file) { - - file->m_base_path = this->m_fpath.parent_path(); - file->m_base_name = this->m_fpath.stem(); - file->m_ext = this->m_fpath.extension(); - - auto pos = file->m_base_name.rfind("_"); - file->m_findex = std::stoi(file->m_base_name.substr(pos + 1)); - pos = file->m_base_name.find("_master_"); - file->m_base_name.erase(pos); -} - -} // namespace aare \ No newline at end of file diff --git a/file_io/src/RawFileHelpers.cpp b/file_io/src/RawFileHelpers.cpp new file mode 100644 index 0000000..a944f9b --- /dev/null +++ b/file_io/src/RawFileHelpers.cpp @@ -0,0 +1,32 @@ +#include "aare/file_io/RawFileFactory.hpp" +#include "aare/core/defs.hpp" +#include "aare/file_io/RawFile.hpp" +#include "aare/file_io/SubFile.hpp" +#include "aare/file_io/helpers.hpp" +#include "aare/utils/logger.hpp" + +#include +#include +#include + +using json = nlohmann::json; + +namespace aare::RawFileHelpers { + + + + + + + + + + + + + + + + + +} // namespace aare \ No newline at end of file diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index 22823cf..4064572 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -7,7 +7,7 @@ namespace aare { -SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) { +SubFile::SubFile(std::filesystem::path fname, DetectorType detector, size_t rows, size_t cols, uint16_t bitdepth) { this->m_rows = rows; this->m_cols = cols; this->m_fname = fname; diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp index 2a0f50a..a4b9216 100644 --- a/python/src/bindings.cpp +++ b/python/src/bindings.cpp @@ -23,7 +23,7 @@ PYBIND11_MODULE(_aare, m) { py::enum_(m, "DetectorType"); py::class_(m, "_Frame") - .def(py::init()) + .def(py::init()) .def("get", &Frame::get) .def_property_readonly("rows", &Frame::rows) .def_property_readonly("cols", &Frame::cols)