diff --git a/examples/json_example.cpp b/examples/json_example.cpp index c630780..1541d93 100644 --- a/examples/json_example.cpp +++ b/examples/json_example.cpp @@ -1,13 +1,13 @@ // Your First C++ Program -#include "aare/ContextManager.hpp" -#include +#include "aare/File.hpp" #include "aare/utils/logger.hpp" +#include #define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" -void test(File *f, int frame_number) { +void test(File& f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl; std::cout << *((uint16_t *)frame.get(1, 0)) << std::endl; @@ -15,13 +15,11 @@ void test(File *f, int frame_number) { } int main() { - ContextManager ctx_manager; auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); - std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"jungfrau"/ "jungfrau_single_master_0.json"); + std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau" / "jungfrau_single_master_0.json"); std::cout << fpath << std::endl; - - File *file = ctx_manager.get_file(fpath); + File file(fpath, "r"); test(file, 0); test(file, 2); test(file, 9); diff --git a/examples/multiport_example.cpp b/examples/multiport_example.cpp index 4152fc0..8b17bee 100644 --- a/examples/multiport_example.cpp +++ b/examples/multiport_example.cpp @@ -1,26 +1,27 @@ // Your First C++ Program -#include "aare/ContextManager.hpp" +#include "aare/File.hpp" #include "aare/utils/logger.hpp" #include +#include #define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" -void test(File *f, int frame_number) { +void test(File &f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl; + std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl; std::cout << *((uint16_t *)frame.get(255, 1023)) << std::endl; std::cout << *((uint16_t *)frame.get(511, 1023)) << std::endl; } int main() { - ContextManager ctx_manager; auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau" / "jungfrau_double_master_0.json"); std::cout << fpath << std::endl; - File *file = ctx_manager.get_file(fpath); + File file(fpath, "r"); test(file, 0); test(file, 9); } \ No newline at end of file diff --git a/examples/mythen_example.cpp b/examples/mythen_example.cpp index 8449d9e..64b5f0e 100644 --- a/examples/mythen_example.cpp +++ b/examples/mythen_example.cpp @@ -1,13 +1,13 @@ // Your First C++ Program -#include "aare/ContextManager.hpp" +#include "aare/File.hpp" #include "aare/utils/logger.hpp" #include #define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" -void test1(File *f, int frame_number) { +void test1(File &f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl; std::cout << *((uint32_t *)frame.get(0, 3839)) << std::endl; @@ -20,29 +20,24 @@ void test1(File *f, int frame_number) { } } -void test2(File *f, int frame_number) { +void test2(File &f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl; - std::cout << *((uint32_t *)frame.get(0, 1280*4 -1)) << std::endl; + std::cout << *((uint32_t *)frame.get(0, 1280 * 4 - 1)) << std::endl; } int main() { - ContextManager ctx_manager; auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); if (PROJECT_ROOT_DIR.empty()) { throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set"); } std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "mythen" / "m3_master_0.json"); - File *file = ctx_manager.get_file(fpath); + File file(fpath, "r"); test1(file, 0); fpath = (PROJECT_ROOT_DIR / "data" / "mythen" / "scan242_master_3.raw"); - file = ctx_manager.get_file(fpath); - test2(file, 0); - - - - + File file2(fpath, "r"); + test2(file2, 0); } \ No newline at end of file diff --git a/examples/numpy_example.cpp b/examples/numpy_example.cpp index 12e57b3..3c71708 100644 --- a/examples/numpy_example.cpp +++ b/examples/numpy_example.cpp @@ -1,12 +1,12 @@ // Your First C++ Program -#include "aare/ContextManager.hpp" +#include "aare/File.hpp" #include #define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" -void test(File *f, int frame_number) { +void test(File& f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl; std::cout << *((uint16_t *)frame.get(1, 0)) << std::endl; @@ -14,13 +14,12 @@ void test(File *f, int frame_number) { } int main() { - ContextManager ctx_manager; auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "numpy" / "test_numpy_file.npy"); std::cout << fpath << std::endl; - File *file = ctx_manager.get_file(fpath); + File file(fpath,"r"); test(file, 0); test(file, 2); test(file, 24); diff --git a/examples/raw_example.cpp b/examples/raw_example.cpp index aba714a..84a5760 100644 --- a/examples/raw_example.cpp +++ b/examples/raw_example.cpp @@ -1,26 +1,25 @@ // Your First C++ Program -#include "aare/ContextManager.hpp" +#include "aare/File.hpp" #include #include "aare/utils/logger.hpp" #define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" -void test(File *f, int frame_number) { +void test(File& f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - Frame frame = f->get_frame(frame_number); + Frame frame = f.iread(frame_number); std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl; std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl; std::cout << *((uint16_t *)frame.get(0, 95)) << std::endl; } int main() { - ContextManager ctx_manager; auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); if (PROJECT_ROOT_DIR.empty()) { throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set"); } std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"moench"/ "moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw"); - File *file = ctx_manager.get_file(fpath); + File file(fpath, "r"); test(file, 0); test(file, 2); test(file, 99); diff --git a/file_io/include/aare/ContextManager.hpp b/file_io/include/aare/ContextManager.hpp deleted file mode 100644 index 231f60f..0000000 --- a/file_io/include/aare/ContextManager.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "aare/File.hpp" -#include "aare/FileFactory.hpp" -#include -#include - - -/** - * @brief A class to manage the context of the files and network connections - * closes the files and network connections when the context is destroyed -*/ -class ContextManager { - public: - ContextManager() = default; - - File *get_file(std::filesystem::path fname) { - auto tmp_file = FileFactory::load_file(fname); - this->files.push_back(tmp_file); - return tmp_file; - } - - // prevent default copying, it can delete the file twice - ContextManager(const ContextManager &) = delete; - ContextManager &operator=(const ContextManager &) = delete; - - ~ContextManager() { - for (auto f : files) { - delete f; - } - } - - private: - std::vector files; - // std::vector connections; -}; \ No newline at end of file diff --git a/file_io/include/aare/File.hpp b/file_io/include/aare/File.hpp index 52c622a..7fc091f 100644 --- a/file_io/include/aare/File.hpp +++ b/file_io/include/aare/File.hpp @@ -1,41 +1,30 @@ -#pragma once - -#include "SubFile.hpp" -#include "aare/Frame.hpp" -#include "aare/defs.hpp" -#include -#include -#include - -class File { - - public: - virtual Frame get_frame(size_t frame_number) =0; - +#include "aare/FileInterface.hpp" +class File : public FileInterface { private: - // comment + FileInterface *file_impl; public: - virtual ~File() = default; - std::filesystem::path fname; - std::filesystem::path base_path; - std::string base_name, ext; - int findex; - size_t total_frames{}; - size_t max_frames_per_file{}; + // options: + // - r reading + // - w writing (overwrites existing file) + // - a appending (appends to existing file) + // TODO! do we need to support w+, r+ and a+? + File(std::filesystem::path fname, std::string mode); + Frame read() override; + std::vector read(size_t n_frames) override; + void read_into(std::byte *image_buf) override; + void read_into(std::byte *image_buf, size_t n_frames) override; + size_t frame_number(size_t frame_index) override; + size_t bytes_per_frame() override; + size_t pixels() override; + void seek(size_t frame_number) override; + size_t tell() override; + size_t total_frames() const override ; + ssize_t rows() const override ; + ssize_t cols() const override ; + ssize_t bitdepth() const override ; + File(File &&other); - std::string version; - DetectorType type; - TimingMode timing_mode; - bool quad{false}; - ssize_t rows{}; - ssize_t cols{}; - ssize_t bitdepth{}; - // File(); - - inline size_t bytes_per_frame() const { return rows * cols * bitdepth / 8; } - inline size_t pixels() const { return rows * cols; } - - // size_t total_frames(); -}; \ No newline at end of file + ~File(); +}; diff --git a/file_io/include/aare/FileFactory.hpp b/file_io/include/aare/FileFactory.hpp index 634b7ae..235de32 100644 --- a/file_io/include/aare/FileFactory.hpp +++ b/file_io/include/aare/FileFactory.hpp @@ -1,23 +1,23 @@ #pragma once #include -#include "aare/File.hpp" +#include "aare/FileInterface.hpp" class FileFactory{ - // Class that will be used to create File objects + // Class that will be used to create FileInterface objects // follows the factory pattern protected: std::filesystem::path m_fpath; public: static FileFactory* get_factory(std::filesystem::path); // virtual int deleteFile() = 0; - static File* load_file(std::filesystem::path p){ + static FileInterface* load_file(std::filesystem::path p){ auto factory = get_factory(p); - File* tmp= factory->load_file(); + FileInterface* tmp= factory->load_file(); delete factory; return tmp; }; - virtual File* load_file()=0;//TODO: add option to load all file to memory or keep it on disk - virtual void parse_metadata(File*)=0; - virtual void parse_fname(File*)=0; + virtual FileInterface* load_file()=0;//TODO: add option to load all file to memory or keep it on disk + virtual void parse_metadata(FileInterface*)=0; + virtual void parse_fname(FileInterface*)=0; virtual ~FileFactory() = default; diff --git a/file_io/include/aare/FileInterface.hpp b/file_io/include/aare/FileInterface.hpp new file mode 100644 index 0000000..5a3c351 --- /dev/null +++ b/file_io/include/aare/FileInterface.hpp @@ -0,0 +1,85 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/defs.hpp" +#include "aare/utils/logger.hpp" +#include +#include +class FileInterface { + public: + friend class FileFactory; + // write one frame + // virtual void write(Frame &frame) = 0; + + // write n_frames frames + // virtual void write(std::vector &frames) = 0; + + // read one frame + virtual Frame read() = 0; + + // read n_frames frames + virtual std::vector read(size_t n_frames) = 0; // Is this the right interface? + + // read one frame into the provided buffer + virtual void read_into(std::byte *image_buf) = 0; + + // read n_frames frame into the provided buffer + virtual void read_into(std::byte *image_buf, size_t n_frames) = 0; + + // read the frame number on position frame_index + virtual size_t frame_number(size_t frame_index) = 0; + + // size of one frame, important fro teh read_into function + virtual size_t bytes_per_frame() = 0; + + // number of pixels in one frame + virtual size_t pixels() = 0; + // goto frame number + virtual void seek(size_t frame_number) = 0; + + // return the position of the file pointer (in number of frames) + virtual size_t tell() = 0; + + // Getter functions + virtual size_t total_frames() const = 0; + virtual ssize_t rows() const = 0; + virtual ssize_t cols() const = 0; + virtual ssize_t bitdepth() const = 0; + + // read one frame at position frame_number + Frame iread(size_t frame_number) { + auto old_pos = tell(); + seek(frame_number); + Frame tmp = read(); + seek(old_pos); + return tmp; + }; + + // read n_frames frames starting at frame_number + std::vector iread(size_t frame_number, size_t n_frames) { + auto old_pos = tell(); + seek(frame_number); + std::vector tmp = read(n_frames); + seek(old_pos); + return tmp; + } + + // function to query the data type of the file + /*virtual DataType dtype = 0; */ + + virtual ~FileInterface(){ + + }; + + public: + std::filesystem::path m_fname; + std::filesystem::path m_base_path; + std::string m_base_name, m_ext; + int m_findex; + size_t m_total_frames{}; + size_t max_frames_per_file{}; + std::string version; + DetectorType m_type; + ssize_t m_rows{}; + ssize_t m_cols{}; + ssize_t m_bitdepth{}; +}; diff --git a/file_io/include/aare/NumpyFile.hpp b/file_io/include/aare/NumpyFile.hpp index 5235764..94e7120 100644 --- a/file_io/include/aare/NumpyFile.hpp +++ b/file_io/include/aare/NumpyFile.hpp @@ -1,17 +1,30 @@ #pragma once -#include "aare/File.hpp" -#include "aare/defs.hpp" +#include "aare/FileInterface.hpp" #include "aare/NumpyHelpers.hpp" +#include "aare/defs.hpp" #include #include - -class NumpyFile : public File { +class NumpyFile : public FileInterface { FILE *fp = nullptr; - + public: + Frame read() override { return get_frame(this->current_frame++); } + + std::vector read(size_t n_frames) override; + void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); } + void read_into(std::byte *image_buf, size_t n_frames) override; + size_t frame_number(size_t frame_index) override { return frame_index; }; + size_t bytes_per_frame() override; + size_t pixels() override; + void seek(size_t frame_number) override { this->current_frame = frame_number; } + size_t tell() override { return this->current_frame; } + size_t total_frames() const override { return header.shape[0]; } + ssize_t rows()const override { return header.shape[1]; } + ssize_t cols()const override { return header.shape[2]; } + ssize_t bitdepth()const override { return header.dtype.itemsize; } + NumpyFile(std::filesystem::path fname); - Frame get_frame(size_t frame_number) override; header_t header{}; static constexpr std::array magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'}; uint8_t major_ver_{}; @@ -20,14 +33,15 @@ class NumpyFile : public File { uint8_t header_len_size{}; const uint8_t magic_string_length{6}; ssize_t header_size{}; - inline ssize_t pixels_per_frame() { - return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies()); - }; - inline ssize_t bytes_per_frame() { return header.dtype.itemsize * pixels_per_frame(); }; - ~NumpyFile(){ + + ~NumpyFile() { if (fp != nullptr) { fclose(fp); } - } + + private: + size_t current_frame{}; + void get_frame_into(size_t, std::byte *); + Frame get_frame(size_t frame_number); }; \ No newline at end of file diff --git a/file_io/include/aare/NumpyFileFactory.hpp b/file_io/include/aare/NumpyFileFactory.hpp index 71c962d..f013cdc 100644 --- a/file_io/include/aare/NumpyFileFactory.hpp +++ b/file_io/include/aare/NumpyFileFactory.hpp @@ -9,12 +9,12 @@ class NumpyFileFactory : public FileFactory { private: std::ifstream f; - void read_data(File *_file); + void read_data(FileInterface *_file); public: NumpyFileFactory(std::filesystem::path fpath); - void parse_metadata(File *_file) override; + void parse_metadata(FileInterface *_file) override; NumpyFile* load_file() override; - void parse_fname(File*){}; + void parse_fname(FileInterface*){}; }; \ No newline at end of file diff --git a/file_io/include/aare/RawFile.hpp b/file_io/include/aare/RawFile.hpp index 7432656..12a216d 100644 --- a/file_io/include/aare/RawFile.hpp +++ b/file_io/include/aare/RawFile.hpp @@ -1,15 +1,32 @@ #pragma once -#include "aare/defs.hpp" +#include "aare/FileInterface.hpp" #include "aare/Frame.hpp" -#include "aare/File.hpp" +#include "aare/SubFile.hpp" +#include "aare/defs.hpp" +class RawFile : public FileInterface { -class RawFile : public File { + using config = RawFileConfig; - using config = RawFileConfig; - public: + public: + Frame read() override { return get_frame(this->current_frame++); }; + std::vector read(size_t n_frames) override; + void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }; + void read_into(std::byte *image_buf, size_t n_frames) override; + size_t frame_number(size_t frame_index) override; + + // size of one frame, important fro teh read_into function + size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; } + + // number of pixels in one frame + size_t pixels() override { return m_rows * m_cols; } + + // goto frame number + void seek(size_t frame_number) { this->current_frame = frame_number; }; + + // return the position of the file pointer (in number of frames) + size_t tell() { return this->current_frame; }; - Frame get_frame(size_t frame_number); size_t n_subfiles; size_t n_subfile_parts; std::vector> subfiles; @@ -17,6 +34,8 @@ class RawFile : public File { xy geometry; std::vector positions; config cfg{0, 0}; + TimingMode timing_mode; + bool quad{false}; inline void set_config(int row, int col) { cfg.module_gap_row = row; @@ -32,11 +51,21 @@ class RawFile : public File { } inline std::filesystem::path master_fname() { - return this->base_path / fmt::format("{}_master_{}{}", this->base_name, this->findex, this->ext); + return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext); } inline std::filesystem::path data_fname(int mod_id, int file_id) { - return this->base_path / fmt::format("{}_d{}_f{}_{}.raw", this->base_name, file_id, mod_id, this->findex); + return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex); } ~RawFile(); + + size_t total_frames() const { return m_total_frames; } + ssize_t rows() const { return m_rows; } + ssize_t cols() const { return m_cols; } + ssize_t bitdepth() const { return m_bitdepth; } + + private: + size_t current_frame{}; + void get_frame_into(size_t frame_number, std::byte *image_buf); + Frame get_frame(size_t frame_number); }; \ No newline at end of file diff --git a/file_io/include/aare/RawFileFactory.hpp b/file_io/include/aare/RawFileFactory.hpp index aa33050..af0aa1b 100644 --- a/file_io/include/aare/RawFileFactory.hpp +++ b/file_io/include/aare/RawFileFactory.hpp @@ -8,9 +8,9 @@ class RawFileFactory : public FileFactory { public: RawFileFactory(std::filesystem::path fpath); virtual RawFile *load_file() override; - void parse_metadata(File *) override; - void parse_fname(File *) override; - void open_subfiles(File *); + void parse_metadata(FileInterface *) override; + void parse_fname(FileInterface *) override; + void open_subfiles(FileInterface *); sls_detector_header read_header(const std::filesystem::path &fname); - void find_geometry(File *); + void find_geometry(FileInterface *); }; diff --git a/file_io/include/aare/SubFile.hpp b/file_io/include/aare/SubFile.hpp index 658302b..2f8ad04 100644 --- a/file_io/include/aare/SubFile.hpp +++ b/file_io/include/aare/SubFile.hpp @@ -35,6 +35,7 @@ class SubFile { SubFile(std::filesystem::path fname,DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth); size_t get_part(std::byte *buffer, int frame_number); + size_t frame_number(int frame_index); // TODO: define the inlines as variables and assign them in constructor inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; } diff --git a/file_io/include/aare/helpers.hpp b/file_io/include/aare/helpers.hpp index a140f58..bf39d55 100644 --- a/file_io/include/aare/helpers.hpp +++ b/file_io/include/aare/helpers.hpp @@ -1,6 +1,6 @@ #pragma once -#include "aare/File.hpp" +#include "aare/FileInterface.hpp" #include #include diff --git a/file_io/src/File.cpp b/file_io/src/File.cpp index 68e2378..6e0b893 100644 --- a/file_io/src/File.cpp +++ b/file_io/src/File.cpp @@ -1,2 +1,34 @@ -// #include "aare/File.hpp" -// template class File; +#include "aare/File.hpp" +#include "aare/FileFactory.hpp" +#include "aare/utils/logger.hpp" + +File::File(std::filesystem::path fname, std::string mode) { + if (mode != "r") { + throw std::runtime_error(LOCATION + " Only read mode is supported"); + } + file_impl = FileFactory::load_file(fname); +} + +Frame File::read() { return file_impl->read(); } +size_t File::total_frames() const { return file_impl->m_total_frames; } +std::vector File::read(size_t n_frames) { return file_impl->read(n_frames); } +void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); } +void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); } +size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); } +size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); } +size_t File::pixels() { return file_impl->pixels(); } +void File::seek(size_t frame_number) { file_impl->seek(frame_number); } +size_t File::tell() { return file_impl->tell(); } +ssize_t File::rows() const { return file_impl->rows(); } +ssize_t File::cols() const { return file_impl->cols(); } +ssize_t File::bitdepth() const { return file_impl->bitdepth(); } +File::~File() { + delete file_impl; +} + +File::File(File &&other) { + file_impl = other.file_impl; + other.file_impl = nullptr; +} + +// write move assignment operator diff --git a/file_io/src/FileFactory.cpp b/file_io/src/FileFactory.cpp index 9aaaece..491875e 100644 --- a/file_io/src/FileFactory.cpp +++ b/file_io/src/FileFactory.cpp @@ -1,8 +1,9 @@ #include "aare/FileFactory.hpp" -#include "aare/File.hpp" +#include "aare/FileInterface.hpp" #include "aare/RawFileFactory.hpp" #include "aare/NumpyFileFactory.hpp" #include "aare/utils/logger.hpp" +#include "aare/utils/logger.hpp" #include FileFactory *FileFactory::get_factory(std::filesystem::path fpath) { @@ -11,6 +12,10 @@ FileFactory *FileFactory::get_factory(std::filesystem::path fpath) { throw std::runtime_error("File does not exist"); } + if (fpath.extension() == ".raw" || fpath.extension() == ".json"){ + aare::logger::info("Loading",fpath.extension(),"file"); + return new RawFileFactory(fpath); + } if (fpath.extension() == ".raw" || fpath.extension() == ".json"){ aare::logger::info("Loading",fpath.extension(),"file"); return new RawFileFactory(fpath); diff --git a/file_io/src/NumpyFile.cpp b/file_io/src/NumpyFile.cpp index fc62424..d57f98a 100644 --- a/file_io/src/NumpyFile.cpp +++ b/file_io/src/NumpyFile.cpp @@ -1,21 +1,48 @@ #include "aare/NumpyFile.hpp" -NumpyFile::NumpyFile(std::filesystem::path fname_){ - this->fname = fname_; - fp = fopen(this->fname.c_str(), "rb"); +NumpyFile::NumpyFile(std::filesystem::path fname_) { + this->m_fname = fname_; + fp = fopen(this->m_fname.c_str(), "rb"); } Frame NumpyFile::get_frame(size_t frame_number) { + Frame frame(header.shape[1], header.shape[2], header.dtype.itemsize*8); + get_frame_into(frame_number,frame._get_data()); + return frame; +} +void NumpyFile::get_frame_into( size_t frame_number,std::byte* image_buf) { if (fp == nullptr) { throw std::runtime_error("File not open"); } if (frame_number > header.shape[0]) { throw std::runtime_error("Frame number out of range"); } - Frame frame = Frame(header.shape[1], header.shape[2], header.dtype.itemsize*8); fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET); - fread(frame._get_data(), bytes_per_frame(), 1, fp); - return frame; + fread(image_buf, bytes_per_frame(), 1, fp); } + + +size_t NumpyFile::pixels() { + return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies()); +}; +size_t NumpyFile::bytes_per_frame() { return header.dtype.itemsize * pixels(); }; + + +std::vector NumpyFile::read(size_t n_frames) { + // TODO: implement this in a more efficient way + std::vector frames; + for (size_t i = 0; i < n_frames; i++) { + frames.push_back(this->get_frame(this->current_frame)); + this->current_frame++; + } + return frames; +} +void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) { + // TODO: implement this in a more efficient way + for (size_t i = 0; i < n_frames; i++) { + this->get_frame_into(this->current_frame++, image_buf); + image_buf += this->bytes_per_frame(); + } +} \ No newline at end of file diff --git a/file_io/src/NumpyFileFactory.cpp b/file_io/src/NumpyFileFactory.cpp index cf6d99f..987a088 100644 --- a/file_io/src/NumpyFileFactory.cpp +++ b/file_io/src/NumpyFileFactory.cpp @@ -3,13 +3,13 @@ NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; } -void NumpyFileFactory::parse_metadata(File *_file) { +void NumpyFileFactory::parse_metadata(FileInterface *_file) { auto file = dynamic_cast(_file); // open ifsteam to file - f = std::ifstream(file->fname, std::ios::binary); + f = std::ifstream(file->m_fname, std::ios::binary); // check if file exists if (!f.is_open()) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", file->fname.c_str())); + throw std::runtime_error(fmt::format("Could not open: {} for reading", file->m_fname.c_str())); } // read magic number std::array tmp{}; diff --git a/file_io/src/RawFile.cpp b/file_io/src/RawFile.cpp index 8b06ac9..cd2b65d 100644 --- a/file_io/src/RawFile.cpp +++ b/file_io/src/RawFile.cpp @@ -2,24 +2,30 @@ #include "aare/utils/logger.hpp" Frame RawFile::get_frame(size_t frame_number) { - if (frame_number > this->total_frames) { - throw std::runtime_error("Frame number out of range"); + auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth); + std::byte *frame_buffer = f._get_data(); + get_frame_into(frame_number, frame_buffer); + return f; +} + +void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { + if (frame_number > this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); } int subfile_id = frame_number / this->max_frames_per_file; // create frame and get its buffer - auto f = Frame(this->rows, this->cols, this->bitdepth); - std::byte *frame_buffer = f._get_data(); if (this->geometry.col == 1) { // get the part from each subfile and copy it to the frame for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); - this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx*part_offset, frame_number % this->max_frames_per_file); + this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, + frame_number % this->max_frames_per_file); } } else { // create a buffer that will hold a the frame part - auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->bitdepth / 8; + auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8; std::byte *part_buffer = new std::byte[bytes_per_part]; for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { @@ -27,16 +33,41 @@ Frame RawFile::get_frame(size_t frame_number) { for (int cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows; auto icol = (part_idx % this->geometry.col) * this->subfile_cols; - auto dest = (irow * this->cols + icol); - dest = dest * this->bitdepth / 8; - memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->bitdepth / 8, - this->subfile_cols * this->bitdepth / 8); + auto dest = (irow * this->m_cols + icol); + dest = dest * this->m_bitdepth / 8; + memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8, + this->subfile_cols * this->m_bitdepth / 8); } } delete[] part_buffer; } +} - return f; +std::vector RawFile::read(size_t n_frames) { + // TODO: implement this in a more efficient way + std::vector frames; + for (size_t i = 0; i < n_frames; i++) { + frames.push_back(this->get_frame(this->current_frame)); + this->current_frame++; + } + return frames; +} +void RawFile::read_into(std::byte *image_buf, size_t n_frames) { + // TODO: implement this in a more efficient way + for (size_t i = 0; i < n_frames; i++) { + this->get_frame_into(this->current_frame++, image_buf); + image_buf += this->bytes_per_frame(); + } +} + + + +size_t RawFile::frame_number(size_t frame_index) { + if (frame_index > this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + int subfile_id = frame_index / this->max_frames_per_file; + return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file); } RawFile::~RawFile() { diff --git a/file_io/src/RawFileFactory.cpp b/file_io/src/RawFileFactory.cpp index 94dd5e0..bb70463 100644 --- a/file_io/src/RawFileFactory.cpp +++ b/file_io/src/RawFileFactory.cpp @@ -17,21 +17,21 @@ RawFileFactory::RawFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; } -void RawFileFactory::parse_metadata(File *_file) { +void RawFileFactory::parse_metadata(FileInterface *_file) { auto file = dynamic_cast(_file); - if (file->ext == ".raw") { + if (file->m_ext == ".raw") { this->parse_raw_metadata(file); - if (file->bitdepth == 0) { - switch (file->type) { + if (file->m_bitdepth == 0) { + switch (file->m_type) { case DetectorType::Eiger: - file->bitdepth = 32; + file->m_bitdepth = 32; break; default: - file->bitdepth = 16; + file->m_bitdepth = 16; } } - } else if (file->ext == ".json") { + } else if (file->m_ext == ".json") { this->parse_json_metadata(file); } else { throw std::runtime_error("Unsupported file type"); @@ -57,7 +57,7 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) { } else if (key == "TimeStamp") { } else if (key == "Detector Type") { - file->type = StringTo(value); + file->m_type = StringTo(value); } else if (key == "Timing Mode") { file->timing_mode = StringTo(value); } else if (key == "Pixels") { @@ -67,9 +67,9 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) { file->subfile_cols = std::stoi(value.substr(1, pos)); file->subfile_rows = std::stoi(value.substr(pos + 1)); } else if (key == "Total Frames") { - file->total_frames = std::stoi(value); + file->m_total_frames = std::stoi(value); } else if (key == "Dynamic Range") { - file->bitdepth = std::stoi(value); + file->m_bitdepth = std::stoi(value); } else if (key == "Quad") { file->quad = (value == "1"); } else if (key == "Max Frames Per File") { @@ -89,32 +89,32 @@ void RawFileFactory::parse_json_metadata(RawFile *file) { double v = j["Version"]; std::cout << "Version: " << v << std::endl; file->version = fmt::format("{:.1f}", v); - file->type = StringTo(j["Detector Type"].get()); + file->m_type = StringTo(j["Detector Type"].get()); file->timing_mode = StringTo(j["Timing Mode"].get()); - file->total_frames = j["Frames in File"]; + file->m_total_frames = j["Frames in File"]; file->subfile_rows = j["Pixels"]["y"]; file->subfile_cols = j["Pixels"]["x"]; file->max_frames_per_file = j["Max Frames Per File"]; try { - file->bitdepth = j.at("Dynamic Range"); + file->m_bitdepth = j.at("Dynamic Range"); } catch (const json::out_of_range &e) { - file->bitdepth = 16; + file->m_bitdepth = 16; } // only Eiger had quad - if (file->type == DetectorType::Eiger) { + if (file->m_type == DetectorType::Eiger) { file->quad = (j["Quad"] == 1); } file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; } -void RawFileFactory::open_subfiles(File *_file) { +void RawFileFactory::open_subfiles(FileInterface *_file) { auto file = dynamic_cast(_file); for (size_t i = 0; i != file->n_subfiles; ++i) { auto v = std::vector(file->n_subfile_parts); for (size_t j = 0; j != file->n_subfile_parts; ++j) { v[j] = - new SubFile(file->data_fname(i, j), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth); + new SubFile(file->data_fname(i, j), file->m_type, file->subfile_rows, file->subfile_cols, file->bitdepth()); } file->subfiles.push_back(v); } @@ -122,7 +122,7 @@ void RawFileFactory::open_subfiles(File *_file) { RawFile *RawFileFactory::load_file() { RawFile *file = new RawFile(); - file->fname = this->m_fpath; + file->m_fname = this->m_fpath; this->parse_fname(file); this->parse_metadata(file); file->find_number_of_subfiles(); @@ -146,7 +146,8 @@ sls_detector_header RawFileFactory::read_header(const std::filesystem::path &fna return h; } -void RawFileFactory::find_geometry(File *_file) { + +void RawFileFactory::find_geometry(FileInterface *_file) { auto file = dynamic_cast(_file); uint16_t r{}; uint16_t c{}; @@ -163,20 +164,20 @@ void RawFileFactory::find_geometry(File *_file) { r++; c++; - file->rows = r * file->subfile_rows; - file->cols = c * file->subfile_cols; + file->m_rows = r * file->subfile_rows; + file->m_cols = c * file->subfile_cols; - file->rows += (r - 1) * file->cfg.module_gap_row; + file->m_rows += (r - 1) * file->cfg.module_gap_row; } -void RawFileFactory::parse_fname(File *file) { +void RawFileFactory::parse_fname(FileInterface *file) { - file->base_path = this->m_fpath.parent_path(); - file->base_name = this->m_fpath.stem(); - file->ext = this->m_fpath.extension(); + file->m_base_path = this->m_fpath.parent_path(); + file->m_base_name = this->m_fpath.stem(); + file->m_ext = this->m_fpath.extension(); - auto pos = file->base_name.rfind("_"); - file->findex = std::stoi(file->base_name.substr(pos + 1)); - pos = file->base_name.find("_master_"); - file->base_name.erase(pos); + auto pos = file->m_base_name.rfind("_"); + file->m_findex = std::stoi(file->m_base_name.substr(pos + 1)); + pos = file->m_base_name.find("_master_"); + file->m_base_name.erase(pos); } diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index ecec98a..915d9f3 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -1,10 +1,8 @@ #include "aare/SubFile.hpp" -#include #include "aare/utils/logger.hpp" +#include // #include - - SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) { this->m_rows = rows; this->m_cols = cols; @@ -12,19 +10,17 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row this->m_bitdepth = bitdepth; this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) { - throw std::runtime_error(LOCATION+"Unsupported detector/bitdepth combination"); + throw std::runtime_error(LOCATION + "Unsupported detector/bitdepth combination"); } - this->read_impl = read_impl_map.at({detector, bitdepth}); - + this->read_impl = read_impl_map.at({detector, bitdepth}); } - size_t SubFile::get_part(std::byte *buffer, int frame_number) { if (frame_number >= n_frames or frame_number < 0) { throw std::runtime_error("Frame number out of range"); } // TODO: find a way to avoid opening and closing the file for each frame - aare::logger::debug(LOCATION,"frame:", frame_number, "file:", m_fname.c_str()); + aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str()); fp = fopen(m_fname.c_str(), "rb"); if (!fp) { throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); @@ -86,3 +82,17 @@ template size_t SubFile::read_impl_flip(std::byte *buffer) { return rc; }; + +size_t SubFile::frame_number(int frame_index) { + sls_detector_header h{}; + FILE *fp = fopen(this->m_fname.c_str(), "r"); + if (!fp) + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); + + size_t rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + fclose(fp); + if (rc != 1) + throw std::runtime_error("Could not read header from file"); + + return h.frameNumber; +}