diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 930311e..67bb44b 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -3,7 +3,10 @@ { "name": "Linux", "includePath": [ - "${workspaceFolder}/**", + "${workspaceFolder}/file_io/**", + "${workspaceFolder}/core/**", + "${workspaceFolder}/tests/**", + "${workspaceFolder}/tests/**", "/usr/include" ], "defines": [], diff --git a/CMakeLists.txt b/CMakeLists.txt index b6fbd68..76ccfa1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ include(GNUInstallDirs) include(FetchContent) -option(AARE_USE_WARNINGS "Eable warnings" ON) +option(AARE_USE_WARNINGS "Eable warnings" OFF) option(AARE_PYTHON_BINDINGS "Build python bindings" ON) option(AARE_TESTS "Build tests" ON) option(AARE_EXAMPLES "Build examples" ON) diff --git a/core/include/aare/Frame.hpp b/core/include/aare/Frame.hpp index 78918a6..b19689a 100644 --- a/core/include/aare/Frame.hpp +++ b/core/include/aare/Frame.hpp @@ -22,6 +22,7 @@ template class Frame{ ssize_t cols; DataType* data; ssize_t bitdepth = sizeof(DataType)*8; + Frame(ssize_t rows, ssize_t cols); Frame(std::byte* fp, ssize_t rows, ssize_t cols); DataType get(int row, int col); diff --git a/core/src/Frame.cpp b/core/src/Frame.cpp index f035fc4..8bed03d 100644 --- a/core/src/Frame.cpp +++ b/core/src/Frame.cpp @@ -8,13 +8,18 @@ Frame::Frame(std::byte* bytes, ssize_t rows, ssize_t cols): std::memcpy(data, bytes, rows*cols*sizeof(DataType)); } +template +Frame::Frame(ssize_t rows, ssize_t cols): + rows(rows), cols(cols) { + data = new DataType[rows*cols]; + } + template DataType Frame::get(int row, int col) { if (row < 0 || row >= rows || col < 0 || col >= cols) { - std::cerr << "Invalid row or column index" << std::endl; - return 0; + throw std::runtime_error("Invalid row or column index"); } return data[row*cols + col]; } diff --git a/data/test_numpy_file.npy b/data/test_numpy_file.npy new file mode 100644 index 0000000..ca1bc7e Binary files /dev/null and b/data/test_numpy_file.npy differ diff --git a/examples/main.cpp b/examples/main.cpp index f4d45cb..66d8af5 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -1,25 +1,30 @@ // Your First C++ Program -#include #include "aare/FileHandler.hpp" +#include -using JFileHandler = FileHandler; -using JFile = File; +using JFileHandler = FileHandler; +using JFile = File; using JFrame = Frame; - -void test(JFileHandler* f,int frame_number){ +void test(JFileHandler *f, int frame_number) { std::cout << "frame number: " << frame_number << std::endl; - JFrame* frame = f->get_frame(frame_number); - std::cout << frame->get(0,0) << ' '; - std::cout << frame->get(0,1) << ' '; - std::cout << frame->get(1,0) << ' '; - std::cout << frame->get(511,1023) << std::endl; + JFrame *frame = f->get_frame(frame_number); + std::cout << frame->get(0, 0) << std::endl; + std::cout << frame->get(0, 1) << std::endl; + std::cout << frame->get(1, 0) << std::endl; + std::cout << frame->get(49, 49) << std::endl; delete frame; } int main() { - std::filesystem::path fpath("/home/bb/github/aare/data/jungfrau_single_master_0.json"); - auto fileHandler = new JFileHandler (fpath); + // std::filesystem::path fpath("/home/bb/github/aare/data/jungfrau_single_master_0.json"); + std::filesystem::path fpath("/home/bb/github/aare/data/test_numpy_file.npy"); + + auto fileHandler = new JFileHandler(fpath); + test(fileHandler, 0); + test(fileHandler, 24); + delete fileHandler; + } diff --git a/file_io/include/aare/File.hpp b/file_io/include/aare/File.hpp index 0a4f628..75c4d28 100644 --- a/file_io/include/aare/File.hpp +++ b/file_io/include/aare/File.hpp @@ -15,6 +15,8 @@ class File { virtual Frame* get_frame(int frame_number) = 0; private: + //comment + public: std::filesystem::path fname; diff --git a/file_io/include/aare/JsonFileFactory.hpp b/file_io/include/aare/JsonFileFactory.hpp index a65453c..4c0ffac 100644 --- a/file_io/include/aare/JsonFileFactory.hpp +++ b/file_io/include/aare/JsonFileFactory.hpp @@ -5,11 +5,10 @@ class JsonFileFactory: public FileFactory private: /* data */ public: + JsonFileFactory(std::filesystem::path fpath); File* load_file() override; void parse_metadata(File*) override; void parse_fname(File*) override; - - JsonFileFactory(std::filesystem::path fpath); void open_subfiles(File*); sls_detector_header read_header(const std::filesystem::path &fname); void find_geometry(File*); diff --git a/file_io/include/aare/NumpyFile.hpp b/file_io/include/aare/NumpyFile.hpp index 1230af9..33ab769 100644 --- a/file_io/include/aare/NumpyFile.hpp +++ b/file_io/include/aare/NumpyFile.hpp @@ -1,9 +1,52 @@ +#pragma once #include "aare/File.hpp" #include "aare/defs.hpp" +#include +#include -template -class NumpyFile : public File -{ +using shape_t = std::vector; + +struct dtype_t { + char byteorder; + char kind; + unsigned int itemsize; + std::string to_string() { + std::stringstream sstm; + sstm << byteorder << kind << itemsize; + return sstm.str(); + } +}; +struct header_t { + dtype_t dtype; + bool fortran_order; + shape_t shape; + std::string to_string() { + std::stringstream sstm; + sstm << "dtype: " << dtype.to_string() << ", fortran_order: " << fortran_order << ' '; + + sstm << "shape: ("; + for (auto item : shape) + sstm << item << ','; + sstm << ')'; + return sstm.str(); + } +}; +template class NumpyFile : public File { + FILE *fp = nullptr; - + public: + NumpyFile(std::filesystem::path fname); + Frame *get_frame(int frame_number) override; + header_t header{}; + static constexpr std::array magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'}; + uint8_t major_ver_{}; + uint8_t minor_ver_{}; + uint32_t header_len{}; + uint8_t header_len_size{}; + const uint8_t magic_string_length{6}; + ssize_t header_size{}; + inline ssize_t pixels_per_frame() { + return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies()); + }; + inline ssize_t bytes_per_frame() { return header.dtype.itemsize * pixels_per_frame(); }; }; \ No newline at end of file diff --git a/file_io/include/aare/NumpyFileFactory.hpp b/file_io/include/aare/NumpyFileFactory.hpp index 3ba3903..f75fcf6 100644 --- a/file_io/include/aare/NumpyFileFactory.hpp +++ b/file_io/include/aare/NumpyFileFactory.hpp @@ -1,6 +1,7 @@ -#include "aare/defs.hpp" +#pragma once #include "aare/FileFactory.hpp" #include "aare/NumpyFile.hpp" +#include "aare/defs.hpp" #include #include #include @@ -8,22 +9,17 @@ #include #include #include +#include template class NumpyFileFactory : public FileFactory { + private: + std::ifstream f; + void read_data(File *_file); + public: NumpyFileFactory(std::filesystem::path fpath); void parse_metadata(File *_file) override; - void open_subfiles(File *_file) override; File *load_file() override; + void parse_fname(File *){}; - uint8_t major_ver() const noexcept { return major_ver_; } - uint8_t minor_ver() const noexcept { return minor_ver_; } - - private: - static constexpr std::array magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'}; - uint8_t major_ver_{}; - uint8_t minor_ver_{}; - uint32_t header_len{}; - uint8_t header_len_size{}; - const uint8_t magic_string_length{6}; }; \ No newline at end of file diff --git a/file_io/src/FileFactory.cpp b/file_io/src/FileFactory.cpp index 9e7ed47..37eaf0c 100644 --- a/file_io/src/FileFactory.cpp +++ b/file_io/src/FileFactory.cpp @@ -1,6 +1,7 @@ #include "aare/FileFactory.hpp" #include "aare/File.hpp" #include "aare/JsonFileFactory.hpp" +#include "aare/NumpyFileFactory.hpp" #include template @@ -20,7 +21,7 @@ FileFactory *FileFactory::get_factory(st // check if extension is numpy else if (fpath.extension() == ".npy") { std::cout << "Loading numpy file" << std::endl; - throw std::runtime_error("Numpy file not implemented"); + return new NumpyFileFactory(fpath); } throw std::runtime_error("Unsupported file type"); diff --git a/file_io/src/JsonFile.cpp b/file_io/src/JsonFile.cpp index c900e74..503b6fa 100644 --- a/file_io/src/JsonFile.cpp +++ b/file_io/src/JsonFile.cpp @@ -2,6 +2,9 @@ template Frame *JsonFile::get_frame(int frame_number) { + if (frame_number > this->total_frames) { + throw std::runtime_error("Frame number out of range"); + } int subfile_id = frame_number / this->max_frames_per_file; std::byte *buffer; size_t frame_size = this->subfiles[subfile_id]->bytes_per_frame(); diff --git a/file_io/src/NumpyFile.cpp b/file_io/src/NumpyFile.cpp index e69de29..78de3d8 100644 --- a/file_io/src/NumpyFile.cpp +++ b/file_io/src/NumpyFile.cpp @@ -0,0 +1,24 @@ + +#include "aare/NumpyFile.hpp" + +template +NumpyFile::NumpyFile(std::filesystem::path fname){ + this->fname = fname; + fp = fopen(fname.c_str(), "rb"); +} + +template +Frame *NumpyFile::get_frame(int frame_number) { + if (fp == nullptr) { + throw std::runtime_error("File not open"); + } + if (frame_number > header.shape[0]) { + throw std::runtime_error("Frame number out of range"); + } + Frame *frame = new Frame(header.shape[1], header.shape[2]); + fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET); + fread(frame->data, sizeof(DataType), pixels_per_frame(), fp); + return frame; +} + +template class NumpyFile; \ No newline at end of file diff --git a/file_io/src/NumpyFileFactory.cpp b/file_io/src/NumpyFileFactory.cpp index fc5d470..fd15970 100644 --- a/file_io/src/NumpyFileFactory.cpp +++ b/file_io/src/NumpyFileFactory.cpp @@ -13,8 +13,8 @@ inline std::string parse_str(const std::string &in) { /** Removes leading and trailing whitespaces */ -inline std::string trim(const std::string& str) { - const std::string whitespace = " \t"; +inline std::string trim(const std::string &str) { + const std::string whitespace = " \t\n"; auto begin = str.find_first_not_of(whitespace); if (begin == std::string::npos) @@ -52,7 +52,6 @@ inline bool parse_bool(const std::string &in) { throw std::runtime_error("Invalid python boolan."); } - inline std::string get_value_from_map(const std::string &mapstr) { size_t sep_pos = mapstr.find_first_of(":"); if (sep_pos == std::string::npos) @@ -63,7 +62,6 @@ inline std::string get_value_from_map(const std::string &mapstr) { } std::unordered_map parse_dict(std::string in, const std::vector &keys) { std::unordered_map map; - if (keys.size() == 0) return map; @@ -113,19 +111,6 @@ std::unordered_map parse_dict(std::string in, const st return map; } - -using shape_t = std::vector; - -struct dtype_t { - char byteorder; - char kind; - unsigned int itemsize; -}; -struct header_t { - dtype_t dtype; - bool fortran_order; - shape_t shape; -}; template inline bool in_array(T val, const std::array &arr) { return std::find(std::begin(arr), std::end(arr), val) != std::end(arr); } @@ -160,14 +145,15 @@ template void NumpyFileFactory::parse_metadata(File *_file) { auto file = dynamic_cast *>(_file); // open ifsteam to file - std::ifstream f(file->fname, std::ios::binary); + f = std::ifstream(file->fname, std::ios::binary); + // check if file exists if (!f.is_open()) { throw std::runtime_error(fmt::format("Could not open: {} for reading", file->fname.c_str())); } // read magic number std::array tmp{}; f.read(tmp.data(), tmp.size()); - if (tmp != NumpyFileFactory::magic_str) { + if (tmp != NumpyFile::magic_str) { for (auto item : tmp) fmt::print("{}, ", int(item)); fmt::print("\n"); @@ -175,29 +161,31 @@ void NumpyFileFactory::parse_metadata(File(&major_ver_), 1); - f.read(reinterpret_cast(&minor_ver_), 1); + f.read(reinterpret_cast(&file->major_ver_), 1); + f.read(reinterpret_cast(&file->minor_ver_), 1); - if (major_ver_ == 1) { - header_len_size = 2; - } else if (major_ver_ == 2) { - header_len_size = 4; + if (file->major_ver_ == 1) { + file->header_len_size = 2; + } else if (file->major_ver_ == 2) { + file->header_len_size = 4; } else { throw std::runtime_error("Unsupported numpy version"); } // read header length - f.read(reinterpret_cast(&header_len), header_len_size); - if ((magic_string_length + 2 + header_len_size + header_len) % 16 != 0) { + f.read(reinterpret_cast(&file->header_len), file->header_len_size); + file->header_size = file->magic_string_length + 2 + file->header_len_size + file->header_len; + if (file->header_size % 16 != 0) { fmt::print("Warning: header length is not a multiple of 16\n"); } // read header - auto buf_v = std::vector(header_len); - f.read(buf_v.data(), header_len); - std::string header(buf_v.data(), header_len); + auto buf_v = std::vector(file->header_len); + f.read(buf_v.data(), file->header_len); + std::string header(buf_v.data(), file->header_len); // parse header std::vector keys{"descr", "fortran_order", "shape"}; + std::cout << "original header: " << '"' << header << '"' << std::endl; auto dict_map = parse_dict(header, keys); if (dict_map.size() == 0) @@ -220,6 +208,23 @@ void NumpyFileFactory::parse_metadata(File(std::stoul(item)); shape.push_back(dim); } + file->header = {dtype, fortran_order, shape}; +} + +template + File* NumpyFileFactory::load_file() { + NumpyFile *file = new NumpyFile(this->fpath); + parse_metadata(file); + NumpyFile *f = dynamic_cast *>(file); + std::cout << "parsed header: " << f->header.to_string() << std::endl; + + if(sizeof(DataType) != f->header.dtype.itemsize){ + std::stringstream s; + s << "Data type size mismatch: " << sizeof(DataType) << " != " << f->header.dtype.itemsize; + throw std::runtime_error(s.str()); + } + return file; +}; +template class NumpyFileFactory; + - // {dtype, fortran_order, shape}; -}; \ No newline at end of file