save numpy files frame by frame (#37)

Co-authored-by: Bechir <bechir.brahem420@gmail.com>

file_io/src/File.cpp

@@ -2,13 +2,11 @@
 #include "aare/FileFactory.hpp"
 #include "aare/utils/logger.hpp"
 
-File::File(std::filesystem::path fname, std::string mode) {
-    if (mode != "r") {
-        throw std::runtime_error(LOCATION + " Only read mode is supported");
-    }
-    file_impl = FileFactory::load_file(fname);
+File::File(std::filesystem::path fname, std::string mode, FileConfig cfg) {
+    file_impl = FileFactory::load_file(fname, mode, cfg);
 }
 
+void File::write(Frame& frame) { file_impl->write(frame); }
 Frame File::read() { return file_impl->read(); }
 size_t File::total_frames() const { return file_impl->total_frames(); }
 std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
@@ -18,17 +16,13 @@ size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(f
 size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
 size_t File::pixels() { return file_impl->pixels(); }
 void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
-size_t File::tell() const{ return file_impl->tell(); }
+size_t File::tell() const { return file_impl->tell(); }
 ssize_t File::rows() const { return file_impl->rows(); }
 ssize_t File::cols() const { return file_impl->cols(); }
 ssize_t File::bitdepth() const { return file_impl->bitdepth(); }
-File::~File() {
-    delete file_impl;
-}
+File::~File() { delete file_impl; }
 
-Frame File::iread(size_t frame_number) {
-    return file_impl->iread(frame_number);
-}
+Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
 
 File::File(File &&other) {
     file_impl = other.file_impl;

file_io/src/FileFactory.cpp

@@ -7,11 +7,6 @@
 #include <iostream>
 
 FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
-    // check if file exists
-    if (!std::filesystem::exists(fpath)) {
-        throw std::runtime_error("File does not exist");
-    }
-
     if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
         aare::logger::debug("Loading",fpath.extension(),"file");
         return new RawFileFactory(fpath);

file_io/src/NumpyFile.cpp

@@ -1,17 +1,45 @@
 
 #include "aare/NumpyFile.hpp"
 
+void NumpyFile::write(Frame &frame) {
+    if (fp == nullptr) {
+        throw std::runtime_error("File not open");
+    }
+    if (not(mode == "w" or mode == "a")) {
+        throw std::runtime_error("File not open for writing");
+    }
+    fseek(fp, 0, SEEK_END);
+    fwrite(frame._get_data(), frame.size(), 1, fp);
+}
+
 NumpyFile::NumpyFile(std::filesystem::path fname_) {
     this->m_fname = fname_;
     fp = fopen(this->m_fname.c_str(), "rb");
 }
+NumpyFile::NumpyFile(FileConfig config, header_t header) {
+    this->mode = "w";
+    this->m_fname = config.fname;
+    this->m_bitdepth = config.dtype.bitdepth();
+    this->m_rows = config.rows;
+    this->m_cols = config.cols;
+    this->header = header;
+    this->header.shape = {0, config.rows, config.cols};
+
+    fp = fopen(this->m_fname.c_str(), "wb");
+    if (!fp) {
+        throw std::runtime_error(fmt::format("Could not open: {} for reading", this->m_fname.c_str()));
+    }
+
+    this->initial_header_len =
+        aare::NumpyHelpers::write_header(std::filesystem::path(this->m_fname.c_str()), this->header);
+}
 
 Frame NumpyFile::get_frame(size_t frame_number) {
-    Frame frame(header.shape[1], header.shape[2], header.dtype.itemsize*8);
-    get_frame_into(frame_number,frame._get_data());
+    Frame frame(header.shape[1], header.shape[2], header.dtype.bitdepth());
+    get_frame_into(frame_number, frame._get_data());
     return frame;
 }
-void NumpyFile::get_frame_into( size_t frame_number,std::byte* image_buf) {
+void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
     if (fp == nullptr) {
         throw std::runtime_error("File not open");
     }
@@ -22,13 +50,10 @@ void NumpyFile::get_frame_into( size_t frame_number,std::byte* image_buf) {
     fread(image_buf, bytes_per_frame(), 1, fp);
 }
 
-
-
 size_t NumpyFile::pixels() {
     return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies<uint64_t>());
 };
-size_t NumpyFile::bytes_per_frame() { return header.dtype.itemsize * pixels(); };
-
+size_t NumpyFile::bytes_per_frame() { return header.dtype.bitdepth() / 8 * pixels(); };
 
 std::vector<Frame> NumpyFile::read(size_t n_frames) {
     // TODO: implement this in a more efficient way
@@ -45,4 +70,30 @@ void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
         this->get_frame_into(this->current_frame++, image_buf);
         image_buf += this->bytes_per_frame();
     }
+}
+
+NumpyFile::~NumpyFile() {
+    if (mode == "w" or mode == "a") {
+        // determine number of frames
+        fseek(fp, 0, SEEK_END);
+        size_t file_size = ftell(fp);
+        size_t data_size = file_size - initial_header_len;
+        size_t n_frames = data_size / bytes_per_frame();
+        // update number of frames in header (first element of shape)
+        this->header.shape[0] = n_frames;
+        fseek(fp, 0, SEEK_SET);
+        // create string stream to contain header
+        std::stringstream ss;
+        aare::NumpyHelpers::write_header(ss, this->header);
+        std::string header_str = ss.str();
+        // write header
+        fwrite(header_str.c_str(), header_str.size(), 1, fp);
+
+        
+    }
+
+    if (fp != nullptr) {
+        fclose(fp);
+        
+    }
 }

file_io/src/NumpyFileFactory.cpp

@@ -1,5 +1,8 @@
 #include "aare/NumpyFileFactory.hpp"
 #include "aare/NumpyHelpers.hpp"
+
+using namespace aare;
+
 NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; }
 void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     auto file = dynamic_cast<NumpyFile *>(_file);
@@ -12,7 +15,7 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     // read magic number
     std::array<char, 6> tmp{};
     f.read(tmp.data(), tmp.size());
-    if (tmp != NumpyFile::magic_str) {
+    if (tmp != aare::NumpyHelpers::magic_str) {
         for (auto item : tmp)
             fmt::print("{}, ", int(item));
         fmt::print("\n");
@@ -32,7 +35,7 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     }
     // read header length
     f.read(reinterpret_cast<char *>(&file->header_len), file->header_len_size);
-    file->header_size = file->magic_string_length + 2 + file->header_len_size + file->header_len;
+    file->header_size = aare::NumpyHelpers::magic_string_length + 2 + file->header_len_size + file->header_len;
     if (file->header_size % 16 != 0) {
         fmt::print("Warning: header length is not a multiple of 16\n");
     }
@@ -46,7 +49,7 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     std::vector<std::string> keys{"descr", "fortran_order", "shape"};
     aare::logger::debug("original header: \"header\"");
 
-    auto dict_map = parse_dict(header, keys);
+    auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
     if (dict_map.size() == 0)
         throw std::runtime_error("invalid dictionary in header");
 
@@ -54,14 +57,14 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     std::string fortran_s = dict_map["fortran_order"];
     std::string shape_s = dict_map["shape"];
 
-    std::string descr = parse_str(descr_s);
-    dtype_t dtype = parse_descr(descr);
+    std::string descr = aare::NumpyHelpers::parse_str(descr_s);
+    aare::DType dtype = aare::NumpyHelpers::parse_descr(descr);
 
     // convert literal Python bool to C++ bool
-    bool fortran_order = parse_bool(fortran_s);
+    bool fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);
 
     // parse the shape tuple
-    auto shape_v = parse_tuple(shape_s);
+    auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
     shape_t shape;
     for (auto item : shape_v) {
         auto dim = static_cast<unsigned long>(std::stoul(item));
@@ -70,8 +73,16 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     file->header = {dtype, fortran_order, shape};
 }
 
-NumpyFile *NumpyFileFactory::load_file() {
+NumpyFile *NumpyFileFactory::load_file_read() {
     NumpyFile *file = new NumpyFile(this->m_fpath);
     parse_metadata(file);
+    return file;
+};
+
+NumpyFile *NumpyFileFactory::load_file_write(FileConfig config) {
+    NumpyFile *file = new NumpyFile(config, {config.dtype, false, {config.rows, config.cols}});
+    
+
+
     return file;
 };

file_io/src/NumpyHelpers.cpp

@@ -1,5 +1,31 @@
+/*
+   28-03-2024 modified by: Bechir Braham <bechir.braham@psi.ch>
+
+   Copyright 2017-2023 Leon Merten Lohse
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
 #include "aare/NumpyHelpers.hpp"
 
+namespace aare::NumpyHelpers {
+
 std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys) {
     std::unordered_map<std::string, std::string> map;
     if (keys.size() == 0)
@@ -51,8 +77,7 @@ std::unordered_map<std::string, std::string> parse_dict(std::string in, const st
     return map;
 }
 
-
-dtype_t parse_descr(std::string typestring) {
+aare::DType parse_descr(std::string typestring) {
     if (typestring.length() < 3) {
         throw std::runtime_error("invalid typestring (length)");
     }
@@ -72,9 +97,9 @@ dtype_t parse_descr(std::string typestring) {
     if (!is_digits(itemsize_s)) {
         throw std::runtime_error("invalid typestring (itemsize)");
     }
-    unsigned int itemsize = std::stoul(itemsize_s);
+    // unsigned int itemsize = std::stoul(itemsize_s);
 
-    return {byteorder_c, kind_c, itemsize};
+    return aare::DType(typestring);
 }
 
 bool parse_bool(const std::string &in) {
@@ -134,4 +159,94 @@ std::string parse_str(const std::string &in) {
         return in.substr(1, in.length() - 2);
 
     throw std::runtime_error("Invalid python string.");
-}
+}
+
+void write_magic(std::ostream &ostream, int version_major, int version_minor) {
+    ostream.write(magic_str.data(), magic_string_length);
+    ostream.put(version_major);
+    ostream.put(version_minor);
+}
+template <typename T> inline std::string write_tuple(const std::vector<T> &v) {
+
+    if (v.size() == 0)
+        return "()";
+    std::ostringstream ss;
+    ss.imbue(std::locale("C"));
+
+    if (v.size() == 1) {
+        ss << "(" << v.front() << ",)";
+    } else {
+        const std::string delimiter = ", ";
+        // v.size() > 1
+        ss << "(";
+        // for (size_t i = 0; i < v.size() - 1; ++i) {
+        //     ss << v[i] << delimiter;
+        // }
+        // ss << v.back();
+        std::copy(v.begin(), v.end()-1, std::ostream_iterator<T>(ss, ", "));
+        ss << v.back();
+        ss << ")";
+    }
+
+    return ss.str();
+}
+
+inline std::string write_boolean(bool b) {
+    if (b)
+        return "True";
+    else
+        return "False";
+}
+
+inline std::string write_header_dict(const std::string &descr, bool fortran_order, const shape_t &shape) {
+    std::string s_fortran_order = write_boolean(fortran_order);
+    std::string shape_s = write_tuple(shape);
+
+    return "{'descr': '" + descr + "', 'fortran_order': " + s_fortran_order + ", 'shape': " + shape_s + ", }";
+}
+
+size_t write_header(std::filesystem::path fname, const header_t &header) {
+    std::ofstream out(fname, std::ios::binary | std::ios::out);
+    return write_header(out, header);
+}
+
+
+size_t write_header(std::ostream &out, const header_t &header) {
+    std::string header_dict = write_header_dict(header.dtype.str(), header.fortran_order, header.shape);
+
+    size_t length = magic_string_length + 2 + 2 + header_dict.length() + 1;
+
+    int version_major = 1;
+    int version_minor = 0;
+    if (length >= 255 * 255) {
+        length = magic_string_length + 2 + 4 + header_dict.length() + 1;
+        version_major = 2;
+        version_minor = 0;
+    }
+    size_t padding_len = 16 - length % 16;
+    std::string padding(padding_len, ' ');
+
+    // write magic
+    write_magic(out, version_major, version_minor);
+
+    // write header length
+    if (version_major == 1 and version_minor == 0) {
+        auto header_len = static_cast<uint16_t>(header_dict.length() + padding.length() + 1);
+
+        std::array<uint8_t, 2> header_len_le16{static_cast<uint8_t>((header_len >> 0) & 0xff),
+                                               static_cast<uint8_t>((header_len >> 8) & 0xff)};
+        out.write(reinterpret_cast<char *>(header_len_le16.data()), 2);
+    } else {
+        auto header_len = static_cast<uint32_t>(header_dict.length() + padding.length() + 1);
+
+        std::array<uint8_t, 4> header_len_le32{
+            static_cast<uint8_t>((header_len >> 0) & 0xff), static_cast<uint8_t>((header_len >> 8) & 0xff),
+            static_cast<uint8_t>((header_len >> 16) & 0xff), static_cast<uint8_t>((header_len >> 24) & 0xff)};
+        out.write(reinterpret_cast<char *>(header_len_le32.data()), 4);
+    }
+
+    out << header_dict << padding << '\n';
+    return length;
+}
+
+} // namespace aare::NumpyHelpers

file_io/src/RawFileFactory.cpp

@@ -119,7 +119,7 @@ void RawFileFactory::open_subfiles(FileInterface *_file) {
     }
 }
 
-RawFile *RawFileFactory::load_file() {
+RawFile *RawFileFactory::load_file_read() {
     RawFile *file = new RawFile();
     file->m_fname = this->m_fpath;
     this->parse_fname(file);