detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2026-02-03 07:48:39 +01:00
Code Issues Actions Packages Releases Activity

save numpy files frame by frame (#37)

Browse Source 
Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham
2024-04-02 11:35:58 +02:00
committed by GitHub
parent 643ea39670
commit 449c5119a4
23 changed files with 357 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
file_io/include/aare/File.hpp
View File

@@ -1,4 +1,6 @@
#include "aare/FileInterface.hpp"
class File {
private:
FileInterface *file_impl;
@@ -9,7 +11,8 @@ class File {
// - w writing (overwrites existing file)
// - a appending (appends to existing file)
// TODO! do we need to support w+, r+ and a+?
File(std::filesystem::path fname, std::string mode);
File(std::filesystem::path fname, std::string mode,FileConfig cfg = {});
void write(Frame& frame);
Frame read();
Frame iread(size_t frame_number);
std::vector<Frame> read(size_t n_frames);

39
file_io/include/aare/FileFactory.hpp
View File

@@ -1,26 +1,37 @@
#pragma once
#include <filesystem>
#include "aare/DType.hpp"
#include "aare/FileInterface.hpp"
class FileFactory{
#include "aare/utils/logger.hpp"
#include <filesystem>
class FileFactory {
// Class that will be used to create FileInterface objects
// follows the factory pattern
protected:
protected:
std::filesystem::path m_fpath;
public:
static FileFactory* get_factory(std::filesystem::path);
public:
static FileFactory *get_factory(std::filesystem::path);
// virtual int deleteFile() = 0;
static FileInterface* load_file(std::filesystem::path p){
static FileInterface *load_file(std::filesystem::path p, std::string mode, FileConfig cfg = {}) {
if ((mode == "r" or mode == "a") and not std::filesystem::exists(p)) {
throw std::runtime_error(LOCATION+"File does not exist");
}
auto factory = get_factory(p);
FileInterface* tmp= factory->load_file();
FileInterface *tmp = nullptr;
if (mode == "r") {
tmp = factory->load_file_read();
} else if (mode == "w") {
tmp = factory->load_file_write(cfg);
}
delete factory;
return tmp;
};
virtual FileInterface* load_file()=0;//TODO: add option to load all file to memory or keep it on disk
virtual void parse_metadata(FileInterface*)=0;
virtual void parse_fname(FileInterface*)=0;
virtual FileInterface *load_file_read() = 0; // TODO: add option to load all file to memory or keep it on disk
virtual FileInterface *load_file_write(FileConfig) = 0;
virtual void parse_metadata(FileInterface *) = 0;
virtual void parse_fname(FileInterface *) = 0;
virtual ~FileFactory() = default;
};

12
file_io/include/aare/FileInterface.hpp
View File

@@ -1,14 +1,23 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include "aare/DType.hpp"
#include "aare/utils/logger.hpp"
#include <filesystem>
#include <vector>
struct FileConfig {
std::filesystem::path fname;
aare::DType dtype = aare::DType(typeid(uint16_t));
uint64_t rows;
uint64_t cols;
xy geometry{1, 1};
};
class FileInterface {
public:
friend class FileFactory;
// write one frame
// virtual void write(Frame &frame) = 0;
virtual void write(Frame &frame) = 0;
// write n_frames frames
// virtual void write(std::vector<Frame> &frames) = 0;
@@ -71,6 +80,7 @@ class FileInterface {
};
public:
std::string mode;
std::filesystem::path m_fname;
std::filesystem::path m_base_path;
std::string m_base_name, m_ext;

16
file_io/include/aare/NumpyFile.hpp
View File

@@ -7,8 +7,11 @@
class NumpyFile : public FileInterface {
FILE *fp = nullptr;
size_t initial_header_len = 0;
public:
void write(Frame &frame) override;
Frame read() override { return get_frame(this->current_frame++); }
std::vector<Frame> read(size_t n_frames) override;
@@ -22,23 +25,18 @@ class NumpyFile : public FileInterface {
size_t total_frames() const override { return header.shape[0]; }
ssize_t rows() const override { return header.shape[1]; }
ssize_t cols() const override { return header.shape[2]; }
ssize_t bitdepth() const override { return header.dtype.itemsize; }
ssize_t bitdepth() const override { return header.dtype.bitdepth(); }
NumpyFile(std::filesystem::path fname);
header_t header{};
static constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
NumpyFile(FileConfig, header_t);
header_t header;
uint8_t major_ver_{};
uint8_t minor_ver_{};
uint32_t header_len{};
uint8_t header_len_size{};
const uint8_t magic_string_length{6};
ssize_t header_size{};
~NumpyFile() {
if (fp != nullptr) {
fclose(fp);
}
}
~NumpyFile();
private:
size_t current_frame{};

5
file_io/include/aare/NumpyFileFactory.hpp
View File

@@ -14,7 +14,8 @@ class NumpyFileFactory : public FileFactory {
public:
NumpyFileFactory(std::filesystem::path fpath);
void parse_metadata(FileInterface *_file) override;
NumpyFile* load_file() override;
void parse_fname(FileInterface*){};
NumpyFile* load_file_read() override;
NumpyFile* load_file_write(FileConfig) override;
void parse_fname(FileInterface*)override{};
};

34
file_io/include/aare/NumpyHelpers.hpp
View File

@@ -4,34 +4,28 @@
#include <array>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <numeric>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>
#include <sstream>
#include <numeric>
#include <iostream>
#include "aare/DType.hpp"
#include "aare/defs.hpp"
using shape_t = std::vector<uint64_t>;
struct dtype_t {
char byteorder;
char kind;
unsigned int itemsize;
std::string to_string() {
std::stringstream sstm;
sstm << byteorder << kind << itemsize;
return sstm.str();
}
};
struct header_t {
dtype_t dtype;
header_t() : dtype(aare::DType(aare::DType::ERROR)), fortran_order(false), shape(shape_t()){};
header_t(aare::DType dtype, bool fortran_order, shape_t shape)
: dtype(dtype), fortran_order(fortran_order), shape(shape){};
aare::DType dtype;
bool fortran_order;
shape_t shape;
std::string to_string() {
std::stringstream sstm;
sstm << "dtype: " << dtype.to_string() << ", fortran_order: " << fortran_order << ' ';
sstm << "dtype: " << dtype.str() << ", fortran_order: " << fortran_order << ' ';
sstm << "shape: (";
for (auto item : shape)
@@ -41,6 +35,9 @@ struct header_t {
}
};
namespace aare::NumpyHelpers {
const constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
const uint8_t magic_string_length{6};
std::string parse_str(const std::string &in);
/**
@@ -61,4 +58,9 @@ template <typename T, size_t N> inline bool in_array(T val, const std::array<T,
}
bool is_digits(const std::string &str);
dtype_t parse_descr(std::string typestring);
aare::DType parse_descr(std::string typestring);
size_t write_header(std::filesystem::path fname, const header_t &header) ;
size_t write_header(std::ostream &out, const header_t &header) ;
bool is_digits(const std::string &str);
} // namespace aare::NumpyHelpers

13
file_io/include/aare/RawFile.hpp
View File

@@ -9,6 +9,7 @@ class RawFile : public FileInterface {
using config = RawFileConfig;
public:
void write(Frame &frame) override{};
Frame read() override { return get_frame(this->current_frame++); };
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
@@ -22,10 +23,10 @@ class RawFile : public FileInterface {
size_t pixels() override { return m_rows * m_cols; }
// goto frame number
void seek(size_t frame_number) { this->current_frame = frame_number; };
void seek(size_t frame_number) override{ this->current_frame = frame_number; };
// return the position of the file pointer (in number of frames)
size_t tell() { return this->current_frame; };
size_t tell() override{ return this->current_frame; };
size_t n_subfiles;
size_t n_subfile_parts;
@@ -59,10 +60,10 @@ class RawFile : public FileInterface {
~RawFile();
size_t total_frames() const { return m_total_frames; }
ssize_t rows() const { return m_rows; }
ssize_t cols() const { return m_cols; }
ssize_t bitdepth() const { return m_bitdepth; }
size_t total_frames() const override { return m_total_frames; }
ssize_t rows() const override { return m_rows; }
ssize_t cols() const override{ return m_cols; }
ssize_t bitdepth() const override{ return m_bitdepth; }
private:
size_t current_frame{};

4
file_io/include/aare/RawFileFactory.hpp
View File

@@ -6,8 +6,10 @@ class RawFileFactory : public FileFactory {
void parse_raw_metadata(RawFile *file);
public:
RawFileFactory(std::filesystem::path fpath);
virtual RawFile *load_file() override;
RawFile *load_file_read() override;
RawFile *load_file_write(FileConfig) override{return new RawFile();};
void parse_metadata(FileInterface *) override;
void parse_fname(FileInterface *) override;
void open_subfiles(FileInterface *);