detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-05 12:30:39 +02:00
Code Issues Actions Packages Releases Activity

add documentation for file_io files

Browse Source
This commit is contained in:
Bechir Braham 2024-04-10 18:08:54 +02:00
parent 530feca830
commit 111b421476
No known key found for this signature in database
GPG Key ID: 7F511B55FD8E9671
15 changed files with 354 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/include/aare/core/DType.hpp
View File

@ -1,5 +1,4 @@
#pragma once
#include "aare/utils/logger.hpp"
#include <cstdint>
#include <string>
#include <typeinfo>

2
core/src/DType.cpp
View File

@ -1,5 +1,7 @@
#include "aare/core/DType.hpp"
#include "aare/utils/logger.hpp"
#include <fmt/format.h>
namespace aare {

28
file_io/include/aare/file_io/File.hpp
View File

@ -2,16 +2,26 @@
#include "aare/file_io/FileInterface.hpp"
namespace aare {
/**
* @brief RAII File class for reading and writing image files in various formats
* wrapper on a FileInterface to abstract the underlying file format
* @note documentation for each function is in the FileInterface class
*/
class File {
private:
FileInterface *file_impl;
public:
// options:
// - r reading
// - w writing (overwrites existing file)
// - a appending (appends to existing file)
// TODO! do we need to support w+, r+ and a+?
/**
* @brief Construct a new File object
* @param fname path to the file
* @param mode file mode (r, w, a)
* @param cfg file configuration
* @throws std::runtime_error if the file cannot be opened
* @throws std::invalid_argument if the file mode is not supported
*
*/
File(std::filesystem::path fname, std::string mode, FileConfig cfg = {});
void write(Frame &frame);
Frame read();
@ -28,8 +38,16 @@ class File {
ssize_t rows() const;
ssize_t cols() const;
ssize_t bitdepth() const;
/**
* @brief Move constructor
* @param other File object to move from
*/
File(File &&other);
/**
* @brief destructor: will only delete the FileInterface object
*/
~File();
};

41
file_io/include/aare/file_io/FileFactory.hpp
View File

@ -1,41 +0,0 @@
#pragma once
#include "aare/core/DType.hpp"
#include "aare/file_io/FileInterface.hpp"
#include "aare/utils/logger.hpp"
#include <filesystem>
namespace aare {
class FileFactory {
// Class that will be used to create FileInterface objects
// follows the factory pattern
protected:
std::filesystem::path m_fpath;
public:
static FileFactory *get_factory(std::filesystem::path);
// virtual int deleteFile() = 0;
static FileInterface *load_file(std::filesystem::path p, std::string mode, FileConfig cfg = {}) {
if ((mode == "r" or mode == "a") and not std::filesystem::exists(p)) {
throw std::runtime_error(LOCATION + "File does not exist");
}
auto factory = get_factory(p);
FileInterface *tmp = nullptr;
if (mode == "r") {
tmp = factory->load_file_read();
} else if (mode == "w") {
tmp = factory->load_file_write(cfg);
}
delete factory;
return tmp;
};
virtual FileInterface *load_file_read() = 0; // TODO: add option to load all file to memory or keep it on disk
virtual FileInterface *load_file_write(FileConfig) = 0;
virtual void parse_metadata(FileInterface *) = 0;
virtual void parse_fname(FileInterface *) = 0;
virtual ~FileFactory() = default;
};
} // namespace aare

111
file_io/include/aare/file_io/FileInterface.hpp
View File

@ -2,12 +2,18 @@
#include "aare/core/DType.hpp"
#include "aare/core/Frame.hpp"
#include "aare/core/defs.hpp"
#include "aare/utils/logger.hpp"
#include <filesystem>
#include <vector>
namespace aare {
/**
* @brief FileConfig structure to store the configuration of a file
* dtype: data type of the file
* rows: number of rows in the file
* cols: number of columns in the file
* geometry: geometry of the file
*/
struct FileConfig {
aare::DType dtype = aare::DType(typeid(uint16_t));
uint64_t rows;
@ -18,48 +24,115 @@ struct FileConfig {
}
bool operator!=(const FileConfig &other) const { return !(*this == other); }
};
/**
* @brief FileInterface class to define the interface for file operations
* @note parent class for NumpyFile and RawFile
* @note all functions are pure virtual and must be implemented by the derived classes
*/
class FileInterface {
public:
friend class FileFactory;
// write one frame
/**
* @brief write a frame to the file
* @param frame frame to write
* @return void
* @throws std::runtime_error if the function is not implemented
*/
virtual void write(Frame &frame) = 0;
// write n_frames frames
/**
* @brief write a vector of frames to the file
* @param frames vector of frames to write
* @return void
*/
// virtual void write(std::vector<Frame> &frames) = 0;
// read one frame
/**
* @brief read one frame from the file at the current position
* @return Frame
*/
virtual Frame read() = 0;
// read n_frames frames
/**
* @brief read n_frames from the file at the current position
* @param n_frames number of frames to read
* @return vector of frames
*/
virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
// read one frame into the provided buffer
/**
* @brief read one frame from the file at the current position and store it in the provided buffer
* @param image_buf buffer to store the frame
* @return void
*/
virtual void read_into(std::byte *image_buf) = 0;
// read n_frames frame into the provided buffer
/**
* @brief read n_frames from the file at the current position and store them in the provided buffer
* @param image_buf buffer to store the frames
* @param n_frames number of frames to read
* @return void
*/
virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
// read the frame number on position frame_index
/**
* @brief get the frame number at the given frame index
* @param frame_index index of the frame
* @return frame number
*/
virtual size_t frame_number(size_t frame_index) = 0;
// size of one frame, important fro teh read_into function
/**
* @brief get the size of one frame in bytes
* @return size of one frame
*/
virtual size_t bytes_per_frame() = 0;
// number of pixels in one frame
/**
* @brief get the number of pixels in one frame
* @return number of pixels in one frame
*/
virtual size_t pixels() = 0;
// goto frame number
/**
* @brief seek to the given frame number
* @param frame_number frame number to seek to
* @return void
*/
virtual void seek(size_t frame_number) = 0;
// return the position of the file pointer (in number of frames)
/**
* @brief get the current position of the file pointer
* @return current position of the file pointer
*/
virtual size_t tell() = 0;
// Getter functions
/**
* @brief get the total number of frames in the file
* @return total number of frames in the file
*/
virtual size_t total_frames() const = 0;
/**
* @brief get the number of rows in the file
* @return number of rows in the file
*/
virtual ssize_t rows() const = 0;
/**
* @brief get the number of columns in the file
* @return number of columns in the file
*/
virtual ssize_t cols() const = 0;
/**
* @brief get the bitdepth of the file
* @return bitdepth of the file
*/
virtual ssize_t bitdepth() const = 0;
// read one frame at position frame_number
/**
* @brief read one frame from the file at the given frame number
* @param frame_number frame number to read
* @return frame
*/
Frame iread(size_t frame_number) {
auto old_pos = tell();
seek(frame_number);
@ -68,7 +141,12 @@ class FileInterface {
return tmp;
};
// read n_frames frames starting at frame_number
/**
* @brief read n_frames from the file starting at the given frame number
* @param frame_number frame number to start reading from
* @param n_frames number of frames to read
* @return vector of frames
*/
std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
auto old_pos = tell();
seek(frame_number);
@ -98,7 +176,6 @@ class FileInterface {
ssize_t m_cols{};
ssize_t m_bitdepth{};
size_t current_frame{};
};
} // namespace aare

29
file_io/include/aare/file_io/NumpyFile.hpp
View File

@ -9,10 +9,23 @@
namespace aare {
/**
* @brief NumpyFile class to read and write numpy files
* @note derived from FileInterface
* @note implements all the pure virtual functions from FileInterface
* @note documentation for the functions can also be found in the FileInterface class
*/
class NumpyFile : public FileInterface {
public:
/**
* @brief NumpyFile constructor
* @param fname path to the numpy file
* @param mode file mode (r, w)
* @param cfg file configuration
*/
NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
void write(Frame &frame) override;
Frame read() override { return get_frame(this->current_frame++); }
@ -29,10 +42,24 @@ class NumpyFile : public FileInterface {
ssize_t cols() const override { return m_header.shape[2]; }
ssize_t bitdepth() const override { return m_header.dtype.bitdepth(); }
/**
* @brief get the data type of the numpy file
* @return DType
*/
DType dtype() const { return m_header.dtype; }
/**
* @brief get the shape of the numpy file
* @return vector of type size_t
*/
std::vector<size_t> shape() const { return m_header.shape; }
// load the full numpy file into a NDArray
/**
* @brief load the numpy file into an NDArray
* @tparam T data type of the NDArray
* @tparam NDim number of dimensions of the NDArray
* @return NDArray<T, NDim>
*/
template <typename T, size_t NDim> NDArray<T, NDim> load() {
NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
fseek(fp, header_size, SEEK_SET);

166
file_io/include/aare/file_io/RawFile.hpp
View File

@ -5,24 +5,42 @@
namespace aare {
/**
* @brief RawFile class to read .raw and .json files
* @note derived from FileInterface
* @note documentation can also be found in the FileInterface class
*/
class RawFile : public FileInterface {
private:
public:
std::filesystem::path m_fname; // TO be made private!
/**
* @brief RawFile constructor
* @param fname path to the file
* @param mode file mode (r, w)
* @param cfg file configuration
*/
RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &cfg = {});
void write(Frame &frame) override { throw std::runtime_error("Not implemented"); };
/**
* @brief write function is not implemented for RawFile
* @param frame frame to write
*/
void write(Frame &frame) override { throw std::runtime_error("Not implemented"); };
Frame read() override { return get_frame(this->current_frame++); };
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
// size of one frame, important fro teh read_into function
/**
* @brief get the number of bytess per frame
* @return size of one frame in bytes
*/
size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
// number of pixels in one frame
/**
* @brief get the number of pixels in the frame
* @return number of pixels
*/
size_t pixels() override { return m_rows * m_cols; }
// goto frame number
@ -31,6 +49,104 @@ class RawFile : public FileInterface {
// return the position of the file pointer (in number of frames)
size_t tell() override { return this->current_frame; };
/**
* @brief check if the file is a master file
* @param fpath path to the file
*/
static bool is_master_file(std::filesystem::path fpath);
/**
* @brief set the module gap row and column
* @param row gap between rows
* @param col gap between columns
*/
inline void set_config(int row, int col) {
cfg.module_gap_row = row;
cfg.module_gap_col = col;
}
// TODO! Deal with fast quad and missing files
/**
* @brief get the number of subfiles for the RawFile
* @return number of subfiles
*/
void find_number_of_subfiles();
/**
* @brief get the master file name path for the RawFile
* @return path to the master file
*/
inline std::filesystem::path master_fname();
/**
* @brief get the data file name path for the RawFile with the given module id and file id
* @param mod_id module id
* @param file_id file id
* @return path to the data file
*/
inline std::filesystem::path data_fname(int mod_id, int file_id);
/**
* @brief destructor: will delete the subfiles
*/
~RawFile();
size_t total_frames() const override { return m_total_frames; }
ssize_t rows() const override { return m_rows; }
ssize_t cols() const override { return m_cols; }
ssize_t bitdepth() const override { return m_bitdepth; }
private:
/**
* @brief read the frame at the given frame number into the image buffer
* @param frame_number frame number to read
* @param image_buf buffer to store the frame
*/
void get_frame_into(size_t frame_number, std::byte *image_buf);
/**
* @brief get the frame at the given frame number
* @param frame_number frame number to read
* @return Frame
*/
Frame get_frame(size_t frame_number);
/**
* @brief parse the file name to get the extension, base name and index
*/
void parse_fname();
/**
* @brief parse the metadata from the file
*/
void parse_metadata();
/**
* @brief parse the metadata of a .raw file
*/
void parse_raw_metadata();
/**
* @brief parse the metadata of a .json file
*/
void parse_json_metadata();
/**
* @brief finds the geometry of the file
*/
void find_geometry();
/**
* @brief read the header of the file
* @param fname path to the data subfile
* @return sls_detector_header
*/
sls_detector_header read_header(const std::filesystem::path &fname);
/**
* @brief open the subfiles
*/
void open_subfiles();
size_t n_subfiles;
size_t n_subfile_parts;
std::vector<std::vector<SubFile *>> subfiles;
@ -40,44 +156,6 @@ class RawFile : public FileInterface {
RawFileConfig cfg{0, 0};
TimingMode timing_mode;
bool quad{false};
inline void set_config(int row, int col) {
cfg.module_gap_row = row;
cfg.module_gap_col = col;
}
// TODO! Deal with fast quad and missing files
void find_number_of_subfiles() {
int n_mod = 0;
while (std::filesystem::exists(data_fname(++n_mod, 0)))
;
n_subfiles = n_mod;
}
inline std::filesystem::path master_fname() {
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
}
inline std::filesystem::path data_fname(int mod_id, int file_id) {
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
}
~RawFile();
size_t total_frames() const override { return m_total_frames; }
ssize_t rows() const override { return m_rows; }
ssize_t cols() const override { return m_cols; }
ssize_t bitdepth() const override { return m_bitdepth; }
private:
void get_frame_into(size_t frame_number, std::byte *image_buf);
Frame get_frame(size_t frame_number);
void parse_fname();
void parse_metadata();
void parse_raw_metadata();
void parse_json_metadata();
void find_geometry();
sls_detector_header read_header(const std::filesystem::path &fname);
void open_subfiles();
};
} // namespace aare

16
file_io/include/aare/file_io/RawFileHelpers.hpp
View File

@ -1,16 +0,0 @@
#pragma once
#include "aare/file_io/RawFile.hpp"
namespace aare::RawFileHelpers {
void parse_json_metadata(RawFile *file);
void parse_raw_metadata(RawFile *file);
RawFile *load_file_read() override;
RawFile *load_file_write(FileConfig) override { return new RawFile(); };
void parse_metadata(FileInterface *) override;
void parse_fname(FileInterface *) override;
void open_subfiles(FileInterface *);
sls_detector_header read_header(const std::filesystem::path &fname);
void find_geometry(FileInterface *);
};

74
file_io/include/aare/file_io/SubFile.hpp
View File

@ -7,17 +7,24 @@
namespace aare {
/**
* @brief Class to read a subfile from a RawFile
*/
class SubFile {
protected:
FILE *fp = nullptr;
ssize_t m_bitdepth;
std::filesystem::path m_fname;
ssize_t m_rows{};
ssize_t m_cols{};
ssize_t n_frames{};
int m_sub_file_index_{};
// pointer to functions that will read frames
/**
* @brief type of the read_impl function pointer
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
using pfunc = size_t (SubFile::*)(std::byte *);
pfunc read_impl = nullptr;
/**
* @brief map to store the read_impl functions for different detectors
* @note the key is a pair of DetectorType and bitdepth
* @note the value is a pointer to the read_impl function specific for the detector
* @note the read_impl function will be set to the appropriate function in the constructor
*/
std::map<std::pair<DetectorType, int>, pfunc> read_impl_map = {
{{DetectorType::Moench, 16}, &SubFile::read_impl_reorder<uint16_t>},
{{DetectorType::Jungfrau, 16}, &SubFile::read_impl_normal},
@ -29,20 +36,59 @@ class SubFile {
};
public:
// pointer to a read_impl function. pointer will be set to the appropriate read_impl function in the constructor
pfunc read_impl = nullptr;
size_t read_impl_normal(std::byte *buffer);
template <typename DataType> size_t read_impl_flip(std::byte *buffer);
template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
/**
* @brief SubFile constructor
* @param fname path to the subfile
* @param detector detector type
* @param rows number of rows in the subfile
* @param cols number of columns in the subfile
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth);
/**
* @brief read the subfile into a buffer
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
size_t read_impl_normal(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes flipped
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_flip(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered
* @param buffer pointer to the buffer to read the data into
* @return number of bytes read
*/
template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
/**
* @brief read the subfile into a buffer with the bytes reordered and flipped
* @param buffer pointer to the buffer to read the data into
* @param frame_number frame number to read
* @return number of bytes read
*/
size_t get_part(std::byte *buffer, int frame_number);
size_t frame_number(int frame_index);
// TODO: define the inlines as variables and assign them in constructor
inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; }
inline size_t pixels_per_part() { return m_rows * m_cols; }
protected:
FILE *fp = nullptr;
ssize_t m_bitdepth;
std::filesystem::path m_fname;
ssize_t m_rows{};
ssize_t m_cols{};
ssize_t n_frames{};
int m_sub_file_index_{};
};
} // namespace aare

11
file_io/include/aare/file_io/helpers.hpp
View File

@ -1,11 +0,0 @@
#pragma once
#include "aare/file_io/FileInterface.hpp"
#include <filesystem>
#include <fmt/core.h>
namespace aare {
bool is_master_file(std::filesystem::path fpath);
}

3
file_io/src/File.cpp
View File

@ -7,6 +7,9 @@
namespace aare {
File::File(std::filesystem::path fname, std::string mode, FileConfig cfg) {
if (mode != "r" && mode != "w" && mode != "a") {
throw std::invalid_argument("Unsupported file mode");
}
if ((mode == "r" or mode == "a") and not std::filesystem::exists(fname)) {
throw std::runtime_error(fmt::format("File does not exist: {}", fname.c_str()));

23
file_io/src/RawFile.cpp
View File

@ -15,7 +15,6 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co
aare::logger::warn(
"In read mode it is not necessary to provide a config, the provided config will be ignored");
}
aare::logger::debug("XXXXXXXXLoading raw file");
parse_fname();
parse_metadata();
find_number_of_subfiles();
@ -49,6 +48,27 @@ sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
throw std::runtime_error("Could not read header from file");
return h;
}
bool RawFile::is_master_file(std::filesystem::path fpath) {
std::string stem = fpath.stem();
if (stem.find("_master_") != std::string::npos)
return true;
else
return false;
}
void RawFile::find_number_of_subfiles() {
int n_mod = 0;
while (std::filesystem::exists(data_fname(++n_mod, 0)))
;
n_subfiles = n_mod;
}
inline std::filesystem::path RawFile::data_fname(int mod_id, int file_id) {
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
}
inline std::filesystem::path RawFile::master_fname() {
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
}
void RawFile::find_geometry() {
uint16_t r{};
@ -163,7 +183,6 @@ void RawFile::parse_fname() {
m_base_path = m_fname.parent_path();
m_base_name = m_fname.stem();
m_ext = m_fname.extension();
aare::logger::debug("EXTEXT", m_ext);
auto pos = m_base_name.rfind("_");
m_findex = std::stoi(m_base_name.substr(pos + 1));
pos = m_base_name.find("_master_");

32
file_io/src/RawFileHelpers.cpp
View File

@ -1,32 +0,0 @@
#include "aare/file_io/RawFileFactory.hpp"
#include "aare/core/defs.hpp"
#include "aare/file_io/RawFile.hpp"
#include "aare/file_io/SubFile.hpp"
#include "aare/file_io/helpers.hpp"
#include "aare/utils/logger.hpp"
#include <fstream>
#include <iostream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
namespace aare::RawFileHelpers {
} // namespace aare

2
file_io/src/SubFile.cpp
View File

@ -16,7 +16,7 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) +
" and bitdepth: " + std::to_string(bitdepth);
throw std::runtime_error(error_msg);
throw std::invalid_argument(error_msg);
}
this->read_impl = read_impl_map.at({detector, bitdepth});
}

13
file_io/src/helpers.cpp
View File

@ -1,13 +0,0 @@
#include "aare/file_io/helpers.hpp"
namespace aare {
bool is_master_file(std::filesystem::path fpath) {
std::string stem = fpath.stem();
if (stem.find("_master_") != std::string::npos)
return true;
else
return false;
}
} // namespace aare