detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-18 02:07:13 +02:00
Code Issues Actions Packages Releases Activity

use clang-tidy (#59)

Browse Source 
* use clang-tidy for Frame.cpp

* fixes for defs.cpp

* clang-tidy 6/45

* clang-tidy for core

* clang-tidy fixes: for hpp File,FileInterface,SubFile.cpp

* ci fixes

* fix build errors

* fix clang-tidy command ci

* fix clang-tidy ci

* clang-tidy for rawfile.cpp

* clang-tidy numpy helpers

* fix ci

* clang-tidy file_io

* clang-tidy file_io and core working

* zmqheader

* clagn-tidy: network_io,file_io,core

* clang-tidy working

* format

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham
2024-04-12 17:35:36 +02:00
committed by GitHub
parent eb7108b837
commit 9dfd388927
44 changed files with 1055 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
file_io/include/aare/file_io/File.hpp
View File

@ -20,9 +20,9 @@ class File {
* @param cfg file configuration
* @throws std::runtime_error if the file cannot be opened
* @throws std::invalid_argument if the file mode is not supported
*
*/
File(std::filesystem::path fname, std::string mode, FileConfig cfg = {});
*
*/
File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg = {});
void write(Frame &frame);
Frame read();
Frame iread(size_t frame_number);
@ -31,23 +31,23 @@ class File {
void read_into(std::byte *image_buf, size_t n_frames);
size_t frame_number(size_t frame_index);
size_t bytes_per_frame();
size_t pixels();
size_t pixels_per_frame();
void seek(size_t frame_number);
size_t tell() const;
size_t total_frames() const;
ssize_t rows() const;
ssize_t cols() const;
ssize_t bitdepth() const;
size_t rows() const;
size_t cols() const;
size_t bitdepth() const;
/**
* @brief Move constructor
* @param other File object to move from
*/
File(File &&other);
*/
File(File &&other) noexcept;
/**
* @brief destructor: will only delete the FileInterface object
*/
*/
~File();
};

36
file_io/include/aare/file_io/FileInterface.hpp
View File

@ -16,8 +16,8 @@ namespace aare {
*/
struct FileConfig {
aare::DType dtype = aare::DType(typeid(uint16_t));
uint64_t rows;
uint64_t cols;
uint64_t rows{};
uint64_t cols{};
xy geometry{1, 1};
bool operator==(const FileConfig &other) const {
return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry;
@ -92,7 +92,7 @@ class FileInterface {
* @brief get the number of pixels in one frame
* @return number of pixels in one frame
*/
virtual size_t pixels() = 0;
virtual size_t pixels_per_frame() = 0;
/**
* @brief seek to the given frame number
@ -116,17 +116,17 @@ class FileInterface {
* @brief get the number of rows in the file
* @return number of rows in the file
*/
virtual ssize_t rows() const = 0;
virtual size_t rows() const = 0;
/**
* @brief get the number of columns in the file
* @return number of columns in the file
*/
virtual ssize_t cols() const = 0;
virtual size_t cols() const = 0;
/**
* @brief get the bitdepth of the file
* @return bitdepth of the file
*/
virtual ssize_t bitdepth() const = 0;
virtual size_t bitdepth() const = 0;
/**
* @brief read one frame from the file at the given frame number
@ -158,23 +158,21 @@ class FileInterface {
// function to query the data type of the file
/*virtual DataType dtype = 0; */
virtual ~FileInterface(){
};
virtual ~FileInterface() = default;
protected:
std::string m_mode;
std::filesystem::path m_fname;
std::filesystem::path m_base_path;
std::string m_base_name, m_ext;
int m_findex;
std::string m_mode{};
std::filesystem::path m_fname{};
std::filesystem::path m_base_path{};
std::string m_base_name{}, m_ext{};
int m_findex{};
size_t m_total_frames{};
size_t max_frames_per_file{};
std::string version;
DetectorType m_type;
ssize_t m_rows{};
ssize_t m_cols{};
ssize_t m_bitdepth{};
std::string version{};
DetectorType m_type{};
size_t m_rows{};
size_t m_cols{};
size_t m_bitdepth{};
size_t current_frame{};
};

26
file_io/include/aare/file_io/NumpyFile.hpp
View File

@ -3,6 +3,7 @@
#include "aare/core/defs.hpp"
#include "aare/file_io/FileInterface.hpp"
#include "aare/file_io/NumpyHelpers.hpp"
#include "aare/utils/logger.hpp"
#include <filesystem>
#include <iostream>
#include <numeric>
@ -24,7 +25,7 @@ class NumpyFile : public FileInterface {
* @param mode file mode (r, w)
* @param cfg file configuration
*/
NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
void write(Frame &frame) override;
Frame read() override { return get_frame(this->current_frame++); }
@ -34,13 +35,13 @@ class NumpyFile : public FileInterface {
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override { return frame_index; };
size_t bytes_per_frame() override;
size_t pixels() override;
size_t pixels_per_frame() override;
void seek(size_t frame_number) override { this->current_frame = frame_number; }
size_t tell() override { return this->current_frame; }
size_t total_frames() const override { return m_header.shape[0]; }
ssize_t rows() const override { return m_header.shape[1]; }
ssize_t cols() const override { return m_header.shape[2]; }
ssize_t bitdepth() const override { return m_header.dtype.bitdepth(); }
size_t rows() const override { return m_header.shape[1]; }
size_t cols() const override { return m_header.shape[2]; }
size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
/**
* @brief get the data type of the numpy file
@ -62,12 +63,17 @@ class NumpyFile : public FileInterface {
*/
template <typename T, size_t NDim> NDArray<T, NDim> load() {
NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
fseek(fp, header_size, SEEK_SET);
fread(arr.data(), sizeof(T), arr.size(), fp);
if (fseek(fp, static_cast<int64_t>(header_size), SEEK_SET)) {
throw std::runtime_error(LOCATION + "Error seeking to the start of the data");
}
size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp);
if (rc != static_cast<size_t>(arr.size())) {
throw std::runtime_error(LOCATION + "Error reading data from file");
}
return arr;
}
~NumpyFile();
~NumpyFile() noexcept override;
private:
FILE *fp = nullptr;
@ -79,9 +85,11 @@ class NumpyFile : public FileInterface {
NumpyHeader m_header;
uint8_t major_ver_{};
uint8_t minor_ver_{};
size_t m_bytes_per_frame{};
size_t m_pixels_per_frame{};
void load_metadata();
void get_frame_into(size_t, std::byte *);
void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
Frame get_frame(size_t frame_number);
};

2
file_io/include/aare/file_io/NumpyHelpers.hpp
View File

@ -51,7 +51,7 @@ template <typename T, size_t N> bool in_array(T val, const std::array<T, N> &arr
bool is_digits(const std::string &str);
aare::DType parse_descr(std::string typestring);
size_t write_header(std::filesystem::path fname, const NumpyHeader &header);
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header);
size_t write_header(std::ostream &out, const NumpyHeader &header);
} // namespace NumpyHelpers

33
file_io/include/aare/file_io/RawFile.hpp
View File

@ -18,13 +18,13 @@ class RawFile : public FileInterface {
* @param mode file mode (r, w)
* @param cfg file configuration
*/
RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &cfg = {});
explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &config = {});
/**
* @brief write function is not implemented for RawFile
* @param frame frame to write
*/
void write(Frame &frame) override { throw std::runtime_error("Not implemented"); };
void write(Frame & /*frame*/) override { throw std::runtime_error("Not implemented"); };
Frame read() override { return get_frame(this->current_frame++); };
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
@ -41,7 +41,7 @@ class RawFile : public FileInterface {
* @brief get the number of pixels in the frame
* @return number of pixels
*/
size_t pixels() override { return m_rows * m_cols; }
size_t pixels_per_frame() override { return m_rows * m_cols; }
// goto frame number
void seek(size_t frame_number) override { this->current_frame = frame_number; };
@ -53,7 +53,7 @@ class RawFile : public FileInterface {
* @brief check if the file is a master file
* @param fpath path to the file
*/
static bool is_master_file(std::filesystem::path fpath);
static bool is_master_file(const std::filesystem::path &fpath);
/**
* @brief set the module gap row and column
@ -83,17 +83,17 @@ class RawFile : public FileInterface {
* @param file_id file id
* @return path to the data file
*/
inline std::filesystem::path data_fname(int mod_id, int file_id);
inline std::filesystem::path data_fname(size_t mod_id, size_t file_id);
/**
* @brief destructor: will delete the subfiles
*/
~RawFile();
~RawFile() override;
size_t total_frames() const override { return m_total_frames; }
ssize_t rows() const override { return m_rows; }
ssize_t cols() const override { return m_cols; }
ssize_t bitdepth() const override { return m_bitdepth; }
size_t rows() const override { return m_rows; }
size_t cols() const override { return m_cols; }
size_t bitdepth() const override { return m_bitdepth; }
private:
/**
@ -101,7 +101,7 @@ class RawFile : public FileInterface {
* @param frame_number frame number to read
* @param image_buf buffer to store the frame
*/
void get_frame_into(size_t frame_number, std::byte *image_buf);
void get_frame_into(size_t frame_number, std::byte *frame_buffer);
/**
* @brief get the frame at the given frame number
@ -140,22 +140,21 @@ class RawFile : public FileInterface {
* @param fname path to the data subfile
* @return sls_detector_header
*/
sls_detector_header read_header(const std::filesystem::path &fname);
static sls_detector_header read_header(const std::filesystem::path &fname);
/**
* @brief open the subfiles
*/
void open_subfiles();
private:
size_t n_subfiles;
size_t n_subfile_parts;
size_t n_subfiles{};
size_t n_subfile_parts{};
std::vector<std::vector<SubFile *>> subfiles;
int subfile_rows, subfile_cols;
xy geometry;
size_t subfile_rows{}, subfile_cols{};
xy geometry{};
std::vector<xy> positions;
RawFileConfig cfg{0, 0};
TimingMode timing_mode;
TimingMode timing_mode{};
bool quad{false};
};

18
file_io/include/aare/file_io/SubFile.hpp
View File

@ -45,7 +45,7 @@ class SubFile {
* @param bitdepth bitdepth of the subfile
* @throws std::invalid_argument if the detector,type pair is not supported
*/
SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth);
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth);
/**
* @brief read the subfile into a buffer
@ -74,20 +74,20 @@ class SubFile {
* @param frame_number frame number to read
* @return number of bytes read
*/
size_t get_part(std::byte *buffer, int frame_number);
size_t frame_number(int frame_index);
size_t get_part(std::byte *buffer, size_t frame_number);
size_t frame_number(size_t frame_index);
// TODO: define the inlines as variables and assign them in constructor
inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; }
inline size_t pixels_per_part() { return m_rows * m_cols; }
inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
inline size_t pixels_per_part() const { return m_rows * m_cols; }
protected:
FILE *fp = nullptr;
ssize_t m_bitdepth;
size_t m_bitdepth;
std::filesystem::path m_fname;
ssize_t m_rows{};
ssize_t m_cols{};
ssize_t n_frames{};
size_t m_rows{};
size_t m_cols{};
size_t n_frames{};
int m_sub_file_index_{};
};

15
file_io/src/File.cpp
View File

@ -6,7 +6,7 @@
namespace aare {
File::File(std::filesystem::path fname, std::string mode, FileConfig cfg) {
File::File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
if (mode != "r" && mode != "w" && mode != "a") {
throw std::invalid_argument("Unsupported file mode");
}
@ -36,20 +36,17 @@ void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
size_t File::pixels() { return file_impl->pixels(); }
size_t File::pixels_per_frame() { return file_impl->pixels_per_frame(); }
void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
size_t File::tell() const { return file_impl->tell(); }
ssize_t File::rows() const { return file_impl->rows(); }
ssize_t File::cols() const { return file_impl->cols(); }
ssize_t File::bitdepth() const { return file_impl->bitdepth(); }
size_t File::rows() const { return file_impl->rows(); }
size_t File::cols() const { return file_impl->cols(); }
size_t File::bitdepth() const { return file_impl->bitdepth(); }
File::~File() { delete file_impl; }
Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
File::File(File &&other) {
file_impl = other.file_impl;
other.file_impl = nullptr;
}
File::File(File &&other) noexcept : file_impl(other.file_impl) { other.file_impl = nullptr; }
// write move assignment operator

103
file_io/src/NumpyFile.cpp
View File

@ -26,6 +26,9 @@ NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode
}
initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header);
}
m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>());
m_bytes_per_frame = m_header.dtype.bitdepth() / 8 * m_pixels_per_frame;
}
void NumpyFile::write(Frame &frame) {
@ -33,10 +36,14 @@ void NumpyFile::write(Frame &frame) {
throw std::runtime_error("File not open");
}
if (not(m_mode == "w" or m_mode == "a")) {
throw std::runtime_error("File not open for writing");
throw std::invalid_argument("File not open for writing");
}
if (fseek(fp, 0, SEEK_END))
throw std::runtime_error("Could not seek to end of file");
size_t const rc = fwrite(frame.data(), frame.size(), 1, fp);
if (rc != 1) {
throw std::runtime_error("Error writing frame to file");
}
fseek(fp, 0, SEEK_END);
fwrite(frame.data(), frame.size(), 1, fp);
}
Frame NumpyFile::get_frame(size_t frame_number) {
@ -49,16 +56,19 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
throw std::runtime_error("File not open");
}
if (frame_number > m_header.shape[0]) {
throw std::runtime_error("Frame number out of range");
throw std::invalid_argument("Frame number out of range");
}
if (fseek(fp, header_size + frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT
throw std::runtime_error("Could not seek to frame");
size_t const rc = fread(image_buf, m_bytes_per_frame, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading frame from file");
}
fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET);
fread(image_buf, bytes_per_frame(), 1, fp);
}
size_t NumpyFile::pixels() {
return std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<uint64_t>());
};
size_t NumpyFile::bytes_per_frame() { return m_header.dtype.bitdepth() / 8 * pixels(); };
size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; };
size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; };
std::vector<Frame> NumpyFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
@ -73,30 +83,39 @@ void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
get_frame_into(current_frame++, image_buf);
image_buf += bytes_per_frame();
image_buf += m_bytes_per_frame;
}
}
NumpyFile::~NumpyFile() {
NumpyFile::~NumpyFile() noexcept {
if (m_mode == "w" or m_mode == "a") {
// determine number of frames
fseek(fp, 0, SEEK_END);
size_t file_size = ftell(fp);
size_t data_size = file_size - initial_header_len;
size_t n_frames = data_size / bytes_per_frame();
if (fseek(fp, 0, SEEK_END)) {
aare::logger::error("Could not seek to end of file");
}
size_t const file_size = ftell(fp);
size_t const data_size = file_size - initial_header_len;
size_t const n_frames = data_size / m_bytes_per_frame;
// update number of frames in header (first element of shape)
m_header.shape[0] = n_frames;
fseek(fp, 0, SEEK_SET);
if (fseek(fp, 0, SEEK_SET)) {
aare::logger::error("Could not seek to beginning of file");
}
// create string stream to contain header
std::stringstream ss;
aare::NumpyHelpers::write_header(ss, m_header);
std::string header_str = ss.str();
std::string const header_str = ss.str();
// write header
fwrite(header_str.c_str(), header_str.size(), 1, fp);
size_t const rc = fwrite(header_str.c_str(), header_str.size(), 1, fp);
if (rc != 1) {
aare::logger::error("Error writing header to numpy file in destructor");
}
}
if (fp != nullptr) {
fclose(fp);
if (fclose(fp)) {
aare::logger::error("Error closing file");
}
}
}
@ -104,17 +123,23 @@ void NumpyFile::load_metadata() {
// read magic number
std::array<char, 6> tmp{};
fread(tmp.data(), tmp.size(), 1, fp);
size_t rc = fread(tmp.data(), tmp.size(), 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading magic number");
}
if (tmp != aare::NumpyHelpers::magic_str) {
for (auto item : tmp)
fmt::print("{}, ", int(item));
fmt::print("{}, ", static_cast<int>(item));
fmt::print("\n");
throw std::runtime_error("Not a numpy file");
}
// read version
fread(reinterpret_cast<char *>(&major_ver_), sizeof(major_ver_), 1, fp);
fread(reinterpret_cast<char *>(&minor_ver_), sizeof(minor_ver_), 1, fp);
rc = fread(reinterpret_cast<char *>(&major_ver_), sizeof(major_ver_), 1, fp);
rc += fread(reinterpret_cast<char *>(&minor_ver_), sizeof(minor_ver_), 1, fp);
if (rc != 2) {
throw std::runtime_error("Error reading numpy version");
}
if (major_ver_ == 1) {
header_len_size = 2;
@ -125,7 +150,10 @@ void NumpyFile::load_metadata() {
}
// read header length
fread(reinterpret_cast<char *>(&header_len), header_len_size, 1, fp);
rc = fread(reinterpret_cast<char *>(&header_len), header_len_size, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading header length");
}
header_size = aare::NumpyHelpers::magic_string_length + 2 + header_len_size + header_len;
if (header_size % 16 != 0) {
fmt::print("Warning: header length is not a multiple of 16\n");
@ -133,31 +161,34 @@ void NumpyFile::load_metadata() {
// read header
std::string header(header_len, '\0');
fread(header.data(), header_len, 1, fp);
rc = fread(header.data(), header_len, 1, fp);
if (rc != 1) {
throw std::runtime_error("Error reading header");
}
// parse header
std::vector<std::string> keys{"descr", "fortran_order", "shape"};
std::vector<std::string> const keys{"descr", "fortran_order", "shape"};
aare::logger::debug("original header: \"header\"");
auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
if (dict_map.size() == 0)
if (dict_map.empty())
throw std::runtime_error("invalid dictionary in header");
std::string descr_s = dict_map["descr"];
std::string fortran_s = dict_map["fortran_order"];
std::string shape_s = dict_map["shape"];
std::string const descr_s = dict_map["descr"];
std::string const fortran_s = dict_map["fortran_order"];
std::string const shape_s = dict_map["shape"];
std::string descr = aare::NumpyHelpers::parse_str(descr_s);
aare::DType dtype = aare::NumpyHelpers::parse_descr(descr);
std::string const descr = aare::NumpyHelpers::parse_str(descr_s);
aare::DType const dtype = aare::NumpyHelpers::parse_descr(descr);
// convert literal Python bool to C++ bool
bool fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);
bool const fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);
// parse the shape tuple
auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
shape_t shape;
for (auto item : shape_v) {
auto dim = static_cast<unsigned long>(std::stoul(item));
for (const auto &item : shape_v) {
auto dim = static_cast<size_t>(std::stoul(item));
shape.push_back(dim);
}
m_header = {dtype, fortran_order, shape};

39
file_io/src/NumpyHelpers.cpp
View File

@ -41,7 +41,7 @@ namespace NumpyHelpers {
std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys) {
std::unordered_map<std::string, std::string> map;
if (keys.size() == 0)
if (keys.empty())
return map;
in = trim(in);
@ -55,12 +55,12 @@ std::unordered_map<std::string, std::string> parse_dict(std::string in, const st
std::vector<std::pair<size_t, std::string>> positions;
for (auto const &key : keys) {
size_t pos = in.find("'" + key + "'");
size_t const pos = in.find("'" + key + "'");
if (pos == std::string::npos)
throw std::runtime_error("Missing '" + key + "' key.");
std::pair<size_t, std::string> position_pair{pos, key};
std::pair<size_t, std::string> const position_pair{pos, key};
positions.push_back(position_pair);
}
@ -69,10 +69,10 @@ std::unordered_map<std::string, std::string> parse_dict(std::string in, const st
for (size_t i = 0; i < positions.size(); ++i) {
std::string raw_value;
size_t begin{positions[i].first};
size_t const begin{positions[i].first};
size_t end{std::string::npos};
std::string key = positions[i].second;
std::string const key = positions[i].second;
if (i + 1 < positions.size())
end = positions[i + 1].first;
@ -104,7 +104,7 @@ aare::DType parse_descr(std::string typestring) {
const char byteorder_c = typestring[0];
const char kind_c = typestring[1];
std::string itemsize_s = typestring.substr(2);
std::string const itemsize_s = typestring.substr(2);
if (!in_array(byteorder_c, endian_chars)) {
throw std::runtime_error("invalid typestring (byteorder)");
@ -130,11 +130,11 @@ bool parse_bool(const std::string &in) {
}
std::string get_value_from_map(const std::string &mapstr) {
size_t sep_pos = mapstr.find_first_of(":");
size_t const sep_pos = mapstr.find_first_of(':');
if (sep_pos == std::string::npos)
return "";
std::string tmp = mapstr.substr(sep_pos + 1);
std::string const tmp = mapstr.substr(sep_pos + 1);
return trim(tmp);
}
@ -180,12 +180,12 @@ std::string parse_str(const std::string &in) {
void write_magic(std::ostream &ostream, int version_major, int version_minor) {
ostream.write(magic_str.data(), magic_string_length);
ostream.put(version_major);
ostream.put(version_minor);
ostream.put(static_cast<char>(version_major));
ostream.put(static_cast<char>(version_minor));
}
template <typename T> inline std::string write_tuple(const std::vector<T> &v) {
if (v.size() == 0)
if (v.empty())
return "()";
std::ostringstream ss;
ss.imbue(std::locale("C"));
@ -211,36 +211,35 @@ template <typename T> inline std::string write_tuple(const std::vector<T> &v) {
inline std::string write_boolean(bool b) {
if (b)
return "True";
else
return "False";
return "False";
}
inline std::string write_header_dict(const std::string &descr, bool fortran_order, const shape_t &shape) {
std::string s_fortran_order = write_boolean(fortran_order);
std::string shape_s = write_tuple(shape);
std::string const s_fortran_order = write_boolean(fortran_order);
std::string const shape_s = write_tuple(shape);
return "{'descr': '" + descr + "', 'fortran_order': " + s_fortran_order + ", 'shape': " + shape_s + ", }";
}
size_t write_header(std::filesystem::path fname, const NumpyHeader &header) {
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header) {
std::ofstream out(fname, std::ios::binary | std::ios::out);
return write_header(out, header);
}
size_t write_header(std::ostream &out, const NumpyHeader &header) {
std::string header_dict = write_header_dict(header.dtype.str(), header.fortran_order, header.shape);
std::string const header_dict = write_header_dict(header.dtype.str(), header.fortran_order, header.shape);
size_t length = magic_string_length + 2 + 2 + header_dict.length() + 1;
int version_major = 1;
int version_minor = 0;
if (length >= 255 * 255) {
if (length >= static_cast<size_t>(255) * 255) {
length = magic_string_length + 2 + 4 + header_dict.length() + 1;
version_major = 2;
version_minor = 0;
}
size_t padding_len = 16 - length % 16;
std::string padding(padding_len, ' ');
size_t const padding_len = 16 - length % 16;
std::string const padding(padding_len, ' ');
// write magic
write_magic(out, version_major, version_minor);

44
file_io/src/RawFile.cpp
View File

@ -30,7 +30,7 @@ void RawFile::open_subfiles() {
for (size_t i = 0; i != n_subfiles; ++i) {
auto v = std::vector<SubFile *>(n_subfile_parts);
for (size_t j = 0; j != n_subfile_parts; ++j) {
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, bitdepth());
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
}
subfiles.push_back(v);
}
@ -42,18 +42,18 @@ sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str()));
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
throw std::runtime_error(LOCATION + "Could not read header from file");
if (fclose(fp)) {
throw std::runtime_error(LOCATION + "Could not close file");
}
return h;
}
bool RawFile::is_master_file(std::filesystem::path fpath) {
std::string stem = fpath.stem();
if (stem.find("_master_") != std::string::npos)
return true;
else
return false;
bool RawFile::is_master_file(const std::filesystem::path &fpath) {
std::string const stem = fpath.stem();
return stem.find("_master_") != std::string::npos;
}
void RawFile::find_number_of_subfiles() {
@ -62,7 +62,7 @@ void RawFile::find_number_of_subfiles() {
;
n_subfiles = n_mod;
}
inline std::filesystem::path RawFile::data_fname(int mod_id, int file_id) {
inline std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) {
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
}
@ -86,10 +86,10 @@ void RawFile::find_geometry() {
r++;
c++;
m_rows = r * subfile_rows;
m_cols = c * subfile_cols;
m_rows = (r * subfile_rows);
m_cols = (c * subfile_cols);
m_rows += (r - 1) * cfg.module_gap_row;
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
}
void RawFile::parse_metadata() {
@ -109,7 +109,7 @@ void RawFile::parse_metadata() {
} else {
throw std::runtime_error(LOCATION + "Unsupported file type");
}
n_subfile_parts = geometry.row * geometry.col;
n_subfile_parts = static_cast<size_t>(geometry.row) * geometry.col;
}
void RawFile::parse_json_metadata() {
@ -141,7 +141,7 @@ void RawFile::parse_raw_metadata() {
for (std::string line; std::getline(ifs, line);) {
if (line == "#Frame Header")
break;
auto pos = line.find(":");
auto pos = line.find(':');
auto key_pos = pos;
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
;
@ -183,7 +183,7 @@ void RawFile::parse_fname() {
m_base_path = m_fname.parent_path();
m_base_name = m_fname.stem();
m_ext = m_fname.extension();
auto pos = m_base_name.rfind("_");
auto pos = m_base_name.rfind('_');
m_findex = std::stoi(m_base_name.substr(pos + 1));
pos = m_base_name.find("_master_");
m_base_name.erase(pos);
@ -200,7 +200,7 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
if (frame_number > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_number / this->max_frames_per_file;
size_t const subfile_id = frame_number / this->max_frames_per_file;
// create frame and get its buffer
if (this->geometry.col == 1) {
@ -214,11 +214,11 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
} else {
// create a buffer that will hold a the frame part
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
std::byte *part_buffer = new std::byte[bytes_per_part];
auto *part_buffer = new std::byte[bytes_per_part];
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file);
for (int cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows;
auto icol = (part_idx % this->geometry.col) * this->subfile_cols;
auto dest = (irow * this->m_cols + icol);
@ -252,13 +252,13 @@ size_t RawFile::frame_number(size_t frame_index) {
if (frame_index > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_index / this->max_frames_per_file;
size_t const subfile_id = frame_index / this->max_frames_per_file;
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
}
RawFile::~RawFile() {
for (auto &vec : subfiles) {
for (auto subfile : vec) {
for (auto *subfile : vec) {
delete subfile;
}
}

62
file_io/src/SubFile.cpp
View File

@ -7,12 +7,10 @@
namespace aare {
SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) {
this->m_rows = rows;
this->m_cols = cols;
this->m_fname = fname;
this->m_bitdepth = bitdepth;
this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth)
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) {
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) +
" and bitdepth: " + std::to_string(bitdepth);
@ -21,8 +19,8 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
this->read_impl = read_impl_map.at({detector, bitdepth});
}
size_t SubFile::get_part(std::byte *buffer, int frame_number) {
if (frame_number >= n_frames or frame_number < 0) {
size_t SubFile::get_part(std::byte *buffer, size_t frame_number) {
if (frame_number >= n_frames) {
throw std::runtime_error("Frame number out of range");
}
// TODO: find a way to avoid opening and closing the file for each frame
@ -31,9 +29,11 @@ size_t SubFile::get_part(std::byte *buffer, int frame_number) {
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
}
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, SEEK_SET);
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT
SEEK_SET);
auto ret = (this->*read_impl)(buffer);
fclose(fp);
if (fclose(fp))
throw std::runtime_error(LOCATION + "Could not close file");
return ret;
}
@ -42,24 +42,24 @@ size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this-
template <typename DataType> size_t SubFile::read_impl_reorder(std::byte *buffer) {
std::vector<DataType> tmp(this->pixels_per_part());
size_t rc = fread(reinterpret_cast<char *>(&tmp[0]), this->bytes_per_part(), 1, this->fp);
size_t const rc = fread(reinterpret_cast<char *>(tmp.data()), this->bytes_per_part(), 1, this->fp);
int adc_nr[32] = {300, 325, 350, 375, 300, 325, 350, 375, 200, 225, 250, 275, 200, 225, 250, 275,
100, 125, 150, 175, 100, 125, 150, 175, 0, 25, 50, 75, 0, 25, 50, 75};
int sc_width = 25;
int nadc = 32;
int pixels_per_sc = 5000;
std::array<int, 32> const adc_nr = {300, 325, 350, 375, 300, 325, 350, 375, 200, 225, 250, 275, 200, 225, 250, 275,
100, 125, 150, 175, 100, 125, 150, 175, 0, 25, 50, 75, 0, 25, 50, 75};
int const sc_width = 25;
int const nadc = 32;
int const pixels_per_sc = 5000;
auto dst = reinterpret_cast<DataType *>(buffer);
int pixel = 0;
for (int i = 0; i != pixels_per_sc; ++i) {
for (int i_adc = 0; i_adc != nadc; ++i_adc) {
int col = adc_nr[i_adc] + (i % sc_width);
int row;
int const col = adc_nr[i_adc] + (i % sc_width);
int row = 0;
if ((i_adc / 4) % 2 == 0)
row = 199 - int(i / sc_width);
row = 199 - (i / sc_width);
else
row = 200 + int(i / sc_width);
row = 200 + (i / sc_width);
dst[col + row * 400] = tmp[pixel];
pixel++;
@ -72,15 +72,15 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
// read to temporary buffer
// TODO! benchmark direct reads
std::vector<std::byte> tmp(this->bytes_per_part());
size_t rc = fread(reinterpret_cast<char *>(&tmp[0]), this->bytes_per_part(), 1, this->fp);
size_t const rc = fread(reinterpret_cast<char *>(tmp.data()), this->bytes_per_part(), 1, this->fp);
// copy to place
const size_t start = this->m_cols * (this->m_rows - 1) * sizeof(DataType);
const size_t row_size = this->m_cols * sizeof(DataType);
auto dst = buffer + start;
auto src = &tmp[0];
auto *dst = buffer + start;
auto *src = tmp.data();
for (int i = 0; i != this->m_rows; ++i) {
for (size_t i = 0; i != this->m_rows; ++i) {
std::memcpy(dst, src, row_size);
dst -= row_size;
src += row_size;
@ -89,16 +89,18 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
return rc;
};
size_t SubFile::frame_number(int frame_index) {
size_t SubFile::frame_number(size_t frame_index) {
sls_detector_header h{};
fp = fopen(this->m_fname.c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET);
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str()));
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
throw std::runtime_error(LOCATION + "Could not read header from file");
if (fclose(fp)) {
throw std::runtime_error(LOCATION + "Could not close file");
}
return h.frameNumber;
}