mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2025-06-19 18:47:13 +02:00
write rawfiles (single file) and read rawfiles in order (#66)
* read subfiles with unordered and missing frames * save work debugging * Revert "save work debugging" This reverts commite791992a05. * Revert "read subfiles with unordered and missing frames" This reverts commit1177fd129d. * throw when two frames have different frame numbers * write single part RawFile (working beta) * correct total number of frames in master file * add new mythen file with syncd frames * read frames with same frame number * clang-tidy fixes, formatting, add tests * improve readability in loop * fix failing tests --------- Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham
@ -11,6 +11,7 @@ namespace aare {
|
||||
class File {
|
||||
private:
|
||||
FileInterface *file_impl;
|
||||
bool is_npy = true;
|
||||
|
||||
public:
|
||||
/**
|
||||
@ -22,8 +23,8 @@ class File {
|
||||
* @throws std::invalid_argument if the file mode is not supported
|
||||
*
|
||||
*/
|
||||
File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg = {});
|
||||
void write(Frame &frame);
|
||||
File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {});
|
||||
void write(Frame &frame, sls_detector_header header = {});
|
||||
Frame read();
|
||||
Frame iread(size_t frame_number);
|
||||
std::vector<Frame> read(size_t n_frames);
|
||||
@ -38,6 +39,7 @@ class File {
|
||||
size_t rows() const;
|
||||
size_t cols() const;
|
||||
size_t bitdepth() const;
|
||||
void set_total_frames(size_t total_frames);
|
||||
|
||||
/**
|
||||
* @brief Move constructor
|
||||
|
||||
@ -15,14 +15,21 @@ namespace aare {
|
||||
* geometry: geometry of the file
|
||||
*/
|
||||
struct FileConfig {
|
||||
aare::DType dtype = aare::DType(typeid(uint16_t));
|
||||
aare::DType dtype{typeid(uint16_t)};
|
||||
uint64_t rows{};
|
||||
uint64_t cols{};
|
||||
xy geometry{1, 1};
|
||||
bool operator==(const FileConfig &other) const {
|
||||
return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry;
|
||||
return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry &&
|
||||
detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file;
|
||||
}
|
||||
bool operator!=(const FileConfig &other) const { return !(*this == other); }
|
||||
|
||||
// rawfile specific
|
||||
std::string version{};
|
||||
xy geometry{1, 1};
|
||||
DetectorType detector_type{DetectorType::Unknown};
|
||||
int max_frames_per_file{};
|
||||
size_t total_frames{};
|
||||
};
|
||||
|
||||
/**
|
||||
@ -38,7 +45,7 @@ class FileInterface {
|
||||
* @return void
|
||||
* @throws std::runtime_error if the function is not implemented
|
||||
*/
|
||||
virtual void write(Frame &frame) = 0;
|
||||
// virtual void write(Frame &frame) = 0;
|
||||
|
||||
/**
|
||||
* @brief write a vector of frames to the file
|
||||
@ -160,6 +167,8 @@ class FileInterface {
|
||||
|
||||
virtual ~FileInterface() = default;
|
||||
|
||||
void set_total_frames(size_t total_frames) { m_total_frames = total_frames; }
|
||||
|
||||
protected:
|
||||
std::string m_mode{};
|
||||
std::filesystem::path m_fname{};
|
||||
|
||||
@ -27,7 +27,7 @@ class NumpyFile : public FileInterface {
|
||||
*/
|
||||
explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
|
||||
|
||||
void write(Frame &frame) override;
|
||||
void write(Frame &frame);
|
||||
Frame read() override { return get_frame(this->current_frame++); }
|
||||
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
|
||||
@ -5,6 +5,19 @@
|
||||
|
||||
namespace aare {
|
||||
|
||||
struct ModuleConfig {
|
||||
int module_gap_row{};
|
||||
int module_gap_col{};
|
||||
|
||||
bool operator==(const ModuleConfig &other) const {
|
||||
if (module_gap_col != other.module_gap_col)
|
||||
return false;
|
||||
if (module_gap_row != other.module_gap_row)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief RawFile class to read .raw and .json files
|
||||
* @note derived from FileInterface
|
||||
@ -18,13 +31,14 @@ class RawFile : public FileInterface {
|
||||
* @param mode file mode (r, w)
|
||||
* @param cfg file configuration
|
||||
*/
|
||||
explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", const FileConfig &config = {});
|
||||
explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r",
|
||||
const FileConfig &config = FileConfig{});
|
||||
|
||||
/**
|
||||
* @brief write function is not implemented for RawFile
|
||||
* @param frame frame to write
|
||||
*/
|
||||
void write([[maybe_unused]] Frame &frame) override { throw std::runtime_error("Not implemented"); };
|
||||
void write(Frame &frame, sls_detector_header header);
|
||||
Frame read() override { return get_frame(this->current_frame++); };
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
|
||||
@ -44,7 +58,15 @@ class RawFile : public FileInterface {
|
||||
size_t pixels_per_frame() override { return m_rows * m_cols; }
|
||||
|
||||
// goto frame number
|
||||
void seek(size_t frame_number) override { this->current_frame = frame_number; };
|
||||
void seek(size_t frame_number) override {
|
||||
// check if the frame number is greater than the total frames
|
||||
// if frame_number == total_frames, then the next read will throw an error
|
||||
if (frame_number > this->total_frames()) {
|
||||
throw std::runtime_error(
|
||||
fmt::format("frame number {} is greater than total frames {}", frame_number, m_total_frames));
|
||||
}
|
||||
this->current_frame = frame_number;
|
||||
};
|
||||
|
||||
// return the position of the file pointer (in number of frames)
|
||||
size_t tell() override { return this->current_frame; };
|
||||
@ -88,7 +110,7 @@ class RawFile : public FileInterface {
|
||||
/**
|
||||
* @brief destructor: will delete the subfiles
|
||||
*/
|
||||
~RawFile() override;
|
||||
~RawFile() noexcept override;
|
||||
|
||||
size_t total_frames() const override { return m_total_frames; }
|
||||
size_t rows() const override { return m_rows; }
|
||||
@ -96,19 +118,20 @@ class RawFile : public FileInterface {
|
||||
size_t bitdepth() const override { return m_bitdepth; }
|
||||
|
||||
private:
|
||||
void write_master_file();
|
||||
/**
|
||||
* @brief read the frame at the given frame number into the image buffer
|
||||
* @brief read the frame at the given frame index into the image buffer
|
||||
* @param frame_number frame number to read
|
||||
* @param image_buf buffer to store the frame
|
||||
*/
|
||||
void get_frame_into(size_t frame_number, std::byte *frame_buffer);
|
||||
void get_frame_into(size_t frame_index, std::byte *frame_buffer);
|
||||
|
||||
/**
|
||||
* @brief get the frame at the given frame number
|
||||
* @brief get the frame at the given frame index
|
||||
* @param frame_number frame number to read
|
||||
* @return Frame
|
||||
*/
|
||||
Frame get_frame(size_t frame_number);
|
||||
Frame get_frame(size_t frame_index);
|
||||
|
||||
/**
|
||||
* @brief parse the file name to get the extension, base name and index
|
||||
@ -146,6 +169,7 @@ class RawFile : public FileInterface {
|
||||
* @brief open the subfiles
|
||||
*/
|
||||
void open_subfiles();
|
||||
void parse_config(const FileConfig &config);
|
||||
|
||||
size_t n_subfiles{};
|
||||
size_t n_subfile_parts{};
|
||||
@ -153,7 +177,7 @@ class RawFile : public FileInterface {
|
||||
size_t subfile_rows{}, subfile_cols{};
|
||||
xy geometry{};
|
||||
std::vector<xy> positions;
|
||||
RawFileConfig cfg{0, 0};
|
||||
ModuleConfig cfg{0, 0};
|
||||
TimingMode timing_mode{};
|
||||
bool quad{false};
|
||||
};
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#pragma once
|
||||
#include "aare/core/Frame.hpp"
|
||||
#include "aare/core/defs.hpp"
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
@ -36,6 +37,7 @@ class SubFile {
|
||||
};
|
||||
|
||||
public:
|
||||
size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
|
||||
/**
|
||||
* @brief SubFile constructor
|
||||
* @param fname path to the subfile
|
||||
@ -45,7 +47,8 @@ class SubFile {
|
||||
* @param bitdepth bitdepth of the subfile
|
||||
* @throws std::invalid_argument if the detector,type pair is not supported
|
||||
*/
|
||||
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth);
|
||||
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
|
||||
const std::string &mode = "r");
|
||||
|
||||
/**
|
||||
* @brief read the subfile into a buffer
|
||||
@ -74,19 +77,22 @@ class SubFile {
|
||||
* @param frame_number frame number to read
|
||||
* @return number of bytes read
|
||||
*/
|
||||
size_t get_part(std::byte *buffer, size_t frame_number);
|
||||
size_t get_part(std::byte *buffer, size_t frame_index);
|
||||
size_t frame_number(size_t frame_index);
|
||||
|
||||
// TODO: define the inlines as variables and assign them in constructor
|
||||
inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
|
||||
inline size_t pixels_per_part() const { return m_rows * m_cols; }
|
||||
|
||||
~SubFile();
|
||||
|
||||
protected:
|
||||
FILE *fp = nullptr;
|
||||
size_t m_bitdepth;
|
||||
std::filesystem::path m_fname;
|
||||
size_t m_rows{};
|
||||
size_t m_cols{};
|
||||
std::string m_mode;
|
||||
size_t n_frames{};
|
||||
int m_sub_file_index_{};
|
||||
};
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
|
||||
namespace aare {
|
||||
|
||||
File::File(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
|
||||
File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg) {
|
||||
if (mode != "r" && mode != "w" && mode != "a") {
|
||||
throw std::invalid_argument("Unsupported file mode");
|
||||
}
|
||||
@ -18,6 +18,7 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf
|
||||
if (fname.extension() == ".raw" || fname.extension() == ".json") {
|
||||
aare::logger::debug("Loading raw file");
|
||||
file_impl = new RawFile(fname, mode, cfg);
|
||||
is_npy = false;
|
||||
}
|
||||
// check if extension is numpy
|
||||
else if (fname.extension() == ".npy") {
|
||||
@ -28,7 +29,14 @@ File::File(const std::filesystem::path &fname, const std::string &mode, FileConf
|
||||
}
|
||||
}
|
||||
|
||||
void File::write(Frame &frame) { file_impl->write(frame); }
|
||||
void File::write(Frame &frame, sls_detector_header header) {
|
||||
if (is_npy) {
|
||||
aare::logger::info("ignoring header for npy file");
|
||||
dynamic_cast<NumpyFile *>(file_impl)->write(frame);
|
||||
} else {
|
||||
dynamic_cast<RawFile *>(file_impl)->write(frame, header);
|
||||
}
|
||||
}
|
||||
Frame File::read() { return file_impl->read(); }
|
||||
size_t File::total_frames() const { return file_impl->total_frames(); }
|
||||
std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
|
||||
@ -42,6 +50,7 @@ size_t File::tell() const { return file_impl->tell(); }
|
||||
size_t File::rows() const { return file_impl->rows(); }
|
||||
size_t File::cols() const { return file_impl->cols(); }
|
||||
size_t File::bitdepth() const { return file_impl->bitdepth(); }
|
||||
void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); }
|
||||
File::~File() { delete file_impl; }
|
||||
|
||||
Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include "aare/file_io/RawFile.hpp"
|
||||
#include "aare/core/defs.hpp"
|
||||
#include "aare/utils/json.hpp"
|
||||
#include "aare/utils/logger.hpp"
|
||||
#include <fmt/format.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
@ -9,8 +10,9 @@ using json = nlohmann::json;
|
||||
namespace aare {
|
||||
|
||||
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) {
|
||||
m_mode = mode;
|
||||
m_fname = fname;
|
||||
if (mode == "r") {
|
||||
if (mode == "r" or mode == "r+") {
|
||||
if (config != FileConfig()) {
|
||||
aare::logger::warn(
|
||||
"In read mode it is not necessary to provide a config, the provided config will be ignored");
|
||||
@ -21,17 +23,108 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co
|
||||
find_geometry();
|
||||
open_subfiles();
|
||||
|
||||
} else if (mode == "w" or mode == "w+") {
|
||||
|
||||
if (std::filesystem::exists(fname)) {
|
||||
// handle mode w as w+ (no overrwriting)
|
||||
throw std::runtime_error(LOCATION + "File already exists");
|
||||
}
|
||||
|
||||
parse_config(config);
|
||||
parse_fname();
|
||||
write_master_file();
|
||||
n_subfiles = 1;
|
||||
n_subfile_parts = 1;
|
||||
subfile_cols = m_cols;
|
||||
subfile_rows = m_rows;
|
||||
open_subfiles();
|
||||
|
||||
} else {
|
||||
throw std::runtime_error(LOCATION + "Unsupported mode");
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::parse_config(const FileConfig &config) {
|
||||
m_bitdepth = config.dtype.bitdepth();
|
||||
m_total_frames = config.total_frames;
|
||||
m_rows = config.rows;
|
||||
m_cols = config.cols;
|
||||
m_type = config.detector_type;
|
||||
max_frames_per_file = config.max_frames_per_file;
|
||||
geometry = config.geometry;
|
||||
version = config.version;
|
||||
subfile_rows = config.geometry.row;
|
||||
subfile_cols = config.geometry.col;
|
||||
|
||||
if (geometry != aare::xy{1, 1}) {
|
||||
throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing");
|
||||
}
|
||||
}
|
||||
void RawFile::write_master_file() {
|
||||
if (m_ext != ".json") {
|
||||
throw std::runtime_error(LOCATION + "only json master files are supported for writing");
|
||||
}
|
||||
std::ofstream ofs(master_fname(), std::ios::binary);
|
||||
std::string ss;
|
||||
ss.reserve(1024);
|
||||
ss += "{\n\t";
|
||||
aare::write_str(ss, "Version", version);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Total Frames", m_total_frames);
|
||||
ss += "\n\t";
|
||||
aare::write_str(ss, "Detector Type", toString(m_type));
|
||||
ss += "\n\t";
|
||||
aare::write_str(ss, "Geometry", geometry.to_string());
|
||||
ss += "\n\t";
|
||||
|
||||
uint64_t img_size = (m_cols * m_rows) / (geometry.col * geometry.row);
|
||||
img_size *= m_bitdepth;
|
||||
aare::write_digit(ss, "Image Size in bytes", img_size);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Max Frames Per File", max_frames_per_file);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Dynamic Range", m_bitdepth);
|
||||
ss += "\n\t";
|
||||
const aare::xy pixels = {m_rows / geometry.row, m_cols / geometry.col};
|
||||
aare::write_str(ss, "Pixels", pixels.to_string());
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Number of rows", m_rows);
|
||||
ss += "\n\t";
|
||||
const std::string tmp = "{\n"
|
||||
" \"Frame Number\": \"8 bytes\",\n"
|
||||
" \"Exposure Length\": \"4 bytes\",\n"
|
||||
" \"Packet Number\": \"4 bytes\",\n"
|
||||
" \"Bunch Id\": \"8 bytes\",\n"
|
||||
" \"Timestamp\": \"8 bytes\",\n"
|
||||
" \"Module Id\": \"2 bytes\",\n"
|
||||
" \"Row\": \"2 bytes\",\n"
|
||||
" \"Column\": \"2 bytes\",\n"
|
||||
" \"Reserved\": \"2 bytes\",\n"
|
||||
" \"Debug\": \"4 bytes\",\n"
|
||||
" \"RoundRNumber\": \"2 bytes\",\n"
|
||||
" \"DetType\": \"1 byte\",\n"
|
||||
" \"Version\": \"1 byte\",\n"
|
||||
" \"Packet Mask\": \"64 bytes\"\n"
|
||||
" }";
|
||||
|
||||
ss += "\"Frame Header Format\":" + tmp + "\n";
|
||||
ss += "}";
|
||||
ofs << ss;
|
||||
ofs.close();
|
||||
}
|
||||
|
||||
void RawFile::open_subfiles() {
|
||||
for (size_t i = 0; i != n_subfiles; ++i) {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts);
|
||||
for (size_t j = 0; j != n_subfile_parts; ++j) {
|
||||
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
|
||||
if (m_mode == "r")
|
||||
for (size_t i = 0; i != n_subfiles; ++i) {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts);
|
||||
for (size_t j = 0; j != n_subfile_parts; ++j) {
|
||||
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
|
||||
}
|
||||
subfiles.push_back(v);
|
||||
}
|
||||
else {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts); // only one subfile is implemented
|
||||
v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w");
|
||||
subfiles.push_back(v);
|
||||
}
|
||||
}
|
||||
@ -173,51 +266,97 @@ void RawFile::parse_raw_metadata() {
|
||||
max_frames_per_file = std::stoi(value);
|
||||
} else if (key == "Geometry") {
|
||||
pos = value.find(',');
|
||||
geometry = {std::stoi(value.substr(1, pos)), std::stoi(value.substr(pos + 1))};
|
||||
const size_t x = static_cast<size_t>(std::stoi(value.substr(1, pos)));
|
||||
const size_t y = static_cast<size_t>(std::stoi(value.substr(pos + 1)));
|
||||
|
||||
geometry = {x, y};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::parse_fname() {
|
||||
bool wrong_format = false;
|
||||
m_base_path = m_fname.parent_path();
|
||||
m_base_name = m_fname.stem();
|
||||
m_ext = m_fname.extension();
|
||||
auto pos = m_base_name.rfind('_');
|
||||
m_findex = std::stoi(m_base_name.substr(pos + 1));
|
||||
pos = m_base_name.find("_master_");
|
||||
m_base_name.erase(pos);
|
||||
try {
|
||||
auto pos = m_base_name.rfind('_');
|
||||
m_findex = std::stoi(m_base_name.substr(pos + 1));
|
||||
} catch (const std::invalid_argument &e) {
|
||||
m_findex = 0;
|
||||
wrong_format = true;
|
||||
}
|
||||
auto pos = m_base_name.find("_master_");
|
||||
if (pos != std::string::npos) {
|
||||
m_base_name.erase(pos);
|
||||
wrong_format = true;
|
||||
}
|
||||
if (wrong_format and (m_mode == "w+" or m_mode == "w")) {
|
||||
aare::logger::warn("Master Filename", m_fname, "is not in the correct format");
|
||||
aare::logger::warn("using", master_fname(), "as the master file");
|
||||
}
|
||||
}
|
||||
|
||||
Frame RawFile::get_frame(size_t frame_number) {
|
||||
Frame RawFile::get_frame(size_t frame_index) {
|
||||
auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth);
|
||||
std::byte *frame_buffer = f.data();
|
||||
get_frame_into(frame_number, frame_buffer);
|
||||
get_frame_into(frame_index, frame_buffer);
|
||||
return f;
|
||||
}
|
||||
|
||||
void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
|
||||
if (frame_number > this->m_total_frames) {
|
||||
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
|
||||
if (frame_index > this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
size_t const subfile_id = frame_number / this->max_frames_per_file;
|
||||
// create frame and get its buffer
|
||||
std::vector<size_t> frame_numbers(this->n_subfile_parts);
|
||||
std::vector<size_t> frame_indices(this->n_subfile_parts, frame_index);
|
||||
|
||||
if (n_subfile_parts != 1) {
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto subfile_id = frame_index / this->max_frames_per_file;
|
||||
frame_numbers[part_idx] =
|
||||
this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file);
|
||||
}
|
||||
// 1. if frame number vector is the same break
|
||||
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) !=
|
||||
frame_numbers.end()) {
|
||||
// 2. find the index of the minimum frame number,
|
||||
auto min_frame_idx =
|
||||
std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end()));
|
||||
// 3. increase its index and update its respective frame number
|
||||
frame_indices[min_frame_idx]++;
|
||||
// 4. if we can't increase its index => throw error
|
||||
if (frame_indices[min_frame_idx] >= this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file;
|
||||
frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number(
|
||||
frame_indices[min_frame_idx] % this->max_frames_per_file);
|
||||
}
|
||||
}
|
||||
|
||||
if (this->geometry.col == 1) {
|
||||
// get the part from each subfile and copy it to the frame
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
|
||||
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
|
||||
frame_number % this->max_frames_per_file);
|
||||
corrected_idx % this->max_frames_per_file);
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
// create a buffer that will hold a the frame part
|
||||
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
|
||||
auto *part_buffer = new std::byte[bytes_per_part];
|
||||
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file);
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
|
||||
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file);
|
||||
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
|
||||
auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows;
|
||||
auto icol = (part_idx % this->geometry.col) * this->subfile_cols;
|
||||
@ -231,6 +370,19 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::write(Frame &frame, sls_detector_header header) {
|
||||
if (m_mode == "r") {
|
||||
throw std::runtime_error(LOCATION + "File is open in read mode");
|
||||
}
|
||||
size_t const subfile_id = this->current_frame / this->max_frames_per_file;
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
|
||||
this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header,
|
||||
this->current_frame % this->max_frames_per_file);
|
||||
}
|
||||
this->current_frame++;
|
||||
}
|
||||
|
||||
std::vector<Frame> RawFile::read(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
@ -256,7 +408,17 @@ size_t RawFile::frame_number(size_t frame_index) {
|
||||
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
|
||||
}
|
||||
|
||||
RawFile::~RawFile() {
|
||||
RawFile::~RawFile() noexcept {
|
||||
|
||||
// update master file
|
||||
if (m_mode == "w" or m_mode == "w+" or m_mode == "r+") {
|
||||
try {
|
||||
write_master_file();
|
||||
} catch (...) {
|
||||
aare::logger::warn(LOCATION + "Could not update master file");
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &vec : subfiles) {
|
||||
for (auto *subfile : vec) {
|
||||
delete subfile;
|
||||
|
||||
@ -7,9 +7,9 @@
|
||||
|
||||
namespace aare {
|
||||
|
||||
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth)
|
||||
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols),
|
||||
n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) {
|
||||
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
|
||||
const std::string &mode)
|
||||
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) {
|
||||
|
||||
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
|
||||
auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) +
|
||||
@ -17,25 +17,49 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size
|
||||
throw std::invalid_argument(error_msg);
|
||||
}
|
||||
this->read_impl = read_impl_map.at({detector, bitdepth});
|
||||
if (std::filesystem::exists(fname)) {
|
||||
n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
|
||||
} else {
|
||||
n_frames = 0;
|
||||
}
|
||||
|
||||
if (mode == "r") {
|
||||
fp = fopen(m_fname.c_str(), "rb");
|
||||
} else {
|
||||
// if file exists, open in read/write mode (without truncating the file)
|
||||
// if file does not exist, open in write mode
|
||||
if (std::filesystem::exists(fname)) {
|
||||
fp = fopen(m_fname.c_str(), "r+b");
|
||||
} else {
|
||||
fp = fopen(m_fname.c_str(), "wb");
|
||||
}
|
||||
}
|
||||
if (fp == nullptr) {
|
||||
throw std::runtime_error(LOCATION + "Could not open file for writing");
|
||||
}
|
||||
}
|
||||
|
||||
size_t SubFile::get_part(std::byte *buffer, size_t frame_number) {
|
||||
if (frame_number >= n_frames) {
|
||||
size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
|
||||
if (frame_index >= n_frames) {
|
||||
throw std::runtime_error("Frame number out of range");
|
||||
}
|
||||
// TODO: find a way to avoid opening and closing the file for each frame
|
||||
aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str());
|
||||
fp = fopen(m_fname.c_str(), "rb");
|
||||
if (!fp) {
|
||||
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
|
||||
}
|
||||
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT
|
||||
aare::logger::debug(LOCATION, "frame:", frame_index, "file:", m_fname.c_str());
|
||||
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT
|
||||
SEEK_SET);
|
||||
auto ret = (this->*read_impl)(buffer);
|
||||
if (fclose(fp))
|
||||
throw std::runtime_error(LOCATION + "Could not close file");
|
||||
return ret;
|
||||
}
|
||||
size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) {
|
||||
if (frame_index > n_frames) {
|
||||
throw std::runtime_error("Frame number out of range");
|
||||
}
|
||||
fseek(fp, static_cast<ssize_t>((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET);
|
||||
auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
|
||||
wc += fwrite(buffer, bytes_per_part(), 1, fp);
|
||||
|
||||
return wc;
|
||||
}
|
||||
|
||||
size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_part(), 1, this->fp); }
|
||||
|
||||
@ -91,18 +115,18 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
|
||||
|
||||
size_t SubFile::frame_number(size_t frame_index) {
|
||||
sls_detector_header h{};
|
||||
fp = fopen(this->m_fname.c_str(), "r");
|
||||
if (!fp)
|
||||
throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str()));
|
||||
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
|
||||
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
|
||||
if (rc != 1)
|
||||
throw std::runtime_error(LOCATION + "Could not read header from file");
|
||||
if (fclose(fp)) {
|
||||
throw std::runtime_error(LOCATION + "Could not close file");
|
||||
}
|
||||
|
||||
return h.frameNumber;
|
||||
}
|
||||
|
||||
SubFile::~SubFile() {
|
||||
if (fp) {
|
||||
fclose(fp);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace aare
|
||||
@ -79,4 +79,36 @@ TEST_CASE("Compare reading from a numpy file with a raw file") {
|
||||
auto npy_frame = npy.read();
|
||||
CHECK(raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read multipart files") {
|
||||
auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
File f(fpath, "r");
|
||||
|
||||
// we know this file has 10 frames check read_multiport.py for the values
|
||||
std::vector<uint16_t> pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116};
|
||||
std::vector<uint16_t> pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833};
|
||||
std::vector<uint16_t> pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142};
|
||||
std::vector<uint16_t> pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171};
|
||||
std::vector<uint16_t> pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598};
|
||||
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
auto frame = f.read();
|
||||
CHECK(frame.rows() == 512);
|
||||
CHECK(frame.cols() == 1024);
|
||||
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
|
||||
CHECK(frame.view<uint16_t>()(0, 1) == pixel_0_1[i]);
|
||||
CHECK(frame.view<uint16_t>()(1, 0) == pixel_1_0[i]);
|
||||
CHECK(frame.view<uint16_t>()(255, 1023) == pixel_255_1023[i]);
|
||||
CHECK(frame.view<uint16_t>()(511, 1023) == pixel_511_1023[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read file with unordered frames") {
|
||||
auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
File f(fpath, "r");
|
||||
REQUIRE_THROWS(f.read());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user