detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2026-02-02 20:44:57 +01:00
Code Issues Actions Packages Releases Activity

conform to file interface (PR#4) (#33)

Browse Source 
* use FileInterface with numpy

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham
2024-03-27 16:33:51 +01:00
committed by GitHub
parent dc9fb51a89
commit d07f867630
22 changed files with 385 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
file_io/src/File.cpp
View File

@@ -1,2 +1,34 @@
// #include "aare/File.hpp"
// template class File<DetectorType::Jungfrau, uint16_t>;
#include "aare/File.hpp"
#include "aare/FileFactory.hpp"
#include "aare/utils/logger.hpp"
File::File(std::filesystem::path fname, std::string mode) {
if (mode != "r") {
throw std::runtime_error(LOCATION + " Only read mode is supported");
}
file_impl = FileFactory::load_file(fname);
}
Frame File::read() { return file_impl->read(); }
size_t File::total_frames() const { return file_impl->m_total_frames; }
std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
size_t File::pixels() { return file_impl->pixels(); }
void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
size_t File::tell() { return file_impl->tell(); }
ssize_t File::rows() const { return file_impl->rows(); }
ssize_t File::cols() const { return file_impl->cols(); }
ssize_t File::bitdepth() const { return file_impl->bitdepth(); }
File::~File() {
delete file_impl;
}
File::File(File &&other) {
file_impl = other.file_impl;
other.file_impl = nullptr;
}
// write move assignment operator

7
file_io/src/FileFactory.cpp
View File

@@ -1,8 +1,9 @@
#include "aare/FileFactory.hpp"
#include "aare/File.hpp"
#include "aare/FileInterface.hpp"
#include "aare/RawFileFactory.hpp"
#include "aare/NumpyFileFactory.hpp"
#include "aare/utils/logger.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
@@ -11,6 +12,10 @@ FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
throw std::runtime_error("File does not exist");
}
if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
aare::logger::info("Loading",fpath.extension(),"file");
return new RawFileFactory(fpath);
}
if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
aare::logger::info("Loading",fpath.extension(),"file");
return new RawFileFactory(fpath);

39
file_io/src/NumpyFile.cpp
View File

@@ -1,21 +1,48 @@
#include "aare/NumpyFile.hpp"
NumpyFile::NumpyFile(std::filesystem::path fname_){
this->fname = fname_;
fp = fopen(this->fname.c_str(), "rb");
NumpyFile::NumpyFile(std::filesystem::path fname_) {
this->m_fname = fname_;
fp = fopen(this->m_fname.c_str(), "rb");
}
Frame NumpyFile::get_frame(size_t frame_number) {
Frame frame(header.shape[1], header.shape[2], header.dtype.itemsize*8);
get_frame_into(frame_number,frame._get_data());
return frame;
}
void NumpyFile::get_frame_into( size_t frame_number,std::byte* image_buf) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
}
if (frame_number > header.shape[0]) {
throw std::runtime_error("Frame number out of range");
}
Frame frame = Frame(header.shape[1], header.shape[2], header.dtype.itemsize*8);
fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET);
fread(frame._get_data(), bytes_per_frame(), 1, fp);
return frame;
fread(image_buf, bytes_per_frame(), 1, fp);
}
size_t NumpyFile::pixels() {
return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies<uint64_t>());
};
size_t NumpyFile::bytes_per_frame() { return header.dtype.itemsize * pixels(); };
std::vector<Frame> NumpyFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
}
return frames;
}
void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
}
}

6
file_io/src/NumpyFileFactory.cpp
View File

@@ -3,13 +3,13 @@
NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) {
this->m_fpath = fpath;
}
void NumpyFileFactory::parse_metadata(File *_file) {
void NumpyFileFactory::parse_metadata(FileInterface *_file) {
auto file = dynamic_cast<NumpyFile*>(_file);
// open ifsteam to file
f = std::ifstream(file->fname, std::ios::binary);
f = std::ifstream(file->m_fname, std::ios::binary);
// check if file exists
if (!f.is_open()) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", file->fname.c_str()));
throw std::runtime_error(fmt::format("Could not open: {} for reading", file->m_fname.c_str()));
}
// read magic number
std::array<char, 6> tmp{};

53
file_io/src/RawFile.cpp
View File

@@ -2,24 +2,30 @@
#include "aare/utils/logger.hpp"
Frame RawFile::get_frame(size_t frame_number) {
if (frame_number > this->total_frames) {
throw std::runtime_error("Frame number out of range");
auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth);
std::byte *frame_buffer = f._get_data();
get_frame_into(frame_number, frame_buffer);
return f;
}
void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
if (frame_number > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_number / this->max_frames_per_file;
// create frame and get its buffer
auto f = Frame(this->rows, this->cols, this->bitdepth);
std::byte *frame_buffer = f._get_data();
if (this->geometry.col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx*part_offset, frame_number % this->max_frames_per_file);
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
frame_number % this->max_frames_per_file);
}
} else {
// create a buffer that will hold a the frame part
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->bitdepth / 8;
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
std::byte *part_buffer = new std::byte[bytes_per_part];
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
@@ -27,16 +33,41 @@ Frame RawFile::get_frame(size_t frame_number) {
for (int cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows;
auto icol = (part_idx % this->geometry.col) * this->subfile_cols;
auto dest = (irow * this->cols + icol);
dest = dest * this->bitdepth / 8;
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->bitdepth / 8,
this->subfile_cols * this->bitdepth / 8);
auto dest = (irow * this->m_cols + icol);
dest = dest * this->m_bitdepth / 8;
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
this->subfile_cols * this->m_bitdepth / 8);
}
}
delete[] part_buffer;
}
}
return f;
std::vector<Frame> RawFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
}
return frames;
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
}
}
size_t RawFile::frame_number(size_t frame_index) {
if (frame_index > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_index / this->max_frames_per_file;
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
}
RawFile::~RawFile() {

61
file_io/src/RawFileFactory.cpp
View File

@@ -17,21 +17,21 @@ RawFileFactory::RawFileFactory(std::filesystem::path fpath) {
this->m_fpath = fpath;
}
void RawFileFactory::parse_metadata(File *_file) {
void RawFileFactory::parse_metadata(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
if (file->ext == ".raw") {
if (file->m_ext == ".raw") {
this->parse_raw_metadata(file);
if (file->bitdepth == 0) {
switch (file->type) {
if (file->m_bitdepth == 0) {
switch (file->m_type) {
case DetectorType::Eiger:
file->bitdepth = 32;
file->m_bitdepth = 32;
break;
default:
file->bitdepth = 16;
file->m_bitdepth = 16;
}
}
} else if (file->ext == ".json") {
} else if (file->m_ext == ".json") {
this->parse_json_metadata(file);
} else {
throw std::runtime_error("Unsupported file type");
@@ -57,7 +57,7 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) {
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
file->type = StringTo<DetectorType>(value);
file->m_type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
file->timing_mode = StringTo<TimingMode>(value);
} else if (key == "Pixels") {
@@ -67,9 +67,9 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) {
file->subfile_cols = std::stoi(value.substr(1, pos));
file->subfile_rows = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
file->total_frames = std::stoi(value);
file->m_total_frames = std::stoi(value);
} else if (key == "Dynamic Range") {
file->bitdepth = std::stoi(value);
file->m_bitdepth = std::stoi(value);
} else if (key == "Quad") {
file->quad = (value == "1");
} else if (key == "Max Frames Per File") {
@@ -89,32 +89,32 @@ void RawFileFactory::parse_json_metadata(RawFile *file) {
double v = j["Version"];
std::cout << "Version: " << v << std::endl;
file->version = fmt::format("{:.1f}", v);
file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file->total_frames = j["Frames in File"];
file->m_total_frames = j["Frames in File"];
file->subfile_rows = j["Pixels"]["y"];
file->subfile_cols = j["Pixels"]["x"];
file->max_frames_per_file = j["Max Frames Per File"];
try {
file->bitdepth = j.at("Dynamic Range");
file->m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
file->bitdepth = 16;
file->m_bitdepth = 16;
}
// only Eiger had quad
if (file->type == DetectorType::Eiger) {
if (file->m_type == DetectorType::Eiger) {
file->quad = (j["Quad"] == 1);
}
file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawFileFactory::open_subfiles(File *_file) {
void RawFileFactory::open_subfiles(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
for (size_t i = 0; i != file->n_subfiles; ++i) {
auto v = std::vector<SubFile *>(file->n_subfile_parts);
for (size_t j = 0; j != file->n_subfile_parts; ++j) {
v[j] =
new SubFile(file->data_fname(i, j), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth);
new SubFile(file->data_fname(i, j), file->m_type, file->subfile_rows, file->subfile_cols, file->bitdepth());
}
file->subfiles.push_back(v);
}
@@ -122,7 +122,7 @@ void RawFileFactory::open_subfiles(File *_file) {
RawFile *RawFileFactory::load_file() {
RawFile *file = new RawFile();
file->fname = this->m_fpath;
file->m_fname = this->m_fpath;
this->parse_fname(file);
this->parse_metadata(file);
file->find_number_of_subfiles();
@@ -146,7 +146,8 @@ sls_detector_header RawFileFactory::read_header(const std::filesystem::path &fna
return h;
}
void RawFileFactory::find_geometry(File *_file) {
void RawFileFactory::find_geometry(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
uint16_t r{};
uint16_t c{};
@@ -163,20 +164,20 @@ void RawFileFactory::find_geometry(File *_file) {
r++;
c++;
file->rows = r * file->subfile_rows;
file->cols = c * file->subfile_cols;
file->m_rows = r * file->subfile_rows;
file->m_cols = c * file->subfile_cols;
file->rows += (r - 1) * file->cfg.module_gap_row;
file->m_rows += (r - 1) * file->cfg.module_gap_row;
}
void RawFileFactory::parse_fname(File *file) {
void RawFileFactory::parse_fname(FileInterface *file) {
file->base_path = this->m_fpath.parent_path();
file->base_name = this->m_fpath.stem();
file->ext = this->m_fpath.extension();
file->m_base_path = this->m_fpath.parent_path();
file->m_base_name = this->m_fpath.stem();
file->m_ext = this->m_fpath.extension();
auto pos = file->base_name.rfind("_");
file->findex = std::stoi(file->base_name.substr(pos + 1));
pos = file->base_name.find("_master_");
file->base_name.erase(pos);
auto pos = file->m_base_name.rfind("_");
file->m_findex = std::stoi(file->m_base_name.substr(pos + 1));
pos = file->m_base_name.find("_master_");
file->m_base_name.erase(pos);
}

26
file_io/src/SubFile.cpp
View File

@@ -1,10 +1,8 @@
#include "aare/SubFile.hpp"
#include <iostream>
#include "aare/utils/logger.hpp"
#include <iostream>
// #include <filesystem>
SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) {
this->m_rows = rows;
this->m_cols = cols;
@@ -12,19 +10,17 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
this->m_bitdepth = bitdepth;
this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
throw std::runtime_error(LOCATION+"Unsupported detector/bitdepth combination");
throw std::runtime_error(LOCATION + "Unsupported detector/bitdepth combination");
}
this->read_impl = read_impl_map.at({detector, bitdepth});
this->read_impl = read_impl_map.at({detector, bitdepth});
}
size_t SubFile::get_part(std::byte *buffer, int frame_number) {
if (frame_number >= n_frames or frame_number < 0) {
throw std::runtime_error("Frame number out of range");
}
// TODO: find a way to avoid opening and closing the file for each frame
aare::logger::debug(LOCATION,"frame:", frame_number, "file:", m_fname.c_str());
aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str());
fp = fopen(m_fname.c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
@@ -86,3 +82,17 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
return rc;
};
size_t SubFile::frame_number(int frame_index) {
sls_detector_header h{};
FILE *fp = fopen(this->m_fname.c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
return h.frameNumber;
}