detectors/aare
25
0
Fork 0
mirror of https://github.com/slsdetectorgroup/aare.git synced 2025-06-05 12:30:39 +02:00
Code Issues Actions Packages Releases Activity

conform to file interface (PR#4) (#33)

Browse Source 
* use FileInterface with numpy

---------

Co-authored-by: Bechir <bechir.brahem420@gmail.com>
This commit is contained in:
Bechir Braham 2024-03-27 16:33:51 +01:00 committed by GitHub
parent dc9fb51a89
commit d07f867630
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 385 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
examples/json_example.cpp
View File

@ -1,13 +1,13 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include <iostream>
#include "aare/File.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File *f, int frame_number) {
void test(File& f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(1, 0)) << std::endl;
@ -15,13 +15,11 @@ void test(File *f, int frame_number) {
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"jungfrau"/ "jungfrau_single_master_0.json");
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau" / "jungfrau_single_master_0.json");
std::cout << fpath << std::endl;
File *file = ctx_manager.get_file(fpath);
File file(fpath, "r");
test(file, 0);
test(file, 2);
test(file, 9);

11
examples/multiport_example.cpp
View File

@ -1,26 +1,27 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include "aare/File.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File *f, int frame_number) {
void test(File &f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(255, 1023)) << std::endl;
std::cout << *((uint16_t *)frame.get(511, 1023)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau" / "jungfrau_double_master_0.json");
std::cout << fpath << std::endl;
File *file = ctx_manager.get_file(fpath);
File file(fpath, "r");
test(file, 0);
test(file, 9);
}

23
examples/mythen_example.cpp
View File

@ -1,13 +1,13 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include "aare/File.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test1(File *f, int frame_number) {
void test1(File &f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 3839)) << std::endl;
@ -20,29 +20,24 @@ void test1(File *f, int frame_number) {
}
}
void test2(File *f, int frame_number) {
void test2(File &f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint32_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1280*4 -1)) << std::endl;
std::cout << *((uint32_t *)frame.get(0, 1280 * 4 - 1)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
if (PROJECT_ROOT_DIR.empty()) {
throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set");
}
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "mythen" / "m3_master_0.json");
File *file = ctx_manager.get_file(fpath);
File file(fpath, "r");
test1(file, 0);
fpath = (PROJECT_ROOT_DIR / "data" / "mythen" / "scan242_master_3.raw");
file = ctx_manager.get_file(fpath);
test2(file, 0);
File file2(fpath, "r");
test2(file2, 0);
}

9
examples/numpy_example.cpp
View File

@ -1,12 +1,12 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include "aare/File.hpp"
#include <iostream>
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File *f, int frame_number) {
void test(File& f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(1, 0)) << std::endl;
@ -14,13 +14,12 @@ void test(File *f, int frame_number) {
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "numpy" / "test_numpy_file.npy");
std::cout << fpath << std::endl;
File *file = ctx_manager.get_file(fpath);
File file(fpath,"r");
test(file, 0);
test(file, 2);
test(file, 24);

9
examples/raw_example.cpp
View File

@ -1,26 +1,25 @@
// Your First C++ Program
#include "aare/ContextManager.hpp"
#include "aare/File.hpp"
#include <iostream>
#include "aare/utils/logger.hpp"
#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR"
void test(File *f, int frame_number) {
void test(File& f, int frame_number) {
std::cout << "frame number: " << frame_number << std::endl;
Frame frame = f->get_frame(frame_number);
Frame frame = f.iread(frame_number);
std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 1)) << std::endl;
std::cout << *((uint16_t *)frame.get(0, 95)) << std::endl;
}
int main() {
ContextManager ctx_manager;
auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR));
if (PROJECT_ROOT_DIR.empty()) {
throw std::runtime_error("environment variable PROJECT_ROOT_DIR is not set");
}
std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" /"moench"/ "moench04_noise_200V_sto_both_100us_no_light_thresh_900_master_0.raw");
File *file = ctx_manager.get_file(fpath);
File file(fpath, "r");
test(file, 0);
test(file, 2);
test(file, 99);

34
file_io/include/aare/ContextManager.hpp
View File

@ -1,34 +0,0 @@
#include "aare/File.hpp"
#include "aare/FileFactory.hpp"
#include <filesystem>
#include <memory>
/**
* @brief A class to manage the context of the files and network connections
* closes the files and network connections when the context is destroyed
*/
class ContextManager {
public:
ContextManager() = default;
File *get_file(std::filesystem::path fname) {
auto tmp_file = FileFactory::load_file(fname);
this->files.push_back(tmp_file);
return tmp_file;
}
// prevent default copying, it can delete the file twice
ContextManager(const ContextManager &) = delete;
ContextManager &operator=(const ContextManager &) = delete;
~ContextManager() {
for (auto f : files) {
delete f;
}
}
private:
std::vector<File *> files;
// std::vector<NetworkConnection*> connections;
};

61
file_io/include/aare/File.hpp
View File

@ -1,41 +1,30 @@
#pragma once
#include "SubFile.hpp"
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include <filesystem>
#include <fmt/core.h>
#include <iostream>
class File {
public:
virtual Frame get_frame(size_t frame_number) =0;
#include "aare/FileInterface.hpp"
class File : public FileInterface {
private:
// comment
FileInterface *file_impl;
public:
virtual ~File() = default;
std::filesystem::path fname;
std::filesystem::path base_path;
std::string base_name, ext;
int findex;
size_t total_frames{};
size_t max_frames_per_file{};
// options:
// - r reading
// - w writing (overwrites existing file)
// - a appending (appends to existing file)
// TODO! do we need to support w+, r+ and a+?
File(std::filesystem::path fname, std::string mode);
Frame read() override;
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override;
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
size_t bytes_per_frame() override;
size_t pixels() override;
void seek(size_t frame_number) override;
size_t tell() override;
size_t total_frames() const override ;
ssize_t rows() const override ;
ssize_t cols() const override ;
ssize_t bitdepth() const override ;
File(File &&other);
std::string version;
DetectorType type;
TimingMode timing_mode;
bool quad{false};
ssize_t rows{};
ssize_t cols{};
ssize_t bitdepth{};
// File();
inline size_t bytes_per_frame() const { return rows * cols * bitdepth / 8; }
inline size_t pixels() const { return rows * cols; }
// size_t total_frames();
};
~File();
};

14
file_io/include/aare/FileFactory.hpp
View File

@ -1,23 +1,23 @@
#pragma once
#include <filesystem>
#include "aare/File.hpp"
#include "aare/FileInterface.hpp"
class FileFactory{
// Class that will be used to create File objects
// Class that will be used to create FileInterface objects
// follows the factory pattern
protected:
std::filesystem::path m_fpath;
public:
static FileFactory* get_factory(std::filesystem::path);
// virtual int deleteFile() = 0;
static File* load_file(std::filesystem::path p){
static FileInterface* load_file(std::filesystem::path p){
auto factory = get_factory(p);
File* tmp= factory->load_file();
FileInterface* tmp= factory->load_file();
delete factory;
return tmp;
};
virtual File* load_file()=0;//TODO: add option to load all file to memory or keep it on disk
virtual void parse_metadata(File*)=0;
virtual void parse_fname(File*)=0;
virtual FileInterface* load_file()=0;//TODO: add option to load all file to memory or keep it on disk
virtual void parse_metadata(FileInterface*)=0;
virtual void parse_fname(FileInterface*)=0;
virtual ~FileFactory() = default;

85
file_io/include/aare/FileInterface.hpp Normal file
View File

@ -0,0 +1,85 @@
#pragma once
#include "aare/Frame.hpp"
#include "aare/defs.hpp"
#include "aare/utils/logger.hpp"
#include <filesystem>
#include <vector>
class FileInterface {
public:
friend class FileFactory;
// write one frame
// virtual void write(Frame &frame) = 0;
// write n_frames frames
// virtual void write(std::vector<Frame> &frames) = 0;
// read one frame
virtual Frame read() = 0;
// read n_frames frames
virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
// read one frame into the provided buffer
virtual void read_into(std::byte *image_buf) = 0;
// read n_frames frame into the provided buffer
virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
// read the frame number on position frame_index
virtual size_t frame_number(size_t frame_index) = 0;
// size of one frame, important fro teh read_into function
virtual size_t bytes_per_frame() = 0;
// number of pixels in one frame
virtual size_t pixels() = 0;
// goto frame number
virtual void seek(size_t frame_number) = 0;
// return the position of the file pointer (in number of frames)
virtual size_t tell() = 0;
// Getter functions
virtual size_t total_frames() const = 0;
virtual ssize_t rows() const = 0;
virtual ssize_t cols() const = 0;
virtual ssize_t bitdepth() const = 0;
// read one frame at position frame_number
Frame iread(size_t frame_number) {
auto old_pos = tell();
seek(frame_number);
Frame tmp = read();
seek(old_pos);
return tmp;
};
// read n_frames frames starting at frame_number
std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
auto old_pos = tell();
seek(frame_number);
std::vector<Frame> tmp = read(n_frames);
seek(old_pos);
return tmp;
}
// function to query the data type of the file
/*virtual DataType dtype = 0; */
virtual ~FileInterface(){
};
public:
std::filesystem::path m_fname;
std::filesystem::path m_base_path;
std::string m_base_name, m_ext;
int m_findex;
size_t m_total_frames{};
size_t max_frames_per_file{};
std::string version;
DetectorType m_type;
ssize_t m_rows{};
ssize_t m_cols{};
ssize_t m_bitdepth{};
};

38
file_io/include/aare/NumpyFile.hpp
View File

@ -1,17 +1,30 @@
#pragma once
#include "aare/File.hpp"
#include "aare/defs.hpp"
#include "aare/FileInterface.hpp"
#include "aare/NumpyHelpers.hpp"
#include "aare/defs.hpp"
#include <iostream>
#include <numeric>
class NumpyFile : public File {
class NumpyFile : public FileInterface {
FILE *fp = nullptr;
public:
Frame read() override { return get_frame(this->current_frame++); }
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override { return frame_index; };
size_t bytes_per_frame() override;
size_t pixels() override;
void seek(size_t frame_number) override { this->current_frame = frame_number; }
size_t tell() override { return this->current_frame; }
size_t total_frames() const override { return header.shape[0]; }
ssize_t rows()const override { return header.shape[1]; }
ssize_t cols()const override { return header.shape[2]; }
ssize_t bitdepth()const override { return header.dtype.itemsize; }
NumpyFile(std::filesystem::path fname);
Frame get_frame(size_t frame_number) override;
header_t header{};
static constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
uint8_t major_ver_{};
@ -20,14 +33,15 @@ class NumpyFile : public File {
uint8_t header_len_size{};
const uint8_t magic_string_length{6};
ssize_t header_size{};
inline ssize_t pixels_per_frame() {
return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies<uint64_t>());
};
inline ssize_t bytes_per_frame() { return header.dtype.itemsize * pixels_per_frame(); };
~NumpyFile(){
~NumpyFile() {
if (fp != nullptr) {
fclose(fp);
}
}
private:
size_t current_frame{};
void get_frame_into(size_t, std::byte *);
Frame get_frame(size_t frame_number);
};

6
file_io/include/aare/NumpyFileFactory.hpp
View File

@ -9,12 +9,12 @@
class NumpyFileFactory : public FileFactory {
private:
std::ifstream f;
void read_data(File *_file);
void read_data(FileInterface *_file);
public:
NumpyFileFactory(std::filesystem::path fpath);
void parse_metadata(File *_file) override;
void parse_metadata(FileInterface *_file) override;
NumpyFile* load_file() override;
void parse_fname(File*){};
void parse_fname(FileInterface*){};
};

45
file_io/include/aare/RawFile.hpp
View File

@ -1,15 +1,32 @@
#pragma once
#include "aare/defs.hpp"
#include "aare/FileInterface.hpp"
#include "aare/Frame.hpp"
#include "aare/File.hpp"
#include "aare/SubFile.hpp"
#include "aare/defs.hpp"
class RawFile : public FileInterface {
class RawFile : public File {
using config = RawFileConfig;
using config = RawFileConfig;
public:
public:
Frame read() override { return get_frame(this->current_frame++); };
std::vector<Frame> read(size_t n_frames) override;
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
void read_into(std::byte *image_buf, size_t n_frames) override;
size_t frame_number(size_t frame_index) override;
// size of one frame, important fro teh read_into function
size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
// number of pixels in one frame
size_t pixels() override { return m_rows * m_cols; }
// goto frame number
void seek(size_t frame_number) { this->current_frame = frame_number; };
// return the position of the file pointer (in number of frames)
size_t tell() { return this->current_frame; };
Frame get_frame(size_t frame_number);
size_t n_subfiles;
size_t n_subfile_parts;
std::vector<std::vector<SubFile *>> subfiles;
@ -17,6 +34,8 @@ class RawFile : public File {
xy geometry;
std::vector<xy> positions;
config cfg{0, 0};
TimingMode timing_mode;
bool quad{false};
inline void set_config(int row, int col) {
cfg.module_gap_row = row;
@ -32,11 +51,21 @@ class RawFile : public File {
}
inline std::filesystem::path master_fname() {
return this->base_path / fmt::format("{}_master_{}{}", this->base_name, this->findex, this->ext);
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
}
inline std::filesystem::path data_fname(int mod_id, int file_id) {
return this->base_path / fmt::format("{}_d{}_f{}_{}.raw", this->base_name, file_id, mod_id, this->findex);
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
}
~RawFile();
size_t total_frames() const { return m_total_frames; }
ssize_t rows() const { return m_rows; }
ssize_t cols() const { return m_cols; }
ssize_t bitdepth() const { return m_bitdepth; }
private:
size_t current_frame{};
void get_frame_into(size_t frame_number, std::byte *image_buf);
Frame get_frame(size_t frame_number);
};

8
file_io/include/aare/RawFileFactory.hpp
View File

@ -8,9 +8,9 @@ class RawFileFactory : public FileFactory {
public:
RawFileFactory(std::filesystem::path fpath);
virtual RawFile *load_file() override;
void parse_metadata(File *) override;
void parse_fname(File *) override;
void open_subfiles(File *);
void parse_metadata(FileInterface *) override;
void parse_fname(FileInterface *) override;
void open_subfiles(FileInterface *);
sls_detector_header read_header(const std::filesystem::path &fname);
void find_geometry(File *);
void find_geometry(FileInterface *);
};

1
file_io/include/aare/SubFile.hpp
View File

@ -35,6 +35,7 @@ class SubFile {
SubFile(std::filesystem::path fname,DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth);
size_t get_part(std::byte *buffer, int frame_number);
size_t frame_number(int frame_index);
// TODO: define the inlines as variables and assign them in constructor
inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; }

2
file_io/include/aare/helpers.hpp
View File

@ -1,6 +1,6 @@
#pragma once
#include "aare/File.hpp"
#include "aare/FileInterface.hpp"
#include <filesystem>
#include <fmt/core.h>

36
file_io/src/File.cpp
View File

@ -1,2 +1,34 @@
// #include "aare/File.hpp"
// template class File<DetectorType::Jungfrau, uint16_t>;
#include "aare/File.hpp"
#include "aare/FileFactory.hpp"
#include "aare/utils/logger.hpp"
File::File(std::filesystem::path fname, std::string mode) {
if (mode != "r") {
throw std::runtime_error(LOCATION + " Only read mode is supported");
}
file_impl = FileFactory::load_file(fname);
}
Frame File::read() { return file_impl->read(); }
size_t File::total_frames() const { return file_impl->m_total_frames; }
std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
size_t File::pixels() { return file_impl->pixels(); }
void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
size_t File::tell() { return file_impl->tell(); }
ssize_t File::rows() const { return file_impl->rows(); }
ssize_t File::cols() const { return file_impl->cols(); }
ssize_t File::bitdepth() const { return file_impl->bitdepth(); }
File::~File() {
delete file_impl;
}
File::File(File &&other) {
file_impl = other.file_impl;
other.file_impl = nullptr;
}
// write move assignment operator

7
file_io/src/FileFactory.cpp
View File

@ -1,8 +1,9 @@
#include "aare/FileFactory.hpp"
#include "aare/File.hpp"
#include "aare/FileInterface.hpp"
#include "aare/RawFileFactory.hpp"
#include "aare/NumpyFileFactory.hpp"
#include "aare/utils/logger.hpp"
#include "aare/utils/logger.hpp"
#include <iostream>
FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
@ -11,6 +12,10 @@ FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
throw std::runtime_error("File does not exist");
}
if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
aare::logger::info("Loading",fpath.extension(),"file");
return new RawFileFactory(fpath);
}
if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
aare::logger::info("Loading",fpath.extension(),"file");
return new RawFileFactory(fpath);

39
file_io/src/NumpyFile.cpp
View File

@ -1,21 +1,48 @@
#include "aare/NumpyFile.hpp"
NumpyFile::NumpyFile(std::filesystem::path fname_){
this->fname = fname_;
fp = fopen(this->fname.c_str(), "rb");
NumpyFile::NumpyFile(std::filesystem::path fname_) {
this->m_fname = fname_;
fp = fopen(this->m_fname.c_str(), "rb");
}
Frame NumpyFile::get_frame(size_t frame_number) {
Frame frame(header.shape[1], header.shape[2], header.dtype.itemsize*8);
get_frame_into(frame_number,frame._get_data());
return frame;
}
void NumpyFile::get_frame_into( size_t frame_number,std::byte* image_buf) {
if (fp == nullptr) {
throw std::runtime_error("File not open");
}
if (frame_number > header.shape[0]) {
throw std::runtime_error("Frame number out of range");
}
Frame frame = Frame(header.shape[1], header.shape[2], header.dtype.itemsize*8);
fseek(fp, header_size + frame_number * bytes_per_frame(), SEEK_SET);
fread(frame._get_data(), bytes_per_frame(), 1, fp);
return frame;
fread(image_buf, bytes_per_frame(), 1, fp);
}
size_t NumpyFile::pixels() {
return std::accumulate(header.shape.begin() + 1, header.shape.end(), 1, std::multiplies<uint64_t>());
};
size_t NumpyFile::bytes_per_frame() { return header.dtype.itemsize * pixels(); };
std::vector<Frame> NumpyFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
}
return frames;
}
void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
}
}

6
file_io/src/NumpyFileFactory.cpp
View File

@ -3,13 +3,13 @@
NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) {
this->m_fpath = fpath;
}
void NumpyFileFactory::parse_metadata(File *_file) {
void NumpyFileFactory::parse_metadata(FileInterface *_file) {
auto file = dynamic_cast<NumpyFile*>(_file);
// open ifsteam to file
f = std::ifstream(file->fname, std::ios::binary);
f = std::ifstream(file->m_fname, std::ios::binary);
// check if file exists
if (!f.is_open()) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", file->fname.c_str()));
throw std::runtime_error(fmt::format("Could not open: {} for reading", file->m_fname.c_str()));
}
// read magic number
std::array<char, 6> tmp{};

53
file_io/src/RawFile.cpp
View File

@ -2,24 +2,30 @@
#include "aare/utils/logger.hpp"
Frame RawFile::get_frame(size_t frame_number) {
if (frame_number > this->total_frames) {
throw std::runtime_error("Frame number out of range");
auto f = Frame(this->m_rows, this->m_cols, this->m_bitdepth);
std::byte *frame_buffer = f._get_data();
get_frame_into(frame_number, frame_buffer);
return f;
}
void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) {
if (frame_number > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_number / this->max_frames_per_file;
// create frame and get its buffer
auto f = Frame(this->rows, this->cols, this->bitdepth);
std::byte *frame_buffer = f._get_data();
if (this->geometry.col == 1) {
// get the part from each subfile and copy it to the frame
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx*part_offset, frame_number % this->max_frames_per_file);
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
frame_number % this->max_frames_per_file);
}
} else {
// create a buffer that will hold a the frame part
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->bitdepth / 8;
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
std::byte *part_buffer = new std::byte[bytes_per_part];
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
@ -27,16 +33,41 @@ Frame RawFile::get_frame(size_t frame_number) {
for (int cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows;
auto icol = (part_idx % this->geometry.col) * this->subfile_cols;
auto dest = (irow * this->cols + icol);
dest = dest * this->bitdepth / 8;
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->bitdepth / 8,
this->subfile_cols * this->bitdepth / 8);
auto dest = (irow * this->m_cols + icol);
dest = dest * this->m_bitdepth / 8;
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
this->subfile_cols * this->m_bitdepth / 8);
}
}
delete[] part_buffer;
}
}
return f;
std::vector<Frame> RawFile::read(size_t n_frames) {
// TODO: implement this in a more efficient way
std::vector<Frame> frames;
for (size_t i = 0; i < n_frames; i++) {
frames.push_back(this->get_frame(this->current_frame));
this->current_frame++;
}
return frames;
}
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
// TODO: implement this in a more efficient way
for (size_t i = 0; i < n_frames; i++) {
this->get_frame_into(this->current_frame++, image_buf);
image_buf += this->bytes_per_frame();
}
}
size_t RawFile::frame_number(size_t frame_index) {
if (frame_index > this->m_total_frames) {
throw std::runtime_error(LOCATION + "Frame number out of range");
}
int subfile_id = frame_index / this->max_frames_per_file;
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
}
RawFile::~RawFile() {

61
file_io/src/RawFileFactory.cpp
View File

@ -17,21 +17,21 @@ RawFileFactory::RawFileFactory(std::filesystem::path fpath) {
this->m_fpath = fpath;
}
void RawFileFactory::parse_metadata(File *_file) {
void RawFileFactory::parse_metadata(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
if (file->ext == ".raw") {
if (file->m_ext == ".raw") {
this->parse_raw_metadata(file);
if (file->bitdepth == 0) {
switch (file->type) {
if (file->m_bitdepth == 0) {
switch (file->m_type) {
case DetectorType::Eiger:
file->bitdepth = 32;
file->m_bitdepth = 32;
break;
default:
file->bitdepth = 16;
file->m_bitdepth = 16;
}
}
} else if (file->ext == ".json") {
} else if (file->m_ext == ".json") {
this->parse_json_metadata(file);
} else {
throw std::runtime_error("Unsupported file type");
@ -57,7 +57,7 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) {
} else if (key == "TimeStamp") {
} else if (key == "Detector Type") {
file->type = StringTo<DetectorType>(value);
file->m_type = StringTo<DetectorType>(value);
} else if (key == "Timing Mode") {
file->timing_mode = StringTo<TimingMode>(value);
} else if (key == "Pixels") {
@ -67,9 +67,9 @@ void RawFileFactory::parse_raw_metadata(RawFile *file) {
file->subfile_cols = std::stoi(value.substr(1, pos));
file->subfile_rows = std::stoi(value.substr(pos + 1));
} else if (key == "Total Frames") {
file->total_frames = std::stoi(value);
file->m_total_frames = std::stoi(value);
} else if (key == "Dynamic Range") {
file->bitdepth = std::stoi(value);
file->m_bitdepth = std::stoi(value);
} else if (key == "Quad") {
file->quad = (value == "1");
} else if (key == "Max Frames Per File") {
@ -89,32 +89,32 @@ void RawFileFactory::parse_json_metadata(RawFile *file) {
double v = j["Version"];
std::cout << "Version: " << v << std::endl;
file->version = fmt::format("{:.1f}", v);
file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
file->total_frames = j["Frames in File"];
file->m_total_frames = j["Frames in File"];
file->subfile_rows = j["Pixels"]["y"];
file->subfile_cols = j["Pixels"]["x"];
file->max_frames_per_file = j["Max Frames Per File"];
try {
file->bitdepth = j.at("Dynamic Range");
file->m_bitdepth = j.at("Dynamic Range");
} catch (const json::out_of_range &e) {
file->bitdepth = 16;
file->m_bitdepth = 16;
}
// only Eiger had quad
if (file->type == DetectorType::Eiger) {
if (file->m_type == DetectorType::Eiger) {
file->quad = (j["Quad"] == 1);
}
file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
}
void RawFileFactory::open_subfiles(File *_file) {
void RawFileFactory::open_subfiles(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
for (size_t i = 0; i != file->n_subfiles; ++i) {
auto v = std::vector<SubFile *>(file->n_subfile_parts);
for (size_t j = 0; j != file->n_subfile_parts; ++j) {
v[j] =
new SubFile(file->data_fname(i, j), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth);
new SubFile(file->data_fname(i, j), file->m_type, file->subfile_rows, file->subfile_cols, file->bitdepth());
}
file->subfiles.push_back(v);
}
@ -122,7 +122,7 @@ void RawFileFactory::open_subfiles(File *_file) {
RawFile *RawFileFactory::load_file() {
RawFile *file = new RawFile();
file->fname = this->m_fpath;
file->m_fname = this->m_fpath;
this->parse_fname(file);
this->parse_metadata(file);
file->find_number_of_subfiles();
@ -146,7 +146,8 @@ sls_detector_header RawFileFactory::read_header(const std::filesystem::path &fna
return h;
}
void RawFileFactory::find_geometry(File *_file) {
void RawFileFactory::find_geometry(FileInterface *_file) {
auto file = dynamic_cast<RawFile *>(_file);
uint16_t r{};
uint16_t c{};
@ -163,20 +164,20 @@ void RawFileFactory::find_geometry(File *_file) {
r++;
c++;
file->rows = r * file->subfile_rows;
file->cols = c * file->subfile_cols;
file->m_rows = r * file->subfile_rows;
file->m_cols = c * file->subfile_cols;
file->rows += (r - 1) * file->cfg.module_gap_row;
file->m_rows += (r - 1) * file->cfg.module_gap_row;
}
void RawFileFactory::parse_fname(File *file) {
void RawFileFactory::parse_fname(FileInterface *file) {
file->base_path = this->m_fpath.parent_path();
file->base_name = this->m_fpath.stem();
file->ext = this->m_fpath.extension();
file->m_base_path = this->m_fpath.parent_path();
file->m_base_name = this->m_fpath.stem();
file->m_ext = this->m_fpath.extension();
auto pos = file->base_name.rfind("_");
file->findex = std::stoi(file->base_name.substr(pos + 1));
pos = file->base_name.find("_master_");
file->base_name.erase(pos);
auto pos = file->m_base_name.rfind("_");
file->m_findex = std::stoi(file->m_base_name.substr(pos + 1));
pos = file->m_base_name.find("_master_");
file->m_base_name.erase(pos);
}

26
file_io/src/SubFile.cpp
View File

@ -1,10 +1,8 @@
#include "aare/SubFile.hpp"
#include <iostream>
#include "aare/utils/logger.hpp"
#include <iostream>
// #include <filesystem>
SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) {
this->m_rows = rows;
this->m_cols = cols;
@ -12,19 +10,17 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
this->m_bitdepth = bitdepth;
this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
throw std::runtime_error(LOCATION+"Unsupported detector/bitdepth combination");
throw std::runtime_error(LOCATION + "Unsupported detector/bitdepth combination");
}
this->read_impl = read_impl_map.at({detector, bitdepth});
this->read_impl = read_impl_map.at({detector, bitdepth});
}
size_t SubFile::get_part(std::byte *buffer, int frame_number) {
if (frame_number >= n_frames or frame_number < 0) {
throw std::runtime_error("Frame number out of range");
}
// TODO: find a way to avoid opening and closing the file for each frame
aare::logger::debug(LOCATION,"frame:", frame_number, "file:", m_fname.c_str());
aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str());
fp = fopen(m_fname.c_str(), "rb");
if (!fp) {
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
@ -86,3 +82,17 @@ template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
return rc;
};
size_t SubFile::frame_number(int frame_index) {
sls_detector_header h{};
FILE *fp = fopen(this->m_fname.c_str(), "r");
if (!fp)
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
fclose(fp);
if (rc != 1)
throw std::runtime_error("Could not read header from file");
return h.frameNumber;
}