diff --git a/data/.gitignore b/data/.gitignore deleted file mode 100644 index e51c42e..0000000 --- a/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.raw diff --git a/data/read_multiport.py b/data/read_multiport.py new file mode 100644 index 0000000..de8a4ae --- /dev/null +++ b/data/read_multiport.py @@ -0,0 +1,80 @@ +import numpy as np +import matplotlib.pyplot as plt +plt.ion() + +header_dt = np.dtype( + [ + ("Frame Number", "u8"), + ("SubFrame Number/ExpLength", "u4"), + ("Packet Number", "u4"), + ("Bunch ID", "u8"), + ("Timestamp", "u8"), + ("Module Id", "u2"), + ("Row", "u2"), + ("Column", "u2"), + ("Reserved", "u2"), + ("Debug", "u4"), + ("Round Robin Number", "u2"), + ("Detector Type", "u1"), + ("Header Version", "u1"), + ("Packets caught mask", "8u8") + ] +) + +# Read three frames from a jungfrau file with a single interface + +frames = 1 +parts = 2 + +frame_cols = 1024 +frame_rows = 512 + +part_cols = 1024 +part_rows = 256 + + +parts_data = np.zeros((frames,parts,part_rows,part_cols), dtype = np.uint16) +data = np.zeros((frames,frame_rows,frame_cols), dtype = np.uint16) +header = np.zeros((frames,parts), dtype = header_dt) + + + + +for frame in range(frames): + + for part in range(parts): + file_name = f'jungfrau_double_d{part}_f{frame}_{0}.raw' + print("Reading file:", file_name) + with open(file_name) as f: + header[frame,part] = np.fromfile(f, dtype=header_dt, count = 1) + parts_data[frame,part] = np.fromfile(f, dtype=np.uint16,count = part_rows*part_cols).reshape(part_rows,part_cols) + + + data[frame] = np.concatenate((parts_data[frame,0],parts_data[frame,1]),axis=0) + + + +# for frame in range(frames): +# print("Frame:", frame) +# print("Data:\n", data[frame]) + +# print(data[0,0,0]) +# print(data[0,0,1]) +# print(data[0,0,50]) +print(data[0,0,0]) +print(data[0,0,1]) +print(data[0,255,1023]) + +print(data[0,511,1023]) +# print() +# print(parts_data[0,0,0,0]) +# print(parts_data[0,0,0,1]) +# print(parts_data[0,0,1,0]) + +# print(data.shape) + + + +#fig, ax = plt.subplots() +#im = ax.imshow(data[0]) +#im.set_clim(2000,4000) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a5f7f23..35358e4 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,5 +1,4 @@ - -set(EXAMPLE_LIST "json_example;logger_example;numpy_example") +set(EXAMPLE_LIST "json_example;logger_example;numpy_example;multiport_example") foreach(example ${EXAMPLE_LIST}) add_executable(${example} ${example}.cpp) target_link_libraries(${example} PUBLIC aare PRIVATE aare_compiler_flags) diff --git a/examples/json_example.cpp b/examples/json_example.cpp index ae668d4..3dad067 100644 --- a/examples/json_example.cpp +++ b/examples/json_example.cpp @@ -28,8 +28,4 @@ int main() { test(file, 9); aare::logger::debug(LOCATION,"Hello", "World"); - - - - } \ No newline at end of file diff --git a/examples/multiport_example.cpp b/examples/multiport_example.cpp new file mode 100644 index 0000000..2ddf86c --- /dev/null +++ b/examples/multiport_example.cpp @@ -0,0 +1,34 @@ +// Your First C++ Program +#include "aare/ContextManager.hpp" +#include +#include "aare/utils/logger.hpp" + +#define AARE_ROOT_DIR_VAR "PROJECT_ROOT_DIR" + +void test(File *f, int frame_number) { + std::cout << "frame number: " << frame_number << std::endl; + Frame frame = f->get_frame(frame_number); + std::cout << *((uint16_t *)frame.get(0, 0)) << std::endl; + std::cout << *((uint16_t *)frame.get(0,1)) << std::endl; + std::cout << *((uint16_t *)frame.get(255, 1023)) << std::endl; + std::cout << *((uint16_t *)frame.get(511, 1023)) << std::endl; + + + +} + +int main() { + ContextManager ctx_manager; + auto PROJECT_ROOT_DIR = std::filesystem::path(getenv(AARE_ROOT_DIR_VAR)); + std::filesystem::path fpath(PROJECT_ROOT_DIR / "data" / "jungfrau_double_master_0.json"); + std::cout << fpath << std::endl; + + + File *file = ctx_manager.get_file(fpath); + test(file, 0); + test(file, 9); + + + + +} \ No newline at end of file diff --git a/file_io/CMakeLists.txt b/file_io/CMakeLists.txt index 474babd..00c2eee 100644 --- a/file_io/CMakeLists.txt +++ b/file_io/CMakeLists.txt @@ -18,7 +18,7 @@ set(SourceFiles add_library(file_io STATIC ${SourceFiles}) target_include_directories(file_io PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) -target_link_libraries(file_io PRIVATE fmt::fmt core nlohmann_json::nlohmann_json aare_compiler_flags) +target_link_libraries(file_io PRIVATE fmt::fmt core utils nlohmann_json::nlohmann_json aare_compiler_flags) if(AARE_PYTHON_BINDINGS) set_property(TARGET file_io PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/file_io/include/aare/JsonFile.hpp b/file_io/include/aare/JsonFile.hpp index 1f8b93a..91d1f42 100644 --- a/file_io/include/aare/JsonFile.hpp +++ b/file_io/include/aare/JsonFile.hpp @@ -10,9 +10,11 @@ class JsonFile : public File { public: Frame get_frame(size_t frame_number); - int n_subfiles; - std::vector subfiles; + size_t n_subfiles; + size_t n_subfile_parts; + std::vector> subfiles; int subfile_rows, subfile_cols; + xy geometry; std::vector positions; config cfg{0, 0}; diff --git a/file_io/include/aare/SubFile.hpp b/file_io/include/aare/SubFile.hpp index d297178..366ef23 100644 --- a/file_io/include/aare/SubFile.hpp +++ b/file_io/include/aare/SubFile.hpp @@ -32,10 +32,10 @@ class SubFile { SubFile(std::filesystem::path fname,DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth); - size_t get_frame(std::byte *buffer, int frame_number); + size_t get_part(std::byte *buffer, int frame_number); // TODO: define the inlines as variables and assign them in constructor - inline size_t bytes_per_frame() { return (m_bitdepth / 8) * m_rows * m_cols; } - inline size_t pixels_per_frame() { return m_rows * m_cols; } + inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; } + inline size_t pixels_per_part() { return m_rows * m_cols; } }; diff --git a/file_io/src/JsonFile.cpp b/file_io/src/JsonFile.cpp index 2b2abdc..15975ac 100644 --- a/file_io/src/JsonFile.cpp +++ b/file_io/src/JsonFile.cpp @@ -1,25 +1,48 @@ #include "aare/JsonFile.hpp" +#include "aare/utils/logger.hpp" Frame JsonFile::get_frame(size_t frame_number) { if (frame_number > this->total_frames) { throw std::runtime_error("Frame number out of range"); } int subfile_id = frame_number / this->max_frames_per_file; - std::byte *buffer; - size_t frame_size = this->subfiles[subfile_id]->bytes_per_frame(); - buffer = new std::byte[frame_size]; - this->subfiles[subfile_id]->get_frame(buffer, frame_number % this->max_frames_per_file); - auto f = Frame(buffer, this->rows, this->cols, this->bitdepth ); + // create frame and get its buffer + auto f = Frame(this->rows, this->cols, this->bitdepth); + std::byte *frame_buffer = f._get_data(); + if (this->geometry.col == 1) { + // get the part from each subfile and copy it to the frame + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); + this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx*part_offset, frame_number % this->max_frames_per_file); + } + + } else { + // create a buffer that will hold a the frame part + auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->bitdepth / 8; + std::byte *part_buffer = new std::byte[bytes_per_part]; + + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file); + for (int cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { + auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows; + auto icol = (part_idx % this->geometry.col) * this->subfile_cols; + auto dest = (irow * this->cols + icol); + dest = dest * this->bitdepth / 8; + memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->bitdepth / 8, + this->subfile_cols * this->bitdepth / 8); + } + } + delete[] part_buffer; + } - delete[] buffer; return f; } JsonFile::~JsonFile() { - for (auto& subfile : subfiles) { - delete subfile; + for (auto &vec : subfiles) { + for (auto subfile : vec) { + delete subfile; + } } } - - diff --git a/file_io/src/JsonFileFactory.cpp b/file_io/src/JsonFileFactory.cpp index 518dcbc..9d58bb6 100644 --- a/file_io/src/JsonFileFactory.cpp +++ b/file_io/src/JsonFileFactory.cpp @@ -3,6 +3,8 @@ #include "aare/SubFile.hpp" #include "aare/defs.hpp" #include "aare/helpers.hpp" +#include "aare/utils/logger.hpp" + #include #include #include @@ -15,8 +17,8 @@ JsonFileFactory::JsonFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; } -void JsonFileFactory::parse_metadata(File*_file) { - auto file = dynamic_cast(_file); +void JsonFileFactory::parse_metadata(File *_file) { + auto file = dynamic_cast(_file); std::ifstream ifs(file->master_fname()); json j; ifs >> j; @@ -26,8 +28,8 @@ void JsonFileFactory::parse_metadata(File*_file) { file->type = StringTo(j["Detector Type"].get()); file->timing_mode = StringTo(j["Timing Mode"].get()); file->total_frames = j["Frames in File"]; - file->subfile_cols = j["Pixels"]["x"]; file->subfile_rows = j["Pixels"]["y"]; + file->subfile_cols = j["Pixels"]["x"]; file->max_frames_per_file = j["Max Frames Per File"]; try { file->bitdepth = j.at("Dynamic Range"); @@ -38,29 +40,36 @@ void JsonFileFactory::parse_metadata(File*_file) { if (file->type == DetectorType::Eiger) { file->quad = (j["Quad"] == 1); } + + file->geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; + file->n_subfile_parts = file->geometry.row * file->geometry.col; } -void JsonFileFactory::open_subfiles(File*_file) { - auto file = dynamic_cast(_file); - for (int i = 0; i != file->n_subfiles; ++i) { - - file->subfiles.push_back( - new SubFile(file->data_fname(i, 0), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth)); +void JsonFileFactory::open_subfiles(File *_file) { + auto file = dynamic_cast(_file); + for (size_t i = 0; i != file->n_subfiles; ++i) { + auto v = std::vector(file->n_subfile_parts); + for (size_t j = 0; j != file->n_subfile_parts; ++j) { + v[j]=new SubFile(file->data_fname(i, j), file->type, file->subfile_rows, + file->subfile_cols, file->bitdepth); + } + file->subfiles.push_back(v); } } -JsonFile* JsonFileFactory::load_file() { - JsonFile* file = new JsonFile(); +JsonFile *JsonFileFactory::load_file() { + JsonFile *file = new JsonFile(); file->fname = this->m_fpath; this->parse_fname(file); this->parse_metadata(file); file->find_number_of_subfiles(); + this->find_geometry(file); this->open_subfiles(file); + return file; } - sls_detector_header JsonFileFactory::read_header(const std::filesystem::path &fname) { sls_detector_header h{}; FILE *fp = fopen(fname.c_str(), "r"); @@ -74,18 +83,20 @@ sls_detector_header JsonFileFactory::read_header(const std::filesystem::path &fn return h; } - -void JsonFileFactory::find_geometry(File* _file) { - auto file = dynamic_cast(_file); +void JsonFileFactory::find_geometry(File *_file) { + auto file = dynamic_cast(_file); uint16_t r{}; uint16_t c{}; - for (int i = 0; i != file->n_subfiles; ++i) { - auto h = this->read_header(file->data_fname(i, 0)); - r = std::max(r, h.row); - c = std::max(c, h.column); + for (size_t i = 0; i < file->n_subfile_parts; i++) { + for (size_t j = 0; j != file->n_subfiles; ++j) { + auto h = this->read_header(file->data_fname(j, i)); + r = std::max(r, h.row); + c = std::max(c, h.column); - file->positions.push_back({h.row, h.column}); + file->positions.push_back({h.row, h.column}); + } } + r++; c++; @@ -95,7 +106,7 @@ void JsonFileFactory::find_geometry(File* _file) { file->rows += (r - 1) * file->cfg.module_gap_row; } -void JsonFileFactory::parse_fname(File* file) { +void JsonFileFactory::parse_fname(File *file) { file->base_path = this->m_fpath.parent_path(); file->base_name = this->m_fpath.stem(); @@ -106,5 +117,3 @@ void JsonFileFactory::parse_fname(File* file) { pos = file->base_name.find("_master_"); file->base_name.erase(pos); } - - diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index af523ea..860688e 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -1,5 +1,6 @@ #include "aare/SubFile.hpp" #include +#include "aare/utils/logger.hpp" // #include @@ -9,38 +10,37 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row this->m_cols = cols; this->m_fname = fname; this->m_bitdepth = bitdepth; - fp = fopen(fname.c_str(), "rb"); - if (fp == nullptr) { - throw std::runtime_error("Could not open file " + fname.string()); - } - std::cout << "File opened" << std::endl; - n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); - std::cout << "Number of frames: " << n_frames << std::endl; - + this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) { throw std::runtime_error("Unsupported detector/bitdepth combination"); } - read_impl = read_impl_map.at({detector, bitdepth}); - - - + this->read_impl = read_impl_map.at({detector, bitdepth}); } -size_t SubFile::get_frame(std::byte *buffer, int frame_number) { + +size_t SubFile::get_part(std::byte *buffer, int frame_number) { if (frame_number >= n_frames or frame_number < 0) { throw std::runtime_error("Frame number out of range"); } - fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_frame()) * frame_number, SEEK_SET); - return (this->*read_impl)(buffer); + // TODO: find a way to avoid opening and closing the file for each frame + aare::logger::debug(LOCATION,"frame:", frame_number, "file:", m_fname.c_str()); + fp = fopen(m_fname.c_str(), "rb"); + if (!fp) { + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); + } + fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, SEEK_SET); + auto ret = (this->*read_impl)(buffer); + fclose(fp); + return ret; } -size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_frame(), 1, this->fp); } +size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_part(), 1, this->fp); } template size_t SubFile::read_impl_reorder(std::byte *buffer) { - std::vector tmp(this->pixels_per_frame()); - size_t rc = fread(reinterpret_cast(&tmp[0]), this->bytes_per_frame(), 1, this->fp); + std::vector tmp(this->pixels_per_part()); + size_t rc = fread(reinterpret_cast(&tmp[0]), this->bytes_per_part(), 1, this->fp); int adc_nr[32] = {300, 325, 350, 375, 300, 325, 350, 375, 200, 225, 250, 275, 200, 225, 250, 275, 100, 125, 150, 175, 100, 125, 150, 175, 0, 25, 50, 75, 0, 25, 50, 75}; @@ -69,8 +69,8 @@ template size_t SubFile::read_impl_flip(std::byte *buffer) { // read to temporary buffer // TODO! benchmark direct reads - std::vector tmp(this->bytes_per_frame()); - size_t rc = fread(reinterpret_cast(&tmp[0]), this->bytes_per_frame(), 1, this->fp); + std::vector tmp(this->bytes_per_part()); + size_t rc = fread(reinterpret_cast(&tmp[0]), this->bytes_per_part(), 1, this->fp); // copy to place const size_t start = this->m_cols * (this->m_rows - 1) * sizeof(DataType);