From 1177fd129d3690db92e9597ccda62598e5a44d41 Mon Sep 17 00:00:00 2001 From: Bechir Braham Date: Mon, 22 Apr 2024 18:15:02 +0200 Subject: [PATCH] read subfiles with unordered and missing frames --- .../include/aare/file_io/FileInterface.hpp | 3 +- file_io/include/aare/file_io/RawFile.hpp | 9 ++- file_io/include/aare/file_io/SubFile.hpp | 9 ++- file_io/src/RawFile.cpp | 23 ++++++-- file_io/src/SubFile.cpp | 59 ++++++++++++------- file_io/test/RawFile.test.cpp | 3 +- 6 files changed, 71 insertions(+), 35 deletions(-) diff --git a/file_io/include/aare/file_io/FileInterface.hpp b/file_io/include/aare/file_io/FileInterface.hpp index ed89f74..123fa72 100644 --- a/file_io/include/aare/file_io/FileInterface.hpp +++ b/file_io/include/aare/file_io/FileInterface.hpp @@ -76,7 +76,8 @@ class FileInterface { virtual void read_into(std::byte *image_buf, size_t n_frames) = 0; /** - * @brief get the frame number at the given frame index + * @brief get the frame number stored in the file at the given frame_index + * @note throws when subfiles have different frame numbers at the same index (note only relevant for RawFile) * @param frame_index index of the frame * @return frame number */ diff --git a/file_io/include/aare/file_io/RawFile.hpp b/file_io/include/aare/file_io/RawFile.hpp index cc7b399..4cfe77d 100644 --- a/file_io/include/aare/file_io/RawFile.hpp +++ b/file_io/include/aare/file_io/RawFile.hpp @@ -25,9 +25,13 @@ class RawFile : public FileInterface { * @param frame frame to write */ void write([[maybe_unused]] Frame &frame) override { throw std::runtime_error("Not implemented"); }; - Frame read() override { return get_frame(this->current_frame++); }; + Frame read() override { + return get_frame(frame_number(this->current_frame++)); + }; std::vector read(size_t n_frames) override; - void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }; + void read_into(std::byte *image_buf) override { + return get_frame_into(frame_number(this->current_frame++), image_buf); + }; void read_into(std::byte *image_buf, size_t n_frames) override; size_t frame_number(size_t frame_index) override; @@ -156,6 +160,7 @@ class RawFile : public FileInterface { RawFileConfig cfg{0, 0}; TimingMode timing_mode{}; bool quad{false}; + size_t m_starting_frame{}; }; } // namespace aare \ No newline at end of file diff --git a/file_io/include/aare/file_io/SubFile.hpp b/file_io/include/aare/file_io/SubFile.hpp index f5b2a3d..f82af39 100644 --- a/file_io/include/aare/file_io/SubFile.hpp +++ b/file_io/include/aare/file_io/SubFile.hpp @@ -45,7 +45,7 @@ class SubFile { * @param bitdepth bitdepth of the subfile * @throws std::invalid_argument if the detector,type pair is not supported */ - SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth); + SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,int subfile_id); /** * @brief read the subfile into a buffer @@ -75,7 +75,9 @@ class SubFile { * @return number of bytes read */ size_t get_part(std::byte *buffer, size_t frame_number); - size_t frame_number(size_t frame_index); + size_t frame_number_in_file(size_t frame_index); + size_t correct_frame_number(size_t frame_number); + ~SubFile() noexcept; // TODO: define the inlines as variables and assign them in constructor inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } @@ -88,7 +90,8 @@ class SubFile { size_t m_rows{}; size_t m_cols{}; size_t n_frames{}; - int m_sub_file_index_{}; + size_t cached_offset{}; + int m_subfile_id{}; }; } // namespace aare \ No newline at end of file diff --git a/file_io/src/RawFile.cpp b/file_io/src/RawFile.cpp index 10720f2..543c9b7 100644 --- a/file_io/src/RawFile.cpp +++ b/file_io/src/RawFile.cpp @@ -24,13 +24,14 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, co } else { throw std::runtime_error(LOCATION + "Unsupported mode"); } + m_starting_frame = this->frame_number(0); } void RawFile::open_subfiles() { for (size_t i = 0; i != n_subfiles; ++i) { auto v = std::vector(n_subfile_parts); for (size_t j = 0; j != n_subfile_parts; ++j) { - v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth, i); } subfiles.push_back(v); } @@ -200,15 +201,15 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { if (frame_number > this->m_total_frames) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - size_t const subfile_id = frame_number / this->max_frames_per_file; + size_t const subfile_id = (frame_number - m_starting_frame) / this->max_frames_per_file; // create frame and get its buffer if (this->geometry.col == 1) { // get the part from each subfile and copy it to the frame for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); - this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, - frame_number % this->max_frames_per_file); + this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, frame_number); } } else { @@ -217,7 +218,7 @@ void RawFile::get_frame_into(size_t frame_number, std::byte *frame_buffer) { auto *part_buffer = new std::byte[bytes_per_part]; for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { - this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number % this->max_frames_per_file); + this->subfiles[subfile_id][part_idx]->get_part(part_buffer, frame_number); for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { auto irow = cur_row + (part_idx / this->geometry.col) * this->subfile_rows; auto icol = (part_idx % this->geometry.col) * this->subfile_cols; @@ -253,7 +254,17 @@ size_t RawFile::frame_number(size_t frame_index) { throw std::runtime_error(LOCATION + "Frame number out of range"); } size_t const subfile_id = frame_index / this->max_frames_per_file; - return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file); + size_t prev_frame_nbr{}; + bool first_time_in_loop = true; + for (auto &subfile_parts : this->subfiles[subfile_id]) { + auto cur_frame_nbr = subfile_parts->frame_number_in_file(frame_index % this->max_frames_per_file); + if ((not first_time_in_loop) && cur_frame_nbr != prev_frame_nbr) { + throw std::runtime_error(LOCATION + "Frame number different in subfiles"); + } + prev_frame_nbr = cur_frame_nbr; + first_time_in_loop = false; + } + return prev_frame_nbr; } RawFile::~RawFile() { diff --git a/file_io/src/SubFile.cpp b/file_io/src/SubFile.cpp index dc1e085..34ee9ca 100644 --- a/file_io/src/SubFile.cpp +++ b/file_io/src/SubFile.cpp @@ -7,9 +7,11 @@ namespace aare { -SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth) +SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + int subfile_id) : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), - n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)) { + n_frames(std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8)), + m_subfile_id(subfile_id) { if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) { auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) + @@ -17,23 +19,36 @@ SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size throw std::invalid_argument(error_msg); } this->read_impl = read_impl_map.at({detector, bitdepth}); + + if (!(fp = fopen(fname.c_str(), "rb"))) { + throw std::runtime_error(fmt::format(LOCATION + "Could not open: {} for reading", fname.c_str())); + } +} + +size_t SubFile::correct_frame_number(size_t frame_number) { + sls_detector_header h{}; + for (size_t offset = cached_offset; offset < n_frames; offset++) { + size_t ret = (frame_number + offset) % n_frames; + fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * ret, SEEK_SET); // NOLINT + size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + if (rc != 1) + throw std::runtime_error(LOCATION + "Could not read header from file"); + if (h.frameNumber == frame_number) { + cached_offset = offset; + return ret; + } + } + aare::logger::error("m_subfile_id:", m_subfile_id); + aare::logger::error(LOCATION, ": frame:", h.frameNumber, "frame_number:", frame_number, "toto", + (h.frameNumber - (m_subfile_id * n_frames))); + throw std::runtime_error(fmt::format(LOCATION + "Could not find frame {} in file", frame_number)); } size_t SubFile::get_part(std::byte *buffer, size_t frame_number) { - if (frame_number >= n_frames) { - throw std::runtime_error("Frame number out of range"); - } - // TODO: find a way to avoid opening and closing the file for each frame aare::logger::debug(LOCATION, "frame:", frame_number, "file:", m_fname.c_str()); - fp = fopen(m_fname.c_str(), "rb"); - if (!fp) { - throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str())); - } - fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_number, // NOLINT - SEEK_SET); + size_t tmp = correct_frame_number(frame_number); + fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * tmp, SEEK_SET); // NOLINT auto ret = (this->*read_impl)(buffer); - if (fclose(fp)) - throw std::runtime_error(LOCATION + "Could not close file"); return ret; } @@ -89,20 +104,20 @@ template size_t SubFile::read_impl_flip(std::byte *buffer) { return rc; }; -size_t SubFile::frame_number(size_t frame_index) { +size_t SubFile::frame_number_in_file(size_t frame_index) { sls_detector_header h{}; - fp = fopen(this->m_fname.c_str(), "r"); - if (!fp) - throw std::runtime_error(LOCATION + fmt::format("Could not open: {} for reading", m_fname.c_str())); fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); if (rc != 1) throw std::runtime_error(LOCATION + "Could not read header from file"); - if (fclose(fp)) { - throw std::runtime_error(LOCATION + "Could not close file"); - } - return h.frameNumber; } +SubFile::~SubFile() noexcept { + if (fp) { + if (fclose(fp)) + aare::logger::error(LOCATION, "Could not close file"); + } +} + } // namespace aare \ No newline at end of file diff --git a/file_io/test/RawFile.test.cpp b/file_io/test/RawFile.test.cpp index 34f2f03..5b8fffd 100644 --- a/file_io/test/RawFile.test.cpp +++ b/file_io/test/RawFile.test.cpp @@ -32,7 +32,7 @@ TEST_CASE("Read frame numbers from a jungfrau raw file") { } } -TEST_CASE("Read data from a jungfrau 500k single port raw file") { +TEST_CASE("Read data from a jungfrau 500k single port raw file", "[debug]") { auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath)); @@ -45,6 +45,7 @@ TEST_CASE("Read data from a jungfrau 500k single port raw file") { CHECK(frame.rows() == 512); CHECK(frame.cols() == 1024); CHECK(frame.view()(0, 0) == pixel_0_0[i]); + // frame.view(); } }