From 2b9a732ae9898e2822e0352b8ec0e62fa27f6559 Mon Sep 17 00:00:00 2001 From: froejdh_e Date: Wed, 30 Apr 2025 16:36:48 +0200 Subject: [PATCH] fixes to frame index --- CMakeLists.txt | 7 +++- include/aare/RawFile.hpp | 4 +- include/aare/logger.hpp | 6 +-- python/examples/play.py | 4 +- python/tests/test_RawSubFile.py | 19 +++++---- src/RawFile.cpp | 43 +++++---------------- src/RawFile.test.cpp | 7 ++-- src/RawSubFile.cpp | 16 ++++++-- src/RawSubFile.test.cpp | 68 +++++++++++++++++++++++++++++++++ 9 files changed, 117 insertions(+), 57 deletions(-) create mode 100644 src/RawSubFile.test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 88f857d..daeb790 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,9 @@ endif() if(AARE_VERBOSE) add_compile_definitions(AARE_VERBOSE) + add_compile_definitions(AARE_LOG_LEVEL=aare::logDEBUG5) +else() + add_compile_definitions(AARE_LOG_LEVEL=aare::logERROR) endif() if(AARE_CUSTOM_ASSERT) @@ -84,8 +87,7 @@ if(AARE_BENCHMARKS) add_subdirectory(benchmarks) endif() -#Set the log level for the library -add_compile_definitions(AARE_LOG_LEVEL=aare::logINFO) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -445,6 +447,7 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.test.cpp ) diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index a0c02d8..1cca1fd 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -45,7 +45,7 @@ class RawFile : public FileInterface { */ RawFile(const std::filesystem::path &fname, const std::string &mode = "r"); - virtual ~RawFile() override; + virtual ~RawFile() override = default; Frame read_frame() override; Frame read_frame(size_t frame_number) override; @@ -103,7 +103,7 @@ class RawFile : public FileInterface { * @return DetectorHeader */ static DetectorHeader read_header(const std::filesystem::path &fname); - + void open_subfiles(); void find_geometry(); }; diff --git a/include/aare/logger.hpp b/include/aare/logger.hpp index e605438..06e6feb 100644 --- a/include/aare/logger.hpp +++ b/include/aare/logger.hpp @@ -61,11 +61,11 @@ enum TLogLevel { class Logger { std::ostringstream os; - TLogLevel level = AARE_LOG_LEVEL; + TLogLevel m_level = AARE_LOG_LEVEL; public: Logger() = default; - explicit Logger(TLogLevel level) : level(level){}; + explicit Logger(TLogLevel level) : m_level(level){}; ~Logger() { // output in the destructor to allow for << syntax os << RESET << '\n'; @@ -103,7 +103,7 @@ class Logger { } std::ostringstream &Get() { - os << Color(level) << "- " << Timestamp() << " " << ToString(level) + os << Color(m_level) << "- " << Timestamp() << " " << ToString(m_level) << ": "; return os; } diff --git a/python/examples/play.py b/python/examples/play.py index fe6d180..0f4feca 100644 --- a/python/examples/play.py +++ b/python/examples/play.py @@ -6,9 +6,9 @@ from aare import RawSubFile, DetectorType, RawFile from pathlib import Path path = Path("/home/l_msdetect/erik/data/aare-test-data/raw/jungfrau/") -# f = RawSubFile(path/"jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16) +f = RawSubFile(path/"jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16) -f = RawFile(path/"jungfrau_single_master_0.json") +# f = RawFile(path/"jungfrau_single_master_0.json") # from aare._aare import ClusterVector_i, Interpolator diff --git a/python/tests/test_RawSubFile.py b/python/tests/test_RawSubFile.py index a5eea91..cdde248 100644 --- a/python/tests/test_RawSubFile.py +++ b/python/tests/test_RawSubFile.py @@ -5,32 +5,35 @@ from aare import RawSubFile, DetectorType @pytest.mark.files def test_read_a_jungfrau_RawSubFile(test_data_path): + + # Starting with f1 there is now 7 frames left in the series of files with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f1_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f: - assert f.frames_in_file == 3 + assert f.frames_in_file == 7 headers, frames = f.read() - assert headers.size == 3 - assert frames.shape == (3, 512, 1024) + assert headers.size == 7 + assert frames.shape == (7, 512, 1024) - # Frame numbers in this file should be 4, 5, 6 - for i,h in zip(range(4,7,1), headers): + + for i,h in zip(range(4,11,1), headers): assert h["frameNumber"] == i # Compare to canned data using numpy data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy") - assert np.all(data[3:6] == frames) + assert np.all(data[3:] == frames) @pytest.mark.files def test_iterate_over_a_jungfrau_RawSubFile(test_data_path): data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy") + # Given the first subfile in a series we can read all frames from f0, f1, f2...fN with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f: i = 0 for header, frame in f: assert header["frameNumber"] == i+1 assert np.all(frame == data[i]) i += 1 - assert i == 3 - assert header["frameNumber"] == 3 \ No newline at end of file + assert i == 10 + assert header["frameNumber"] == 10 \ No newline at end of file diff --git a/src/RawFile.cpp b/src/RawFile.cpp index b1262fd..acd5fee 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -1,6 +1,7 @@ #include "aare/RawFile.hpp" #include "aare/PixelMap.hpp" #include "aare/defs.hpp" +#include "aare/logger.hpp" #include "aare/geo_helpers.hpp" #include @@ -183,7 +184,7 @@ size_t RawFile::bytes_per_pixel() const { } void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) { - + LOG(logDEBUG) << "RawFile::get_frame_into(" << frame_index << ")"; if (frame_index >= total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } @@ -195,36 +196,27 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect if (n_modules() != 1) { //if we have more than one module for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) { - // auto subfile_id = frame_index / m_master.max_frames_per_file(); - // if (subfile_id >= n_subfiles) { - // throw std::runtime_error(LOCATION + - // " Subfile out of range. Possible missing data."); - // } - - //Check and open the correct subfile??? - - // frame_numbers[part_idx] = - // subfiles[subfile_id][part_idx]->frame_number( - // frame_index % m_master.max_frames_per_file()); frame_numbers[part_idx] = m_subfiles[part_idx]->frame_number(frame_index); } + // 1. if frame number vector is the same break while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) != frame_numbers.end()) { + // 2. find the index of the minimum frame number, auto min_frame_idx = std::distance( frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end())); + // 3. increase its index and update its respective frame number frame_indices[min_frame_idx]++; + // 4. if we can't increase its index => throw error if (frame_indices[min_frame_idx] >= total_frames()) { throw std::runtime_error(LOCATION + "Frame number out of range"); } - auto subfile_id = - frame_indices[min_frame_idx] / m_master.max_frames_per_file(); frame_numbers[min_frame_idx] = m_subfiles[min_frame_idx]->frame_number(frame_indices[min_frame_idx]); @@ -236,16 +228,14 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) { auto corrected_idx = frame_indices[part_idx]; - - // This is where we start writing auto offset = (m_geometry.module_pixel_0[part_idx].origin_y * m_geometry.pixels_x + m_geometry.module_pixel_0[part_idx].origin_x)*m_master.bitdepth()/8; if (m_geometry.module_pixel_0[part_idx].origin_x!=0) - throw std::runtime_error(LOCATION + "Implementation error. x pos not 0."); + throw std::runtime_error(LOCATION + " Implementation error. x pos not 0."); - //TODO! Risk for out of range access + //TODO! What if the files don't match? m_subfiles[part_idx]->seek(corrected_idx); m_subfiles[part_idx]->read_into(frame_buffer + offset, header); if (header) @@ -256,10 +246,10 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect //TODO! should we read row by row? // create a buffer large enough to hold a full module - auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() * m_master.bitdepth() / 8; // TODO! replace with image_size_in_bytes + auto *part_buffer = new std::byte[bytes_per_part]; // TODO! if we have many submodules we should reorder them on the module @@ -269,8 +259,6 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect auto pos = m_geometry.module_pixel_0[part_idx]; auto corrected_idx = frame_indices[part_idx]; - - m_subfiles[part_idx]->seek(corrected_idx); m_subfiles[part_idx]->read_into(part_buffer, header); if(header) @@ -312,18 +300,5 @@ size_t RawFile::frame_number(size_t frame_index) { return m_subfiles[0]->frame_number(frame_index); } -RawFile::~RawFile() { - - // // TODO! Fix this, for file closing - // for (auto &vec : subfiles) { - // for (auto *subfile : vec) { - // delete subfile; - // } - // } -} - - - - } // namespace aare \ No newline at end of file diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index 5f9b2e1..9109985 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -99,11 +99,11 @@ TEST_CASE("Read frame numbers from a raw file", "[.integration]") { } } -TEST_CASE("Compare reading from a numpy file with a raw file", "[.integration]") { - auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; +TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") { + auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json"; REQUIRE(std::filesystem::exists(fpath_raw)); - auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy"; + auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; REQUIRE(std::filesystem::exists(fpath_npy)); File raw(fpath_raw, "r"); @@ -113,6 +113,7 @@ TEST_CASE("Compare reading from a numpy file with a raw file", "[.integration]") CHECK(npy.total_frames() == 10); for (size_t i = 0; i < 10; ++i) { + CHECK(raw.tell() == i); auto raw_frame = raw.read_frame(); auto npy_frame = npy.read_frame(); CHECK((raw_frame.view() == npy_frame.view())); diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp index e7c13c2..01ef48c 100644 --- a/src/RawSubFile.cpp +++ b/src/RawSubFile.cpp @@ -31,7 +31,7 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname, parse_fname(fname); scan_files(); - open_file(m_offset); // open the first file + open_file(m_current_file_index); // open the first file } void RawSubFile::seek(size_t frame_index) { @@ -54,10 +54,13 @@ void RawSubFile::seek(size_t frame_index) { } size_t RawSubFile::tell() { - return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame()); + LOG(logDEBUG) << "RawSubFile::tell():" << m_current_frame_index; + return m_current_frame_index; } void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { + LOG(logDEBUG) << "RawSubFile::read_into()"; + if (header) { m_file.read(reinterpret_cast(header), sizeof(DetectorHeader)); } else { @@ -91,6 +94,13 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) { if (m_file.fail()){ throw std::runtime_error(LOCATION + ifstream_error_msg(m_file)); } + + ++ m_current_frame_index; + if (m_current_frame_index >= m_last_frame_in_file[m_current_file_index] && + (m_current_frame_index < m_total_frames)) { + ++m_current_file_index; + open_file(m_current_file_index); + } } void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) { @@ -163,7 +173,7 @@ std::filesystem::path RawSubFile::fpath(size_t file_index) const { void RawSubFile::open_file(size_t file_index) { m_file.close(); - auto fname = fpath(file_index); + auto fname = fpath(file_index+m_offset); LOG(logDEBUG) << "RawSubFile::open_file(): " << fname.string(); m_file.open(fname, std::ios::binary); if (!m_file.is_open()) { diff --git a/src/RawSubFile.test.cpp b/src/RawSubFile.test.cpp new file mode 100644 index 0000000..10e3a75 --- /dev/null +++ b/src/RawSubFile.test.cpp @@ -0,0 +1,68 @@ +#include "aare/RawSubFile.hpp" +#include "aare/File.hpp" +#include "aare/NDArray.hpp" +#include +#include "test_config.hpp" + +using namespace aare; + +TEST_CASE("Read frames directly from a RawSubFile", "[.files]"){ + auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_d0_f0_0.raw"; + REQUIRE(std::filesystem::exists(fpath_raw)); + + RawSubFile f(fpath_raw, DetectorType::Jungfrau, 512, 1024, 16); + REQUIRE(f.rows() == 512); + REQUIRE(f.cols() == 1024); + REQUIRE(f.pixels_per_frame() == 512 * 1024); + REQUIRE(f.bytes_per_frame() == 512 * 1024 * 2); + REQUIRE(f.bytes_per_pixel() == 2); + + + auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); + + //Numpy file with the same data to use as reference + File npy(fpath_npy, "r"); + + CHECK(f.frames_in_file() == 10); + CHECK(npy.total_frames() == 10); + + + DetectorHeader header{}; + NDArray image({static_cast(f.rows()), static_cast(f.cols())}); + for (size_t i = 0; i < 10; ++i) { + CHECK(f.tell() == i); + f.read_into(image.buffer(), &header); + auto npy_frame = npy.read_frame(); + CHECK((image.view() == npy_frame.view())); + } +} + +TEST_CASE("Read frames directly from a RawSubFile starting at the second file", "[.files]"){ + auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_d0_f1_0.raw"; + REQUIRE(std::filesystem::exists(fpath_raw)); + + RawSubFile f(fpath_raw, DetectorType::Jungfrau, 512, 1024, 16); + + + auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); + + //Numpy file with the same data to use as reference + File npy(fpath_npy, "r"); + npy.seek(3); + + CHECK(f.frames_in_file() == 7); + CHECK(npy.total_frames() == 10); + + + DetectorHeader header{}; + NDArray image({static_cast(f.rows()), static_cast(f.cols())}); + for (size_t i = 0; i < 7; ++i) { + CHECK(f.tell() == i); + f.read_into(image.buffer(), &header); + CHECK(header.frameNumber == i + 4); + auto npy_frame = npy.read_frame(); + CHECK((image.view() == npy_frame.view())); + } +} \ No newline at end of file