fixes to frame index

CMakeLists.txt

@@ -74,6 +74,9 @@ endif()
 
 if(AARE_VERBOSE)
     add_compile_definitions(AARE_VERBOSE)
+    add_compile_definitions(AARE_LOG_LEVEL=aare::logDEBUG5)
+else()
+    add_compile_definitions(AARE_LOG_LEVEL=aare::logERROR)
 endif()
 
 if(AARE_CUSTOM_ASSERT)
@@ -84,8 +87,7 @@ if(AARE_BENCHMARKS)
     add_subdirectory(benchmarks)
 endif()
 
-#Set the log level for the library
+
-add_compile_definitions(AARE_LOG_LEVEL=aare::logINFO)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -445,6 +447,7 @@ if(AARE_TESTS)
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/src/RawSubFile.test.cpp
             ${CMAKE_CURRENT_SOURCE_DIR}/src/utils/task.test.cpp
 
         )

include/aare/RawFile.hpp

@@ -45,7 +45,7 @@ class RawFile : public FileInterface {
 
      */
     RawFile(const std::filesystem::path &fname, const std::string &mode = "r");
-    virtual ~RawFile() override;
+    virtual ~RawFile() override = default;
 
     Frame read_frame() override;
     Frame read_frame(size_t frame_number) override;
@@ -103,7 +103,7 @@ class RawFile : public FileInterface {
      * @return DetectorHeader
      */
     static DetectorHeader read_header(const std::filesystem::path &fname);
-    
+
     void open_subfiles();
     void find_geometry();
 };

include/aare/logger.hpp

@@ -61,11 +61,11 @@ enum TLogLevel {
 
 class Logger {
     std::ostringstream os;
-    TLogLevel level = AARE_LOG_LEVEL;
+    TLogLevel m_level = AARE_LOG_LEVEL;
 
   public:
     Logger() = default;
-    explicit Logger(TLogLevel level) : level(level){};
+    explicit Logger(TLogLevel level) : m_level(level){};
     ~Logger() {
         // output in the destructor to allow for << syntax
         os << RESET << '\n';
@@ -103,7 +103,7 @@ class Logger {
     }
 
     std::ostringstream &Get() {
-        os << Color(level) << "- " << Timestamp() << " " << ToString(level)
+        os << Color(m_level) << "- " << Timestamp() << " " << ToString(m_level)
            << ": ";
         return os;
     }

python/examples/play.py

@@ -6,9 +6,9 @@ from aare import RawSubFile, DetectorType, RawFile
 
 from pathlib import Path
 path = Path("/home/l_msdetect/erik/data/aare-test-data/raw/jungfrau/")
-# f = RawSubFile(path/"jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16)
+f = RawSubFile(path/"jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16)
 
-f = RawFile(path/"jungfrau_single_master_0.json")
+# f = RawFile(path/"jungfrau_single_master_0.json")
 
 
 # from aare._aare import ClusterVector_i, Interpolator

python/tests/test_RawSubFile.py

@@ -5,32 +5,35 @@ from aare import RawSubFile, DetectorType
 
 @pytest.mark.files
 def test_read_a_jungfrau_RawSubFile(test_data_path):
+
+    # Starting with f1 there is now 7 frames left in the series of files
     with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f1_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f:
-        assert f.frames_in_file == 3
+        assert f.frames_in_file == 7
 
         headers, frames = f.read()
 
-    assert headers.size == 3
+    assert headers.size == 7
-    assert frames.shape == (3, 512, 1024)
+    assert frames.shape == (7, 512, 1024)
     
-    # Frame numbers in this file should be 4, 5, 6
+
-    for i,h in zip(range(4,7,1), headers):
+    for i,h in zip(range(4,11,1), headers):
         assert h["frameNumber"] == i
 
     # Compare to canned data using numpy
     data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy")
-    assert np.all(data[3:6] == frames)
+    assert np.all(data[3:] == frames)
 
 @pytest.mark.files
 def test_iterate_over_a_jungfrau_RawSubFile(test_data_path):
 
     data = np.load(test_data_path / "raw/jungfrau/jungfrau_single_0.npy")
 
+    # Given the first subfile in a series we can read all frames from f0, f1, f2...fN
     with RawSubFile(test_data_path / "raw/jungfrau/jungfrau_single_d0_f0_0.raw", DetectorType.Jungfrau, 512, 1024, 16) as f:
         i = 0
         for header, frame in f:
             assert header["frameNumber"] == i+1
             assert np.all(frame == data[i])
             i += 1
-        assert i == 3
+        assert i == 10
-        assert header["frameNumber"] == 3
+        assert header["frameNumber"] == 10

src/RawFile.cpp

@@ -1,6 +1,7 @@
 #include "aare/RawFile.hpp"
 #include "aare/PixelMap.hpp"
 #include "aare/defs.hpp"
+#include "aare/logger.hpp"
 #include "aare/geo_helpers.hpp"
 
 #include <fmt/format.h>
@@ -183,7 +184,7 @@ size_t RawFile::bytes_per_pixel() const {
 }
 
 void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, DetectorHeader *header) {
-    
+    LOG(logDEBUG) << "RawFile::get_frame_into(" << frame_index << ")";
     if (frame_index >= total_frames()) {
         throw std::runtime_error(LOCATION + "Frame number out of range");
     }
@@ -195,36 +196,27 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
 
     if (n_modules() != 1) { //if we have more than one module
         for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) {
-            // auto subfile_id = frame_index / m_master.max_frames_per_file();
-            // if (subfile_id >= n_subfiles) {
-            //     throw std::runtime_error(LOCATION +
-            //                              " Subfile out of range. Possible missing data.");
-            // }
-
-            //Check and open the correct subfile???
-
-            // frame_numbers[part_idx] =
-            //     subfiles[subfile_id][part_idx]->frame_number(
-            //         frame_index % m_master.max_frames_per_file());
             frame_numbers[part_idx] = m_subfiles[part_idx]->frame_number(frame_index); 
         }
+
         // 1. if frame number vector is the same break
         while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(),
                                   std::not_equal_to<>()) !=
                frame_numbers.end()) {
+
             // 2. find the index of the minimum frame number,
             auto min_frame_idx = std::distance(
                 frame_numbers.begin(),
                 std::min_element(frame_numbers.begin(), frame_numbers.end()));
+
             // 3. increase its index and update its respective frame number
             frame_indices[min_frame_idx]++;
+
             // 4. if we can't increase its index => throw error
             if (frame_indices[min_frame_idx] >= total_frames()) {
                 throw std::runtime_error(LOCATION +
                                          "Frame number out of range");
             }
-            auto subfile_id =
-                frame_indices[min_frame_idx] / m_master.max_frames_per_file();
 
             frame_numbers[min_frame_idx] =
                 m_subfiles[min_frame_idx]->frame_number(frame_indices[min_frame_idx]);
@@ -236,16 +228,14 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
         for (size_t part_idx = 0; part_idx != n_modules(); ++part_idx) {
             auto corrected_idx = frame_indices[part_idx];
       
-
-
             // This is where we start writing
             auto offset = (m_geometry.module_pixel_0[part_idx].origin_y * m_geometry.pixels_x +
                 m_geometry.module_pixel_0[part_idx].origin_x)*m_master.bitdepth()/8;
 
             if (m_geometry.module_pixel_0[part_idx].origin_x!=0)
-                throw std::runtime_error(LOCATION + "Implementation error. x pos not 0.");
+                throw std::runtime_error(LOCATION + " Implementation error. x pos not 0.");
             
-            //TODO! Risk for out of range access
+            //TODO! What if the files don't match?
             m_subfiles[part_idx]->seek(corrected_idx);
             m_subfiles[part_idx]->read_into(frame_buffer + offset, header);
             if (header)
@@ -256,10 +246,10 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
         //TODO! should we read row by row?
 
         // create a buffer large enough to hold a full module
-
         auto bytes_per_part = m_master.pixels_y() * m_master.pixels_x() *
                               m_master.bitdepth() /
                               8; // TODO! replace with image_size_in_bytes
+
         auto *part_buffer = new std::byte[bytes_per_part];
 
         // TODO! if we have many submodules we should reorder them on the module
@@ -269,8 +259,6 @@ void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer, Detect
             auto pos = m_geometry.module_pixel_0[part_idx];
             auto corrected_idx = frame_indices[part_idx];
 
-
-
             m_subfiles[part_idx]->seek(corrected_idx);
             m_subfiles[part_idx]->read_into(part_buffer, header);
             if(header)
@@ -312,18 +300,5 @@ size_t RawFile::frame_number(size_t frame_index) {
     return m_subfiles[0]->frame_number(frame_index);
 }
 
-RawFile::~RawFile() {
-
-    // // TODO! Fix this, for file closing
-    // for (auto &vec : subfiles) {
-    //     for (auto *subfile : vec) {
-    //         delete subfile;
-    //     }
-    // }
-}
-
-
-
-
 
 } // namespace aare

src/RawFile.test.cpp

@@ -99,11 +99,11 @@ TEST_CASE("Read frame numbers from a raw file", "[.integration]") {
     }
 }
 
-TEST_CASE("Compare reading from a numpy file with a raw file", "[.integration]") {
+TEST_CASE("Compare reading from a numpy file with a raw file", "[.files]") {
-    auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_master_0.json";
     REQUIRE(std::filesystem::exists(fpath_raw));
 
-    auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy";
+    auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy";
     REQUIRE(std::filesystem::exists(fpath_npy));
 
     File raw(fpath_raw, "r");
@@ -113,6 +113,7 @@ TEST_CASE("Compare reading from a numpy file with a raw file", "[.integration]")
     CHECK(npy.total_frames() == 10);
 
     for (size_t i = 0; i < 10; ++i) {
+        CHECK(raw.tell() == i);
         auto raw_frame = raw.read_frame();
         auto npy_frame = npy.read_frame();
         CHECK((raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>()));

src/RawSubFile.cpp

@@ -31,7 +31,7 @@ RawSubFile::RawSubFile(const std::filesystem::path &fname,
 
     parse_fname(fname);
     scan_files();
-    open_file(m_offset); // open the first file
+    open_file(m_current_file_index); // open the first file
 }
 
 void RawSubFile::seek(size_t frame_index) {
@@ -54,10 +54,13 @@ void RawSubFile::seek(size_t frame_index) {
 }
 
 size_t RawSubFile::tell() {
-    return m_file.tellg() / (sizeof(DetectorHeader) + bytes_per_frame());
+    LOG(logDEBUG) << "RawSubFile::tell():" << m_current_frame_index;
+    return m_current_frame_index;
 }
 
 void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
+    LOG(logDEBUG) << "RawSubFile::read_into()";
+
     if (header) {
         m_file.read(reinterpret_cast<char *>(header), sizeof(DetectorHeader));
     } else {
@@ -91,6 +94,13 @@ void RawSubFile::read_into(std::byte *image_buf, DetectorHeader *header) {
     if (m_file.fail()){
         throw std::runtime_error(LOCATION + ifstream_error_msg(m_file));
     }
+
+    ++ m_current_frame_index;
+    if (m_current_frame_index >= m_last_frame_in_file[m_current_file_index] &&
+        (m_current_frame_index < m_total_frames)) {
+        ++m_current_file_index;
+        open_file(m_current_file_index);
+    }
 }
 
 void RawSubFile::read_into(std::byte *image_buf, size_t n_frames, DetectorHeader *header) {
@@ -163,7 +173,7 @@ std::filesystem::path RawSubFile::fpath(size_t file_index) const {
 
 void RawSubFile::open_file(size_t file_index) {
     m_file.close();
-    auto fname = fpath(file_index);
+    auto fname = fpath(file_index+m_offset);
     LOG(logDEBUG) << "RawSubFile::open_file():  " << fname.string();
     m_file.open(fname, std::ios::binary);
     if (!m_file.is_open()) {

src/RawSubFile.test.cpp

@@ -0,0 +1,68 @@
+#include "aare/RawSubFile.hpp"
+#include "aare/File.hpp"
+#include "aare/NDArray.hpp"
+#include <catch2/catch_test_macros.hpp>
+#include "test_config.hpp"
+
+using namespace aare;
+
+TEST_CASE("Read frames directly from a RawSubFile", "[.files]"){
+    auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_d0_f0_0.raw";
+    REQUIRE(std::filesystem::exists(fpath_raw));
+
+    RawSubFile f(fpath_raw, DetectorType::Jungfrau, 512, 1024, 16);
+    REQUIRE(f.rows() == 512);
+    REQUIRE(f.cols() == 1024);
+    REQUIRE(f.pixels_per_frame() == 512 * 1024);
+    REQUIRE(f.bytes_per_frame() == 512 * 1024 * 2);
+    REQUIRE(f.bytes_per_pixel() == 2);
+
+
+    auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy";
+    REQUIRE(std::filesystem::exists(fpath_npy));
+
+    //Numpy file with the same data to use as reference
+    File npy(fpath_npy, "r");
+
+    CHECK(f.frames_in_file() == 10);
+    CHECK(npy.total_frames() == 10);
+
+
+    DetectorHeader header{};
+    NDArray<uint16_t, 2> image({static_cast<ssize_t>(f.rows()), static_cast<ssize_t>(f.cols())});
+    for (size_t i = 0; i < 10; ++i) {
+        CHECK(f.tell() == i);
+        f.read_into(image.buffer(), &header);
+        auto npy_frame = npy.read_frame();
+        CHECK((image.view() == npy_frame.view<uint16_t>()));
+    }
+}
+
+TEST_CASE("Read frames directly from a RawSubFile starting at the second file", "[.files]"){
+    auto fpath_raw = test_data_path() / "raw/jungfrau" / "jungfrau_single_d0_f1_0.raw";
+    REQUIRE(std::filesystem::exists(fpath_raw));
+
+    RawSubFile f(fpath_raw, DetectorType::Jungfrau, 512, 1024, 16);
+
+
+    auto fpath_npy = test_data_path() / "raw/jungfrau" / "jungfrau_single_0.npy";
+    REQUIRE(std::filesystem::exists(fpath_npy));
+
+    //Numpy file with the same data to use as reference
+    File npy(fpath_npy, "r");
+    npy.seek(3);
+
+    CHECK(f.frames_in_file() == 7);
+    CHECK(npy.total_frames() == 10);
+
+
+    DetectorHeader header{};
+    NDArray<uint16_t, 2> image({static_cast<ssize_t>(f.rows()), static_cast<ssize_t>(f.cols())});
+    for (size_t i = 0; i < 7; ++i) {
+        CHECK(f.tell() == i);
+        f.read_into(image.buffer(), &header);
+        CHECK(header.frameNumber == i + 4);
+        auto npy_frame = npy.read_frame();
+        CHECK((image.view() == npy_frame.view<uint16_t>()));
+    }
+}