Merge pull request #36 from slsdetectorgroup/file-tests

added first tests for reading files

core/include/aare/NDView.hpp

@@ -68,6 +68,16 @@ template <typename T, ssize_t Ndim=2> class NDView {
     T &operator()(ssize_t i) { return buffer_[i]; }
     T &operator[](ssize_t i) { return buffer_[i]; }
 
+    bool operator==(const NDView &other) const {
+        if (size_ != other.size_)
+            return false;
+        for (ssize_t i = 0; i != size_; ++i) {
+            if (buffer_[i] != other.buffer_[i])
+                return false;
+        }
+        return true;
+    }
+
     NDView &operator+=(const T val) { return elemenwise(val, std::plus<T>()); }
     NDView &operator-=(const T val) { return elemenwise(val, std::minus<T>()); }
     NDView &operator*=(const T val) { return elemenwise(val, std::multiplies<T>()); }

core/test/NDView.test.cpp

@@ -176,3 +176,19 @@ TEST_CASE("Retrieve shape"){
     REQUIRE(data.shape()[1] == 4);
 
 }
+
+TEST_CASE("compare two views"){
+    std::vector<int> vec1;
+    for (int i = 0; i != 12; ++i) {
+        vec1.push_back(i);
+    }
+    NDView<int,2> view1(vec1.data(), Shape<2>{3,4});
+
+    std::vector<int> vec2;
+    for (int i = 0; i != 12; ++i) {
+        vec2.push_back(i);
+    }
+    NDView<int,2> view2(vec2.data(), Shape<2>{3,4});
+
+    REQUIRE(view1 == view2);
+}

data/jungfrau/read_frame.py

@@ -24,7 +24,7 @@ header_dt = np.dtype(
 # Read three frames from a jungfrau file with a single interface
 rows = 512
 cols = 1024
-frames = 3
+frames = 10
 
 data = np.zeros((frames,rows,cols), dtype = np.uint16)
 header = np.zeros(frames, dtype = header_dt)
@@ -32,17 +32,18 @@ for file_id in range(4):
     file_name = 'jungfrau_single_d0_f{}_0.raw'.format(file_id)
     print("Reading file:", file_name)
     with open(file_name) as f:
-        for i in range(frames if file_id != 3 else 1):
+        for i in range(3 if file_id != 3 else 1):
-            header[i] = np.fromfile(f, dtype=header_dt, count = 1)
+            header[i+file_id*3] = np.fromfile(f, dtype=header_dt, count = 1)
-            data[i] = np.fromfile(f, dtype=np.uint16,count = rows*cols).reshape(rows,cols)
+            data[i+file_id*3] = np.fromfile(f, dtype=np.uint16,count = rows*cols).reshape(rows,cols)
 
 
-    for i in range(frames if file_id != 3 else 1 ):
+    # for i in range(frames if file_id != 3 else 1 ):
-        print("frame:",i)
+    #     print("frame:",i)
-        print(data[i][0,0],data[i][0,1],data[i][1,0],data[i][rows-1,cols-1])
+    #     print(header[i][0,0],data[i][0,1],data[i][1,0],data[i][rows-1,cols-1])
-        print("")
+    #     print("")
 
 
+print(header[1]["Frame Number"])
 #fig, ax = plt.subplots()
 #im = ax.imshow(data[0])
 #im.set_clim(2000,4000)

data/scripts/read_first_frame_number.py

@@ -0,0 +1,29 @@
+import numpy as np
+from pathlib import Path
+
+header_dt = np.dtype(
+    [
+        ("Frame Number", "u8"),
+        ("SubFrame Number/ExpLength", "u4"),
+        ("Packet Number", "u4"),
+        ("Bunch ID", "u8"),
+        ("Timestamp", "u8"),
+        ("Module Id", "u2"),
+        ("Row", "u2"),
+        ("Column", "u2"),
+        ("Reserved", "u2"),
+        ("Debug", "u4"),
+        ("Round Robin Number", "u2"),
+        ("Detector Type", "u1"),
+        ("Header Version", "u1"),
+        ("Packets caught mask", "8u8")
+    ]
+)
+
+
+
+with open("data/eiger/eiger_500k_16bit_d0_f0_0.raw", "rb") as f:
+    for i in range(3):
+        frame_number = np.fromfile(f, dtype=header_dt, count=1)["Frame Number"][0]
+        print(frame_number)
+        f.seek(262144,1)

file_io/CMakeLists.txt

@@ -32,7 +32,8 @@ endif()
 
 if(AARE_TESTS)
     set(TestSources
-            ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/test/NumpyHelpers.test.cpp
+            ${CMAKE_CURRENT_SOURCE_DIR}/test/RawFile.test.cpp
         )
     target_sources(tests PRIVATE ${TestSources} )
     target_link_libraries(tests PRIVATE core file_io)

file_io/include/aare/SubFile.hpp

@@ -2,8 +2,8 @@
 #include "aare/defs.hpp"
 #include <cstdint>
 #include <filesystem>
-#include <variant>
 #include <map>
+#include <variant>
 
 class SubFile {
   protected:
@@ -20,9 +20,11 @@ class SubFile {
         {{DetectorType::Moench, 16}, &SubFile::read_impl_reorder<uint16_t>},
         {{DetectorType::Jungfrau, 16}, &SubFile::read_impl_normal},
         {{DetectorType::ChipTestBoard, 16}, &SubFile::read_impl_normal},
-        {{DetectorType::Mythen3, 32}, &SubFile::read_impl_normal}
+        {{DetectorType::Mythen3, 32}, &SubFile::read_impl_normal},
-    };
+        {{DetectorType::Eiger, 32}, &SubFile::read_impl_normal},
+        {{DetectorType::Eiger, 16}, &SubFile::read_impl_normal}
 
+    };
 
   public:
     // pointer to a read_impl function. pointer will be set to the appropriate read_impl function in the constructor
@@ -31,7 +33,6 @@ class SubFile {
     template <typename DataType> size_t read_impl_flip(std::byte *buffer);
     template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
 
-
     SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth);
 
     size_t get_part(std::byte *buffer, int frame_number);
@@ -40,5 +41,4 @@ class SubFile {
     // TODO: define the inlines as variables and assign them in constructor
     inline size_t bytes_per_part() { return (m_bitdepth / 8) * m_rows * m_cols; }
     inline size_t pixels_per_part() { return m_rows * m_cols; }
-
 };

file_io/src/File.cpp

@@ -10,7 +10,7 @@ File::File(std::filesystem::path fname, std::string mode) {
 }
 
 Frame File::read() { return file_impl->read(); }
-size_t File::total_frames() const { return file_impl->m_total_frames; }
+size_t File::total_frames() const { return file_impl->total_frames(); }
 std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
 void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
 void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }

file_io/src/FileFactory.cpp

@@ -13,16 +13,16 @@ FileFactory *FileFactory::get_factory(std::filesystem::path fpath) {
     }
 
     if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
-        aare::logger::info("Loading",fpath.extension(),"file");
+        aare::logger::debug("Loading",fpath.extension(),"file");
         return new RawFileFactory(fpath);
     } 
     if (fpath.extension() == ".raw" || fpath.extension() == ".json"){
-        aare::logger::info("Loading",fpath.extension(),"file");
+        aare::logger::debug("Loading",fpath.extension(),"file");
         return new RawFileFactory(fpath);
     } 
     // check if extension is numpy
     else if (fpath.extension() == ".npy") {
-        aare::logger::info("Loading numpy file");
+        aare::logger::debug("Loading numpy file");
         return new NumpyFileFactory(fpath);
     }
 

file_io/src/NumpyFileFactory.cpp

@@ -1,8 +1,6 @@
 #include "aare/NumpyFileFactory.hpp"
 #include "aare/NumpyHelpers.hpp"
-NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) {
+NumpyFileFactory::NumpyFileFactory(std::filesystem::path fpath) { this->m_fpath = fpath; }
-    this->m_fpath = fpath;
-}
 void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     auto file = dynamic_cast<NumpyFile *>(_file);
     // open ifsteam to file
@@ -46,7 +44,7 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
     // parse header
 
     std::vector<std::string> keys{"descr", "fortran_order", "shape"};
-    std::cout << "original header: " << '"' << header << '"' << std::endl;
+    aare::logger::debug("original header: \"header\"");
 
     auto dict_map = parse_dict(header, keys);
     if (dict_map.size() == 0)
@@ -75,8 +73,5 @@ void NumpyFileFactory::parse_metadata(FileInterface *_file) {
 NumpyFile *NumpyFileFactory::load_file() {
     NumpyFile *file = new NumpyFile(this->m_fpath);
     parse_metadata(file);
-    std::cout << "parsed header: " << file->header.to_string() << std::endl;      
     return file;
 };
-
-

file_io/src/RawFileFactory.cpp

@@ -87,7 +87,6 @@ void RawFileFactory::parse_json_metadata(RawFile *file) {
     json j;
     ifs >> j;
     double v = j["Version"];
-    std::cout << "Version: " << v << std::endl;
     file->version = fmt::format("{:.1f}", v);
     file->m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
     file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());

file_io/src/SubFile.cpp

@@ -10,7 +10,8 @@ SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t row
     this->m_bitdepth = bitdepth;
     this->n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
     if (read_impl_map.find({detector, bitdepth}) == read_impl_map.end()) {
-        throw std::runtime_error(LOCATION + "Unsupported detector/bitdepth combination");
+        auto error_msg = LOCATION + "No read_impl function found for detector: " + toString(detector) + " and bitdepth: " + std::to_string(bitdepth);
+        throw std::runtime_error(error_msg);
     }
     this->read_impl = read_impl_map.at({detector, bitdepth});
 }
@@ -88,7 +89,7 @@ size_t SubFile::frame_number(int frame_index) {
     FILE *fp = fopen(this->m_fname.c_str(), "r");
     if (!fp)
         throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.c_str()));
-
+    fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET);
     size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
     fclose(fp);
     if (rc != 1)

file_io/test/RawFile.test.cpp

@@ -0,0 +1,81 @@
+#include "aare/File.hpp"
+#include "aare/utils/logger.hpp"
+#include <catch2/catch_test_macros.hpp>
+#include <filesystem>
+
+#include "test_config.hpp"
+
+
+TEST_CASE("Read number of frames from a jungfrau raw file") {
+
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    File f(fpath, "r");
+    REQUIRE(f.total_frames() == 10);
+}
+
+TEST_CASE("Read frame numbers from a jungfrau raw file") {
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    File f(fpath, "r");
+
+    // we know this file has 10 frames with frame numbers 1 to 10
+    // f0 1,2,3
+    // f1 4,5,6
+    // f2 7,8,9
+    // f3 10
+    for (size_t i = 0; i < 10; i++) {
+        CHECK(f.frame_number(i) == i + 1);
+    }
+}
+
+TEST_CASE("Read data from a jungfrau 500k single port raw file") {
+    auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    File f(fpath, "r");
+
+    // we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102
+    std::vector<uint16_t> pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102};
+    for (size_t i = 0; i < 10; i++) {
+        auto frame = f.read();
+        CHECK(frame.rows() == 512);
+        CHECK(frame.cols() == 1024);
+        CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
+    }
+}
+
+TEST_CASE("Read frame numbers from a raw file") {
+    auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath));
+
+    // we know this file has 3 frames with frame numbers 14, 15, 16
+    std::vector<size_t> frame_numbers = {14, 15, 16};
+
+    File f(fpath, "r");
+    for (size_t i = 0; i < 3; i++) {
+        CHECK(f.frame_number(i) == frame_numbers[i]);
+    }
+}
+
+TEST_CASE("Compare reading from a numpy file with a raw file") {
+    auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
+    REQUIRE(std::filesystem::exists(fpath_raw));
+
+    auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy";
+    REQUIRE(std::filesystem::exists(fpath_npy));
+
+    File raw(fpath_raw, "r");
+    File npy(fpath_npy, "r");
+
+    CHECK(raw.total_frames() == 10);
+    CHECK(npy.total_frames() == 10);
+
+    for (size_t i = 0; i < 10; ++i) {
+        auto raw_frame = raw.read();
+        auto npy_frame = npy.read();
+        CHECK(raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>());
+    }
+}

tests/CMakeLists.txt

@@ -32,7 +32,7 @@ target_sources(tests PRIVATE ${TestSources} )
 
 #Work around to remove, this is not the way to do it =) 
 # target_include_directories(tests PRIVATE ${CMAKE_SOURCE_DIR}/include/common)
-target_link_libraries(tests PRIVATE core aare_compiler_flags)
+target_link_libraries(tests PRIVATE core aare_compiler_flags utils)
 
 catch_discover_tests(tests
     # WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/data

utils/include/aare/utils/logger.hpp

@@ -37,7 +37,7 @@ class Logger {
     std::streambuf *error_buf = std::cerr.rdbuf();
     std::ostream *standard_output;
     std::ostream *error_output;
-    LOGGING_LEVEL VERBOSITY_LEVEL = LOGGING_LEVEL::DEBUG;
+    LOGGING_LEVEL VERBOSITY_LEVEL = LOGGING_LEVEL::INFO;
 
     std::ofstream out_file;