implemented flatfield creation added CustomFile class for Mythen files

CMakeLists.txt

@@ -367,6 +367,7 @@ set(PUBLICHEADERS
     include/aare/ClusterFile.hpp
     include/aare/CtbRawFile.hpp
     include/aare/ClusterVector.hpp
+    include/aare/CustomFiles.hpp
     include/aare/decode.hpp
     include/aare/defs.hpp
     include/aare/Dtype.hpp
@@ -399,6 +400,7 @@ set(PUBLICHEADERS
 set(SourceFiles
     ${CMAKE_CURRENT_SOURCE_DIR}/src/AngleCalibration.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/CustomFiles.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.cpp

include/aare/CustomFiles.hpp

@@ -0,0 +1,59 @@
+#include "Dtype.hpp"
+#include "FileInterface.hpp"
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace aare {
+
+// TODO a lot to overload
+class CustomMythenFile : public FileInterface {
+
+  public:
+    CustomMythenFile(const std::string &filename, const ssize_t rows,
+                     const ssize_t cols = 1, const std::string &mode = "r");
+
+    ~CustomMythenFile();
+
+    Frame read_frame() override;
+
+    Frame read_frame(size_t frame_number) override;
+
+    std::vector<Frame> read_n(size_t n_frames) override;
+
+    void read_into(std::byte *image_buf) override;
+
+    void read_into(std::byte *image_buf, size_t n_frames) override;
+
+    size_t frame_number(size_t frame_index) override;
+
+    size_t bytes_per_frame() override;
+
+    size_t pixels_per_frame() override;
+
+    void seek(size_t frame_number) override;
+
+    size_t tell() override;
+
+    size_t total_frames() const override;
+
+    size_t rows() const override;
+    size_t cols() const override;
+
+    size_t bitdepth() const override;
+
+    DetectorType detector_type() const override;
+
+  private:
+    std::string m_filename{};
+    std::ifstream m_file{};
+    ssize_t m_num_strips{};
+    // uint8_t m_num_counts{}; TODO extend
+    ssize_t m_rows{};
+    ssize_t m_cols{};
+    static const Dtype m_dtype;
+    static const DetectorType det_type = DetectorType::Mythen3;
+};
+} // namespace aare

include/aare/FlatField.hpp

@@ -7,17 +7,19 @@
 #include <cmath>
 #include <cstdint>
 
+#include <filesystem>
 #include <fstream>
 #include <iostream>
 #include <memory>
 #include <sstream>
 #include <string>
 
+#include "File.hpp"
 #include "MythenDetectorSpecifications.hpp"
 #include "NDArray.hpp"
 
 namespace aare {
-// TODO maybe template now its uint32
+
 class FlatField {
 
   public:
@@ -28,38 +30,116 @@ class FlatField {
             std::array<ssize_t, 1>{mythen_detector->num_strips()}, 0);
     }
 
-    void read_flatfield_from_file(const std::string &filename) {
+    /**
-
+     * @brief sums up the photon counts for multiple acquisitions
-        std::string word;
+     * @param file_path: path to filesystem - the filesystem should contain
-        uint32_t strip_number{};
+     * multiple acqisitions for different detector angles acquired using
+     * slsReceiver and mythen3Detector //TODO: constructor needs to be the same
+     * - ugly
+     */
+    // TODO unsure about design - maybe scientist has to give file with paths
+    // one wants to accumulate - what to do with strange file format?
+    // TODO: adjust for different counters!!!!
+    void create_flatfield_from_rawfilesystem(
+        const std::filesystem::path &file_path) {
 
         try {
-            std::ifstream file(filename, std::ios_base::in);
+            for (const auto &file_in_path :
-            if (!file.good()) {
+                 std::filesystem::recursive_directory_iterator(file_path)) {
-                throw std::logic_error("file does not exist");
+                if (file_in_path.is_regular_file()) {
+                    std::string filename =
+                        file_in_path.path().filename().string();
+                    if (filename.find("master") != std::string::npos) {
+                        File f(filename);
+                        auto frames = f.read_n(f.total_frames());
+                        for (const auto &frame : frames) {
+                            if (frame.rows() * frame.cols() !=
+                                mythen_detector->num_strips()) {
+                                throw std::runtime_error(fmt::format(
+                                    "sizes mismatch. Expect a size of "
+                                    "{} - frame has a size of {}",
+                                    mythen_detector->num_strips(),
+                                    frame.rows() * frame.cols()));
                             }
-
+                            for (ssize_t row = 0; row < frame.rows(); ++row)
-            std::stringstream file_buffer;
+                                for (ssize_t col = 0; col < frame.cols();
-            file_buffer << file.rdbuf();
+                                     ++col) {
-
+                                    flat_field(row * frame.cols() + col) +=
-            while (file_buffer >> word) {
+                                        *reinterpret_cast<uint32_t *>(
-
+                                            frame.pixel_ptr(
-                strip_number = std::stoi(word);
+                                                row,
-
+                                                col)); // TODO inefficient as
-                file_buffer >> word;
+                                                       // one has to copy twice
-                if (!mythen_detector->get_bad_channels()[strip_number])
+                                                       // into frame and into
-                    flat_field[strip_number] = std::stod(word);
+                                                       // flat_field
                                 }
-
+                        }
-            file.close();
+                    }
+                }
+            }
+        } catch (const std::filesystem::filesystem_error &e) {
+            std::cerr << "Filesystem error: " << e.what()
+                      << '\n'; // TODO replace with log
         } catch (const std::exception &e) {
-            std::cerr << "Error: " << e.what() << std::endl;
+            std::cerr << "Runtime error: " << e.what() << '\n';
         }
     }
 
+    /**
+     * @brief sums up the photon counts for multiple acquisitions
+     * @param filelist: path to file that stores the file paths to the aquired
+     * data the list should contain multiple acquisitions for different detector
+     * angles
+     * @tparam CustomFile: Fileclass that inherits from aare::FileInterface
+     * class needs to overload read_frame() //TODO: constructor needs to be the
+     * same - ugly
+     */
+    template <class CustomFile>
+    void create_flatfield_from_filelist(const std::filesystem::path &filelist) {
+        std::ifstream file_filelist(filelist);
+
+        try {
+            std::string filename;
+            while (std::getline(file_filelist, filename)) {
+                CustomFile file(filename, mythen_detector->num_strips(), 1);
+                Frame frame = file.read_frame();
+                if (frame.rows() * frame.cols() !=
+                    mythen_detector->num_strips()) {
+                    throw std::runtime_error(
+                        fmt::format("sizes mismatch. Expect a size of "
+                                    "{} - frame has a size of {}",
+                                    mythen_detector->num_strips(),
+                                    frame.rows() * frame.cols()));
+                }
+                for (ssize_t row = 0; row < frame.rows(); ++row)
+                    for (ssize_t col = 0; col < frame.cols(); ++col) {
+                        flat_field(row * frame.cols() + col) +=
+                            *reinterpret_cast<uint32_t *>(frame.pixel_ptr(
+                                row,
+                                col)); // TODO inefficient as one has to copy
+                                       // twice into frame and into flat_field
+                    }
+            }
+            file_filelist.close();
+        } catch (const std::exception &e) {
+            std::cerr << "Error: " << e.what() << '\n';
+        }
+    }
+
+    /**
+     * @tparam CustomFile: Fileclass that inherits from aare::FileInterface
+     * class needs to overload read_into()
+     */
+    template <class CustomFile>
+    void read_flatfield_from_file(const std::string &filename) {
+        CustomFile file(filename, mythen_detector->num_strips(), 1);
+        file.read_into(reinterpret_cast<std::byte *>(flat_field.data()));
+    }
+
     NDView<uint32_t, 1> get_flatfield() const { return flat_field.view(); }
 
-    double calculate_mean(double tolerance = 0.001) const {
+    // TODO: remove tolerance
+    double calculate_mean(double tolerance) const {
         auto [sum, count] = std::accumulate(
             flat_field.begin(), flat_field.end(),
             std::make_pair<double, ssize_t>(0.0, 0),
@@ -73,7 +153,7 @@ class FlatField {
     }
 
     NDArray<double, 1>
-    inverse_normalized_flatfield(double tolerance = 0.001) const {
+    inverse_normalized_flatfield(double tolerance = 0.0001) const {
         double mean = calculate_mean(tolerance);
 
         NDArray<double, 1> inverse_normalized_flatfield(flat_field.shape());
@@ -92,7 +172,7 @@ class FlatField {
         // maybe store as member variable access with view
     }
 
-    NDArray<double, 1> normalized_flatfield(double tolerance = 0.001) const {
+    NDArray<double, 1> normalized_flatfield(double tolerance = 0.0001) const {
         double mean = calculate_mean(tolerance);
 
         NDArray<double, 1> normalized_flatfield(flat_field.shape());

include/aare/NDArray.hpp

@@ -446,6 +446,7 @@ NDArray<T, Ndim> load_non_binary_file(const std::string &filename,
                                       const std::array<ssize_t, Ndim> shape) {
     std::string word;
     NDArray<T, Ndim> array(shape);
+
     try {
         std::ifstream file(filename, std::ios_base::in);
         if (!file.good()) {
@@ -456,7 +457,7 @@ NDArray<T, Ndim> load_non_binary_file(const std::string &filename,
         file_buffer << file.rdbuf();
 
         ssize_t counter = 0;
-        while (file_buffer >> word && counter < size) {
+        while (file_buffer >> word && counter < array.size()) {
             array[counter] = static_cast<T>(
                 std::stod(word)); // TODO change for different Types
             ++counter;

src/AngleCalibration.test.cpp

@@ -4,6 +4,8 @@
  ***********************************************/
 
 #include "aare/AngleCalibration.hpp"
+#include "aare/CustomFiles.hpp"
+#include "aare/FlatField.hpp"
 
 #include <filesystem>
 
@@ -104,7 +106,7 @@ TEST_CASE("read flatfield", "[.anglecalibration][.flatfield][.files]") {
 
     REQUIRE(std::filesystem::exists(flatfield_filename));
 
-    flatfield.read_flatfield_from_file(flatfield_filename);
+    flatfield.read_flatfield_from_file<CustomMythenFile>(flatfield_filename);
 
     auto flatfield_data = flatfield.get_flatfield();
 
@@ -145,7 +147,8 @@ TEST_CASE("compare result with python code", "[.anglecalibration] [.files]") {
 
     REQUIRE(std::filesystem::exists(flatfield_filename));
 
-    flat_field_ptr->read_flatfield_from_file(flatfield_filename);
+    flat_field_ptr->read_flatfield_from_file<CustomMythenFile>(
+        flatfield_filename);
 
     std::shared_ptr<MythenFileReader> mythen_file_reader_ptr =
         std::make_shared<MythenFileReader>(fpath,

src/CustomFiles.cpp

@@ -0,0 +1,89 @@
+#include "aare/CustomFiles.hpp"
+
+namespace aare {
+
+CustomMythenFile::CustomMythenFile(const std::string &filename,
+                                   const ssize_t rows, const ssize_t cols,
+                                   const std::string &mode)
+    : m_filename(filename), m_rows(rows), m_cols(cols) {
+
+    m_mode = mode;
+    if (m_mode == "r") {
+        try {
+            m_file.open(m_filename, std::ios_base::in);
+            if (!m_file.good()) {
+                throw std::logic_error("file does not exist");
+            }
+        } catch (std::exception &e) {
+
+            std::cerr << "Error: " << e.what()
+                      << std::endl; // TODO replace with log
+        }
+
+    } else {
+        throw std::runtime_error(LOCATION +
+                                 "Unsupported mode. Can only read RawFiles.");
+    }
+}
+
+CustomMythenFile::~CustomMythenFile() { m_file.close(); }
+
+Frame CustomMythenFile::read_frame() {
+    auto f = Frame(m_rows, m_cols, m_dtype);
+    uint32_t *frame_buffer = reinterpret_cast<uint32_t *>(f.data());
+    uint32_t strip_index, photon_count;
+    while (m_file >> strip_index >> photon_count) {
+        *frame_buffer = photon_count;
+        ++frame_buffer;
+    }
+    return f;
+}
+
+void CustomMythenFile::read_into(std::byte *image_buf) {
+    uint32_t strip_index, photon_count;
+    while (m_file >> strip_index >> photon_count) {
+        std::memcpy(image_buf, &photon_count, sizeof(photon_count));
+        image_buf += sizeof(photon_count);
+    }
+}
+
+size_t CustomMythenFile::bytes_per_frame() {
+    return m_num_strips * m_dtype.bytes(); // TODO do i want m_counts?
+}
+
+Frame CustomMythenFile::read_frame(size_t frame_number) {
+    return read_frame(); // maybe give count as frame_number
+}
+
+std::vector<Frame> CustomMythenFile::read_n(size_t n_frames) {
+    std::vector<Frame> vec;
+    vec.reserve(1);
+    vec.push_back(read_frame());
+    return vec; // std::vector<Frame>{read_frame()};
+}
+
+void CustomMythenFile::read_into(std::byte *image_buf, size_t n_frames) {
+    read_into(image_buf);
+}
+
+size_t CustomMythenFile::frame_number(size_t frame_index) { return 1; }
+
+size_t CustomMythenFile::pixels_per_frame() { return m_rows * m_cols; }
+
+void CustomMythenFile::seek(size_t frame_number) {}
+
+size_t CustomMythenFile::tell() { return 1; }
+
+size_t CustomMythenFile::total_frames() const { return 1; }
+
+size_t CustomMythenFile::rows() const { return m_rows; }
+
+size_t CustomMythenFile::cols() const { return m_cols; }
+
+size_t CustomMythenFile::bitdepth() const { return m_dtype.bitdepth(); }
+
+DetectorType CustomMythenFile::detector_type() const { return det_type; }
+
+const Dtype CustomMythenFile::m_dtype(Dtype::TypeIndex::UINT32);
+
+} // namespace aare