new folder structure

file_io/CMakeLists.txt

@@ -0,0 +1,28 @@
+
+
+set(SourceFiles
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/FileFactory.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/helpers.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/JsonFile.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/JsonFileFactory.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/SubFile.cpp
+)
+
+add_library(file_io STATIC ${SourceFiles})
+target_include_directories(file_io PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
+target_link_libraries(file_io  PUBLIC fmt::fmt core nlohmann_json::nlohmann_json)
+set_property(TARGET file_io PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+
+# if(AARE_BUILD_TESTS)
+#     set(TestSources
+#             ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp
+#         )
+#     target_sources(tests PRIVATE ${TestSources} )
+
+#     #Work around to remove, this is not the way to do it =) 
+#     # target_include_directories(tests PRIVATE ${CMAKE_SOURCE_DIR}/include/common)
+#     target_link_libraries(tests PRIVATE file_io)
+
+# endif()

file_io/include/aare/File.hpp

@@ -0,0 +1,80 @@
+#pragma once
+
+#include "aare/defs.hpp"
+#include "aare/Frame.hpp"
+#include "SubFile.hpp"
+
+#include <filesystem>
+#include <fmt/core.h>
+#include <iostream>
+
+struct RawFileConfig {
+    int module_gap_row{};
+    int module_gap_col{};
+
+    bool operator==(const RawFileConfig &other) const {
+        if (module_gap_col != other.module_gap_col)
+            return false;
+        if (module_gap_row != other.module_gap_row)
+            return false;
+        return true;
+    }
+};
+template <DetectorType detector, typename DataType>
+class File {
+
+  public:
+    virtual Frame<DataType>* get_frame(int frame_number) = 0;
+
+  private:
+    using config = RawFileConfig;
+
+  public:
+    std::vector<SubFile*> subfiles;
+    std::filesystem::path fname;
+    std::filesystem::path base_path;
+    std::string base_name, ext;
+    int findex, n_subfiles;
+    size_t total_frames{};
+    size_t max_frames_per_file{};
+
+    std::string version;
+    DetectorType type;
+    TimingMode timing_mode;
+    int subfile_rows, subfile_cols;
+    bool quad{false};
+
+    ssize_t rows{};
+    ssize_t cols{};
+    uint8_t bitdepth{};
+    using data_type = uint16_t;
+    std::vector<xy> positions;
+
+    config cfg{0, 0};
+    // File();
+    ~File();
+
+    inline size_t bytes_per_frame() const { return rows * cols * bitdepth / 8; }
+    inline size_t pixels() const { return rows * cols; }
+    inline void set_config(int row, int col) {
+        cfg.module_gap_row = row;
+        cfg.module_gap_col = col;
+    }
+
+    // TODO! Deal with fast quad and missing files
+    void find_number_of_subfiles() {
+        int n_mod = 0;
+        while (std::filesystem::exists(data_fname(++n_mod, 0)))
+            ;
+        n_subfiles = n_mod;
+    }
+
+    inline std::filesystem::path master_fname() {
+        return base_path / fmt::format("{}_master_{}{}", base_name, findex, ext);
+    }
+    inline std::filesystem::path data_fname(int mod_id, int file_id) {
+        return base_path / fmt::format("{}_d{}_f{}_{}.raw", base_name, file_id, mod_id, findex);
+    }
+
+    // size_t total_frames();
+};

file_io/include/aare/FileFactory.hpp

@@ -0,0 +1,26 @@
+#pragma once
+#include <filesystem>
+#include "aare/File.hpp"
+
+template <DetectorType detector,typename DataType>
+class FileFactory{
+    // Class that will be used to create File objects
+    // follows the factory pattern
+    protected:
+    std::filesystem::path fpath;
+public:
+    static FileFactory<detector,DataType>* get_factory(std::filesystem::path);
+    // virtual int deleteFile() = 0;
+    virtual File<detector,DataType>* load_file()=0;//TODO: add option to load all file to memory or keep it on disk
+    virtual void parse_metadata(File<detector,DataType>*)=0;
+    
+    
+    void find_geometry(File<detector,DataType>*);
+    void parse_fname(File<detector,DataType>*);
+
+    sls_detector_header read_header(const std::filesystem::path &fname);
+
+
+
+};
+

file_io/include/aare/FileHandler.hpp

@@ -0,0 +1,29 @@
+#include <filesystem>
+#include "aare/FileFactory.hpp"
+#include "aare/File.hpp"
+
+template <DetectorType detector,typename DataType>
+class FileHandler{
+    private:
+    std::filesystem::path fpath;
+    FileFactory<detector,DataType>* fileFactory;
+    File<detector,DataType>* f;
+
+    public:
+    FileHandler<detector,DataType>(std::filesystem::path fpath){
+        this->fpath = fpath;
+         this->fileFactory= FileFactory<detector,DataType>::get_factory(fpath);
+         this->f= fileFactory->load_file();
+         delete fileFactory;
+    }
+
+    Frame<DataType>* get_frame(int index){
+         return  f->get_frame(index);
+    }
+
+    
+
+    ~FileHandler(){
+        delete f;
+    }
+};

file_io/include/aare/JsonFile.hpp

@@ -0,0 +1,9 @@
+#pragma once
+#include "aare/File.hpp"
+#include "aare/Frame.hpp"
+#include "aare/defs.hpp"
+template <DetectorType detector, typename DataType>
+class JsonFile : public File<detector, DataType> {
+
+    Frame<DataType> *get_frame(int frame_number);
+};

file_io/include/aare/JsonFileFactory.hpp

@@ -0,0 +1,16 @@
+#include "aare/FileFactory.hpp"
+template <DetectorType detector,typename DataType>
+class JsonFileFactory: public FileFactory<detector,DataType>
+{
+private:
+    /* data */
+public:
+    File<detector,DataType>* load_file() override;
+    void parse_metadata(File<detector,DataType>*) override;
+    JsonFileFactory<detector,DataType>(std::filesystem::path fpath);
+    void open_subfiles(File<detector,DataType>*);
+    
+
+
+};
+

file_io/include/aare/RawFileFactory.hpp

@@ -0,0 +1,9 @@
+#include "aare/File.hpp"
+#include <filesystem>
+template<DetectorType detector,typename DataType>  
+class RawFileFactory{
+    public:
+        // RawFileFactory();
+        // ~RawFileFactory();
+        File<detector,DataType> loadFile(std::filesystem::path fpath);
+};

file_io/include/aare/SubFile.hpp

@@ -0,0 +1,32 @@
+#pragma once
+#include "aare/defs.hpp"
+#include <cstdint>
+#include <filesystem>
+#include <variant>
+
+class SubFile {
+  protected:
+    FILE *fp = nullptr;
+    uint16_t bitdepth;
+
+  public:
+    // pointer to a read_impl function. pointer will be set to the appropriate read_impl function in the constructor
+    size_t (SubFile::*read_impl)(std::byte *buffer) = nullptr;
+    size_t read_impl_normal(std::byte *buffer);
+    template <typename DataType> size_t  read_impl_flip(std::byte *buffer);
+    template <typename DataType> size_t  read_impl_reorder(std::byte *buffer);
+
+
+    SubFile(std::filesystem::path fname,DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth);
+
+    size_t get_frame(std::byte *buffer, int frame_number);
+
+    // TODO: define the inlines as variables and assign them in constructor
+    inline size_t bytes_per_frame() { return (bitdepth / 8) * rows * cols; }
+    inline size_t pixels_per_frame() { return rows * cols; }
+    std::filesystem::path fname;
+    ssize_t rows{};
+    ssize_t cols{};
+    ssize_t n_frames{};
+    int sub_file_index_{};
+};

file_io/include/aare/helpers.hpp

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "aare/File.hpp"
+#include <filesystem>
+#include <fmt/core.h>
+
+bool is_master_file(std::filesystem::path fpath);
+

file_io/src/File.cpp

@@ -0,0 +1,10 @@
+#include "aare/File.hpp"
+
+template <DetectorType detector, typename DataType>
+File<detector,DataType>::~File<detector,DataType>() {
+    for (auto& subfile : subfiles) {
+        delete subfile;
+    }
+}
+
+template class File<DetectorType::Jungfrau, uint16_t>;

file_io/src/FileFactory.cpp

@@ -0,0 +1,75 @@
+#include "aare/FileFactory.hpp"
+#include "aare/File.hpp"
+#include "aare/JsonFileFactory.hpp"
+#include <iostream>
+
+template <DetectorType detector, typename DataType>
+FileFactory<detector, DataType> *FileFactory<detector, DataType>::get_factory(std::filesystem::path fpath) {
+    // check if file exists
+    if (!std::filesystem::exists(fpath)) {
+        throw std::runtime_error("File does not exist");
+    }
+
+    if (fpath.extension() == ".raw") {
+        std::cout << "Loading raw file" << std::endl;
+        throw std::runtime_error("Raw file not implemented");
+    } else if (fpath.extension() == ".json") {
+        std::cout << "Loading json file" << std::endl;
+        return new JsonFileFactory<detector, DataType>(fpath);
+    }
+    // check if extension is numpy
+    else if (fpath.extension() == ".npy") {
+        std::cout << "Loading numpy file" << std::endl;
+        throw std::runtime_error("Numpy file not implemented");
+    }
+
+    throw std::runtime_error("Unsupported file type");
+}
+
+template <DetectorType detector, typename DataType>
+void FileFactory<detector, DataType>::parse_fname(File<detector, DataType> *file) {
+    file->base_path = fpath.parent_path();
+    file->base_name = fpath.stem();
+    file->ext = fpath.extension();
+
+    auto pos = file->base_name.rfind("_");
+    file->findex = std::stoi(file->base_name.substr(pos + 1));
+    pos = file->base_name.find("_master_");
+    file->base_name.erase(pos);
+}
+
+template <DetectorType detector, typename DataType>
+sls_detector_header FileFactory<detector, DataType>::read_header(const std::filesystem::path &fname) {
+    sls_detector_header h{};
+    FILE *fp = fopen(fname.c_str(), "r");
+    if (!fp)
+        throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.c_str()));
+
+    size_t rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
+    fclose(fp);
+    if (rc != 1)
+        throw std::runtime_error("Could not read header from file");
+    return h;
+}
+
+template <DetectorType detector, typename DataType>
+void FileFactory<detector, DataType>::find_geometry(File<detector, DataType> *file) {
+    uint16_t r{};
+    uint16_t c{};
+    for (int i = 0; i != file->n_subfiles; ++i) {
+        auto h = this->read_header(file->data_fname(i, 0));
+        r = std::max(r, h.row);
+        c = std::max(c, h.column);
+
+        file->positions.push_back({h.row, h.column});
+    }
+    r++;
+    c++;
+
+    file->rows = r * file->subfile_rows;
+    file->cols = c * file->subfile_cols;
+
+    file->rows += (r - 1) * file->cfg.module_gap_row;
+}
+
+template class FileFactory<DetectorType::Jungfrau, uint16_t>;

file_io/src/JsonFile.cpp

@@ -0,0 +1,19 @@
+#include "aare/JsonFile.hpp"
+#include <typeinfo>
+
+template <DetectorType detector, typename DataType>
+Frame<DataType> *JsonFile<detector, DataType>::get_frame(int frame_number) {
+    int subfile_id = frame_number / this->max_frames_per_file;
+    std::byte *buffer;
+    size_t frame_size = this->subfiles[subfile_id]->bytes_per_frame();
+    buffer = new std::byte[frame_size];
+
+    this->subfiles[subfile_id]->get_frame(buffer, frame_number % this->max_frames_per_file);
+
+    auto f = new Frame<DataType>(buffer, this->rows, this->cols);
+
+    delete[] buffer;
+    return f;
+}
+
+template class JsonFile<DetectorType::Jungfrau, uint16_t>;

file_io/src/JsonFileFactory.cpp

@@ -0,0 +1,76 @@
+#include "aare/JsonFileFactory.hpp"
+#include "aare/JsonFile.hpp"
+#include "aare/SubFile.hpp"
+#include "aare/defs.hpp"
+#include "aare/helpers.hpp"
+#include <fstream>
+#include <iostream>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+template <DetectorType detector,typename DataType>
+JsonFileFactory<detector,DataType>::JsonFileFactory(std::filesystem::path fpath) {
+    if (not is_master_file(fpath))
+        throw std::runtime_error("Json file is not a master file");
+    this->fpath = fpath;
+}
+
+template <DetectorType detector,typename DataType>
+void JsonFileFactory<detector,DataType>::parse_metadata(File<detector,DataType> *file) {
+    std::cout << "Parsing metadata" << std::endl;
+    std::ifstream ifs(file->master_fname());
+    json j;
+    ifs >> j;
+    double v = j["Version"];
+    std::cout << "Version: " << v << std::endl;
+    file->version = fmt::format("{:.1f}", v);
+    file->type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
+    if (file->type != detector)
+        throw std::runtime_error("Detector type mismatch: file type (" + toString<DetectorType>(file->type) +
+                                 ") != specified type (" + toString<DetectorType>(detector) + ")");
+    file->timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
+    file->total_frames = j["Frames in File"];
+    file->subfile_cols = j["Pixels"]["x"];
+    file->subfile_rows = j["Pixels"]["y"];
+    file->max_frames_per_file = j["Max Frames Per File"];
+    try {
+        file->bitdepth = j.at("Dynamic Range");
+    } catch (const json::out_of_range &e) {
+        std::cerr << "master file does not have Dynamic Range. Defaulting to 16 bit" << '\n';
+        file->bitdepth = 16;
+    }
+    if (file->bitdepth != sizeof(DataType) * 8)
+        throw std::runtime_error("Bitdepth mismatch: file bitdepth (" + std::to_string(file->bitdepth) +
+                                 ") != specified bitdepth (" + std::to_string(sizeof(DataType) * 8) + ")");
+
+    // only Eiger had quad
+    if (file->type == DetectorType::Eiger) {
+        file->quad = (j["Quad"] == 1);
+    }
+}
+
+template <DetectorType detector,typename DataType>
+void JsonFileFactory<detector,DataType>::open_subfiles(File<detector,DataType> *file) {
+    for (int i = 0; i != file->n_subfiles; ++i) {
+
+        file->subfiles.push_back(
+            new SubFile(file->data_fname(i, 0), file->type, file->subfile_rows, file->subfile_cols, file->bitdepth));
+    }
+}
+
+template <DetectorType detector,typename DataType>
+File<detector,DataType> *JsonFileFactory<detector,DataType>::load_file() {
+    std::cout << "Loading json file" << std::endl;
+    JsonFile<detector,DataType> *file = new JsonFile<detector,DataType>();
+    file->fname = this->fpath;
+    this->parse_fname(file);
+    this->parse_metadata(file);
+    file->find_number_of_subfiles();
+    this->find_geometry(file);
+    this->open_subfiles(file);
+
+    return file;
+}
+
+template class JsonFileFactory<DetectorType::Jungfrau, uint16_t>;

file_io/src/SubFile.cpp

@@ -0,0 +1,92 @@
+#include "aare/SubFile.hpp"
+#include <iostream>
+// #include <filesystem>
+
+/**
+ * SubFile methods
+ *
+ *
+ */
+
+SubFile::SubFile(std::filesystem::path fname, DetectorType detector, ssize_t rows, ssize_t cols, uint16_t bitdepth) {
+    this->rows = rows;
+    this->cols = cols;
+    this->fname = fname;
+    this->bitdepth = bitdepth;
+    fp = fopen(fname.c_str(), "rb");
+    if (fp == nullptr) {
+        throw std::runtime_error("Could not open file " + fname.string());
+    }
+    std::cout << "File opened" << std::endl;
+    n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
+    std::cout << "Number of frames: " << n_frames << std::endl;
+    
+    if (detector == DetectorType::Moench) {
+        read_impl = &SubFile::read_impl_reorder<uint16_t>;
+    } else if (detector == DetectorType::Jungfrau) {
+        read_impl = &SubFile::read_impl_normal;
+    }
+    else {
+        throw std::runtime_error("Detector type not implemented");
+    }
+}
+
+size_t SubFile::get_frame(std::byte *buffer, int frame_number) {
+    if (frame_number >= n_frames or frame_number < 0) {
+        throw std::runtime_error("Frame number out of range");
+    }
+    fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_frame()) * frame_number, SEEK_SET);
+    return (this->*read_impl)(buffer);
+}
+
+size_t SubFile::read_impl_normal(std::byte *buffer) { return fread(buffer, this->bytes_per_frame(), 1, this->fp); }
+
+template <typename DataType> size_t SubFile::read_impl_reorder(std::byte *buffer) {
+
+    std::vector<DataType> tmp(this->pixels_per_frame());
+    size_t rc = fread(reinterpret_cast<char *>(&tmp[0]), this->bytes_per_frame(), 1, this->fp);
+
+    int adc_nr[32] = {300, 325, 350, 375, 300, 325, 350, 375, 200, 225, 250, 275, 200, 225, 250, 275,
+                      100, 125, 150, 175, 100, 125, 150, 175, 0,   25,  50,  75,  0,   25,  50,  75};
+    int sc_width = 25;
+    int nadc = 32;
+    int pixels_per_sc = 5000;
+
+    auto dst = reinterpret_cast<DataType *>(buffer);
+    int pixel = 0;
+    for (int i = 0; i != pixels_per_sc; ++i) {
+        for (int i_adc = 0; i_adc != nadc; ++i_adc) {
+            int col = adc_nr[i_adc] + (i % sc_width);
+            int row;
+            if ((i_adc / 4) % 2 == 0)
+                row = 199 - int(i / sc_width);
+            else
+                row = 200 + int(i / sc_width);
+
+            dst[col + row * 400] = tmp[pixel];
+            pixel++;
+        }
+    }
+    return rc;
+};
+template <typename DataType> size_t SubFile::read_impl_flip(std::byte *buffer) {
+
+    // read to temporary buffer
+    // TODO! benchmark direct reads
+    std::vector<std::byte> tmp(this->bytes_per_frame());
+    size_t rc = fread(reinterpret_cast<char *>(&tmp[0]), this->bytes_per_frame(), 1, this->fp);
+
+    // copy to place
+    const size_t start = this->cols * (this->rows - 1) * sizeof(DataType);
+    const size_t row_size = this->cols * sizeof(DataType);
+    auto dst = buffer + start;
+    auto src = &tmp[0];
+
+    for (int i = 0; i != this->rows; ++i) {
+        memcpy(dst, src, row_size);
+        dst -= row_size;
+        src += row_size;
+    }
+
+    return rc;
+};

file_io/src/helpers.cpp

@@ -0,0 +1,11 @@
+#include "aare/helpers.hpp"
+
+
+bool is_master_file(std::filesystem::path fpath) {
+    std::string stem = fpath.stem();
+    if (stem.find("_master_") != std::string::npos)
+        return true;
+    else
+        return false;
+}
+