diff --git a/CMakeLists.txt b/CMakeLists.txt index db6501c..1fae609 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,7 +56,6 @@ endif() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(AARE_FETCH_ZMQ) - FetchContent_Declare( libzmq GIT_REPOSITORY https://github.com/zeromq/libzmq.git @@ -70,7 +69,6 @@ if(AARE_FETCH_ZMQ) FetchContent_Populate(libzmq) add_subdirectory(${libzmq_SOURCE_DIR} ${libzmq_BINARY_DIR} EXCLUDE_FROM_ALL) endif() - else() find_package(ZeroMQ 4 REQUIRED) endif() @@ -91,10 +89,15 @@ else() find_package(fmt 6 REQUIRED) endif() +find_package(nlohmann_json 3.11.3 REQUIRED) add_library(aare_compiler_flags INTERFACE) target_compile_features(aare_compiler_flags INTERFACE cxx_std_17) +if(AARE_PYTHON_BINDINGS) + add_subdirectory(python) +endif() + ################# # MSVC specific # ################# @@ -145,33 +148,6 @@ else() -D_GLIBCXX_DEBUG_PEDANTIC ) - if (NOT AARE_PYTHON_BINDINGS) - target_compile_options( - aare_compiler_flags - INTERFACE - -fdiagnostics-parseable-fixits - # -fdiagnostics-generate-patch - -fdiagnostics-show-template-tree - -fsanitize=address,undefined,pointer-compare - -fno-sanitize-recover - # -D_FORTIFY_SOURCE=2 # not needed for debug builds - # -fstack-protector # cause errors wih folly? (ProducerConsumerQueue.hpp) - -fno-omit-frame-pointer - ) - - target_link_libraries( - aare_compiler_flags - INTERFACE - -fdiagnostics-parseable-fixits - # -fdiagnostics-generate-patch - -fdiagnostics-show-template-tree - -fsanitize=address,undefined,pointer-compare - -fno-sanitize-recover - # -D_FORTIFY_SOURCE=2 - -fno-omit-frame-pointer - ) - endif() - endif() if(AARE_USE_WARNINGS) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 45aa42a..9a16ebd 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -16,6 +16,9 @@ set(SPHINX_SOURCE_FILES src/NDView.rst src/Frame.rst src/Dtype.rst + src/ClusterFinder.rst + src/Pedestal.rst + src/VarClusterFinder.rst ) foreach(filename ${SPHINX_SOURCE_FILES}) diff --git a/docs/src/ClusterFinder.rst b/docs/src/ClusterFinder.rst new file mode 100644 index 0000000..1411dbd --- /dev/null +++ b/docs/src/ClusterFinder.rst @@ -0,0 +1,5 @@ +ClusterFinder +============= + + +.. doxygenfile:: ClusterFinder.hpp \ No newline at end of file diff --git a/docs/src/Pedestal.rst b/docs/src/Pedestal.rst new file mode 100644 index 0000000..ca991c3 --- /dev/null +++ b/docs/src/Pedestal.rst @@ -0,0 +1,5 @@ +Pedestal +============= + + +.. doxygenfile:: Pedestal.hpp \ No newline at end of file diff --git a/docs/src/VarClusterFinder.rst b/docs/src/VarClusterFinder.rst new file mode 100644 index 0000000..e3839bd --- /dev/null +++ b/docs/src/VarClusterFinder.rst @@ -0,0 +1,5 @@ +VarClusterFinder +==================== + + +.. doxygenfile:: VarClusterFinder.hpp \ No newline at end of file diff --git a/docs/src/index.rst b/docs/src/index.rst index 12640d3..1e45daa 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -12,4 +12,7 @@ AARE NDArray NDView Frame - Dtype \ No newline at end of file + Dtype + ClusterFinder + Pedestal + VarClusterFinder \ No newline at end of file diff --git a/include/aare/ClusterFileV2.hpp b/include/aare/ClusterFileV2.hpp new file mode 100644 index 0000000..99f5976 --- /dev/null +++ b/include/aare/ClusterFileV2.hpp @@ -0,0 +1,148 @@ +#pragma once +#include "aare/core/defs.hpp" +#include +#include +#include + +namespace aare { +struct ClusterHeader { + int32_t frame_number; + int32_t n_clusters; + std::string to_string() const { + return "frame_number: " + std::to_string(frame_number) + ", n_clusters: " + std::to_string(n_clusters); + } +}; + +struct ClusterV2_ { + int16_t x; + int16_t y; + std::array data; + std::string to_string(bool detailed = false) const { + if (detailed) { + std::string data_str = "["; + for (auto &d : data) { + data_str += std::to_string(d) + ", "; + } + data_str += "]"; + return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", data: " + data_str; + } + return "x: " + std::to_string(x) + ", y: " + std::to_string(y); + } +}; + +struct ClusterV2 { + ClusterV2_ cluster; + int32_t frame_number; + std::string to_string() const { + return "frame_number: " + std::to_string(frame_number) + ", " + cluster.to_string(); + } +}; + +/** + * @brief + * important not: fp always points to the clusters header and does not point to individual clusters + * + */ +class ClusterFileV2 { + std::filesystem::path m_fpath; + std::string m_mode; + FILE *fp{nullptr}; + + void check_open(){ + if (!fp) + throw std::runtime_error(fmt::format("File: {} not open", m_fpath.string())); + } + + public: + ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode): m_fpath(fpath), m_mode(mode) { + if (m_mode != "r" && m_mode != "w") + throw std::invalid_argument("mode must be 'r' or 'w'"); + if (m_mode == "r" && !std::filesystem::exists(m_fpath)) + throw std::invalid_argument("File does not exist"); + if (mode == "r") { + fp = fopen(fpath.string().c_str(), "rb"); + } else if (mode == "w") { + if (std::filesystem::exists(fpath)) { + fp = fopen(fpath.string().c_str(), "r+b"); + } else { + fp = fopen(fpath.string().c_str(), "wb"); + } + } + if (fp == nullptr) { + throw std::runtime_error("Failed to open file"); + } + } + ~ClusterFileV2() { close(); } + std::vector read() { + check_open(); + + ClusterHeader header; + fread(&header, sizeof(ClusterHeader), 1, fp); + std::vector clusters_(header.n_clusters); + fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp); + std::vector clusters; + for (auto &c : clusters_) { + ClusterV2 cluster; + cluster.cluster = std::move(c); + cluster.frame_number = header.frame_number; + clusters.push_back(cluster); + } + + return clusters; + } + std::vector> read(int n_frames) { + std::vector> clusters; + for (int i = 0; i < n_frames; i++) { + clusters.push_back(read()); + } + return clusters; + } + + size_t write(std::vector const &clusters) { + check_open(); + if (m_mode != "w") + throw std::runtime_error("File not opened in write mode"); + if (clusters.empty()) + return 0; + + ClusterHeader header; + header.frame_number = clusters[0].frame_number; + header.n_clusters = clusters.size(); + fwrite(&header, sizeof(ClusterHeader), 1, fp); + for (auto &c : clusters) { + fwrite(&c.cluster, sizeof(ClusterV2_), 1, fp); + } + return clusters.size(); + } + + size_t write(std::vector> const &clusters) { + check_open(); + if (m_mode != "w") + throw std::runtime_error("File not opened in write mode"); + + size_t n_clusters = 0; + for (auto &c : clusters) { + n_clusters += write(c); + } + return n_clusters; + } + + int seek_to_begin() { return fseek(fp, 0, SEEK_SET); } + int seek_to_end() { return fseek(fp, 0, SEEK_END); } + + int32_t frame_number() { + auto pos = ftell(fp); + ClusterHeader header; + fread(&header, sizeof(ClusterHeader), 1, fp); + fseek(fp, pos, SEEK_SET); + return header.frame_number; + } + + void close() { + if (fp) { + fclose(fp); + fp = nullptr; + } + } +}; +} // namespace aare \ No newline at end of file diff --git a/include/aare/File.hpp b/include/aare/File.hpp new file mode 100644 index 0000000..6286cd2 --- /dev/null +++ b/include/aare/File.hpp @@ -0,0 +1,59 @@ +#pragma once +#include "aare/FileInterface.hpp" + + +namespace aare { + +/** + * @brief RAII File class for reading and writing image files in various formats + * wrapper on a FileInterface to abstract the underlying file format + * @note documentation for each function is in the FileInterface class + */ +class File { + private: + FileInterface *file_impl; + bool is_npy; + + public: + /** + * @brief Construct a new File object + * @param fname path to the file + * @param mode file mode (r, w, a) + * @param cfg file configuration + * @throws std::runtime_error if the file cannot be opened + * @throws std::invalid_argument if the file mode is not supported + * + */ + File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {}); + void write(Frame &frame, sls_detector_header header = {}); + Frame read(); + Frame iread(size_t frame_number); + std::vector read(size_t n_frames); + void read_into(std::byte *image_buf); + void read_into(std::byte *image_buf, size_t n_frames); + size_t frame_number(size_t frame_index); + size_t bytes_per_frame(); + size_t pixels_per_frame(); + void seek(size_t frame_number); + size_t tell() const; + size_t total_frames() const; + size_t rows() const; + size_t cols() const; + size_t bitdepth() const; + void set_total_frames(size_t total_frames); + DetectorType detector_type() const; + xy geometry() const; + + /** + * @brief Move constructor + * @param other File object to move from + */ + File(File &&other) noexcept; + + /** + * @brief destructor: will only delete the FileInterface object + */ + ~File(); +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/FileInterface.hpp b/include/aare/FileInterface.hpp new file mode 100644 index 0000000..97e0d19 --- /dev/null +++ b/include/aare/FileInterface.hpp @@ -0,0 +1,196 @@ +#pragma once +#include "aare/Dtype.hpp" +#include "aare/Frame.hpp" +#include "aare/defs.hpp" + +#include +#include + +namespace aare { + +/** + * @brief FileConfig structure to store the configuration of a file + * dtype: data type of the file + * rows: number of rows in the file + * cols: number of columns in the file + * geometry: geometry of the file + */ +struct FileConfig { + aare::Dtype dtype{typeid(uint16_t)}; + uint64_t rows{}; + uint64_t cols{}; + bool operator==(const FileConfig &other) const { + return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry && + detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file; + } + bool operator!=(const FileConfig &other) const { return !(*this == other); } + + // rawfile specific + std::string version{}; + xy geometry{1, 1}; + DetectorType detector_type{DetectorType::Unknown}; + int max_frames_per_file{}; + size_t total_frames{}; + std::string to_string() const { + return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) + + ", geometry: " + geometry.to_string() + ", detector_type: " + toString(detector_type) + + ", max_frames_per_file: " + std::to_string(max_frames_per_file) + + ", total_frames: " + std::to_string(total_frames) + " }"; + } +}; + +/** + * @brief FileInterface class to define the interface for file operations + * @note parent class for NumpyFile and RawFile + * @note all functions are pure virtual and must be implemented by the derived classes + */ +class FileInterface { + public: + /** + * @brief write a frame to the file + * @param frame frame to write + * @return void + * @throws std::runtime_error if the function is not implemented + */ + // virtual void write(Frame &frame) = 0; + + /** + * @brief write a vector of frames to the file + * @param frames vector of frames to write + * @return void + */ + // virtual void write(std::vector &frames) = 0; + + /** + * @brief read one frame from the file at the current position + * @return Frame + */ + virtual Frame read() = 0; + + /** + * @brief read n_frames from the file at the current position + * @param n_frames number of frames to read + * @return vector of frames + */ + virtual std::vector read(size_t n_frames) = 0; // Is this the right interface? + + /** + * @brief read one frame from the file at the current position and store it in the provided buffer + * @param image_buf buffer to store the frame + * @return void + */ + virtual void read_into(std::byte *image_buf) = 0; + + /** + * @brief read n_frames from the file at the current position and store them in the provided buffer + * @param image_buf buffer to store the frames + * @param n_frames number of frames to read + * @return void + */ + virtual void read_into(std::byte *image_buf, size_t n_frames) = 0; + + /** + * @brief get the frame number at the given frame index + * @param frame_index index of the frame + * @return frame number + */ + virtual size_t frame_number(size_t frame_index) = 0; + + /** + * @brief get the size of one frame in bytes + * @return size of one frame + */ + virtual size_t bytes_per_frame() = 0; + + /** + * @brief get the number of pixels in one frame + * @return number of pixels in one frame + */ + virtual size_t pixels_per_frame() = 0; + + /** + * @brief seek to the given frame number + * @param frame_number frame number to seek to + * @return void + */ + virtual void seek(size_t frame_number) = 0; + + /** + * @brief get the current position of the file pointer + * @return current position of the file pointer + */ + virtual size_t tell() = 0; + + /** + * @brief get the total number of frames in the file + * @return total number of frames in the file + */ + virtual size_t total_frames() const = 0; + /** + * @brief get the number of rows in the file + * @return number of rows in the file + */ + virtual size_t rows() const = 0; + /** + * @brief get the number of columns in the file + * @return number of columns in the file + */ + virtual size_t cols() const = 0; + /** + * @brief get the bitdepth of the file + * @return bitdepth of the file + */ + virtual size_t bitdepth() const = 0; + + /** + * @brief read one frame from the file at the given frame number + * @param frame_number frame number to read + * @return frame + */ + Frame iread(size_t frame_number) { + auto old_pos = tell(); + seek(frame_number); + Frame tmp = read(); + seek(old_pos); + return tmp; + }; + + /** + * @brief read n_frames from the file starting at the given frame number + * @param frame_number frame number to start reading from + * @param n_frames number of frames to read + * @return vector of frames + */ + std::vector iread(size_t frame_number, size_t n_frames) { + auto old_pos = tell(); + seek(frame_number); + std::vector tmp = read(n_frames); + seek(old_pos); + return tmp; + } + DetectorType detector_type() const { return m_type; } + + // function to query the data type of the file + /*virtual DataType dtype = 0; */ + + virtual ~FileInterface() = default; + + void set_total_frames(size_t total_frames) { m_total_frames = total_frames; } + + protected: + std::string m_mode{}; + std::filesystem::path m_fname{}; + std::filesystem::path m_base_path{}; + std::string m_base_name{}, m_ext{}; + int m_findex{}; + size_t m_total_frames{}; + size_t max_frames_per_file{}; + std::string version{}; + DetectorType m_type{DetectorType::Unknown}; + size_t m_rows{}; + size_t m_cols{}; + size_t m_bitdepth{}; + size_t current_frame{}; +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/NumpyFile.hpp b/include/aare/NumpyFile.hpp new file mode 100644 index 0000000..0e4ea41 --- /dev/null +++ b/include/aare/NumpyFile.hpp @@ -0,0 +1,112 @@ +#pragma once +#include "aare/Dtype.hpp" +#include "aare/defs.hpp" +#include "aare/FileInterface.hpp" +#include "aare/NumpyHelpers.hpp" + + +#include +#include +#include + +namespace aare { + + + +/** + * @brief NumpyFile class to read and write numpy files + * @note derived from FileInterface + * @note implements all the pure virtual functions from FileInterface + * @note documentation for the functions can also be found in the FileInterface class + */ +class NumpyFile : public FileInterface { + + public: + /** + * @brief NumpyFile constructor + * @param fname path to the numpy file + * @param mode file mode (r, w) + * @param cfg file configuration + */ + explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {}); + + void write(Frame &frame); + Frame read() override { return get_frame(this->current_frame++); } + + std::vector read(size_t n_frames) override; + void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); } + void read_into(std::byte *image_buf, size_t n_frames) override; + size_t frame_number(size_t frame_index) override { return frame_index; }; + size_t bytes_per_frame() override; + size_t pixels_per_frame() override; + void seek(size_t frame_number) override { this->current_frame = frame_number; } + size_t tell() override { return this->current_frame; } + size_t total_frames() const override { return m_header.shape[0]; } + size_t rows() const override { return m_header.shape[1]; } + size_t cols() const override { return m_header.shape[2]; } + size_t bitdepth() const override { return m_header.dtype.bitdepth(); } + + /** + * @brief get the data type of the numpy file + * @return DType + */ + Dtype dtype() const { return m_header.dtype; } + + /** + * @brief get the shape of the numpy file + * @return vector of type size_t + */ + std::vector shape() const { return m_header.shape; } + + /** + * @brief load the numpy file into an NDArray + * @tparam T data type of the NDArray + * @tparam NDim number of dimensions of the NDArray + * @return NDArray + */ + template NDArray load() { + NDArray arr(make_shape(m_header.shape)); + if (fseek(fp, static_cast(header_size), SEEK_SET)) { + throw std::runtime_error(LOCATION + "Error seeking to the start of the data"); + } + size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp); + if (rc != static_cast(arr.size())) { + throw std::runtime_error(LOCATION + "Error reading data from file"); + } + return arr; + } + template void write(NDView &frame) { + write_impl(frame.data(), frame.total_bytes()); + } + template void write(NDArray &frame) { + write_impl(frame.data(), frame.total_bytes()); + } + template void write(NDView &&frame) { + write_impl(frame.data(), frame.total_bytes()); + } + template void write(NDArray &&frame) { + write_impl(frame.data(), frame.total_bytes()); + } + + ~NumpyFile() noexcept override; + + private: + FILE *fp = nullptr; + size_t initial_header_len = 0; + size_t current_frame{}; + uint32_t header_len{}; + uint8_t header_len_size{}; + size_t header_size{}; + NumpyHeader m_header; + uint8_t major_ver_{}; + uint8_t minor_ver_{}; + size_t m_bytes_per_frame{}; + size_t m_pixels_per_frame{}; + + void load_metadata(); + void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/); + Frame get_frame(size_t frame_number); + void write_impl(void *data, uint64_t size); +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/NumpyHelpers.hpp b/include/aare/NumpyHelpers.hpp new file mode 100644 index 0000000..8ed0ec7 --- /dev/null +++ b/include/aare/NumpyHelpers.hpp @@ -0,0 +1,55 @@ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aare/Dtype.hpp" +#include "aare/defs.hpp" + +namespace aare { + +struct NumpyHeader { + Dtype dtype{aare::Dtype::ERROR}; + bool fortran_order{false}; + std::vector shape{}; + std::string to_string() const; +}; + +namespace NumpyHelpers { + +const constexpr std::array magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'}; +const uint8_t magic_string_length{6}; + +std::string parse_str(const std::string &in); +/** + Removes leading and trailing whitespaces + */ +std::string trim(const std::string &str); + +std::vector parse_tuple(std::string in); + +bool parse_bool(const std::string &in); + +std::string get_value_from_map(const std::string &mapstr); + +std::unordered_map parse_dict(std::string in, const std::vector &keys); + +template bool in_array(T val, const std::array &arr) { + return std::find(std::begin(arr), std::end(arr), val) != std::end(arr); +} +bool is_digits(const std::string &str); + +aare::Dtype parse_descr(std::string typestring); +size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header); +size_t write_header(std::ostream &out, const NumpyHeader &header); + +} // namespace NumpyHelpers +} // namespace aare \ No newline at end of file diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp new file mode 100644 index 0000000..7de4aef --- /dev/null +++ b/include/aare/RawFile.hpp @@ -0,0 +1,186 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/FileInterface.hpp" +#include "aare/SubFile.hpp" + +namespace aare { + +struct ModuleConfig { + int module_gap_row{}; + int module_gap_col{}; + + bool operator==(const ModuleConfig &other) const { + if (module_gap_col != other.module_gap_col) + return false; + if (module_gap_row != other.module_gap_row) + return false; + return true; + } +}; + +/** + * @brief RawFile class to read .raw and .json files + * @note derived from FileInterface + * @note documentation can also be found in the FileInterface class + */ +class RawFile : public FileInterface { + public: + /** + * @brief RawFile constructor + * @param fname path to the file + * @param mode file mode (r, w) + * @param cfg file configuration + */ + explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r", + const FileConfig &config = FileConfig{}); + + /** + * @brief write function is not implemented for RawFile + * @param frame frame to write + */ + void write(Frame &frame, sls_detector_header header); + Frame read() override { return get_frame(this->current_frame++); }; + std::vector read(size_t n_frames) override; + void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }; + void read_into(std::byte *image_buf, size_t n_frames) override; + size_t frame_number(size_t frame_index) override; + + /** + * @brief get the number of bytess per frame + * @return size of one frame in bytes + */ + size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; } + + /** + * @brief get the number of pixels in the frame + * @return number of pixels + */ + size_t pixels_per_frame() override { return m_rows * m_cols; } + + // goto frame index + void seek(size_t frame_index) override { + // check if the frame number is greater than the total frames + // if frame_number == total_frames, then the next read will throw an error + if (frame_index > this->total_frames()) { + throw std::runtime_error( + fmt::format("frame number {} is greater than total frames {}", frame_index, m_total_frames)); + } + this->current_frame = frame_index; + }; + + // return the position of the file pointer (in number of frames) + size_t tell() override { return this->current_frame; }; + + /** + * @brief check if the file is a master file + * @param fpath path to the file + */ + static bool is_master_file(const std::filesystem::path &fpath); + + /** + * @brief set the module gap row and column + * @param row gap between rows + * @param col gap between columns + */ + inline void set_config(int row, int col) { + cfg.module_gap_row = row; + cfg.module_gap_col = col; + } + // TODO! Deal with fast quad and missing files + + /** + * @brief get the number of subfiles for the RawFile + * @return number of subfiles + */ + void find_number_of_subfiles(); + + /** + * @brief get the master file name path for the RawFile + * @return path to the master file + */ + inline std::filesystem::path master_fname(); + /** + * @brief get the data file name path for the RawFile with the given module id and file id + * @param mod_id module id + * @param file_id file id + * @return path to the data file + */ + inline std::filesystem::path data_fname(size_t mod_id, size_t file_id); + + /** + * @brief destructor: will delete the subfiles + */ + ~RawFile() noexcept override; + + size_t total_frames() const override { return m_total_frames; } + size_t rows() const override { return m_rows; } + size_t cols() const override { return m_cols; } + size_t bitdepth() const override { return m_bitdepth; } + xy geometry() { return m_geometry; } + + private: + void write_master_file(); + /** + * @brief read the frame at the given frame index into the image buffer + * @param frame_number frame number to read + * @param image_buf buffer to store the frame + */ + void get_frame_into(size_t frame_index, std::byte *frame_buffer); + + /** + * @brief get the frame at the given frame index + * @param frame_number frame number to read + * @return Frame + */ + Frame get_frame(size_t frame_index); + + /** + * @brief parse the file name to get the extension, base name and index + */ + void parse_fname(); + + /** + * @brief parse the metadata from the file + */ + void parse_metadata(); + + /** + * @brief parse the metadata of a .raw file + */ + void parse_raw_metadata(); + + /** + * @brief parse the metadata of a .json file + */ + void parse_json_metadata(); + + /** + * @brief finds the geometry of the file + */ + void find_geometry(); + + /** + * @brief read the header of the file + * @param fname path to the data subfile + * @return sls_detector_header + */ + static sls_detector_header read_header(const std::filesystem::path &fname); + + /** + * @brief open the subfiles + */ + void open_subfiles(); + void parse_config(const FileConfig &config); + + size_t n_subfiles{}; + size_t n_subfile_parts{}; + std::vector> subfiles; + size_t subfile_rows{}, subfile_cols{}; + xy m_geometry{}; + std::vector positions; + ModuleConfig cfg{0, 0}; + TimingMode timing_mode{}; + bool quad{false}; +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/SubFile.hpp b/include/aare/SubFile.hpp new file mode 100644 index 0000000..16bdc41 --- /dev/null +++ b/include/aare/SubFile.hpp @@ -0,0 +1,77 @@ +#pragma once +#include "aare/Frame.hpp" +#include "aare/defs.hpp" + +#include +#include +#include +#include + +namespace aare { + +/** + * @brief Class to read a subfile from a RawFile + */ +class SubFile { + public: + size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index); + /** + * @brief SubFile constructor + * @param fname path to the subfile + * @param detector detector type + * @param rows number of rows in the subfile + * @param cols number of columns in the subfile + * @param bitdepth bitdepth of the subfile + * @throws std::invalid_argument if the detector,type pair is not supported + */ + SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode = "r"); + + /** + * @brief read the subfile into a buffer + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + size_t read_impl_normal(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes flipped + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + template size_t read_impl_flip(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes reordered + * @param buffer pointer to the buffer to read the data into + * @return number of bytes read + */ + template size_t read_impl_reorder(std::byte *buffer); + + /** + * @brief read the subfile into a buffer with the bytes reordered and flipped + * @param buffer pointer to the buffer to read the data into + * @param frame_number frame number to read + * @return number of bytes read + */ + size_t get_part(std::byte *buffer, size_t frame_index); + size_t frame_number(size_t frame_index); + + // TODO: define the inlines as variables and assign them in constructor + inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; } + inline size_t pixels_per_part() const { return m_rows * m_cols; } + + ~SubFile(); + + protected: + FILE *fp = nullptr; + size_t m_bitdepth; + std::filesystem::path m_fname; + size_t m_rows{}; + size_t m_cols{}; + std::string m_mode; + size_t n_frames{}; + int m_sub_file_index_{}; +}; + +} // namespace aare \ No newline at end of file diff --git a/include/aare/VariableSizeClusterFinder.hpp b/include/aare/VarClusterFinder.hpp similarity index 93% rename from include/aare/VariableSizeClusterFinder.hpp rename to include/aare/VarClusterFinder.hpp index 88e9016..dae45cb 100644 --- a/include/aare/VariableSizeClusterFinder.hpp +++ b/include/aare/VarClusterFinder.hpp @@ -10,7 +10,7 @@ const int MAX_CLUSTER_SIZE = 200; namespace aare { -template class ClusterFinder { +template class VarClusterFinder { public: struct Hit { int16_t size{}; @@ -49,7 +49,7 @@ template class ClusterFinder { int check_neighbours(int i, int j); public: - ClusterFinder(image_shape shape, T threshold) + VarClusterFinder(image_shape shape, T threshold) : shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) { hits.reserve(2000); } @@ -112,7 +112,7 @@ template class ClusterFinder { } } }; -template int ClusterFinder::check_neighbours(int i, int j) { +template int VarClusterFinder::check_neighbours(int i, int j) { std::vector neighbour_labels; for (int k = 0; k < 4; ++k) { @@ -144,7 +144,7 @@ template int ClusterFinder::check_neighbours(int i, int j) { } } -template void ClusterFinder::find_clusters(NDView img) { +template void VarClusterFinder::find_clusters(NDView img) { original_ = img; labeled_ = 0; peripheral_labeled_ = 0; @@ -156,7 +156,7 @@ template void ClusterFinder::find_clusters(NDView img) { store_clusters(); } -template void ClusterFinder::find_clusters_X(NDView img) { +template void VarClusterFinder::find_clusters_X(NDView img) { original_ = img; int clusterIndex = 0; for (int i = 0; i < shape_[0]; ++i) { @@ -175,7 +175,7 @@ template void ClusterFinder::find_clusters_X(NDView img) { h_size.clear(); } -template void ClusterFinder::rec_FillHit(int clusterIndex, int i, int j) { +template void VarClusterFinder::rec_FillHit(int clusterIndex, int i, int j) { // printf("original_(%d, %d)=%f\n", i, j, original_(i,j)); // printf("h_size[%d].size=%d\n", clusterIndex, h_size[clusterIndex].size); if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE) { @@ -213,7 +213,7 @@ template void ClusterFinder::rec_FillHit(int clusterIndex, int i } } -template void ClusterFinder::single_pass(NDView img) { +template void VarClusterFinder::single_pass(NDView img) { original_ = img; labeled_ = 0; current_label = 0; @@ -224,7 +224,7 @@ template void ClusterFinder::single_pass(NDView img) { // store_clusters(); } -template void ClusterFinder::first_pass() { +template void VarClusterFinder::first_pass() { for (int i = 0; i < original_.size(); ++i) { if (use_noise_map) @@ -248,7 +248,7 @@ template void ClusterFinder::first_pass() { } } -template void ClusterFinder::second_pass() { +template void VarClusterFinder::second_pass() { for (int64_t i = 0; i != labeled_.size(); ++i) { auto current_label = labeled_(i); @@ -265,7 +265,7 @@ template void ClusterFinder::second_pass() { } } -template void ClusterFinder::store_clusters() { +template void VarClusterFinder::store_clusters() { // Accumulate hit information in a map // Do we always have monotonic increasing diff --git a/include/aare/json.hpp b/include/aare/json.hpp new file mode 100644 index 0000000..2afe7f7 --- /dev/null +++ b/include/aare/json.hpp @@ -0,0 +1,68 @@ +#pragma once +#include +#include +#include +#include + +// helper functions to write json +// append to string for better performance (not tested) + +namespace aare { + +/** + * @brief write a digit to a string + * takes key and value and outputs->"key": value, + * @tparam T type of value (int, uint32_t, ...) + * @param s string to append to + * @param key key to write + * @param value value to write + * @return void + * @note + * - can't use concepts here because we are using c++17 + */ +template inline void write_digit(std::string &s, const std::string &key, const T &value) { + s += "\""; + s += key; + s += "\": "; + s += std::to_string(value); + s += ", "; +} +inline void write_str(std::string &s, const std::string &key, const std::string &value) { + s += "\""; + s += key; + s += "\": \""; + s += value; + s += "\", "; +} +inline void write_map(std::string &s, const std::string &key, const std::map &value) { + s += "\""; + s += key; + s += "\": {"; + for (const auto &kv : value) { + write_str(s, kv.first, kv.second); + } + // remove last comma or trailing spaces + for (size_t i = s.size() - 1; i > 0; i--) { + if ((s[i] == ',') || (s[i] == ' ')) { + s.pop_back(); + } else + break; + } + s += "}, "; +} + +template void write_array(std::string &s, const std::string &key, const std::array &value) { + s += "\""; + s += key; + s += "\": ["; + + for (size_t i = 0; i < N - 1; i++) { + s += std::to_string(value[i]); + s += ", "; + } + s += std::to_string(value[N - 1]); + + s += "], "; +} + +} // namespace aare diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000..9dd1e34 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,17 @@ +if(AARE_FETCH_PYBIND11) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v2.11.0 + ) + FetchContent_MakeAvailable(pybind11) +else() + find_package(pybind11 2.11 REQUIRED) +endif() + + +pybind11_add_module(_aare src/module.cpp) +set_target_properties(_aare PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) +target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags) diff --git a/python/src/module.cpp b/python/src/module.cpp new file mode 100644 index 0000000..6379aa2 --- /dev/null +++ b/python/src/module.cpp @@ -0,0 +1,9 @@ + + + +#include + +namespace py = pybind11; + +PYBIND11_MODULE(_aare, m) { +} \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ac6aec..351be37 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,15 +4,20 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/defs.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Frame.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NumpyFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/RawFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/SubFile.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NumpyHelpers.cpp ) add_library(aare_core STATIC ${SourceFiles}) target_include_directories(aare_core PUBLIC ${CMAKE_SOURCE_DIR}/include) -target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags ) +target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags nlohmann_json::nlohmann_json) if (AARE_PYTHON_BINDINGS) -set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON) + set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON) endif() if(AARE_TESTS) @@ -25,11 +30,13 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/NDView.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ClusterFinder.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Pedestal.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NumpyFile.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/NumpyHelpers.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/RawFile.test.cpp # ${CMAKE_CURRENT_SOURCE_DIR}/test/CircularFifo.test.cpp # ${CMAKE_CURRENT_SOURCE_DIR}/test/wrappers.test.cpp # ${CMAKE_CURRENT_SOURCE_DIR}/test/Transforms.test.cpp ) target_sources(tests PRIVATE ${TestSources} ) - target_link_libraries(tests PRIVATE aare_core) endif() \ No newline at end of file diff --git a/src/File.cpp b/src/File.cpp new file mode 100644 index 0000000..5a7c198 --- /dev/null +++ b/src/File.cpp @@ -0,0 +1,70 @@ +#include "aare/File.hpp" +#include "aare/NumpyFile.hpp" +#include "aare/RawFile.hpp" + +#include + +namespace aare { + +File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg) + : file_impl(nullptr), is_npy(true) { + if (mode != "r" && mode != "w" && mode != "a") { + throw std::invalid_argument("Unsupported file mode"); + } + + if ((mode == "r" || mode == "a") && !std::filesystem::exists(fname)) { + throw std::runtime_error(fmt::format("File does not exist: {}", fname.string())); + } + + if (fname.extension() == ".raw" || fname.extension() == ".json") { + // aare::logger::debug("Loading raw file"); + file_impl = new RawFile(fname, mode, cfg); + is_npy = false; + } + // check if extension is numpy + else if (fname.extension() == ".npy") { + // aare::logger::debug("Loading numpy file"); + file_impl = new NumpyFile(fname, mode, cfg); + } else { + throw std::runtime_error("Unsupported file type"); + } +} + +void File::write(Frame &frame, sls_detector_header header) { + if (is_npy) { + // aare::logger::info("ignoring header for npy file"); + dynamic_cast(file_impl)->write(frame); + } else { + dynamic_cast(file_impl)->write(frame, header); + } +} +Frame File::read() { return file_impl->read(); } +size_t File::total_frames() const { return file_impl->total_frames(); } +std::vector File::read(size_t n_frames) { return file_impl->read(n_frames); } +void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); } +void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); } +size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); } +size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); } +size_t File::pixels_per_frame() { return file_impl->pixels_per_frame(); } +void File::seek(size_t frame_number) { file_impl->seek(frame_number); } +size_t File::tell() const { return file_impl->tell(); } +size_t File::rows() const { return file_impl->rows(); } +size_t File::cols() const { return file_impl->cols(); } +size_t File::bitdepth() const { return file_impl->bitdepth(); } +void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); } +File::~File() { delete file_impl; } +DetectorType File::detector_type() const { return file_impl->detector_type(); } +xy File::geometry() const { + if (is_npy) { + return {1, 1}; + } + return reinterpret_cast(file_impl)->geometry(); +} + +Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); } + +File::File(File &&other) noexcept : file_impl(other.file_impl), is_npy(other.is_npy) { other.file_impl = nullptr; } + +// write move assignment operator + +} // namespace aare \ No newline at end of file diff --git a/src/NumpyFile.cpp b/src/NumpyFile.cpp new file mode 100644 index 0000000..8ec9fd5 --- /dev/null +++ b/src/NumpyFile.cpp @@ -0,0 +1,200 @@ + +#include "aare/NumpyFile.hpp" +#include "aare/NumpyHelpers.hpp" + +namespace aare { + + + +NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) { + // TODO! add opts to constructor + m_fname = fname; + m_mode = mode; + if (mode == "r") { + fp = fopen(m_fname.string().c_str(), "rb"); + if (!fp) { + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string())); + } + load_metadata(); + } else if (mode == "w") { + m_bitdepth = cfg.dtype.bitdepth(); + m_rows = cfg.rows; + m_cols = cfg.cols; + m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}}; + m_header.shape = {0, cfg.rows, cfg.cols}; + fp = fopen(m_fname.string().c_str(), "wb"); + if (!fp) { + throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string())); + } + initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header); + } + m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>()); + + m_bytes_per_frame = m_header.dtype.bitdepth() / 8 * m_pixels_per_frame; +} +void NumpyFile::write(Frame &frame) { write_impl(frame.data(), frame.bytes()); } +void NumpyFile::write_impl(void *data, uint64_t size) { + + if (fp == nullptr) { + throw std::runtime_error("File not open"); + } + if (!(m_mode == "w" || m_mode == "a")) { + throw std::invalid_argument("File not open for writing"); + } + if (fseek(fp, 0, SEEK_END)) + throw std::runtime_error("Could not seek to end of file"); + size_t const rc = fwrite(data, size, 1, fp); + if (rc != 1) { + throw std::runtime_error("Error writing frame to file"); + } + + m_header.shape[0]++; +} + +Frame NumpyFile::get_frame(size_t frame_number) { + Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype); + get_frame_into(frame_number, frame.data()); + return frame; +} +void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) { + if (fp == nullptr) { + throw std::runtime_error("File not open"); + } + if (frame_number > m_header.shape[0]) { + throw std::invalid_argument("Frame number out of range"); + } + if (fseek(fp, header_size + frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT + throw std::runtime_error("Could not seek to frame"); + + size_t const rc = fread(image_buf, m_bytes_per_frame, 1, fp); + if (rc != 1) { + throw std::runtime_error("Error reading frame from file"); + } +} + +size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; }; +size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; }; + +std::vector NumpyFile::read(size_t n_frames) { + // TODO: implement this in a more efficient way + std::vector frames; + for (size_t i = 0; i < n_frames; i++) { + frames.push_back(get_frame(current_frame)); + current_frame++; + } + return frames; +} +void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) { + // TODO: implement this in a more efficient way + for (size_t i = 0; i < n_frames; i++) { + get_frame_into(current_frame++, image_buf); + image_buf += m_bytes_per_frame; + } +} + +NumpyFile::~NumpyFile() noexcept { + if (m_mode == "w" || m_mode == "a") { + // determine number of frames + if (fseek(fp, 0, SEEK_END)) { + std::cout << "Could not seek to end of file" << std::endl; + } + size_t const file_size = ftell(fp); + size_t const data_size = file_size - initial_header_len; + size_t const n_frames = data_size / m_bytes_per_frame; + // update number of frames in header (first element of shape) + m_header.shape[0] = n_frames; + if (fseek(fp, 0, SEEK_SET)) { + std::cout << "Could not seek to beginning of file" << std::endl; + } + // create string stream to contain header + std::stringstream ss; + aare::NumpyHelpers::write_header(ss, m_header); + std::string const header_str = ss.str(); + // write header + size_t const rc = fwrite(header_str.c_str(), header_str.size(), 1, fp); + if (rc != 1) { + std::cout << "Error writing header to numpy file in destructor" << std::endl; + } + } + + if (fp != nullptr) { + if (fclose(fp)) { + std::cout << "Error closing file" << std::endl; + } + } +} + +void NumpyFile::load_metadata() { + + // read magic number + std::array tmp{}; + size_t rc = fread(tmp.data(), tmp.size(), 1, fp); + if (rc != 1) { + throw std::runtime_error("Error reading magic number"); + } + if (tmp != aare::NumpyHelpers::magic_str) { + for (auto item : tmp) + fmt::print("{}, ", static_cast(item)); + fmt::print("\n"); + throw std::runtime_error("Not a numpy file"); + } + + // read version + rc = fread(reinterpret_cast(&major_ver_), sizeof(major_ver_), 1, fp); + rc += fread(reinterpret_cast(&minor_ver_), sizeof(minor_ver_), 1, fp); + if (rc != 2) { + throw std::runtime_error("Error reading numpy version"); + } + + if (major_ver_ == 1) { + header_len_size = 2; + } else if (major_ver_ == 2) { + header_len_size = 4; + } else { + throw std::runtime_error("Unsupported numpy version"); + } + + // read header length + rc = fread(reinterpret_cast(&header_len), header_len_size, 1, fp); + if (rc != 1) { + throw std::runtime_error("Error reading header length"); + } + header_size = aare::NumpyHelpers::magic_string_length + 2 + header_len_size + header_len; + if (header_size % 16 != 0) { + fmt::print("Warning: header length is not a multiple of 16\n"); + } + + // read header + std::string header(header_len, '\0'); + rc = fread(header.data(), header_len, 1, fp); + if (rc != 1) { + throw std::runtime_error("Error reading header"); + } + + // parse header + std::vector const keys{"descr", "fortran_order", "shape"}; + auto dict_map = aare::NumpyHelpers::parse_dict(header, keys); + if (dict_map.empty()) + throw std::runtime_error("invalid dictionary in header"); + + std::string const descr_s = dict_map["descr"]; + std::string const fortran_s = dict_map["fortran_order"]; + std::string const shape_s = dict_map["shape"]; + + std::string const descr = aare::NumpyHelpers::parse_str(descr_s); + aare::Dtype const dtype = aare::NumpyHelpers::parse_descr(descr); + + // convert literal Python bool to C++ bool + bool const fortran_order = aare::NumpyHelpers::parse_bool(fortran_s); + + // parse the shape tuple + auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s); + std::vector shape; + for (const auto &item : shape_v) { + auto dim = static_cast(std::stoul(item)); + shape.push_back(dim); + } + m_header = {dtype, fortran_order, shape}; +} + +} // namespace aare \ No newline at end of file diff --git a/src/NumpyFile.test.cpp b/src/NumpyFile.test.cpp new file mode 100644 index 0000000..daa3903 --- /dev/null +++ b/src/NumpyFile.test.cpp @@ -0,0 +1,50 @@ +#include "aare/NumpyFile.hpp" +#include "aare/NDArray.hpp" + +#include +#include "test_config.hpp" + +using aare::Dtype; +using aare::NumpyFile; +TEST_CASE("Read a 1D numpy file with int32 data type") { + + auto fpath = test_data_path() / "numpy" / "test_1d_int32.npy"; + REQUIRE(std::filesystem::exists(fpath)); + + NumpyFile f(fpath); + + // we know the file contains 10 elements of np.int32 containing values 0-9 + REQUIRE(f.dtype() == Dtype::INT32); + REQUIRE(f.shape() == std::vector{10}); + + // use the load function to read the full file into a NDArray + auto data = f.load(); + for (int32_t i = 0; i < 10; i++) { + REQUIRE(data(i) == i); + } +} + +TEST_CASE("Read a 3D numpy file with np.double data type") { + + auto fpath = test_data_path() / "numpy" / "test_3d_double.npy"; + REQUIRE(std::filesystem::exists(fpath)); + + NumpyFile f(fpath); + + // we know the file contains 10 elements of np.int32 containing values 0-9 + REQUIRE(f.dtype() == Dtype::DOUBLE); + REQUIRE(f.shape() == std::vector{3, 2, 5}); + + // use the load function to read the full file into a NDArray + // numpy code to generate the array + // arr2[0,0,0] = 1.0 + // arr2[0,0,1] = 2.0 + // arr2[0,1,0] = 72.0 + // arr2[2,0,4] = 63.0 + + auto data = f.load(); + REQUIRE(data(0, 0, 0) == 1.0); + REQUIRE(data(0, 0, 1) == 2.0); + REQUIRE(data(0, 1, 0) == 72.0); + REQUIRE(data(2, 0, 4) == 63.0); +} \ No newline at end of file diff --git a/src/NumpyHelpers.cpp b/src/NumpyHelpers.cpp new file mode 100644 index 0000000..b8414d9 --- /dev/null +++ b/src/NumpyHelpers.cpp @@ -0,0 +1,269 @@ +/* + 28-03-2024 modified by: Bechir Braham + + Copyright 2017-2023 Leon Merten Lohse + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +#include "aare/NumpyHelpers.hpp" +#include + +namespace aare { + +std::string NumpyHeader::to_string() const { + std::stringstream sstm; + sstm << "dtype: " << dtype.to_string() << ", fortran_order: " << fortran_order << ' '; + sstm << "shape: ("; + for (auto item : shape) + sstm << item << ','; + sstm << ')'; + return sstm.str(); +} + + +namespace NumpyHelpers { + +std::unordered_map parse_dict(std::string in, const std::vector &keys) { + std::unordered_map map; + if (keys.empty()) + return map; + + in = trim(in); + + // unwrap dictionary + if ((in.front() == '{') && (in.back() == '}')) + in = in.substr(1, in.length() - 2); + else + throw std::runtime_error("Not a Python dictionary."); + + std::vector> positions; + + for (auto const &key : keys) { + size_t const pos = in.find("'" + key + "'"); + + if (pos == std::string::npos) + throw std::runtime_error("Missing '" + key + "' key."); + + std::pair const position_pair{pos, key}; + positions.push_back(position_pair); + } + + // sort by position in dict + std::sort(positions.begin(), positions.end()); + + for (size_t i = 0; i < positions.size(); ++i) { + std::string raw_value; + size_t const begin{positions[i].first}; + size_t end{std::string::npos}; + + std::string const key = positions[i].second; + + if (i + 1 < positions.size()) + end = positions[i + 1].first; + + raw_value = in.substr(begin, end - begin); + + raw_value = trim(raw_value); + + if (raw_value.back() == ',') + raw_value.pop_back(); + + map[key] = get_value_from_map(raw_value); + } + + return map; +} + +aare::Dtype parse_descr(std::string typestring) { + + if (typestring.length() < 3) { + throw std::runtime_error("invalid typestring (length)"); + } + + constexpr char little_endian_char = '<'; + constexpr char big_endian_char = '>'; + constexpr char no_endian_char = '|'; + constexpr std::array endian_chars = {little_endian_char, big_endian_char, no_endian_char}; + constexpr std::array numtype_chars = {'f', 'i', 'u', 'c'}; + + const char byteorder_c = typestring[0]; + const char kind_c = typestring[1]; + std::string const itemsize_s = typestring.substr(2); + + if (!in_array(byteorder_c, endian_chars)) { + throw std::runtime_error("invalid typestring (byteorder)"); + } + + if (!in_array(kind_c, numtype_chars)) { + throw std::runtime_error("invalid typestring (kind)"); + } + + if (!is_digits(itemsize_s)) { + throw std::runtime_error("invalid typestring (itemsize)"); + } + + return aare::Dtype(typestring); +} + +bool parse_bool(const std::string &in) { + if (in == "True") + return true; + if (in == "False") + return false; + throw std::runtime_error("Invalid python boolean."); +} + +std::string get_value_from_map(const std::string &mapstr) { + size_t const sep_pos = mapstr.find_first_of(':'); + if (sep_pos == std::string::npos) + return ""; + + std::string const tmp = mapstr.substr(sep_pos + 1); + return trim(tmp); +} + +bool is_digits(const std::string &str) { return std::all_of(str.begin(), str.end(), ::isdigit); } + +std::vector parse_tuple(std::string in) { + std::vector v; + const char separator = ','; + + in = trim(in); + + if ((in.front() == '(') && (in.back() == ')')) + in = in.substr(1, in.length() - 2); + else + throw std::runtime_error("Invalid Python tuple."); + + std::istringstream iss(in); + + for (std::string token; std::getline(iss, token, separator);) { + v.push_back(token); + } + + return v; +} + +std::string trim(const std::string &str) { + const std::string whitespace = " \t\n"; + auto begin = str.find_first_not_of(whitespace); + + if (begin == std::string::npos) + return ""; + + auto end = str.find_last_not_of(whitespace); + return str.substr(begin, end - begin + 1); +} + +std::string parse_str(const std::string &in) { + if ((in.front() == '\'') && (in.back() == '\'')) + return in.substr(1, in.length() - 2); + + throw std::runtime_error("Invalid python string."); +} + +void write_magic(std::ostream &ostream, int version_major, int version_minor) { + ostream.write(magic_str.data(), magic_string_length); + ostream.put(static_cast(version_major)); + ostream.put(static_cast(version_minor)); +} +template inline std::string write_tuple(const std::vector &v) { + + if (v.empty()) + return "()"; + std::ostringstream ss; + ss.imbue(std::locale("C")); + + if (v.size() == 1) { + ss << "(" << v.front() << ",)"; + } else { + const std::string delimiter = ", "; + // v.size() > 1 + ss << "("; + // for (size_t i = 0; i < v.size() - 1; ++i) { + // ss << v[i] << delimiter; + // } + // ss << v.back(); + std::copy(v.begin(), v.end() - 1, std::ostream_iterator(ss, ", ")); + ss << v.back(); + ss << ")"; + } + + return ss.str(); +} + +inline std::string write_boolean(bool b) { + if (b) + return "True"; + return "False"; +} + +inline std::string write_header_dict(const std::string &descr, bool fortran_order, const std::vector &shape) { + std::string const s_fortran_order = write_boolean(fortran_order); + std::string const shape_s = write_tuple(shape); + + return "{'descr': '" + descr + "', 'fortran_order': " + s_fortran_order + ", 'shape': " + shape_s + ", }"; +} + +size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header) { + std::ofstream out(fname, std::ios::binary | std::ios::out); + return write_header(out, header); +} + +size_t write_header(std::ostream &out, const NumpyHeader &header) { + std::string const header_dict = write_header_dict(header.dtype.to_string(), header.fortran_order, header.shape); + + size_t length = magic_string_length + 2 + 2 + header_dict.length() + 1; + + int version_major = 1; + int version_minor = 0; + if (length >= static_cast(255) * 255) { + length = magic_string_length + 2 + 4 + header_dict.length() + 1; + version_major = 2; + version_minor = 0; + } + size_t const padding_len = 16 - length % 16; + std::string const padding(padding_len, ' '); + + // write magic + write_magic(out, version_major, version_minor); + + // write header length + if (version_major == 1 && version_minor == 0) { + auto header_len = static_cast(header_dict.length() + padding.length() + 1); + + std::array header_len_le16{static_cast((header_len >> 0) & 0xff), + static_cast((header_len >> 8) & 0xff)}; + out.write(reinterpret_cast(header_len_le16.data()), 2); + } else { + auto header_len = static_cast(header_dict.length() + padding.length() + 1); + + std::array header_len_le32{ + static_cast((header_len >> 0) & 0xff), static_cast((header_len >> 8) & 0xff), + static_cast((header_len >> 16) & 0xff), static_cast((header_len >> 24) & 0xff)}; + out.write(reinterpret_cast(header_len_le32.data()), 4); + } + + out << header_dict << padding << '\n'; + return length; +} + +} // namespace NumpyHelpers +} // namespace aare diff --git a/src/NumpyHelpers.test.cpp b/src/NumpyHelpers.test.cpp new file mode 100644 index 0000000..36fcfe6 --- /dev/null +++ b/src/NumpyHelpers.test.cpp @@ -0,0 +1,62 @@ +#include "aare/NumpyHelpers.hpp" //Is this really a public header? +#include + +using namespace aare::NumpyHelpers; + +TEST_CASE("is_digits with a few standard cases") { + REQUIRE(is_digits("")); + REQUIRE(is_digits("123")); + REQUIRE(is_digits("0")); + REQUIRE_FALSE(is_digits("hej123")); + REQUIRE_FALSE(is_digits("a")); + REQUIRE_FALSE(is_digits(" ")); + REQUIRE_FALSE(is_digits("abcdef")); +} + +TEST_CASE("Check for quotes and return stripped string") { + REQUIRE(parse_str("'hej'") == "hej"); + REQUIRE(parse_str("'hej hej'") == "hej hej"); + REQUIRE(parse_str("''") == ""); +} + +TEST_CASE("parsing a string without quotes throws") { REQUIRE_THROWS(parse_str("hej")); } + +TEST_CASE("trim whitespace") { + REQUIRE(trim(" hej ") == "hej"); + REQUIRE(trim("hej") == "hej"); + REQUIRE(trim(" hej") == "hej"); + REQUIRE(trim("hej ") == "hej"); + REQUIRE(trim(" ") == ""); + REQUIRE(trim(" \thej hej ") == "hej hej"); +} + +TEST_CASE("parse data type descriptions") { + REQUIRE(parse_descr("{1, 2, 3})); + REQUIRE_FALSE(in_array(4, std::array{1, 2, 3})); + REQUIRE(in_array(1, std::array{1})); + REQUIRE_FALSE(in_array(1, std::array{})); +} + +TEST_CASE("Parse numpy dict") { + std::string in = "{'descr': ' keys{"descr", "fortran_order", "shape"}; + auto map = parse_dict(in, keys); + REQUIRE(map["descr"] == "' +#include + +using json = nlohmann::json; + +namespace aare { + +RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) { + m_mode = mode; + m_fname = fname; + if (mode == "r" || mode == "r+") { + if (config != FileConfig()) { + // aare::logger::warn( + // "In read mode it is not necessary to provide a config, the provided config will be ignored"); + } + parse_fname(); + parse_metadata(); + find_number_of_subfiles(); + find_geometry(); + open_subfiles(); + + } else if (mode == "w" || mode == "w+") { + + if (std::filesystem::exists(fname)) { + // handle mode w as w+ (no overrwriting) + throw std::runtime_error(LOCATION + "File already exists"); + } + + parse_config(config); + parse_fname(); + write_master_file(); + n_subfiles = 1; + n_subfile_parts = 1; + subfile_cols = m_cols; + subfile_rows = m_rows; + open_subfiles(); + + } else { + throw std::runtime_error(LOCATION + "Unsupported mode"); + } +} + +void RawFile::parse_config(const FileConfig &config) { + m_bitdepth = config.dtype.bitdepth(); + m_total_frames = config.total_frames; + m_rows = config.rows; + m_cols = config.cols; + m_type = config.detector_type; + max_frames_per_file = config.max_frames_per_file; + m_geometry = config.geometry; + version = config.version; + subfile_rows = config.geometry.row; + subfile_cols = config.geometry.col; + + if (m_geometry != aare::xy{1, 1}) { + throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing"); + } +} +void RawFile::write_master_file() { + if (m_ext != ".json") { + throw std::runtime_error(LOCATION + "only json master files are supported for writing"); + } + std::ofstream ofs(master_fname(), std::ios::binary); + std::string ss; + ss.reserve(1024); + ss += "{\n\t"; + aare::write_str(ss, "Version", version); + ss += "\n\t"; + aare::write_digit(ss, "Total Frames", m_total_frames); + ss += "\n\t"; + aare::write_str(ss, "Detector Type", toString(m_type)); + ss += "\n\t"; + aare::write_str(ss, "Geometry", m_geometry.to_string()); + ss += "\n\t"; + + uint64_t img_size = (m_cols * m_rows) / (static_cast(m_geometry.col * m_geometry.row)); + img_size *= m_bitdepth; + aare::write_digit(ss, "Image Size in bytes", img_size); + ss += "\n\t"; + aare::write_digit(ss, "Max Frames Per File", max_frames_per_file); + ss += "\n\t"; + aare::write_digit(ss, "Dynamic Range", m_bitdepth); + ss += "\n\t"; + const aare::xy pixels = {static_cast(m_rows / m_geometry.row), + static_cast(m_cols / m_geometry.col)}; + aare::write_str(ss, "Pixels", pixels.to_string()); + ss += "\n\t"; + aare::write_digit(ss, "Number of rows", m_rows); + ss += "\n\t"; + const std::string tmp = "{\n" + " \"Frame Number\": \"8 bytes\",\n" + " \"Exposure Length\": \"4 bytes\",\n" + " \"Packet Number\": \"4 bytes\",\n" + " \"Bunch Id\": \"8 bytes\",\n" + " \"Timestamp\": \"8 bytes\",\n" + " \"Module Id\": \"2 bytes\",\n" + " \"Row\": \"2 bytes\",\n" + " \"Column\": \"2 bytes\",\n" + " \"Reserved\": \"2 bytes\",\n" + " \"Debug\": \"4 bytes\",\n" + " \"RoundRNumber\": \"2 bytes\",\n" + " \"DetType\": \"1 byte\",\n" + " \"Version\": \"1 byte\",\n" + " \"Packet Mask\": \"64 bytes\"\n" + " }"; + + ss += "\"Frame Header Format\":" + tmp + "\n"; + ss += "}"; + ofs << ss; + ofs.close(); +} + +void RawFile::open_subfiles() { + if (m_mode == "r") + for (size_t i = 0; i != n_subfiles; ++i) { + auto v = std::vector(n_subfile_parts); + for (size_t j = 0; j != n_subfile_parts; ++j) { + v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth); + } + subfiles.push_back(v); + } + else { + auto v = std::vector(n_subfile_parts); // only one subfile is implemented + v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w"); + subfiles.push_back(v); + } +} + +sls_detector_header RawFile::read_header(const std::filesystem::path &fname) { + sls_detector_header h{}; + FILE *fp = fopen(fname.string().c_str(), "r"); + if (!fp) + throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string())); + + size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + if (rc != 1) + throw std::runtime_error(LOCATION + "Could not read header from file"); + if (fclose(fp)) { + throw std::runtime_error(LOCATION + "Could not close file"); + } + + return h; +} +bool RawFile::is_master_file(const std::filesystem::path &fpath) { + std::string const stem = fpath.stem().string(); + return stem.find("_master_") != std::string::npos; +} + +void RawFile::find_number_of_subfiles() { + int n_mod = 0; + while (std::filesystem::exists(data_fname(++n_mod, 0))) + ; + n_subfiles = n_mod; +} +inline std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) { + return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex); +} + +inline std::filesystem::path RawFile::master_fname() { + return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext); +} + +void RawFile::find_geometry() { + uint16_t r{}; + uint16_t c{}; + for (size_t i = 0; i < n_subfile_parts; i++) { + for (size_t j = 0; j != n_subfiles; ++j) { + auto h = this->read_header(data_fname(j, i)); + r = std::max(r, h.row); + c = std::max(c, h.column); + + positions.push_back({h.row, h.column}); + } + } + + r++; + c++; + + m_rows = (r * subfile_rows); + m_cols = (c * subfile_cols); + + m_rows += static_cast((r - 1) * cfg.module_gap_row); +} + +void RawFile::parse_metadata() { + if (m_ext == ".raw") { + parse_raw_metadata(); + if (m_bitdepth == 0) { + switch (m_type) { + case DetectorType::Eiger: + m_bitdepth = 32; + break; + default: + m_bitdepth = 16; + } + } + } else if (m_ext == ".json") { + parse_json_metadata(); + } else { + throw std::runtime_error(LOCATION + "Unsupported file type"); + } + n_subfile_parts = static_cast(m_geometry.row) * m_geometry.col; +} + +void RawFile::parse_json_metadata() { + std::ifstream ifs(master_fname()); + json j; + ifs >> j; + double v = j["Version"]; + version = fmt::format("{:.1f}", v); + m_type = StringTo(j["Detector Type"].get()); + timing_mode = StringTo(j["Timing Mode"].get()); + m_total_frames = j["Frames in File"]; + subfile_rows = j["Pixels"]["y"]; + subfile_cols = j["Pixels"]["x"]; + max_frames_per_file = j["Max Frames Per File"]; + try { + m_bitdepth = j.at("Dynamic Range"); + } catch (const json::out_of_range &e) { + m_bitdepth = 16; + } + // only Eiger had quad + if (m_type == DetectorType::Eiger) { + quad = (j["Quad"] == 1); + } + + m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]}; +} +void RawFile::parse_raw_metadata() { + std::ifstream ifs(master_fname()); + for (std::string line; std::getline(ifs, line);) { + if (line == "#Frame Header") + break; + auto pos = line.find(':'); + auto key_pos = pos; + while (key_pos != std::string::npos && std::isspace(line[--key_pos])) + ; + if (key_pos != std::string::npos) { + auto key = line.substr(0, key_pos + 1); + auto value = line.substr(pos + 2); + // do the actual parsing + if (key == "Version") { + version = value; + } else if (key == "TimeStamp") { + + } else if (key == "Detector Type") { + m_type = StringTo(value); + } else if (key == "Timing Mode") { + timing_mode = StringTo(value); + } else if (key == "Pixels") { + // Total number of pixels cannot be found yet looking at + // submodule + pos = value.find(','); + subfile_cols = std::stoi(value.substr(1, pos)); + subfile_rows = std::stoi(value.substr(pos + 1)); + } else if (key == "Total Frames") { + m_total_frames = std::stoi(value); + } else if (key == "Dynamic Range") { + m_bitdepth = std::stoi(value); + } else if (key == "Quad") { + quad = (value == "1"); + } else if (key == "Max Frames Per File") { + max_frames_per_file = std::stoi(value); + } else if (key == "Geometry") { + pos = value.find(','); + m_geometry = {static_cast(std::stoi(value.substr(1, pos))), + static_cast(std::stoi(value.substr(pos + 1)))}; + } + } + } +} + +void RawFile::parse_fname() { + bool wrong_format = false; + m_base_path = m_fname.parent_path().string(); + m_base_name = m_fname.stem().string(); + m_ext = m_fname.extension().string(); + try { + auto pos = m_base_name.rfind('_'); + m_findex = std::stoi(m_base_name.substr(pos + 1)); + } catch (const std::invalid_argument &e) { + m_findex = 0; + wrong_format = true; + } + auto pos = m_base_name.find("_master_"); + if (pos != std::string::npos) { + m_base_name.erase(pos); + wrong_format = true; + } + if (wrong_format && (m_mode == "w+" || m_mode == "w")) { + // aare::logger::warn("Master Filename", m_fname, "is not in the correct format"); + // aare::logger::warn("using", master_fname(), "as the master file"); + } +} + +Frame RawFile::get_frame(size_t frame_index) { + auto f = Frame(this->m_rows, this->m_cols, Dtype::from_bitdepth(this->m_bitdepth)); + std::byte *frame_buffer = f.data(); + get_frame_into(frame_index, frame_buffer); + return f; +} + +void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) { + if (frame_index > this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + std::vector frame_numbers(this->n_subfile_parts); + std::vector frame_indices(this->n_subfile_parts, frame_index); + + if (n_subfile_parts != 1) { + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto subfile_id = frame_index / this->max_frames_per_file; + frame_numbers[part_idx] = + this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file); + } + // 1. if frame number vector is the same break + while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) != + frame_numbers.end()) { + // 2. find the index of the minimum frame number, + auto min_frame_idx = + std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end())); + // 3. increase its index and update its respective frame number + frame_indices[min_frame_idx]++; + // 4. if we can't increase its index => throw error + if (frame_indices[min_frame_idx] >= this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file; + frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number( + frame_indices[min_frame_idx] % this->max_frames_per_file); + } + } + + if (this->m_geometry.col == 1) { + // get the part from each subfile and copy it to the frame + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; + auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part(); + this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset, + corrected_idx % this->max_frames_per_file); + } + + } else { + + // create a buffer that will hold a the frame part + auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8; + auto *part_buffer = new std::byte[bytes_per_part]; + + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + auto corrected_idx = frame_indices[part_idx]; + auto subfile_id = corrected_idx / this->max_frames_per_file; + + this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file); + for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) { + auto irow = cur_row + (part_idx / this->m_geometry.col) * this->subfile_rows; + auto icol = (part_idx % this->m_geometry.col) * this->subfile_cols; + auto dest = (irow * this->m_cols + icol); + dest = dest * this->m_bitdepth / 8; + memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8, + this->subfile_cols * this->m_bitdepth / 8); + } + } + delete[] part_buffer; + } +} + +void RawFile::write(Frame &frame, sls_detector_header header) { + if (m_mode == "r") { + throw std::runtime_error(LOCATION + "File is open in read mode"); + } + size_t const subfile_id = this->current_frame / this->max_frames_per_file; + for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) { + + this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header, + this->current_frame % this->max_frames_per_file); + } + this->current_frame++; +} + +std::vector RawFile::read(size_t n_frames) { + // TODO: implement this in a more efficient way + std::vector frames; + for (size_t i = 0; i < n_frames; i++) { + frames.push_back(this->get_frame(this->current_frame)); + this->current_frame++; + } + return frames; +} +void RawFile::read_into(std::byte *image_buf, size_t n_frames) { + // TODO: implement this in a more efficient way + for (size_t i = 0; i < n_frames; i++) { + this->get_frame_into(this->current_frame++, image_buf); + image_buf += this->bytes_per_frame(); + } +} + +size_t RawFile::frame_number(size_t frame_index) { + if (frame_index > this->m_total_frames) { + throw std::runtime_error(LOCATION + "Frame number out of range"); + } + size_t const subfile_id = frame_index / this->max_frames_per_file; + return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file); +} + +RawFile::~RawFile() noexcept { + + // update master file + if (m_mode == "w" || m_mode == "w+" || m_mode == "r+") { + try { + write_master_file(); + } catch (...) { + // aare::logger::warn(LOCATION + "Could not update master file"); + } + } + + for (auto &vec : subfiles) { + for (auto *subfile : vec) { + delete subfile; + } + } +} + +} // namespace aare \ No newline at end of file diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp new file mode 100644 index 0000000..9e99ce7 --- /dev/null +++ b/src/RawFile.test.cpp @@ -0,0 +1,114 @@ +#include "aare/File.hpp" + +#include +#include + +#include "test_config.hpp" + +using aare::File; + +TEST_CASE("Read number of frames from a jungfrau raw file") { + + auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + REQUIRE(f.total_frames() == 10); +} + +TEST_CASE("Read frame numbers from a jungfrau raw file") { + auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + + // we know this file has 10 frames with frame numbers 1 to 10 + // f0 1,2,3 + // f1 4,5,6 + // f2 7,8,9 + // f3 10 + for (size_t i = 0; i < 10; i++) { + CHECK(f.frame_number(i) == i + 1); + } +} + +TEST_CASE("Read data from a jungfrau 500k single port raw file") { + auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + + // we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102 + std::vector pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102}; + for (size_t i = 0; i < 10; i++) { + auto frame = f.read(); + CHECK(frame.rows() == 512); + CHECK(frame.cols() == 1024); + CHECK(frame.view()(0, 0) == pixel_0_0[i]); + } +} + +TEST_CASE("Read frame numbers from a raw file") { + auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + // we know this file has 3 frames with frame numbers 14, 15, 16 + std::vector frame_numbers = {14, 15, 16}; + + File f(fpath, "r"); + for (size_t i = 0; i < 3; i++) { + CHECK(f.frame_number(i) == frame_numbers[i]); + } +} + +TEST_CASE("Compare reading from a numpy file with a raw file") { + auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json"; + REQUIRE(std::filesystem::exists(fpath_raw)); + + auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy"; + REQUIRE(std::filesystem::exists(fpath_npy)); + + File raw(fpath_raw, "r"); + File npy(fpath_npy, "r"); + + CHECK(raw.total_frames() == 10); + CHECK(npy.total_frames() == 10); + + for (size_t i = 0; i < 10; ++i) { + auto raw_frame = raw.read(); + auto npy_frame = npy.read(); + CHECK((raw_frame.view() == npy_frame.view())); + } +} + +TEST_CASE("Read multipart files") { + auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json"; + REQUIRE(std::filesystem::exists(fpath)); + + File f(fpath, "r"); + + // we know this file has 10 frames check read_multiport.py for the values + std::vector pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116}; + std::vector pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833}; + std::vector pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142}; + std::vector pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171}; + std::vector pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598}; + + for (size_t i = 0; i < 10; i++) { + auto frame = f.read(); + CHECK(frame.rows() == 512); + CHECK(frame.cols() == 1024); + CHECK(frame.view()(0, 0) == pixel_0_0[i]); + CHECK(frame.view()(0, 1) == pixel_0_1[i]); + CHECK(frame.view()(1, 0) == pixel_1_0[i]); + CHECK(frame.view()(255, 1023) == pixel_255_1023[i]); + CHECK(frame.view()(511, 1023) == pixel_511_1023[i]); + } +} + +TEST_CASE("Read file with unordered frames") { + auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw"; + REQUIRE(std::filesystem::exists(fpath)); + File f(fpath, "r"); + REQUIRE_THROWS((f.read())); +} diff --git a/src/SubFile.cpp b/src/SubFile.cpp new file mode 100644 index 0000000..9c6c64a --- /dev/null +++ b/src/SubFile.cpp @@ -0,0 +1,71 @@ +#include "aare/SubFile.hpp" + +#include // memcpy +#include +#include + + +namespace aare { + +SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth, + const std::string &mode) + : m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) { + + if (std::filesystem::exists(fname)) { + n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8); + } else { + n_frames = 0; + } + + if (mode == "r") { + fp = fopen(m_fname.string().c_str(), "rb"); + } else { + // if file exists, open in read/write mode (without truncating the file) + // if file does not exist, open in write mode + if (std::filesystem::exists(fname)) { + fp = fopen(m_fname.string().c_str(), "r+b"); + } else { + fp = fopen(m_fname.string().c_str(), "wb"); + } + } + if (fp == nullptr) { + throw std::runtime_error(LOCATION + "Could not open file for writing"); + } +} + +size_t SubFile::get_part(std::byte *buffer, size_t frame_index) { + if (frame_index >= n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT + SEEK_SET); + return fread(buffer, this->bytes_per_part(), 1, this->fp); +} +size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) { + if (frame_index > n_frames) { + throw std::runtime_error("Frame number out of range"); + } + fseek(fp, static_cast((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET); + auto wc = fwrite(reinterpret_cast(&header), sizeof(header), 1, fp); + wc += fwrite(buffer, bytes_per_part(), 1, fp); + + return wc; +} + +size_t SubFile::frame_number(size_t frame_index) { + sls_detector_header h{}; + fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT + size_t const rc = fread(reinterpret_cast(&h), sizeof(h), 1, fp); + if (rc != 1) + throw std::runtime_error(LOCATION + "Could not read header from file"); + + return h.frameNumber; +} + +SubFile::~SubFile() { + if (fp) { + fclose(fp); + } +} + +} // namespace aare \ No newline at end of file