mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2025-06-03 19:40:40 +02:00
Files and structure for python interface
This commit is contained in:
Erik Fröjdh
parent
5d643dc133
commit
a4fb217e3f
@ -56,7 +56,6 @@ endif()
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
if(AARE_FETCH_ZMQ)
|
||||
|
||||
FetchContent_Declare(
|
||||
libzmq
|
||||
GIT_REPOSITORY https://github.com/zeromq/libzmq.git
|
||||
@ -70,7 +69,6 @@ if(AARE_FETCH_ZMQ)
|
||||
FetchContent_Populate(libzmq)
|
||||
add_subdirectory(${libzmq_SOURCE_DIR} ${libzmq_BINARY_DIR} EXCLUDE_FROM_ALL)
|
||||
endif()
|
||||
|
||||
else()
|
||||
find_package(ZeroMQ 4 REQUIRED)
|
||||
endif()
|
||||
@ -91,10 +89,15 @@ else()
|
||||
find_package(fmt 6 REQUIRED)
|
||||
endif()
|
||||
|
||||
find_package(nlohmann_json 3.11.3 REQUIRED)
|
||||
|
||||
add_library(aare_compiler_flags INTERFACE)
|
||||
target_compile_features(aare_compiler_flags INTERFACE cxx_std_17)
|
||||
|
||||
if(AARE_PYTHON_BINDINGS)
|
||||
add_subdirectory(python)
|
||||
endif()
|
||||
|
||||
#################
|
||||
# MSVC specific #
|
||||
#################
|
||||
@ -145,33 +148,6 @@ else()
|
||||
-D_GLIBCXX_DEBUG_PEDANTIC
|
||||
)
|
||||
|
||||
if (NOT AARE_PYTHON_BINDINGS)
|
||||
target_compile_options(
|
||||
aare_compiler_flags
|
||||
INTERFACE
|
||||
-fdiagnostics-parseable-fixits
|
||||
# -fdiagnostics-generate-patch
|
||||
-fdiagnostics-show-template-tree
|
||||
-fsanitize=address,undefined,pointer-compare
|
||||
-fno-sanitize-recover
|
||||
# -D_FORTIFY_SOURCE=2 # not needed for debug builds
|
||||
# -fstack-protector # cause errors wih folly? (ProducerConsumerQueue.hpp)
|
||||
-fno-omit-frame-pointer
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
aare_compiler_flags
|
||||
INTERFACE
|
||||
-fdiagnostics-parseable-fixits
|
||||
# -fdiagnostics-generate-patch
|
||||
-fdiagnostics-show-template-tree
|
||||
-fsanitize=address,undefined,pointer-compare
|
||||
-fno-sanitize-recover
|
||||
# -D_FORTIFY_SOURCE=2
|
||||
-fno-omit-frame-pointer
|
||||
)
|
||||
endif()
|
||||
|
||||
endif()
|
||||
|
||||
if(AARE_USE_WARNINGS)
|
||||
|
||||
@ -16,6 +16,9 @@ set(SPHINX_SOURCE_FILES
|
||||
src/NDView.rst
|
||||
src/Frame.rst
|
||||
src/Dtype.rst
|
||||
src/ClusterFinder.rst
|
||||
src/Pedestal.rst
|
||||
src/VarClusterFinder.rst
|
||||
)
|
||||
|
||||
foreach(filename ${SPHINX_SOURCE_FILES})
|
||||
|
||||
5
docs/src/ClusterFinder.rst
Normal file
5
docs/src/ClusterFinder.rst
Normal file
@ -0,0 +1,5 @@
|
||||
ClusterFinder
|
||||
=============
|
||||
|
||||
|
||||
.. doxygenfile:: ClusterFinder.hpp
|
||||
5
docs/src/Pedestal.rst
Normal file
5
docs/src/Pedestal.rst
Normal file
@ -0,0 +1,5 @@
|
||||
Pedestal
|
||||
=============
|
||||
|
||||
|
||||
.. doxygenfile:: Pedestal.hpp
|
||||
5
docs/src/VarClusterFinder.rst
Normal file
5
docs/src/VarClusterFinder.rst
Normal file
@ -0,0 +1,5 @@
|
||||
VarClusterFinder
|
||||
====================
|
||||
|
||||
|
||||
.. doxygenfile:: VarClusterFinder.hpp
|
||||
@ -12,4 +12,7 @@ AARE
|
||||
NDArray
|
||||
NDView
|
||||
Frame
|
||||
Dtype
|
||||
Dtype
|
||||
ClusterFinder
|
||||
Pedestal
|
||||
VarClusterFinder
|
||||
148
include/aare/ClusterFileV2.hpp
Normal file
148
include/aare/ClusterFileV2.hpp
Normal file
@ -0,0 +1,148 @@
|
||||
#pragma once
|
||||
#include "aare/core/defs.hpp"
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <fmt/format.h>
|
||||
|
||||
namespace aare {
|
||||
struct ClusterHeader {
|
||||
int32_t frame_number;
|
||||
int32_t n_clusters;
|
||||
std::string to_string() const {
|
||||
return "frame_number: " + std::to_string(frame_number) + ", n_clusters: " + std::to_string(n_clusters);
|
||||
}
|
||||
};
|
||||
|
||||
struct ClusterV2_ {
|
||||
int16_t x;
|
||||
int16_t y;
|
||||
std::array<int32_t, 9> data;
|
||||
std::string to_string(bool detailed = false) const {
|
||||
if (detailed) {
|
||||
std::string data_str = "[";
|
||||
for (auto &d : data) {
|
||||
data_str += std::to_string(d) + ", ";
|
||||
}
|
||||
data_str += "]";
|
||||
return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", data: " + data_str;
|
||||
}
|
||||
return "x: " + std::to_string(x) + ", y: " + std::to_string(y);
|
||||
}
|
||||
};
|
||||
|
||||
struct ClusterV2 {
|
||||
ClusterV2_ cluster;
|
||||
int32_t frame_number;
|
||||
std::string to_string() const {
|
||||
return "frame_number: " + std::to_string(frame_number) + ", " + cluster.to_string();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief
|
||||
* important not: fp always points to the clusters header and does not point to individual clusters
|
||||
*
|
||||
*/
|
||||
class ClusterFileV2 {
|
||||
std::filesystem::path m_fpath;
|
||||
std::string m_mode;
|
||||
FILE *fp{nullptr};
|
||||
|
||||
void check_open(){
|
||||
if (!fp)
|
||||
throw std::runtime_error(fmt::format("File: {} not open", m_fpath.string()));
|
||||
}
|
||||
|
||||
public:
|
||||
ClusterFileV2(std::filesystem::path const &fpath, std::string const &mode): m_fpath(fpath), m_mode(mode) {
|
||||
if (m_mode != "r" && m_mode != "w")
|
||||
throw std::invalid_argument("mode must be 'r' or 'w'");
|
||||
if (m_mode == "r" && !std::filesystem::exists(m_fpath))
|
||||
throw std::invalid_argument("File does not exist");
|
||||
if (mode == "r") {
|
||||
fp = fopen(fpath.string().c_str(), "rb");
|
||||
} else if (mode == "w") {
|
||||
if (std::filesystem::exists(fpath)) {
|
||||
fp = fopen(fpath.string().c_str(), "r+b");
|
||||
} else {
|
||||
fp = fopen(fpath.string().c_str(), "wb");
|
||||
}
|
||||
}
|
||||
if (fp == nullptr) {
|
||||
throw std::runtime_error("Failed to open file");
|
||||
}
|
||||
}
|
||||
~ClusterFileV2() { close(); }
|
||||
std::vector<ClusterV2> read() {
|
||||
check_open();
|
||||
|
||||
ClusterHeader header;
|
||||
fread(&header, sizeof(ClusterHeader), 1, fp);
|
||||
std::vector<ClusterV2_> clusters_(header.n_clusters);
|
||||
fread(clusters_.data(), sizeof(ClusterV2_), header.n_clusters, fp);
|
||||
std::vector<ClusterV2> clusters;
|
||||
for (auto &c : clusters_) {
|
||||
ClusterV2 cluster;
|
||||
cluster.cluster = std::move(c);
|
||||
cluster.frame_number = header.frame_number;
|
||||
clusters.push_back(cluster);
|
||||
}
|
||||
|
||||
return clusters;
|
||||
}
|
||||
std::vector<std::vector<ClusterV2>> read(int n_frames) {
|
||||
std::vector<std::vector<ClusterV2>> clusters;
|
||||
for (int i = 0; i < n_frames; i++) {
|
||||
clusters.push_back(read());
|
||||
}
|
||||
return clusters;
|
||||
}
|
||||
|
||||
size_t write(std::vector<ClusterV2> const &clusters) {
|
||||
check_open();
|
||||
if (m_mode != "w")
|
||||
throw std::runtime_error("File not opened in write mode");
|
||||
if (clusters.empty())
|
||||
return 0;
|
||||
|
||||
ClusterHeader header;
|
||||
header.frame_number = clusters[0].frame_number;
|
||||
header.n_clusters = clusters.size();
|
||||
fwrite(&header, sizeof(ClusterHeader), 1, fp);
|
||||
for (auto &c : clusters) {
|
||||
fwrite(&c.cluster, sizeof(ClusterV2_), 1, fp);
|
||||
}
|
||||
return clusters.size();
|
||||
}
|
||||
|
||||
size_t write(std::vector<std::vector<ClusterV2>> const &clusters) {
|
||||
check_open();
|
||||
if (m_mode != "w")
|
||||
throw std::runtime_error("File not opened in write mode");
|
||||
|
||||
size_t n_clusters = 0;
|
||||
for (auto &c : clusters) {
|
||||
n_clusters += write(c);
|
||||
}
|
||||
return n_clusters;
|
||||
}
|
||||
|
||||
int seek_to_begin() { return fseek(fp, 0, SEEK_SET); }
|
||||
int seek_to_end() { return fseek(fp, 0, SEEK_END); }
|
||||
|
||||
int32_t frame_number() {
|
||||
auto pos = ftell(fp);
|
||||
ClusterHeader header;
|
||||
fread(&header, sizeof(ClusterHeader), 1, fp);
|
||||
fseek(fp, pos, SEEK_SET);
|
||||
return header.frame_number;
|
||||
}
|
||||
|
||||
void close() {
|
||||
if (fp) {
|
||||
fclose(fp);
|
||||
fp = nullptr;
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace aare
|
||||
59
include/aare/File.hpp
Normal file
59
include/aare/File.hpp
Normal file
@ -0,0 +1,59 @@
|
||||
#pragma once
|
||||
#include "aare/FileInterface.hpp"
|
||||
|
||||
|
||||
namespace aare {
|
||||
|
||||
/**
|
||||
* @brief RAII File class for reading and writing image files in various formats
|
||||
* wrapper on a FileInterface to abstract the underlying file format
|
||||
* @note documentation for each function is in the FileInterface class
|
||||
*/
|
||||
class File {
|
||||
private:
|
||||
FileInterface *file_impl;
|
||||
bool is_npy;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a new File object
|
||||
* @param fname path to the file
|
||||
* @param mode file mode (r, w, a)
|
||||
* @param cfg file configuration
|
||||
* @throws std::runtime_error if the file cannot be opened
|
||||
* @throws std::invalid_argument if the file mode is not supported
|
||||
*
|
||||
*/
|
||||
File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {});
|
||||
void write(Frame &frame, sls_detector_header header = {});
|
||||
Frame read();
|
||||
Frame iread(size_t frame_number);
|
||||
std::vector<Frame> read(size_t n_frames);
|
||||
void read_into(std::byte *image_buf);
|
||||
void read_into(std::byte *image_buf, size_t n_frames);
|
||||
size_t frame_number(size_t frame_index);
|
||||
size_t bytes_per_frame();
|
||||
size_t pixels_per_frame();
|
||||
void seek(size_t frame_number);
|
||||
size_t tell() const;
|
||||
size_t total_frames() const;
|
||||
size_t rows() const;
|
||||
size_t cols() const;
|
||||
size_t bitdepth() const;
|
||||
void set_total_frames(size_t total_frames);
|
||||
DetectorType detector_type() const;
|
||||
xy geometry() const;
|
||||
|
||||
/**
|
||||
* @brief Move constructor
|
||||
* @param other File object to move from
|
||||
*/
|
||||
File(File &&other) noexcept;
|
||||
|
||||
/**
|
||||
* @brief destructor: will only delete the FileInterface object
|
||||
*/
|
||||
~File();
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
196
include/aare/FileInterface.hpp
Normal file
196
include/aare/FileInterface.hpp
Normal file
@ -0,0 +1,196 @@
|
||||
#pragma once
|
||||
#include "aare/Dtype.hpp"
|
||||
#include "aare/Frame.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
|
||||
namespace aare {
|
||||
|
||||
/**
|
||||
* @brief FileConfig structure to store the configuration of a file
|
||||
* dtype: data type of the file
|
||||
* rows: number of rows in the file
|
||||
* cols: number of columns in the file
|
||||
* geometry: geometry of the file
|
||||
*/
|
||||
struct FileConfig {
|
||||
aare::Dtype dtype{typeid(uint16_t)};
|
||||
uint64_t rows{};
|
||||
uint64_t cols{};
|
||||
bool operator==(const FileConfig &other) const {
|
||||
return dtype == other.dtype && rows == other.rows && cols == other.cols && geometry == other.geometry &&
|
||||
detector_type == other.detector_type && max_frames_per_file == other.max_frames_per_file;
|
||||
}
|
||||
bool operator!=(const FileConfig &other) const { return !(*this == other); }
|
||||
|
||||
// rawfile specific
|
||||
std::string version{};
|
||||
xy geometry{1, 1};
|
||||
DetectorType detector_type{DetectorType::Unknown};
|
||||
int max_frames_per_file{};
|
||||
size_t total_frames{};
|
||||
std::string to_string() const {
|
||||
return "{ dtype: " + dtype.to_string() + ", rows: " + std::to_string(rows) + ", cols: " + std::to_string(cols) +
|
||||
", geometry: " + geometry.to_string() + ", detector_type: " + toString(detector_type) +
|
||||
", max_frames_per_file: " + std::to_string(max_frames_per_file) +
|
||||
", total_frames: " + std::to_string(total_frames) + " }";
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief FileInterface class to define the interface for file operations
|
||||
* @note parent class for NumpyFile and RawFile
|
||||
* @note all functions are pure virtual and must be implemented by the derived classes
|
||||
*/
|
||||
class FileInterface {
|
||||
public:
|
||||
/**
|
||||
* @brief write a frame to the file
|
||||
* @param frame frame to write
|
||||
* @return void
|
||||
* @throws std::runtime_error if the function is not implemented
|
||||
*/
|
||||
// virtual void write(Frame &frame) = 0;
|
||||
|
||||
/**
|
||||
* @brief write a vector of frames to the file
|
||||
* @param frames vector of frames to write
|
||||
* @return void
|
||||
*/
|
||||
// virtual void write(std::vector<Frame> &frames) = 0;
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the current position
|
||||
* @return Frame
|
||||
*/
|
||||
virtual Frame read() = 0;
|
||||
|
||||
/**
|
||||
* @brief read n_frames from the file at the current position
|
||||
* @param n_frames number of frames to read
|
||||
* @return vector of frames
|
||||
*/
|
||||
virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the current position and store it in the provided buffer
|
||||
* @param image_buf buffer to store the frame
|
||||
* @return void
|
||||
*/
|
||||
virtual void read_into(std::byte *image_buf) = 0;
|
||||
|
||||
/**
|
||||
* @brief read n_frames from the file at the current position and store them in the provided buffer
|
||||
* @param image_buf buffer to store the frames
|
||||
* @param n_frames number of frames to read
|
||||
* @return void
|
||||
*/
|
||||
virtual void read_into(std::byte *image_buf, size_t n_frames) = 0;
|
||||
|
||||
/**
|
||||
* @brief get the frame number at the given frame index
|
||||
* @param frame_index index of the frame
|
||||
* @return frame number
|
||||
*/
|
||||
virtual size_t frame_number(size_t frame_index) = 0;
|
||||
|
||||
/**
|
||||
* @brief get the size of one frame in bytes
|
||||
* @return size of one frame
|
||||
*/
|
||||
virtual size_t bytes_per_frame() = 0;
|
||||
|
||||
/**
|
||||
* @brief get the number of pixels in one frame
|
||||
* @return number of pixels in one frame
|
||||
*/
|
||||
virtual size_t pixels_per_frame() = 0;
|
||||
|
||||
/**
|
||||
* @brief seek to the given frame number
|
||||
* @param frame_number frame number to seek to
|
||||
* @return void
|
||||
*/
|
||||
virtual void seek(size_t frame_number) = 0;
|
||||
|
||||
/**
|
||||
* @brief get the current position of the file pointer
|
||||
* @return current position of the file pointer
|
||||
*/
|
||||
virtual size_t tell() = 0;
|
||||
|
||||
/**
|
||||
* @brief get the total number of frames in the file
|
||||
* @return total number of frames in the file
|
||||
*/
|
||||
virtual size_t total_frames() const = 0;
|
||||
/**
|
||||
* @brief get the number of rows in the file
|
||||
* @return number of rows in the file
|
||||
*/
|
||||
virtual size_t rows() const = 0;
|
||||
/**
|
||||
* @brief get the number of columns in the file
|
||||
* @return number of columns in the file
|
||||
*/
|
||||
virtual size_t cols() const = 0;
|
||||
/**
|
||||
* @brief get the bitdepth of the file
|
||||
* @return bitdepth of the file
|
||||
*/
|
||||
virtual size_t bitdepth() const = 0;
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the given frame number
|
||||
* @param frame_number frame number to read
|
||||
* @return frame
|
||||
*/
|
||||
Frame iread(size_t frame_number) {
|
||||
auto old_pos = tell();
|
||||
seek(frame_number);
|
||||
Frame tmp = read();
|
||||
seek(old_pos);
|
||||
return tmp;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief read n_frames from the file starting at the given frame number
|
||||
* @param frame_number frame number to start reading from
|
||||
* @param n_frames number of frames to read
|
||||
* @return vector of frames
|
||||
*/
|
||||
std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
|
||||
auto old_pos = tell();
|
||||
seek(frame_number);
|
||||
std::vector<Frame> tmp = read(n_frames);
|
||||
seek(old_pos);
|
||||
return tmp;
|
||||
}
|
||||
DetectorType detector_type() const { return m_type; }
|
||||
|
||||
// function to query the data type of the file
|
||||
/*virtual DataType dtype = 0; */
|
||||
|
||||
virtual ~FileInterface() = default;
|
||||
|
||||
void set_total_frames(size_t total_frames) { m_total_frames = total_frames; }
|
||||
|
||||
protected:
|
||||
std::string m_mode{};
|
||||
std::filesystem::path m_fname{};
|
||||
std::filesystem::path m_base_path{};
|
||||
std::string m_base_name{}, m_ext{};
|
||||
int m_findex{};
|
||||
size_t m_total_frames{};
|
||||
size_t max_frames_per_file{};
|
||||
std::string version{};
|
||||
DetectorType m_type{DetectorType::Unknown};
|
||||
size_t m_rows{};
|
||||
size_t m_cols{};
|
||||
size_t m_bitdepth{};
|
||||
size_t current_frame{};
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
112
include/aare/NumpyFile.hpp
Normal file
112
include/aare/NumpyFile.hpp
Normal file
@ -0,0 +1,112 @@
|
||||
#pragma once
|
||||
#include "aare/Dtype.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
#include "aare/FileInterface.hpp"
|
||||
#include "aare/NumpyHelpers.hpp"
|
||||
|
||||
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
|
||||
namespace aare {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @brief NumpyFile class to read and write numpy files
|
||||
* @note derived from FileInterface
|
||||
* @note implements all the pure virtual functions from FileInterface
|
||||
* @note documentation for the functions can also be found in the FileInterface class
|
||||
*/
|
||||
class NumpyFile : public FileInterface {
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief NumpyFile constructor
|
||||
* @param fname path to the numpy file
|
||||
* @param mode file mode (r, w)
|
||||
* @param cfg file configuration
|
||||
*/
|
||||
explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
|
||||
|
||||
void write(Frame &frame);
|
||||
Frame read() override { return get_frame(this->current_frame++); }
|
||||
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
|
||||
void read_into(std::byte *image_buf, size_t n_frames) override;
|
||||
size_t frame_number(size_t frame_index) override { return frame_index; };
|
||||
size_t bytes_per_frame() override;
|
||||
size_t pixels_per_frame() override;
|
||||
void seek(size_t frame_number) override { this->current_frame = frame_number; }
|
||||
size_t tell() override { return this->current_frame; }
|
||||
size_t total_frames() const override { return m_header.shape[0]; }
|
||||
size_t rows() const override { return m_header.shape[1]; }
|
||||
size_t cols() const override { return m_header.shape[2]; }
|
||||
size_t bitdepth() const override { return m_header.dtype.bitdepth(); }
|
||||
|
||||
/**
|
||||
* @brief get the data type of the numpy file
|
||||
* @return DType
|
||||
*/
|
||||
Dtype dtype() const { return m_header.dtype; }
|
||||
|
||||
/**
|
||||
* @brief get the shape of the numpy file
|
||||
* @return vector of type size_t
|
||||
*/
|
||||
std::vector<size_t> shape() const { return m_header.shape; }
|
||||
|
||||
/**
|
||||
* @brief load the numpy file into an NDArray
|
||||
* @tparam T data type of the NDArray
|
||||
* @tparam NDim number of dimensions of the NDArray
|
||||
* @return NDArray<T, NDim>
|
||||
*/
|
||||
template <typename T, size_t NDim> NDArray<T, NDim> load() {
|
||||
NDArray<T, NDim> arr(make_shape<NDim>(m_header.shape));
|
||||
if (fseek(fp, static_cast<int64_t>(header_size), SEEK_SET)) {
|
||||
throw std::runtime_error(LOCATION + "Error seeking to the start of the data");
|
||||
}
|
||||
size_t rc = fread(arr.data(), sizeof(T), arr.size(), fp);
|
||||
if (rc != static_cast<size_t>(arr.size())) {
|
||||
throw std::runtime_error(LOCATION + "Error reading data from file");
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &frame) {
|
||||
write_impl(frame.data(), frame.total_bytes());
|
||||
}
|
||||
template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &frame) {
|
||||
write_impl(frame.data(), frame.total_bytes());
|
||||
}
|
||||
template <typename A, typename TYPENAME, A Ndim> void write(NDView<TYPENAME, Ndim> &&frame) {
|
||||
write_impl(frame.data(), frame.total_bytes());
|
||||
}
|
||||
template <typename A, typename TYPENAME, A Ndim> void write(NDArray<TYPENAME, Ndim> &&frame) {
|
||||
write_impl(frame.data(), frame.total_bytes());
|
||||
}
|
||||
|
||||
~NumpyFile() noexcept override;
|
||||
|
||||
private:
|
||||
FILE *fp = nullptr;
|
||||
size_t initial_header_len = 0;
|
||||
size_t current_frame{};
|
||||
uint32_t header_len{};
|
||||
uint8_t header_len_size{};
|
||||
size_t header_size{};
|
||||
NumpyHeader m_header;
|
||||
uint8_t major_ver_{};
|
||||
uint8_t minor_ver_{};
|
||||
size_t m_bytes_per_frame{};
|
||||
size_t m_pixels_per_frame{};
|
||||
|
||||
void load_metadata();
|
||||
void get_frame_into(size_t /*frame_number*/, std::byte * /*image_buf*/);
|
||||
Frame get_frame(size_t frame_number);
|
||||
void write_impl(void *data, uint64_t size);
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
55
include/aare/NumpyHelpers.hpp
Normal file
55
include/aare/NumpyHelpers.hpp
Normal file
@ -0,0 +1,55 @@
|
||||
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "aare/Dtype.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
|
||||
namespace aare {
|
||||
|
||||
struct NumpyHeader {
|
||||
Dtype dtype{aare::Dtype::ERROR};
|
||||
bool fortran_order{false};
|
||||
std::vector<size_t> shape{};
|
||||
std::string to_string() const;
|
||||
};
|
||||
|
||||
namespace NumpyHelpers {
|
||||
|
||||
const constexpr std::array<char, 6> magic_str{'\x93', 'N', 'U', 'M', 'P', 'Y'};
|
||||
const uint8_t magic_string_length{6};
|
||||
|
||||
std::string parse_str(const std::string &in);
|
||||
/**
|
||||
Removes leading and trailing whitespaces
|
||||
*/
|
||||
std::string trim(const std::string &str);
|
||||
|
||||
std::vector<std::string> parse_tuple(std::string in);
|
||||
|
||||
bool parse_bool(const std::string &in);
|
||||
|
||||
std::string get_value_from_map(const std::string &mapstr);
|
||||
|
||||
std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys);
|
||||
|
||||
template <typename T, size_t N> bool in_array(T val, const std::array<T, N> &arr) {
|
||||
return std::find(std::begin(arr), std::end(arr), val) != std::end(arr);
|
||||
}
|
||||
bool is_digits(const std::string &str);
|
||||
|
||||
aare::Dtype parse_descr(std::string typestring);
|
||||
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header);
|
||||
size_t write_header(std::ostream &out, const NumpyHeader &header);
|
||||
|
||||
} // namespace NumpyHelpers
|
||||
} // namespace aare
|
||||
186
include/aare/RawFile.hpp
Normal file
186
include/aare/RawFile.hpp
Normal file
@ -0,0 +1,186 @@
|
||||
#pragma once
|
||||
#include "aare/Frame.hpp"
|
||||
#include "aare/FileInterface.hpp"
|
||||
#include "aare/SubFile.hpp"
|
||||
|
||||
namespace aare {
|
||||
|
||||
struct ModuleConfig {
|
||||
int module_gap_row{};
|
||||
int module_gap_col{};
|
||||
|
||||
bool operator==(const ModuleConfig &other) const {
|
||||
if (module_gap_col != other.module_gap_col)
|
||||
return false;
|
||||
if (module_gap_row != other.module_gap_row)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief RawFile class to read .raw and .json files
|
||||
* @note derived from FileInterface
|
||||
* @note documentation can also be found in the FileInterface class
|
||||
*/
|
||||
class RawFile : public FileInterface {
|
||||
public:
|
||||
/**
|
||||
* @brief RawFile constructor
|
||||
* @param fname path to the file
|
||||
* @param mode file mode (r, w)
|
||||
* @param cfg file configuration
|
||||
*/
|
||||
explicit RawFile(const std::filesystem::path &fname, const std::string &mode = "r",
|
||||
const FileConfig &config = FileConfig{});
|
||||
|
||||
/**
|
||||
* @brief write function is not implemented for RawFile
|
||||
* @param frame frame to write
|
||||
*/
|
||||
void write(Frame &frame, sls_detector_header header);
|
||||
Frame read() override { return get_frame(this->current_frame++); };
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
|
||||
void read_into(std::byte *image_buf, size_t n_frames) override;
|
||||
size_t frame_number(size_t frame_index) override;
|
||||
|
||||
/**
|
||||
* @brief get the number of bytess per frame
|
||||
* @return size of one frame in bytes
|
||||
*/
|
||||
size_t bytes_per_frame() override { return m_rows * m_cols * m_bitdepth / 8; }
|
||||
|
||||
/**
|
||||
* @brief get the number of pixels in the frame
|
||||
* @return number of pixels
|
||||
*/
|
||||
size_t pixels_per_frame() override { return m_rows * m_cols; }
|
||||
|
||||
// goto frame index
|
||||
void seek(size_t frame_index) override {
|
||||
// check if the frame number is greater than the total frames
|
||||
// if frame_number == total_frames, then the next read will throw an error
|
||||
if (frame_index > this->total_frames()) {
|
||||
throw std::runtime_error(
|
||||
fmt::format("frame number {} is greater than total frames {}", frame_index, m_total_frames));
|
||||
}
|
||||
this->current_frame = frame_index;
|
||||
};
|
||||
|
||||
// return the position of the file pointer (in number of frames)
|
||||
size_t tell() override { return this->current_frame; };
|
||||
|
||||
/**
|
||||
* @brief check if the file is a master file
|
||||
* @param fpath path to the file
|
||||
*/
|
||||
static bool is_master_file(const std::filesystem::path &fpath);
|
||||
|
||||
/**
|
||||
* @brief set the module gap row and column
|
||||
* @param row gap between rows
|
||||
* @param col gap between columns
|
||||
*/
|
||||
inline void set_config(int row, int col) {
|
||||
cfg.module_gap_row = row;
|
||||
cfg.module_gap_col = col;
|
||||
}
|
||||
// TODO! Deal with fast quad and missing files
|
||||
|
||||
/**
|
||||
* @brief get the number of subfiles for the RawFile
|
||||
* @return number of subfiles
|
||||
*/
|
||||
void find_number_of_subfiles();
|
||||
|
||||
/**
|
||||
* @brief get the master file name path for the RawFile
|
||||
* @return path to the master file
|
||||
*/
|
||||
inline std::filesystem::path master_fname();
|
||||
/**
|
||||
* @brief get the data file name path for the RawFile with the given module id and file id
|
||||
* @param mod_id module id
|
||||
* @param file_id file id
|
||||
* @return path to the data file
|
||||
*/
|
||||
inline std::filesystem::path data_fname(size_t mod_id, size_t file_id);
|
||||
|
||||
/**
|
||||
* @brief destructor: will delete the subfiles
|
||||
*/
|
||||
~RawFile() noexcept override;
|
||||
|
||||
size_t total_frames() const override { return m_total_frames; }
|
||||
size_t rows() const override { return m_rows; }
|
||||
size_t cols() const override { return m_cols; }
|
||||
size_t bitdepth() const override { return m_bitdepth; }
|
||||
xy geometry() { return m_geometry; }
|
||||
|
||||
private:
|
||||
void write_master_file();
|
||||
/**
|
||||
* @brief read the frame at the given frame index into the image buffer
|
||||
* @param frame_number frame number to read
|
||||
* @param image_buf buffer to store the frame
|
||||
*/
|
||||
void get_frame_into(size_t frame_index, std::byte *frame_buffer);
|
||||
|
||||
/**
|
||||
* @brief get the frame at the given frame index
|
||||
* @param frame_number frame number to read
|
||||
* @return Frame
|
||||
*/
|
||||
Frame get_frame(size_t frame_index);
|
||||
|
||||
/**
|
||||
* @brief parse the file name to get the extension, base name and index
|
||||
*/
|
||||
void parse_fname();
|
||||
|
||||
/**
|
||||
* @brief parse the metadata from the file
|
||||
*/
|
||||
void parse_metadata();
|
||||
|
||||
/**
|
||||
* @brief parse the metadata of a .raw file
|
||||
*/
|
||||
void parse_raw_metadata();
|
||||
|
||||
/**
|
||||
* @brief parse the metadata of a .json file
|
||||
*/
|
||||
void parse_json_metadata();
|
||||
|
||||
/**
|
||||
* @brief finds the geometry of the file
|
||||
*/
|
||||
void find_geometry();
|
||||
|
||||
/**
|
||||
* @brief read the header of the file
|
||||
* @param fname path to the data subfile
|
||||
* @return sls_detector_header
|
||||
*/
|
||||
static sls_detector_header read_header(const std::filesystem::path &fname);
|
||||
|
||||
/**
|
||||
* @brief open the subfiles
|
||||
*/
|
||||
void open_subfiles();
|
||||
void parse_config(const FileConfig &config);
|
||||
|
||||
size_t n_subfiles{};
|
||||
size_t n_subfile_parts{};
|
||||
std::vector<std::vector<SubFile *>> subfiles;
|
||||
size_t subfile_rows{}, subfile_cols{};
|
||||
xy m_geometry{};
|
||||
std::vector<xy> positions;
|
||||
ModuleConfig cfg{0, 0};
|
||||
TimingMode timing_mode{};
|
||||
bool quad{false};
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
77
include/aare/SubFile.hpp
Normal file
77
include/aare/SubFile.hpp
Normal file
@ -0,0 +1,77 @@
|
||||
#pragma once
|
||||
#include "aare/Frame.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <variant>
|
||||
|
||||
namespace aare {
|
||||
|
||||
/**
|
||||
* @brief Class to read a subfile from a RawFile
|
||||
*/
|
||||
class SubFile {
|
||||
public:
|
||||
size_t write_part(std::byte *buffer, sls_detector_header header, size_t frame_index);
|
||||
/**
|
||||
* @brief SubFile constructor
|
||||
* @param fname path to the subfile
|
||||
* @param detector detector type
|
||||
* @param rows number of rows in the subfile
|
||||
* @param cols number of columns in the subfile
|
||||
* @param bitdepth bitdepth of the subfile
|
||||
* @throws std::invalid_argument if the detector,type pair is not supported
|
||||
*/
|
||||
SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
|
||||
const std::string &mode = "r");
|
||||
|
||||
/**
|
||||
* @brief read the subfile into a buffer
|
||||
* @param buffer pointer to the buffer to read the data into
|
||||
* @return number of bytes read
|
||||
*/
|
||||
size_t read_impl_normal(std::byte *buffer);
|
||||
|
||||
/**
|
||||
* @brief read the subfile into a buffer with the bytes flipped
|
||||
* @param buffer pointer to the buffer to read the data into
|
||||
* @return number of bytes read
|
||||
*/
|
||||
template <typename DataType> size_t read_impl_flip(std::byte *buffer);
|
||||
|
||||
/**
|
||||
* @brief read the subfile into a buffer with the bytes reordered
|
||||
* @param buffer pointer to the buffer to read the data into
|
||||
* @return number of bytes read
|
||||
*/
|
||||
template <typename DataType> size_t read_impl_reorder(std::byte *buffer);
|
||||
|
||||
/**
|
||||
* @brief read the subfile into a buffer with the bytes reordered and flipped
|
||||
* @param buffer pointer to the buffer to read the data into
|
||||
* @param frame_number frame number to read
|
||||
* @return number of bytes read
|
||||
*/
|
||||
size_t get_part(std::byte *buffer, size_t frame_index);
|
||||
size_t frame_number(size_t frame_index);
|
||||
|
||||
// TODO: define the inlines as variables and assign them in constructor
|
||||
inline size_t bytes_per_part() const { return (m_bitdepth / 8) * m_rows * m_cols; }
|
||||
inline size_t pixels_per_part() const { return m_rows * m_cols; }
|
||||
|
||||
~SubFile();
|
||||
|
||||
protected:
|
||||
FILE *fp = nullptr;
|
||||
size_t m_bitdepth;
|
||||
std::filesystem::path m_fname;
|
||||
size_t m_rows{};
|
||||
size_t m_cols{};
|
||||
std::string m_mode;
|
||||
size_t n_frames{};
|
||||
int m_sub_file_index_{};
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
@ -10,7 +10,7 @@
|
||||
const int MAX_CLUSTER_SIZE = 200;
|
||||
namespace aare {
|
||||
|
||||
template <typename T> class ClusterFinder {
|
||||
template <typename T> class VarClusterFinder {
|
||||
public:
|
||||
struct Hit {
|
||||
int16_t size{};
|
||||
@ -49,7 +49,7 @@ template <typename T> class ClusterFinder {
|
||||
int check_neighbours(int i, int j);
|
||||
|
||||
public:
|
||||
ClusterFinder(image_shape shape, T threshold)
|
||||
VarClusterFinder(image_shape shape, T threshold)
|
||||
: shape_(shape), labeled_(shape, 0), peripheral_labeled_(shape, 0), binary_(shape), threshold_(threshold) {
|
||||
hits.reserve(2000);
|
||||
}
|
||||
@ -112,7 +112,7 @@ template <typename T> class ClusterFinder {
|
||||
}
|
||||
}
|
||||
};
|
||||
template <typename T> int ClusterFinder<T>::check_neighbours(int i, int j) {
|
||||
template <typename T> int VarClusterFinder<T>::check_neighbours(int i, int j) {
|
||||
std::vector<int> neighbour_labels;
|
||||
|
||||
for (int k = 0; k < 4; ++k) {
|
||||
@ -144,7 +144,7 @@ template <typename T> int ClusterFinder<T>::check_neighbours(int i, int j) {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::find_clusters(NDView<T, 2> img) {
|
||||
template <typename T> void VarClusterFinder<T>::find_clusters(NDView<T, 2> img) {
|
||||
original_ = img;
|
||||
labeled_ = 0;
|
||||
peripheral_labeled_ = 0;
|
||||
@ -156,7 +156,7 @@ template <typename T> void ClusterFinder<T>::find_clusters(NDView<T, 2> img) {
|
||||
store_clusters();
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
|
||||
template <typename T> void VarClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
|
||||
original_ = img;
|
||||
int clusterIndex = 0;
|
||||
for (int i = 0; i < shape_[0]; ++i) {
|
||||
@ -175,7 +175,7 @@ template <typename T> void ClusterFinder<T>::find_clusters_X(NDView<T, 2> img) {
|
||||
h_size.clear();
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
|
||||
template <typename T> void VarClusterFinder<T>::rec_FillHit(int clusterIndex, int i, int j) {
|
||||
// printf("original_(%d, %d)=%f\n", i, j, original_(i,j));
|
||||
// printf("h_size[%d].size=%d\n", clusterIndex, h_size[clusterIndex].size);
|
||||
if (h_size[clusterIndex].size < MAX_CLUSTER_SIZE) {
|
||||
@ -213,7 +213,7 @@ template <typename T> void ClusterFinder<T>::rec_FillHit(int clusterIndex, int i
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::single_pass(NDView<T, 2> img) {
|
||||
template <typename T> void VarClusterFinder<T>::single_pass(NDView<T, 2> img) {
|
||||
original_ = img;
|
||||
labeled_ = 0;
|
||||
current_label = 0;
|
||||
@ -224,7 +224,7 @@ template <typename T> void ClusterFinder<T>::single_pass(NDView<T, 2> img) {
|
||||
// store_clusters();
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::first_pass() {
|
||||
template <typename T> void VarClusterFinder<T>::first_pass() {
|
||||
|
||||
for (int i = 0; i < original_.size(); ++i) {
|
||||
if (use_noise_map)
|
||||
@ -248,7 +248,7 @@ template <typename T> void ClusterFinder<T>::first_pass() {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::second_pass() {
|
||||
template <typename T> void VarClusterFinder<T>::second_pass() {
|
||||
|
||||
for (int64_t i = 0; i != labeled_.size(); ++i) {
|
||||
auto current_label = labeled_(i);
|
||||
@ -265,7 +265,7 @@ template <typename T> void ClusterFinder<T>::second_pass() {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T> void ClusterFinder<T>::store_clusters() {
|
||||
template <typename T> void VarClusterFinder<T>::store_clusters() {
|
||||
|
||||
// Accumulate hit information in a map
|
||||
// Do we always have monotonic increasing
|
||||
68
include/aare/json.hpp
Normal file
68
include/aare/json.hpp
Normal file
@ -0,0 +1,68 @@
|
||||
#pragma once
|
||||
#include <array>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// helper functions to write json
|
||||
// append to string for better performance (not tested)
|
||||
|
||||
namespace aare {
|
||||
|
||||
/**
|
||||
* @brief write a digit to a string
|
||||
* takes key and value and outputs->"key": value,
|
||||
* @tparam T type of value (int, uint32_t, ...)
|
||||
* @param s string to append to
|
||||
* @param key key to write
|
||||
* @param value value to write
|
||||
* @return void
|
||||
* @note
|
||||
* - can't use concepts here because we are using c++17
|
||||
*/
|
||||
template <typename T> inline void write_digit(std::string &s, const std::string &key, const T &value) {
|
||||
s += "\"";
|
||||
s += key;
|
||||
s += "\": ";
|
||||
s += std::to_string(value);
|
||||
s += ", ";
|
||||
}
|
||||
inline void write_str(std::string &s, const std::string &key, const std::string &value) {
|
||||
s += "\"";
|
||||
s += key;
|
||||
s += "\": \"";
|
||||
s += value;
|
||||
s += "\", ";
|
||||
}
|
||||
inline void write_map(std::string &s, const std::string &key, const std::map<std::string, std::string> &value) {
|
||||
s += "\"";
|
||||
s += key;
|
||||
s += "\": {";
|
||||
for (const auto &kv : value) {
|
||||
write_str(s, kv.first, kv.second);
|
||||
}
|
||||
// remove last comma or trailing spaces
|
||||
for (size_t i = s.size() - 1; i > 0; i--) {
|
||||
if ((s[i] == ',') || (s[i] == ' ')) {
|
||||
s.pop_back();
|
||||
} else
|
||||
break;
|
||||
}
|
||||
s += "}, ";
|
||||
}
|
||||
|
||||
template <typename T, int N> void write_array(std::string &s, const std::string &key, const std::array<T, N> &value) {
|
||||
s += "\"";
|
||||
s += key;
|
||||
s += "\": [";
|
||||
|
||||
for (size_t i = 0; i < N - 1; i++) {
|
||||
s += std::to_string(value[i]);
|
||||
s += ", ";
|
||||
}
|
||||
s += std::to_string(value[N - 1]);
|
||||
|
||||
s += "], ";
|
||||
}
|
||||
|
||||
} // namespace aare
|
||||
17
python/CMakeLists.txt
Normal file
17
python/CMakeLists.txt
Normal file
@ -0,0 +1,17 @@
|
||||
if(AARE_FETCH_PYBIND11)
|
||||
FetchContent_Declare(
|
||||
pybind11
|
||||
GIT_REPOSITORY https://github.com/pybind/pybind11
|
||||
GIT_TAG v2.11.0
|
||||
)
|
||||
FetchContent_MakeAvailable(pybind11)
|
||||
else()
|
||||
find_package(pybind11 2.11 REQUIRED)
|
||||
endif()
|
||||
|
||||
|
||||
pybind11_add_module(_aare src/module.cpp)
|
||||
set_target_properties(_aare PROPERTIES
|
||||
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
|
||||
)
|
||||
target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags)
|
||||
9
python/src/module.cpp
Normal file
9
python/src/module.cpp
Normal file
@ -0,0 +1,9 @@
|
||||
|
||||
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
PYBIND11_MODULE(_aare, m) {
|
||||
}
|
||||
@ -4,15 +4,20 @@ set(SourceFiles
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/defs.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Dtype.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Frame.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/File.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NumpyFile.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/RawFile.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/SubFile.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NumpyHelpers.cpp
|
||||
)
|
||||
|
||||
|
||||
add_library(aare_core STATIC ${SourceFiles})
|
||||
target_include_directories(aare_core PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags )
|
||||
target_link_libraries(aare_core PUBLIC fmt::fmt PRIVATE aare_compiler_flags nlohmann_json::nlohmann_json)
|
||||
|
||||
if (AARE_PYTHON_BINDINGS)
|
||||
set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON)
|
||||
set_property(TARGET aare_core PROPERTY POSITION_INDEPENDENT_CODE ON)
|
||||
endif()
|
||||
|
||||
if(AARE_TESTS)
|
||||
@ -25,11 +30,13 @@ if(AARE_TESTS)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NDView.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/ClusterFinder.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Pedestal.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NumpyFile.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/NumpyHelpers.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/RawFile.test.cpp
|
||||
# ${CMAKE_CURRENT_SOURCE_DIR}/test/CircularFifo.test.cpp
|
||||
# ${CMAKE_CURRENT_SOURCE_DIR}/test/wrappers.test.cpp
|
||||
# ${CMAKE_CURRENT_SOURCE_DIR}/test/Transforms.test.cpp
|
||||
|
||||
)
|
||||
target_sources(tests PRIVATE ${TestSources} )
|
||||
target_link_libraries(tests PRIVATE aare_core)
|
||||
endif()
|
||||
70
src/File.cpp
Normal file
70
src/File.cpp
Normal file
@ -0,0 +1,70 @@
|
||||
#include "aare/File.hpp"
|
||||
#include "aare/NumpyFile.hpp"
|
||||
#include "aare/RawFile.hpp"
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
namespace aare {
|
||||
|
||||
File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg)
|
||||
: file_impl(nullptr), is_npy(true) {
|
||||
if (mode != "r" && mode != "w" && mode != "a") {
|
||||
throw std::invalid_argument("Unsupported file mode");
|
||||
}
|
||||
|
||||
if ((mode == "r" || mode == "a") && !std::filesystem::exists(fname)) {
|
||||
throw std::runtime_error(fmt::format("File does not exist: {}", fname.string()));
|
||||
}
|
||||
|
||||
if (fname.extension() == ".raw" || fname.extension() == ".json") {
|
||||
// aare::logger::debug("Loading raw file");
|
||||
file_impl = new RawFile(fname, mode, cfg);
|
||||
is_npy = false;
|
||||
}
|
||||
// check if extension is numpy
|
||||
else if (fname.extension() == ".npy") {
|
||||
// aare::logger::debug("Loading numpy file");
|
||||
file_impl = new NumpyFile(fname, mode, cfg);
|
||||
} else {
|
||||
throw std::runtime_error("Unsupported file type");
|
||||
}
|
||||
}
|
||||
|
||||
void File::write(Frame &frame, sls_detector_header header) {
|
||||
if (is_npy) {
|
||||
// aare::logger::info("ignoring header for npy file");
|
||||
dynamic_cast<NumpyFile *>(file_impl)->write(frame);
|
||||
} else {
|
||||
dynamic_cast<RawFile *>(file_impl)->write(frame, header);
|
||||
}
|
||||
}
|
||||
Frame File::read() { return file_impl->read(); }
|
||||
size_t File::total_frames() const { return file_impl->total_frames(); }
|
||||
std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
|
||||
void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
|
||||
void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
|
||||
size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
|
||||
size_t File::bytes_per_frame() { return file_impl->bytes_per_frame(); }
|
||||
size_t File::pixels_per_frame() { return file_impl->pixels_per_frame(); }
|
||||
void File::seek(size_t frame_number) { file_impl->seek(frame_number); }
|
||||
size_t File::tell() const { return file_impl->tell(); }
|
||||
size_t File::rows() const { return file_impl->rows(); }
|
||||
size_t File::cols() const { return file_impl->cols(); }
|
||||
size_t File::bitdepth() const { return file_impl->bitdepth(); }
|
||||
void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); }
|
||||
File::~File() { delete file_impl; }
|
||||
DetectorType File::detector_type() const { return file_impl->detector_type(); }
|
||||
xy File::geometry() const {
|
||||
if (is_npy) {
|
||||
return {1, 1};
|
||||
}
|
||||
return reinterpret_cast<RawFile *>(file_impl)->geometry();
|
||||
}
|
||||
|
||||
Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
|
||||
|
||||
File::File(File &&other) noexcept : file_impl(other.file_impl), is_npy(other.is_npy) { other.file_impl = nullptr; }
|
||||
|
||||
// write move assignment operator
|
||||
|
||||
} // namespace aare
|
||||
200
src/NumpyFile.cpp
Normal file
200
src/NumpyFile.cpp
Normal file
@ -0,0 +1,200 @@
|
||||
|
||||
#include "aare/NumpyFile.hpp"
|
||||
#include "aare/NumpyHelpers.hpp"
|
||||
|
||||
namespace aare {
|
||||
|
||||
|
||||
|
||||
NumpyFile::NumpyFile(const std::filesystem::path &fname, const std::string &mode, FileConfig cfg) {
|
||||
// TODO! add opts to constructor
|
||||
m_fname = fname;
|
||||
m_mode = mode;
|
||||
if (mode == "r") {
|
||||
fp = fopen(m_fname.string().c_str(), "rb");
|
||||
if (!fp) {
|
||||
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
|
||||
}
|
||||
load_metadata();
|
||||
} else if (mode == "w") {
|
||||
m_bitdepth = cfg.dtype.bitdepth();
|
||||
m_rows = cfg.rows;
|
||||
m_cols = cfg.cols;
|
||||
m_header = {cfg.dtype, false, {cfg.rows, cfg.cols}};
|
||||
m_header.shape = {0, cfg.rows, cfg.cols};
|
||||
fp = fopen(m_fname.string().c_str(), "wb");
|
||||
if (!fp) {
|
||||
throw std::runtime_error(fmt::format("Could not open: {} for reading", m_fname.string()));
|
||||
}
|
||||
initial_header_len = aare::NumpyHelpers::write_header(std::filesystem::path(m_fname.c_str()), m_header);
|
||||
}
|
||||
m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, std::multiplies<>());
|
||||
|
||||
m_bytes_per_frame = m_header.dtype.bitdepth() / 8 * m_pixels_per_frame;
|
||||
}
|
||||
void NumpyFile::write(Frame &frame) { write_impl(frame.data(), frame.bytes()); }
|
||||
void NumpyFile::write_impl(void *data, uint64_t size) {
|
||||
|
||||
if (fp == nullptr) {
|
||||
throw std::runtime_error("File not open");
|
||||
}
|
||||
if (!(m_mode == "w" || m_mode == "a")) {
|
||||
throw std::invalid_argument("File not open for writing");
|
||||
}
|
||||
if (fseek(fp, 0, SEEK_END))
|
||||
throw std::runtime_error("Could not seek to end of file");
|
||||
size_t const rc = fwrite(data, size, 1, fp);
|
||||
if (rc != 1) {
|
||||
throw std::runtime_error("Error writing frame to file");
|
||||
}
|
||||
|
||||
m_header.shape[0]++;
|
||||
}
|
||||
|
||||
Frame NumpyFile::get_frame(size_t frame_number) {
|
||||
Frame frame(m_header.shape[1], m_header.shape[2], m_header.dtype);
|
||||
get_frame_into(frame_number, frame.data());
|
||||
return frame;
|
||||
}
|
||||
void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
|
||||
if (fp == nullptr) {
|
||||
throw std::runtime_error("File not open");
|
||||
}
|
||||
if (frame_number > m_header.shape[0]) {
|
||||
throw std::invalid_argument("Frame number out of range");
|
||||
}
|
||||
if (fseek(fp, header_size + frame_number * m_bytes_per_frame, SEEK_SET)) // NOLINT
|
||||
throw std::runtime_error("Could not seek to frame");
|
||||
|
||||
size_t const rc = fread(image_buf, m_bytes_per_frame, 1, fp);
|
||||
if (rc != 1) {
|
||||
throw std::runtime_error("Error reading frame from file");
|
||||
}
|
||||
}
|
||||
|
||||
size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; };
|
||||
size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; };
|
||||
|
||||
std::vector<Frame> NumpyFile::read(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
frames.push_back(get_frame(current_frame));
|
||||
current_frame++;
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
get_frame_into(current_frame++, image_buf);
|
||||
image_buf += m_bytes_per_frame;
|
||||
}
|
||||
}
|
||||
|
||||
NumpyFile::~NumpyFile() noexcept {
|
||||
if (m_mode == "w" || m_mode == "a") {
|
||||
// determine number of frames
|
||||
if (fseek(fp, 0, SEEK_END)) {
|
||||
std::cout << "Could not seek to end of file" << std::endl;
|
||||
}
|
||||
size_t const file_size = ftell(fp);
|
||||
size_t const data_size = file_size - initial_header_len;
|
||||
size_t const n_frames = data_size / m_bytes_per_frame;
|
||||
// update number of frames in header (first element of shape)
|
||||
m_header.shape[0] = n_frames;
|
||||
if (fseek(fp, 0, SEEK_SET)) {
|
||||
std::cout << "Could not seek to beginning of file" << std::endl;
|
||||
}
|
||||
// create string stream to contain header
|
||||
std::stringstream ss;
|
||||
aare::NumpyHelpers::write_header(ss, m_header);
|
||||
std::string const header_str = ss.str();
|
||||
// write header
|
||||
size_t const rc = fwrite(header_str.c_str(), header_str.size(), 1, fp);
|
||||
if (rc != 1) {
|
||||
std::cout << "Error writing header to numpy file in destructor" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
if (fp != nullptr) {
|
||||
if (fclose(fp)) {
|
||||
std::cout << "Error closing file" << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void NumpyFile::load_metadata() {
|
||||
|
||||
// read magic number
|
||||
std::array<char, 6> tmp{};
|
||||
size_t rc = fread(tmp.data(), tmp.size(), 1, fp);
|
||||
if (rc != 1) {
|
||||
throw std::runtime_error("Error reading magic number");
|
||||
}
|
||||
if (tmp != aare::NumpyHelpers::magic_str) {
|
||||
for (auto item : tmp)
|
||||
fmt::print("{}, ", static_cast<int>(item));
|
||||
fmt::print("\n");
|
||||
throw std::runtime_error("Not a numpy file");
|
||||
}
|
||||
|
||||
// read version
|
||||
rc = fread(reinterpret_cast<char *>(&major_ver_), sizeof(major_ver_), 1, fp);
|
||||
rc += fread(reinterpret_cast<char *>(&minor_ver_), sizeof(minor_ver_), 1, fp);
|
||||
if (rc != 2) {
|
||||
throw std::runtime_error("Error reading numpy version");
|
||||
}
|
||||
|
||||
if (major_ver_ == 1) {
|
||||
header_len_size = 2;
|
||||
} else if (major_ver_ == 2) {
|
||||
header_len_size = 4;
|
||||
} else {
|
||||
throw std::runtime_error("Unsupported numpy version");
|
||||
}
|
||||
|
||||
// read header length
|
||||
rc = fread(reinterpret_cast<char *>(&header_len), header_len_size, 1, fp);
|
||||
if (rc != 1) {
|
||||
throw std::runtime_error("Error reading header length");
|
||||
}
|
||||
header_size = aare::NumpyHelpers::magic_string_length + 2 + header_len_size + header_len;
|
||||
if (header_size % 16 != 0) {
|
||||
fmt::print("Warning: header length is not a multiple of 16\n");
|
||||
}
|
||||
|
||||
// read header
|
||||
std::string header(header_len, '\0');
|
||||
rc = fread(header.data(), header_len, 1, fp);
|
||||
if (rc != 1) {
|
||||
throw std::runtime_error("Error reading header");
|
||||
}
|
||||
|
||||
// parse header
|
||||
std::vector<std::string> const keys{"descr", "fortran_order", "shape"};
|
||||
auto dict_map = aare::NumpyHelpers::parse_dict(header, keys);
|
||||
if (dict_map.empty())
|
||||
throw std::runtime_error("invalid dictionary in header");
|
||||
|
||||
std::string const descr_s = dict_map["descr"];
|
||||
std::string const fortran_s = dict_map["fortran_order"];
|
||||
std::string const shape_s = dict_map["shape"];
|
||||
|
||||
std::string const descr = aare::NumpyHelpers::parse_str(descr_s);
|
||||
aare::Dtype const dtype = aare::NumpyHelpers::parse_descr(descr);
|
||||
|
||||
// convert literal Python bool to C++ bool
|
||||
bool const fortran_order = aare::NumpyHelpers::parse_bool(fortran_s);
|
||||
|
||||
// parse the shape tuple
|
||||
auto shape_v = aare::NumpyHelpers::parse_tuple(shape_s);
|
||||
std::vector<size_t> shape;
|
||||
for (const auto &item : shape_v) {
|
||||
auto dim = static_cast<size_t>(std::stoul(item));
|
||||
shape.push_back(dim);
|
||||
}
|
||||
m_header = {dtype, fortran_order, shape};
|
||||
}
|
||||
|
||||
} // namespace aare
|
||||
50
src/NumpyFile.test.cpp
Normal file
50
src/NumpyFile.test.cpp
Normal file
@ -0,0 +1,50 @@
|
||||
#include "aare/NumpyFile.hpp"
|
||||
#include "aare/NDArray.hpp"
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include "test_config.hpp"
|
||||
|
||||
using aare::Dtype;
|
||||
using aare::NumpyFile;
|
||||
TEST_CASE("Read a 1D numpy file with int32 data type") {
|
||||
|
||||
auto fpath = test_data_path() / "numpy" / "test_1d_int32.npy";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
NumpyFile f(fpath);
|
||||
|
||||
// we know the file contains 10 elements of np.int32 containing values 0-9
|
||||
REQUIRE(f.dtype() == Dtype::INT32);
|
||||
REQUIRE(f.shape() == std::vector<size_t>{10});
|
||||
|
||||
// use the load function to read the full file into a NDArray
|
||||
auto data = f.load<int32_t, 1>();
|
||||
for (int32_t i = 0; i < 10; i++) {
|
||||
REQUIRE(data(i) == i);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read a 3D numpy file with np.double data type") {
|
||||
|
||||
auto fpath = test_data_path() / "numpy" / "test_3d_double.npy";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
NumpyFile f(fpath);
|
||||
|
||||
// we know the file contains 10 elements of np.int32 containing values 0-9
|
||||
REQUIRE(f.dtype() == Dtype::DOUBLE);
|
||||
REQUIRE(f.shape() == std::vector<size_t>{3, 2, 5});
|
||||
|
||||
// use the load function to read the full file into a NDArray
|
||||
// numpy code to generate the array
|
||||
// arr2[0,0,0] = 1.0
|
||||
// arr2[0,0,1] = 2.0
|
||||
// arr2[0,1,0] = 72.0
|
||||
// arr2[2,0,4] = 63.0
|
||||
|
||||
auto data = f.load<double, 3>();
|
||||
REQUIRE(data(0, 0, 0) == 1.0);
|
||||
REQUIRE(data(0, 0, 1) == 2.0);
|
||||
REQUIRE(data(0, 1, 0) == 72.0);
|
||||
REQUIRE(data(2, 0, 4) == 63.0);
|
||||
}
|
||||
269
src/NumpyHelpers.cpp
Normal file
269
src/NumpyHelpers.cpp
Normal file
@ -0,0 +1,269 @@
|
||||
/*
|
||||
28-03-2024 modified by: Bechir Braham <bechir.braham@psi.ch>
|
||||
|
||||
Copyright 2017-2023 Leon Merten Lohse
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "aare/NumpyHelpers.hpp"
|
||||
#include <iterator>
|
||||
|
||||
namespace aare {
|
||||
|
||||
std::string NumpyHeader::to_string() const {
|
||||
std::stringstream sstm;
|
||||
sstm << "dtype: " << dtype.to_string() << ", fortran_order: " << fortran_order << ' ';
|
||||
sstm << "shape: (";
|
||||
for (auto item : shape)
|
||||
sstm << item << ',';
|
||||
sstm << ')';
|
||||
return sstm.str();
|
||||
}
|
||||
|
||||
|
||||
namespace NumpyHelpers {
|
||||
|
||||
std::unordered_map<std::string, std::string> parse_dict(std::string in, const std::vector<std::string> &keys) {
|
||||
std::unordered_map<std::string, std::string> map;
|
||||
if (keys.empty())
|
||||
return map;
|
||||
|
||||
in = trim(in);
|
||||
|
||||
// unwrap dictionary
|
||||
if ((in.front() == '{') && (in.back() == '}'))
|
||||
in = in.substr(1, in.length() - 2);
|
||||
else
|
||||
throw std::runtime_error("Not a Python dictionary.");
|
||||
|
||||
std::vector<std::pair<size_t, std::string>> positions;
|
||||
|
||||
for (auto const &key : keys) {
|
||||
size_t const pos = in.find("'" + key + "'");
|
||||
|
||||
if (pos == std::string::npos)
|
||||
throw std::runtime_error("Missing '" + key + "' key.");
|
||||
|
||||
std::pair<size_t, std::string> const position_pair{pos, key};
|
||||
positions.push_back(position_pair);
|
||||
}
|
||||
|
||||
// sort by position in dict
|
||||
std::sort(positions.begin(), positions.end());
|
||||
|
||||
for (size_t i = 0; i < positions.size(); ++i) {
|
||||
std::string raw_value;
|
||||
size_t const begin{positions[i].first};
|
||||
size_t end{std::string::npos};
|
||||
|
||||
std::string const key = positions[i].second;
|
||||
|
||||
if (i + 1 < positions.size())
|
||||
end = positions[i + 1].first;
|
||||
|
||||
raw_value = in.substr(begin, end - begin);
|
||||
|
||||
raw_value = trim(raw_value);
|
||||
|
||||
if (raw_value.back() == ',')
|
||||
raw_value.pop_back();
|
||||
|
||||
map[key] = get_value_from_map(raw_value);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
aare::Dtype parse_descr(std::string typestring) {
|
||||
|
||||
if (typestring.length() < 3) {
|
||||
throw std::runtime_error("invalid typestring (length)");
|
||||
}
|
||||
|
||||
constexpr char little_endian_char = '<';
|
||||
constexpr char big_endian_char = '>';
|
||||
constexpr char no_endian_char = '|';
|
||||
constexpr std::array<char, 3> endian_chars = {little_endian_char, big_endian_char, no_endian_char};
|
||||
constexpr std::array<char, 4> numtype_chars = {'f', 'i', 'u', 'c'};
|
||||
|
||||
const char byteorder_c = typestring[0];
|
||||
const char kind_c = typestring[1];
|
||||
std::string const itemsize_s = typestring.substr(2);
|
||||
|
||||
if (!in_array(byteorder_c, endian_chars)) {
|
||||
throw std::runtime_error("invalid typestring (byteorder)");
|
||||
}
|
||||
|
||||
if (!in_array(kind_c, numtype_chars)) {
|
||||
throw std::runtime_error("invalid typestring (kind)");
|
||||
}
|
||||
|
||||
if (!is_digits(itemsize_s)) {
|
||||
throw std::runtime_error("invalid typestring (itemsize)");
|
||||
}
|
||||
|
||||
return aare::Dtype(typestring);
|
||||
}
|
||||
|
||||
bool parse_bool(const std::string &in) {
|
||||
if (in == "True")
|
||||
return true;
|
||||
if (in == "False")
|
||||
return false;
|
||||
throw std::runtime_error("Invalid python boolean.");
|
||||
}
|
||||
|
||||
std::string get_value_from_map(const std::string &mapstr) {
|
||||
size_t const sep_pos = mapstr.find_first_of(':');
|
||||
if (sep_pos == std::string::npos)
|
||||
return "";
|
||||
|
||||
std::string const tmp = mapstr.substr(sep_pos + 1);
|
||||
return trim(tmp);
|
||||
}
|
||||
|
||||
bool is_digits(const std::string &str) { return std::all_of(str.begin(), str.end(), ::isdigit); }
|
||||
|
||||
std::vector<std::string> parse_tuple(std::string in) {
|
||||
std::vector<std::string> v;
|
||||
const char separator = ',';
|
||||
|
||||
in = trim(in);
|
||||
|
||||
if ((in.front() == '(') && (in.back() == ')'))
|
||||
in = in.substr(1, in.length() - 2);
|
||||
else
|
||||
throw std::runtime_error("Invalid Python tuple.");
|
||||
|
||||
std::istringstream iss(in);
|
||||
|
||||
for (std::string token; std::getline(iss, token, separator);) {
|
||||
v.push_back(token);
|
||||
}
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
std::string trim(const std::string &str) {
|
||||
const std::string whitespace = " \t\n";
|
||||
auto begin = str.find_first_not_of(whitespace);
|
||||
|
||||
if (begin == std::string::npos)
|
||||
return "";
|
||||
|
||||
auto end = str.find_last_not_of(whitespace);
|
||||
return str.substr(begin, end - begin + 1);
|
||||
}
|
||||
|
||||
std::string parse_str(const std::string &in) {
|
||||
if ((in.front() == '\'') && (in.back() == '\''))
|
||||
return in.substr(1, in.length() - 2);
|
||||
|
||||
throw std::runtime_error("Invalid python string.");
|
||||
}
|
||||
|
||||
void write_magic(std::ostream &ostream, int version_major, int version_minor) {
|
||||
ostream.write(magic_str.data(), magic_string_length);
|
||||
ostream.put(static_cast<char>(version_major));
|
||||
ostream.put(static_cast<char>(version_minor));
|
||||
}
|
||||
template <typename T> inline std::string write_tuple(const std::vector<T> &v) {
|
||||
|
||||
if (v.empty())
|
||||
return "()";
|
||||
std::ostringstream ss;
|
||||
ss.imbue(std::locale("C"));
|
||||
|
||||
if (v.size() == 1) {
|
||||
ss << "(" << v.front() << ",)";
|
||||
} else {
|
||||
const std::string delimiter = ", ";
|
||||
// v.size() > 1
|
||||
ss << "(";
|
||||
// for (size_t i = 0; i < v.size() - 1; ++i) {
|
||||
// ss << v[i] << delimiter;
|
||||
// }
|
||||
// ss << v.back();
|
||||
std::copy(v.begin(), v.end() - 1, std::ostream_iterator<T>(ss, ", "));
|
||||
ss << v.back();
|
||||
ss << ")";
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
inline std::string write_boolean(bool b) {
|
||||
if (b)
|
||||
return "True";
|
||||
return "False";
|
||||
}
|
||||
|
||||
inline std::string write_header_dict(const std::string &descr, bool fortran_order, const std::vector<size_t> &shape) {
|
||||
std::string const s_fortran_order = write_boolean(fortran_order);
|
||||
std::string const shape_s = write_tuple(shape);
|
||||
|
||||
return "{'descr': '" + descr + "', 'fortran_order': " + s_fortran_order + ", 'shape': " + shape_s + ", }";
|
||||
}
|
||||
|
||||
size_t write_header(const std::filesystem::path &fname, const NumpyHeader &header) {
|
||||
std::ofstream out(fname, std::ios::binary | std::ios::out);
|
||||
return write_header(out, header);
|
||||
}
|
||||
|
||||
size_t write_header(std::ostream &out, const NumpyHeader &header) {
|
||||
std::string const header_dict = write_header_dict(header.dtype.to_string(), header.fortran_order, header.shape);
|
||||
|
||||
size_t length = magic_string_length + 2 + 2 + header_dict.length() + 1;
|
||||
|
||||
int version_major = 1;
|
||||
int version_minor = 0;
|
||||
if (length >= static_cast<size_t>(255) * 255) {
|
||||
length = magic_string_length + 2 + 4 + header_dict.length() + 1;
|
||||
version_major = 2;
|
||||
version_minor = 0;
|
||||
}
|
||||
size_t const padding_len = 16 - length % 16;
|
||||
std::string const padding(padding_len, ' ');
|
||||
|
||||
// write magic
|
||||
write_magic(out, version_major, version_minor);
|
||||
|
||||
// write header length
|
||||
if (version_major == 1 && version_minor == 0) {
|
||||
auto header_len = static_cast<uint16_t>(header_dict.length() + padding.length() + 1);
|
||||
|
||||
std::array<uint8_t, 2> header_len_le16{static_cast<uint8_t>((header_len >> 0) & 0xff),
|
||||
static_cast<uint8_t>((header_len >> 8) & 0xff)};
|
||||
out.write(reinterpret_cast<char *>(header_len_le16.data()), 2);
|
||||
} else {
|
||||
auto header_len = static_cast<uint32_t>(header_dict.length() + padding.length() + 1);
|
||||
|
||||
std::array<uint8_t, 4> header_len_le32{
|
||||
static_cast<uint8_t>((header_len >> 0) & 0xff), static_cast<uint8_t>((header_len >> 8) & 0xff),
|
||||
static_cast<uint8_t>((header_len >> 16) & 0xff), static_cast<uint8_t>((header_len >> 24) & 0xff)};
|
||||
out.write(reinterpret_cast<char *>(header_len_le32.data()), 4);
|
||||
}
|
||||
|
||||
out << header_dict << padding << '\n';
|
||||
return length;
|
||||
}
|
||||
|
||||
} // namespace NumpyHelpers
|
||||
} // namespace aare
|
||||
62
src/NumpyHelpers.test.cpp
Normal file
62
src/NumpyHelpers.test.cpp
Normal file
@ -0,0 +1,62 @@
|
||||
#include "aare/NumpyHelpers.hpp" //Is this really a public header?
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
using namespace aare::NumpyHelpers;
|
||||
|
||||
TEST_CASE("is_digits with a few standard cases") {
|
||||
REQUIRE(is_digits(""));
|
||||
REQUIRE(is_digits("123"));
|
||||
REQUIRE(is_digits("0"));
|
||||
REQUIRE_FALSE(is_digits("hej123"));
|
||||
REQUIRE_FALSE(is_digits("a"));
|
||||
REQUIRE_FALSE(is_digits(" "));
|
||||
REQUIRE_FALSE(is_digits("abcdef"));
|
||||
}
|
||||
|
||||
TEST_CASE("Check for quotes and return stripped string") {
|
||||
REQUIRE(parse_str("'hej'") == "hej");
|
||||
REQUIRE(parse_str("'hej hej'") == "hej hej");
|
||||
REQUIRE(parse_str("''") == "");
|
||||
}
|
||||
|
||||
TEST_CASE("parsing a string without quotes throws") { REQUIRE_THROWS(parse_str("hej")); }
|
||||
|
||||
TEST_CASE("trim whitespace") {
|
||||
REQUIRE(trim(" hej ") == "hej");
|
||||
REQUIRE(trim("hej") == "hej");
|
||||
REQUIRE(trim(" hej") == "hej");
|
||||
REQUIRE(trim("hej ") == "hej");
|
||||
REQUIRE(trim(" ") == "");
|
||||
REQUIRE(trim(" \thej hej ") == "hej hej");
|
||||
}
|
||||
|
||||
TEST_CASE("parse data type descriptions") {
|
||||
REQUIRE(parse_descr("<i1") == aare::Dtype::INT8);
|
||||
REQUIRE(parse_descr("<i2") == aare::Dtype::INT16);
|
||||
REQUIRE(parse_descr("<i4") == aare::Dtype::INT32);
|
||||
REQUIRE(parse_descr("<i8") == aare::Dtype::INT64);
|
||||
|
||||
REQUIRE(parse_descr("<u1") == aare::Dtype::UINT8);
|
||||
REQUIRE(parse_descr("<u2") == aare::Dtype::UINT16);
|
||||
REQUIRE(parse_descr("<u4") == aare::Dtype::UINT32);
|
||||
REQUIRE(parse_descr("<u8") == aare::Dtype::UINT64);
|
||||
|
||||
REQUIRE(parse_descr("<f4") == aare::Dtype::FLOAT);
|
||||
REQUIRE(parse_descr("<f8") == aare::Dtype::DOUBLE);
|
||||
}
|
||||
|
||||
TEST_CASE("is element in array") {
|
||||
REQUIRE(in_array(1, std::array<int, 3>{1, 2, 3}));
|
||||
REQUIRE_FALSE(in_array(4, std::array<int, 3>{1, 2, 3}));
|
||||
REQUIRE(in_array(1, std::array<int, 1>{1}));
|
||||
REQUIRE_FALSE(in_array(1, std::array<int, 0>{}));
|
||||
}
|
||||
|
||||
TEST_CASE("Parse numpy dict") {
|
||||
std::string in = "{'descr': '<f4', 'fortran_order': False, 'shape': (3, 4)}";
|
||||
std::vector<std::string> keys{"descr", "fortran_order", "shape"};
|
||||
auto map = parse_dict(in, keys);
|
||||
REQUIRE(map["descr"] == "'<f4'");
|
||||
REQUIRE(map["fortran_order"] == "False");
|
||||
REQUIRE(map["shape"] == "(3, 4)");
|
||||
}
|
||||
428
src/RawFile.cpp
Normal file
428
src/RawFile.cpp
Normal file
@ -0,0 +1,428 @@
|
||||
#include "aare/RawFile.hpp"
|
||||
#include "aare/defs.hpp"
|
||||
#include "aare/json.hpp"
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
namespace aare {
|
||||
|
||||
RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode, const FileConfig &config) {
|
||||
m_mode = mode;
|
||||
m_fname = fname;
|
||||
if (mode == "r" || mode == "r+") {
|
||||
if (config != FileConfig()) {
|
||||
// aare::logger::warn(
|
||||
// "In read mode it is not necessary to provide a config, the provided config will be ignored");
|
||||
}
|
||||
parse_fname();
|
||||
parse_metadata();
|
||||
find_number_of_subfiles();
|
||||
find_geometry();
|
||||
open_subfiles();
|
||||
|
||||
} else if (mode == "w" || mode == "w+") {
|
||||
|
||||
if (std::filesystem::exists(fname)) {
|
||||
// handle mode w as w+ (no overrwriting)
|
||||
throw std::runtime_error(LOCATION + "File already exists");
|
||||
}
|
||||
|
||||
parse_config(config);
|
||||
parse_fname();
|
||||
write_master_file();
|
||||
n_subfiles = 1;
|
||||
n_subfile_parts = 1;
|
||||
subfile_cols = m_cols;
|
||||
subfile_rows = m_rows;
|
||||
open_subfiles();
|
||||
|
||||
} else {
|
||||
throw std::runtime_error(LOCATION + "Unsupported mode");
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::parse_config(const FileConfig &config) {
|
||||
m_bitdepth = config.dtype.bitdepth();
|
||||
m_total_frames = config.total_frames;
|
||||
m_rows = config.rows;
|
||||
m_cols = config.cols;
|
||||
m_type = config.detector_type;
|
||||
max_frames_per_file = config.max_frames_per_file;
|
||||
m_geometry = config.geometry;
|
||||
version = config.version;
|
||||
subfile_rows = config.geometry.row;
|
||||
subfile_cols = config.geometry.col;
|
||||
|
||||
if (m_geometry != aare::xy{1, 1}) {
|
||||
throw std::runtime_error(LOCATION + "Only geometry {1,1} files are supported for writing");
|
||||
}
|
||||
}
|
||||
void RawFile::write_master_file() {
|
||||
if (m_ext != ".json") {
|
||||
throw std::runtime_error(LOCATION + "only json master files are supported for writing");
|
||||
}
|
||||
std::ofstream ofs(master_fname(), std::ios::binary);
|
||||
std::string ss;
|
||||
ss.reserve(1024);
|
||||
ss += "{\n\t";
|
||||
aare::write_str(ss, "Version", version);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Total Frames", m_total_frames);
|
||||
ss += "\n\t";
|
||||
aare::write_str(ss, "Detector Type", toString(m_type));
|
||||
ss += "\n\t";
|
||||
aare::write_str(ss, "Geometry", m_geometry.to_string());
|
||||
ss += "\n\t";
|
||||
|
||||
uint64_t img_size = (m_cols * m_rows) / (static_cast<size_t>(m_geometry.col * m_geometry.row));
|
||||
img_size *= m_bitdepth;
|
||||
aare::write_digit(ss, "Image Size in bytes", img_size);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Max Frames Per File", max_frames_per_file);
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Dynamic Range", m_bitdepth);
|
||||
ss += "\n\t";
|
||||
const aare::xy pixels = {static_cast<uint32_t>(m_rows / m_geometry.row),
|
||||
static_cast<uint32_t>(m_cols / m_geometry.col)};
|
||||
aare::write_str(ss, "Pixels", pixels.to_string());
|
||||
ss += "\n\t";
|
||||
aare::write_digit(ss, "Number of rows", m_rows);
|
||||
ss += "\n\t";
|
||||
const std::string tmp = "{\n"
|
||||
" \"Frame Number\": \"8 bytes\",\n"
|
||||
" \"Exposure Length\": \"4 bytes\",\n"
|
||||
" \"Packet Number\": \"4 bytes\",\n"
|
||||
" \"Bunch Id\": \"8 bytes\",\n"
|
||||
" \"Timestamp\": \"8 bytes\",\n"
|
||||
" \"Module Id\": \"2 bytes\",\n"
|
||||
" \"Row\": \"2 bytes\",\n"
|
||||
" \"Column\": \"2 bytes\",\n"
|
||||
" \"Reserved\": \"2 bytes\",\n"
|
||||
" \"Debug\": \"4 bytes\",\n"
|
||||
" \"RoundRNumber\": \"2 bytes\",\n"
|
||||
" \"DetType\": \"1 byte\",\n"
|
||||
" \"Version\": \"1 byte\",\n"
|
||||
" \"Packet Mask\": \"64 bytes\"\n"
|
||||
" }";
|
||||
|
||||
ss += "\"Frame Header Format\":" + tmp + "\n";
|
||||
ss += "}";
|
||||
ofs << ss;
|
||||
ofs.close();
|
||||
}
|
||||
|
||||
void RawFile::open_subfiles() {
|
||||
if (m_mode == "r")
|
||||
for (size_t i = 0; i != n_subfiles; ++i) {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts);
|
||||
for (size_t j = 0; j != n_subfile_parts; ++j) {
|
||||
v[j] = new SubFile(data_fname(i, j), m_type, subfile_rows, subfile_cols, m_bitdepth);
|
||||
}
|
||||
subfiles.push_back(v);
|
||||
}
|
||||
else {
|
||||
auto v = std::vector<SubFile *>(n_subfile_parts); // only one subfile is implemented
|
||||
v[0] = new SubFile(data_fname(0, 0), m_type, m_rows, m_cols, m_bitdepth, "w");
|
||||
subfiles.push_back(v);
|
||||
}
|
||||
}
|
||||
|
||||
sls_detector_header RawFile::read_header(const std::filesystem::path &fname) {
|
||||
sls_detector_header h{};
|
||||
FILE *fp = fopen(fname.string().c_str(), "r");
|
||||
if (!fp)
|
||||
throw std::runtime_error(fmt::format("Could not open: {} for reading", fname.string()));
|
||||
|
||||
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
|
||||
if (rc != 1)
|
||||
throw std::runtime_error(LOCATION + "Could not read header from file");
|
||||
if (fclose(fp)) {
|
||||
throw std::runtime_error(LOCATION + "Could not close file");
|
||||
}
|
||||
|
||||
return h;
|
||||
}
|
||||
bool RawFile::is_master_file(const std::filesystem::path &fpath) {
|
||||
std::string const stem = fpath.stem().string();
|
||||
return stem.find("_master_") != std::string::npos;
|
||||
}
|
||||
|
||||
void RawFile::find_number_of_subfiles() {
|
||||
int n_mod = 0;
|
||||
while (std::filesystem::exists(data_fname(++n_mod, 0)))
|
||||
;
|
||||
n_subfiles = n_mod;
|
||||
}
|
||||
inline std::filesystem::path RawFile::data_fname(size_t mod_id, size_t file_id) {
|
||||
return this->m_base_path / fmt::format("{}_d{}_f{}_{}.raw", this->m_base_name, file_id, mod_id, this->m_findex);
|
||||
}
|
||||
|
||||
inline std::filesystem::path RawFile::master_fname() {
|
||||
return this->m_base_path / fmt::format("{}_master_{}{}", this->m_base_name, this->m_findex, this->m_ext);
|
||||
}
|
||||
|
||||
void RawFile::find_geometry() {
|
||||
uint16_t r{};
|
||||
uint16_t c{};
|
||||
for (size_t i = 0; i < n_subfile_parts; i++) {
|
||||
for (size_t j = 0; j != n_subfiles; ++j) {
|
||||
auto h = this->read_header(data_fname(j, i));
|
||||
r = std::max(r, h.row);
|
||||
c = std::max(c, h.column);
|
||||
|
||||
positions.push_back({h.row, h.column});
|
||||
}
|
||||
}
|
||||
|
||||
r++;
|
||||
c++;
|
||||
|
||||
m_rows = (r * subfile_rows);
|
||||
m_cols = (c * subfile_cols);
|
||||
|
||||
m_rows += static_cast<size_t>((r - 1) * cfg.module_gap_row);
|
||||
}
|
||||
|
||||
void RawFile::parse_metadata() {
|
||||
if (m_ext == ".raw") {
|
||||
parse_raw_metadata();
|
||||
if (m_bitdepth == 0) {
|
||||
switch (m_type) {
|
||||
case DetectorType::Eiger:
|
||||
m_bitdepth = 32;
|
||||
break;
|
||||
default:
|
||||
m_bitdepth = 16;
|
||||
}
|
||||
}
|
||||
} else if (m_ext == ".json") {
|
||||
parse_json_metadata();
|
||||
} else {
|
||||
throw std::runtime_error(LOCATION + "Unsupported file type");
|
||||
}
|
||||
n_subfile_parts = static_cast<size_t>(m_geometry.row) * m_geometry.col;
|
||||
}
|
||||
|
||||
void RawFile::parse_json_metadata() {
|
||||
std::ifstream ifs(master_fname());
|
||||
json j;
|
||||
ifs >> j;
|
||||
double v = j["Version"];
|
||||
version = fmt::format("{:.1f}", v);
|
||||
m_type = StringTo<DetectorType>(j["Detector Type"].get<std::string>());
|
||||
timing_mode = StringTo<TimingMode>(j["Timing Mode"].get<std::string>());
|
||||
m_total_frames = j["Frames in File"];
|
||||
subfile_rows = j["Pixels"]["y"];
|
||||
subfile_cols = j["Pixels"]["x"];
|
||||
max_frames_per_file = j["Max Frames Per File"];
|
||||
try {
|
||||
m_bitdepth = j.at("Dynamic Range");
|
||||
} catch (const json::out_of_range &e) {
|
||||
m_bitdepth = 16;
|
||||
}
|
||||
// only Eiger had quad
|
||||
if (m_type == DetectorType::Eiger) {
|
||||
quad = (j["Quad"] == 1);
|
||||
}
|
||||
|
||||
m_geometry = {j["Geometry"]["y"], j["Geometry"]["x"]};
|
||||
}
|
||||
void RawFile::parse_raw_metadata() {
|
||||
std::ifstream ifs(master_fname());
|
||||
for (std::string line; std::getline(ifs, line);) {
|
||||
if (line == "#Frame Header")
|
||||
break;
|
||||
auto pos = line.find(':');
|
||||
auto key_pos = pos;
|
||||
while (key_pos != std::string::npos && std::isspace(line[--key_pos]))
|
||||
;
|
||||
if (key_pos != std::string::npos) {
|
||||
auto key = line.substr(0, key_pos + 1);
|
||||
auto value = line.substr(pos + 2);
|
||||
// do the actual parsing
|
||||
if (key == "Version") {
|
||||
version = value;
|
||||
} else if (key == "TimeStamp") {
|
||||
|
||||
} else if (key == "Detector Type") {
|
||||
m_type = StringTo<DetectorType>(value);
|
||||
} else if (key == "Timing Mode") {
|
||||
timing_mode = StringTo<TimingMode>(value);
|
||||
} else if (key == "Pixels") {
|
||||
// Total number of pixels cannot be found yet looking at
|
||||
// submodule
|
||||
pos = value.find(',');
|
||||
subfile_cols = std::stoi(value.substr(1, pos));
|
||||
subfile_rows = std::stoi(value.substr(pos + 1));
|
||||
} else if (key == "Total Frames") {
|
||||
m_total_frames = std::stoi(value);
|
||||
} else if (key == "Dynamic Range") {
|
||||
m_bitdepth = std::stoi(value);
|
||||
} else if (key == "Quad") {
|
||||
quad = (value == "1");
|
||||
} else if (key == "Max Frames Per File") {
|
||||
max_frames_per_file = std::stoi(value);
|
||||
} else if (key == "Geometry") {
|
||||
pos = value.find(',');
|
||||
m_geometry = {static_cast<uint32_t>(std::stoi(value.substr(1, pos))),
|
||||
static_cast<uint32_t>(std::stoi(value.substr(pos + 1)))};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::parse_fname() {
|
||||
bool wrong_format = false;
|
||||
m_base_path = m_fname.parent_path().string();
|
||||
m_base_name = m_fname.stem().string();
|
||||
m_ext = m_fname.extension().string();
|
||||
try {
|
||||
auto pos = m_base_name.rfind('_');
|
||||
m_findex = std::stoi(m_base_name.substr(pos + 1));
|
||||
} catch (const std::invalid_argument &e) {
|
||||
m_findex = 0;
|
||||
wrong_format = true;
|
||||
}
|
||||
auto pos = m_base_name.find("_master_");
|
||||
if (pos != std::string::npos) {
|
||||
m_base_name.erase(pos);
|
||||
wrong_format = true;
|
||||
}
|
||||
if (wrong_format && (m_mode == "w+" || m_mode == "w")) {
|
||||
// aare::logger::warn("Master Filename", m_fname, "is not in the correct format");
|
||||
// aare::logger::warn("using", master_fname(), "as the master file");
|
||||
}
|
||||
}
|
||||
|
||||
Frame RawFile::get_frame(size_t frame_index) {
|
||||
auto f = Frame(this->m_rows, this->m_cols, Dtype::from_bitdepth(this->m_bitdepth));
|
||||
std::byte *frame_buffer = f.data();
|
||||
get_frame_into(frame_index, frame_buffer);
|
||||
return f;
|
||||
}
|
||||
|
||||
void RawFile::get_frame_into(size_t frame_index, std::byte *frame_buffer) {
|
||||
if (frame_index > this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
std::vector<size_t> frame_numbers(this->n_subfile_parts);
|
||||
std::vector<size_t> frame_indices(this->n_subfile_parts, frame_index);
|
||||
|
||||
if (n_subfile_parts != 1) {
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto subfile_id = frame_index / this->max_frames_per_file;
|
||||
frame_numbers[part_idx] =
|
||||
this->subfiles[subfile_id][part_idx]->frame_number(frame_index % this->max_frames_per_file);
|
||||
}
|
||||
// 1. if frame number vector is the same break
|
||||
while (std::adjacent_find(frame_numbers.begin(), frame_numbers.end(), std::not_equal_to<>()) !=
|
||||
frame_numbers.end()) {
|
||||
// 2. find the index of the minimum frame number,
|
||||
auto min_frame_idx =
|
||||
std::distance(frame_numbers.begin(), std::min_element(frame_numbers.begin(), frame_numbers.end()));
|
||||
// 3. increase its index and update its respective frame number
|
||||
frame_indices[min_frame_idx]++;
|
||||
// 4. if we can't increase its index => throw error
|
||||
if (frame_indices[min_frame_idx] >= this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
auto subfile_id = frame_indices[min_frame_idx] / this->max_frames_per_file;
|
||||
frame_numbers[min_frame_idx] = this->subfiles[subfile_id][min_frame_idx]->frame_number(
|
||||
frame_indices[min_frame_idx] % this->max_frames_per_file);
|
||||
}
|
||||
}
|
||||
|
||||
if (this->m_geometry.col == 1) {
|
||||
// get the part from each subfile and copy it to the frame
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
auto part_offset = this->subfiles[subfile_id][part_idx]->bytes_per_part();
|
||||
this->subfiles[subfile_id][part_idx]->get_part(frame_buffer + part_idx * part_offset,
|
||||
corrected_idx % this->max_frames_per_file);
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
// create a buffer that will hold a the frame part
|
||||
auto bytes_per_part = this->subfile_rows * this->subfile_cols * this->m_bitdepth / 8;
|
||||
auto *part_buffer = new std::byte[bytes_per_part];
|
||||
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
auto corrected_idx = frame_indices[part_idx];
|
||||
auto subfile_id = corrected_idx / this->max_frames_per_file;
|
||||
|
||||
this->subfiles[subfile_id][part_idx]->get_part(part_buffer, corrected_idx % this->max_frames_per_file);
|
||||
for (size_t cur_row = 0; cur_row < (this->subfile_rows); cur_row++) {
|
||||
auto irow = cur_row + (part_idx / this->m_geometry.col) * this->subfile_rows;
|
||||
auto icol = (part_idx % this->m_geometry.col) * this->subfile_cols;
|
||||
auto dest = (irow * this->m_cols + icol);
|
||||
dest = dest * this->m_bitdepth / 8;
|
||||
memcpy(frame_buffer + dest, part_buffer + cur_row * this->subfile_cols * this->m_bitdepth / 8,
|
||||
this->subfile_cols * this->m_bitdepth / 8);
|
||||
}
|
||||
}
|
||||
delete[] part_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
void RawFile::write(Frame &frame, sls_detector_header header) {
|
||||
if (m_mode == "r") {
|
||||
throw std::runtime_error(LOCATION + "File is open in read mode");
|
||||
}
|
||||
size_t const subfile_id = this->current_frame / this->max_frames_per_file;
|
||||
for (size_t part_idx = 0; part_idx != this->n_subfile_parts; ++part_idx) {
|
||||
|
||||
this->subfiles[subfile_id][part_idx]->write_part(frame.data(), header,
|
||||
this->current_frame % this->max_frames_per_file);
|
||||
}
|
||||
this->current_frame++;
|
||||
}
|
||||
|
||||
std::vector<Frame> RawFile::read(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
frames.push_back(this->get_frame(this->current_frame));
|
||||
this->current_frame++;
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
void RawFile::read_into(std::byte *image_buf, size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
this->get_frame_into(this->current_frame++, image_buf);
|
||||
image_buf += this->bytes_per_frame();
|
||||
}
|
||||
}
|
||||
|
||||
size_t RawFile::frame_number(size_t frame_index) {
|
||||
if (frame_index > this->m_total_frames) {
|
||||
throw std::runtime_error(LOCATION + "Frame number out of range");
|
||||
}
|
||||
size_t const subfile_id = frame_index / this->max_frames_per_file;
|
||||
return this->subfiles[subfile_id][0]->frame_number(frame_index % this->max_frames_per_file);
|
||||
}
|
||||
|
||||
RawFile::~RawFile() noexcept {
|
||||
|
||||
// update master file
|
||||
if (m_mode == "w" || m_mode == "w+" || m_mode == "r+") {
|
||||
try {
|
||||
write_master_file();
|
||||
} catch (...) {
|
||||
// aare::logger::warn(LOCATION + "Could not update master file");
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &vec : subfiles) {
|
||||
for (auto *subfile : vec) {
|
||||
delete subfile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace aare
|
||||
114
src/RawFile.test.cpp
Normal file
114
src/RawFile.test.cpp
Normal file
@ -0,0 +1,114 @@
|
||||
#include "aare/File.hpp"
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <filesystem>
|
||||
|
||||
#include "test_config.hpp"
|
||||
|
||||
using aare::File;
|
||||
|
||||
TEST_CASE("Read number of frames from a jungfrau raw file") {
|
||||
|
||||
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
File f(fpath, "r");
|
||||
REQUIRE(f.total_frames() == 10);
|
||||
}
|
||||
|
||||
TEST_CASE("Read frame numbers from a jungfrau raw file") {
|
||||
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
File f(fpath, "r");
|
||||
|
||||
// we know this file has 10 frames with frame numbers 1 to 10
|
||||
// f0 1,2,3
|
||||
// f1 4,5,6
|
||||
// f2 7,8,9
|
||||
// f3 10
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
CHECK(f.frame_number(i) == i + 1);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read data from a jungfrau 500k single port raw file") {
|
||||
auto fpath = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
File f(fpath, "r");
|
||||
|
||||
// we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102
|
||||
std::vector<uint16_t> pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102};
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
auto frame = f.read();
|
||||
CHECK(frame.rows() == 512);
|
||||
CHECK(frame.cols() == 1024);
|
||||
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read frame numbers from a raw file") {
|
||||
auto fpath = test_data_path() / "eiger" / "eiger_500k_16bit_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
// we know this file has 3 frames with frame numbers 14, 15, 16
|
||||
std::vector<size_t> frame_numbers = {14, 15, 16};
|
||||
|
||||
File f(fpath, "r");
|
||||
for (size_t i = 0; i < 3; i++) {
|
||||
CHECK(f.frame_number(i) == frame_numbers[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Compare reading from a numpy file with a raw file") {
|
||||
auto fpath_raw = test_data_path() / "jungfrau" / "jungfrau_single_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath_raw));
|
||||
|
||||
auto fpath_npy = test_data_path() / "jungfrau" / "jungfrau_single_0.npy";
|
||||
REQUIRE(std::filesystem::exists(fpath_npy));
|
||||
|
||||
File raw(fpath_raw, "r");
|
||||
File npy(fpath_npy, "r");
|
||||
|
||||
CHECK(raw.total_frames() == 10);
|
||||
CHECK(npy.total_frames() == 10);
|
||||
|
||||
for (size_t i = 0; i < 10; ++i) {
|
||||
auto raw_frame = raw.read();
|
||||
auto npy_frame = npy.read();
|
||||
CHECK((raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>()));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read multipart files") {
|
||||
auto fpath = test_data_path() / "jungfrau" / "jungfrau_double_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
File f(fpath, "r");
|
||||
|
||||
// we know this file has 10 frames check read_multiport.py for the values
|
||||
std::vector<uint16_t> pixel_0_0 = {2099, 2121, 2108, 2084, 2084, 2118, 2066, 2108, 2112, 2116};
|
||||
std::vector<uint16_t> pixel_0_1 = {2842, 2796, 2865, 2798, 2805, 2817, 2852, 2789, 2792, 2833};
|
||||
std::vector<uint16_t> pixel_255_1023 = {2149, 2037, 2115, 2102, 2118, 2090, 2036, 2071, 2073, 2142};
|
||||
std::vector<uint16_t> pixel_511_1023 = {3231, 3169, 3167, 3162, 3168, 3160, 3171, 3171, 3169, 3171};
|
||||
std::vector<uint16_t> pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598};
|
||||
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
auto frame = f.read();
|
||||
CHECK(frame.rows() == 512);
|
||||
CHECK(frame.cols() == 1024);
|
||||
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
|
||||
CHECK(frame.view<uint16_t>()(0, 1) == pixel_0_1[i]);
|
||||
CHECK(frame.view<uint16_t>()(1, 0) == pixel_1_0[i]);
|
||||
CHECK(frame.view<uint16_t>()(255, 1023) == pixel_255_1023[i]);
|
||||
CHECK(frame.view<uint16_t>()(511, 1023) == pixel_511_1023[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("Read file with unordered frames") {
|
||||
auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
File f(fpath, "r");
|
||||
REQUIRE_THROWS((f.read()));
|
||||
}
|
||||
71
src/SubFile.cpp
Normal file
71
src/SubFile.cpp
Normal file
@ -0,0 +1,71 @@
|
||||
#include "aare/SubFile.hpp"
|
||||
|
||||
#include <cstring> // memcpy
|
||||
#include <fmt/core.h>
|
||||
#include <iostream>
|
||||
|
||||
|
||||
namespace aare {
|
||||
|
||||
SubFile::SubFile(const std::filesystem::path &fname, DetectorType detector, size_t rows, size_t cols, size_t bitdepth,
|
||||
const std::string &mode)
|
||||
: m_bitdepth(bitdepth), m_fname(fname), m_rows(rows), m_cols(cols), m_mode(mode) {
|
||||
|
||||
if (std::filesystem::exists(fname)) {
|
||||
n_frames = std::filesystem::file_size(fname) / (sizeof(sls_detector_header) + rows * cols * bitdepth / 8);
|
||||
} else {
|
||||
n_frames = 0;
|
||||
}
|
||||
|
||||
if (mode == "r") {
|
||||
fp = fopen(m_fname.string().c_str(), "rb");
|
||||
} else {
|
||||
// if file exists, open in read/write mode (without truncating the file)
|
||||
// if file does not exist, open in write mode
|
||||
if (std::filesystem::exists(fname)) {
|
||||
fp = fopen(m_fname.string().c_str(), "r+b");
|
||||
} else {
|
||||
fp = fopen(m_fname.string().c_str(), "wb");
|
||||
}
|
||||
}
|
||||
if (fp == nullptr) {
|
||||
throw std::runtime_error(LOCATION + "Could not open file for writing");
|
||||
}
|
||||
}
|
||||
|
||||
size_t SubFile::get_part(std::byte *buffer, size_t frame_index) {
|
||||
if (frame_index >= n_frames) {
|
||||
throw std::runtime_error("Frame number out of range");
|
||||
}
|
||||
fseek(fp, sizeof(sls_detector_header) + (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, // NOLINT
|
||||
SEEK_SET);
|
||||
return fread(buffer, this->bytes_per_part(), 1, this->fp);
|
||||
}
|
||||
size_t SubFile::write_part(std::byte *buffer, sls_detector_header header, size_t frame_index) {
|
||||
if (frame_index > n_frames) {
|
||||
throw std::runtime_error("Frame number out of range");
|
||||
}
|
||||
fseek(fp, static_cast<int64_t>((sizeof(sls_detector_header) + bytes_per_part()) * frame_index), SEEK_SET);
|
||||
auto wc = fwrite(reinterpret_cast<char *>(&header), sizeof(header), 1, fp);
|
||||
wc += fwrite(buffer, bytes_per_part(), 1, fp);
|
||||
|
||||
return wc;
|
||||
}
|
||||
|
||||
size_t SubFile::frame_number(size_t frame_index) {
|
||||
sls_detector_header h{};
|
||||
fseek(fp, (sizeof(sls_detector_header) + bytes_per_part()) * frame_index, SEEK_SET); // NOLINT
|
||||
size_t const rc = fread(reinterpret_cast<char *>(&h), sizeof(h), 1, fp);
|
||||
if (rc != 1)
|
||||
throw std::runtime_error(LOCATION + "Could not read header from file");
|
||||
|
||||
return h.frameNumber;
|
||||
}
|
||||
|
||||
SubFile::~SubFile() {
|
||||
if (fp) {
|
||||
fclose(fp);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace aare
|
||||
Loading…
x
Reference in New Issue
Block a user