mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2025-04-21 06:10:02 +02:00
file reading
This commit is contained in:
parent
be019b9769
commit
f754e0f769
2
.github/workflows/build_pkg.yml
vendored
2
.github/workflows/build_pkg.yml
vendored
@ -13,7 +13,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [ubuntu-latest, ] # macos-12, windows-2019]
|
||||
python-version: ["3.11", "3.12", "3.13",]
|
||||
python-version: ["3.11", "3.12",]
|
||||
|
||||
runs-on: ${{ matrix.platform }}
|
||||
|
||||
|
@ -59,7 +59,8 @@ if(AARE_FETCH_ZMQ)
|
||||
# Fetchcontent_Declare is deprecated need to find a way to update this
|
||||
# for now setting the policy to old is enough
|
||||
if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30")
|
||||
cmake_policy(SET CMP0169 OLD)
|
||||
cmake_policy(SET CMP0169 OLD)
|
||||
endif()
|
||||
FetchContent_Declare(
|
||||
libzmq
|
||||
GIT_REPOSITORY https://github.com/zeromq/libzmq.git
|
||||
|
@ -1,6 +1,6 @@
|
||||
package:
|
||||
name: aare
|
||||
version: 2024.10.29.dev0 #TODO! how to not duplicate this?
|
||||
version: 2024.10.30.dev0 #TODO! how to not duplicate this?
|
||||
|
||||
source:
|
||||
path: ..
|
||||
|
@ -12,7 +12,6 @@ namespace aare {
|
||||
class File {
|
||||
private:
|
||||
FileInterface *file_impl;
|
||||
bool is_npy;
|
||||
|
||||
public:
|
||||
/**
|
||||
@ -24,14 +23,16 @@ class File {
|
||||
* @throws std::invalid_argument if the file mode is not supported
|
||||
*
|
||||
*/
|
||||
File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg = {});
|
||||
void write(Frame &frame, sls_detector_header header = {});
|
||||
Frame read();
|
||||
Frame iread(size_t frame_number);
|
||||
std::vector<Frame> read(size_t n_frames);
|
||||
File(const std::filesystem::path &fname, const std::string &mode="r", const FileConfig &cfg = {});
|
||||
|
||||
Frame read_frame(); //!< read one frame from the file at the current position
|
||||
Frame read_frame(size_t frame_number); //!< read the frame at the position given by frame number
|
||||
std::vector<Frame> read_n(size_t n_frames); //!< read n_frames from the file at the current position
|
||||
|
||||
void read_into(std::byte *image_buf);
|
||||
void read_into(std::byte *image_buf, size_t n_frames);
|
||||
size_t frame_number(size_t frame_index);
|
||||
|
||||
size_t frame_number(size_t frame_index); //!< get the frame number at the given frame index
|
||||
size_t bytes_per_frame();
|
||||
size_t pixels_per_frame();
|
||||
void seek(size_t frame_number);
|
||||
@ -43,17 +44,8 @@ class File {
|
||||
size_t bytes_per_pixel() const;
|
||||
void set_total_frames(size_t total_frames);
|
||||
DetectorType detector_type() const;
|
||||
xy geometry() const;
|
||||
|
||||
/**
|
||||
* @brief Move constructor
|
||||
* @param other File object to move from
|
||||
*/
|
||||
File(File &&other) noexcept;
|
||||
|
||||
/**
|
||||
* @brief destructor: will only delete the FileInterface object
|
||||
*/
|
||||
~File();
|
||||
};
|
||||
|
||||
|
@ -47,32 +47,24 @@ struct FileConfig {
|
||||
class FileInterface {
|
||||
public:
|
||||
/**
|
||||
* @brief write a frame to the file
|
||||
* @param frame frame to write
|
||||
* @return void
|
||||
* @throws std::runtime_error if the function is not implemented
|
||||
*/
|
||||
// virtual void write(Frame &frame) = 0;
|
||||
|
||||
/**
|
||||
* @brief write a vector of frames to the file
|
||||
* @param frames vector of frames to write
|
||||
* @return void
|
||||
*/
|
||||
// virtual void write(std::vector<Frame> &frames) = 0;
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the current position
|
||||
* @brief one frame from the file at the current position
|
||||
* @return Frame
|
||||
*/
|
||||
virtual Frame read() = 0;
|
||||
virtual Frame read_frame() = 0;
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the given frame number
|
||||
* @param frame_number frame number to read
|
||||
* @return frame
|
||||
*/
|
||||
virtual Frame read_frame(size_t frame_number) = 0;
|
||||
|
||||
/**
|
||||
* @brief read n_frames from the file at the current position
|
||||
* @param n_frames number of frames to read
|
||||
* @return vector of frames
|
||||
*/
|
||||
virtual std::vector<Frame> read(size_t n_frames) = 0; // Is this the right interface?
|
||||
virtual std::vector<Frame> read_n(size_t n_frames) = 0; // Is this the right interface?
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the current position and store it in the provided buffer
|
||||
@ -142,32 +134,7 @@ class FileInterface {
|
||||
*/
|
||||
virtual size_t bitdepth() const = 0;
|
||||
|
||||
/**
|
||||
* @brief read one frame from the file at the given frame number
|
||||
* @param frame_number frame number to read
|
||||
* @return frame
|
||||
*/
|
||||
Frame iread(size_t frame_number) {
|
||||
auto old_pos = tell();
|
||||
seek(frame_number);
|
||||
Frame tmp = read();
|
||||
seek(old_pos);
|
||||
return tmp;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief read n_frames from the file starting at the given frame number
|
||||
* @param frame_number frame number to start reading from
|
||||
* @param n_frames number of frames to read
|
||||
* @return vector of frames
|
||||
*/
|
||||
std::vector<Frame> iread(size_t frame_number, size_t n_frames) {
|
||||
auto old_pos = tell();
|
||||
seek(frame_number);
|
||||
std::vector<Frame> tmp = read(n_frames);
|
||||
seek(old_pos);
|
||||
return tmp;
|
||||
}
|
||||
DetectorType detector_type() const { return m_type; }
|
||||
|
||||
// function to query the data type of the file
|
||||
@ -175,8 +142,6 @@ class FileInterface {
|
||||
|
||||
virtual ~FileInterface() = default;
|
||||
|
||||
void set_total_frames(size_t total_frames) { m_total_frames = total_frames; }
|
||||
|
||||
protected:
|
||||
std::string m_mode{};
|
||||
std::filesystem::path m_fname{};
|
||||
|
@ -31,9 +31,10 @@ class NumpyFile : public FileInterface {
|
||||
explicit NumpyFile(const std::filesystem::path &fname, const std::string &mode = "r", FileConfig cfg = {});
|
||||
|
||||
void write(Frame &frame);
|
||||
Frame read() override { return get_frame(this->current_frame++); }
|
||||
Frame read_frame() override { return get_frame(this->current_frame++); }
|
||||
Frame read_frame(size_t frame_number) override { return get_frame(frame_number); }
|
||||
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
std::vector<Frame> read_n(size_t n_frames) override;
|
||||
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); }
|
||||
void read_into(std::byte *image_buf, size_t n_frames) override;
|
||||
size_t frame_number(size_t frame_index) override { return frame_index; };
|
||||
|
@ -39,8 +39,12 @@ class RawFile : public FileInterface {
|
||||
* @param frame frame to write
|
||||
*/
|
||||
void write(Frame &frame, sls_detector_header header);
|
||||
Frame read() override { return get_frame(this->current_frame++); };
|
||||
std::vector<Frame> read(size_t n_frames) override;
|
||||
Frame read_frame() override { return get_frame(this->current_frame++); };
|
||||
Frame read_frame(size_t frame_number) override{
|
||||
seek(frame_number);
|
||||
return read_frame();
|
||||
}
|
||||
std::vector<Frame> read_n(size_t n_frames) override;
|
||||
void read_into(std::byte *image_buf) override { return get_frame_into(this->current_frame++, image_buf); };
|
||||
void read_into(std::byte *image_buf, size_t n_frames) override;
|
||||
size_t frame_number(size_t frame_index) override;
|
||||
|
@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
|
||||
|
||||
[project]
|
||||
name = "aare"
|
||||
version = "2024.10.29.dev0"
|
||||
version = "2024.10.30.dev0"
|
||||
|
||||
[tool.scikit-build]
|
||||
cmake.verbose = true
|
||||
|
@ -16,25 +16,12 @@ namespace py = pybind11;
|
||||
using namespace::aare;
|
||||
|
||||
void define_file_io_bindings(py::module &m) {
|
||||
py::class_<xy>(m, "xy")
|
||||
.def(py::init<>())
|
||||
.def(py::init<uint32_t, uint32_t>())
|
||||
.def_readwrite("row", &xy::row)
|
||||
.def_readwrite("col", &xy::col)
|
||||
.def("__eq__", &xy::operator==)
|
||||
.def("__ne__", &xy::operator!=)
|
||||
.def("__repr__",
|
||||
[](const xy &a) { return "<xy: row=" + std::to_string(a.row) + ", col=" + std::to_string(a.col) + ">"; });
|
||||
|
||||
|
||||
py::class_<File>(m, "File")
|
||||
.def(py::init([](const std::filesystem::path &fname) { return File(fname, "r", {}); }))
|
||||
.def(
|
||||
py::init([](const std::filesystem::path &fname, const std::string &mode) { return File(fname, mode, {}); }))
|
||||
.def(py::init<const std::filesystem::path &, const std::string &, const FileConfig &>())
|
||||
// .def("read", py::overload_cast<>(&File::read))
|
||||
// .def("read", py::overload_cast<size_t>(&File::read))
|
||||
.def("iread", py::overload_cast<size_t>(&File::iread),py::call_guard<py::gil_scoped_release>())
|
||||
|
||||
.def("frame_number", &File::frame_number)
|
||||
.def_property_readonly("bytes_per_frame", &File::bytes_per_frame)
|
||||
.def_property_readonly("pixels_per_frame", &File::pixels_per_frame)
|
||||
@ -44,10 +31,8 @@ void define_file_io_bindings(py::module &m) {
|
||||
.def_property_readonly("rows", &File::rows)
|
||||
.def_property_readonly("cols", &File::cols)
|
||||
.def_property_readonly("bitdepth", &File::bitdepth)
|
||||
.def_property_readonly("bytes_per_pixel", &File::bytes_per_pixel)
|
||||
.def_property_readonly("detector_type", &File::detector_type)
|
||||
.def_property_readonly("geometry", &File::geometry,
|
||||
py::call_guard<py::scoped_ostream_redirect, py::scoped_estream_redirect>())
|
||||
// .def("set_total_frames", &File::set_total_frames)
|
||||
.def("read_frame", [](File &self) {
|
||||
const uint8_t item_size = self.bytes_per_pixel();
|
||||
py::array image;
|
||||
@ -64,6 +49,42 @@ void define_file_io_bindings(py::module &m) {
|
||||
}
|
||||
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
|
||||
return image;
|
||||
})
|
||||
.def("read_frame", [](File &self, size_t frame_number) {
|
||||
self.seek(frame_number);
|
||||
const uint8_t item_size = self.bytes_per_pixel();
|
||||
py::array image;
|
||||
std::vector<ssize_t> shape;
|
||||
shape.reserve(2);
|
||||
shape.push_back(self.rows());
|
||||
shape.push_back(self.cols());
|
||||
if (item_size == 1) {
|
||||
image = py::array_t<uint8_t>(shape);
|
||||
} else if (item_size == 2) {
|
||||
image = py::array_t<uint16_t>(shape);
|
||||
} else if (item_size == 4) {
|
||||
image = py::array_t<uint32_t>(shape);
|
||||
}
|
||||
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()));
|
||||
return image;
|
||||
})
|
||||
.def("read_n", [](File &self, size_t n_frames) {
|
||||
const uint8_t item_size = self.bytes_per_pixel();
|
||||
py::array image;
|
||||
std::vector<ssize_t> shape;
|
||||
shape.reserve(3);
|
||||
shape.push_back(n_frames);
|
||||
shape.push_back(self.rows());
|
||||
shape.push_back(self.cols());
|
||||
if (item_size == 1) {
|
||||
image = py::array_t<uint8_t>(shape);
|
||||
} else if (item_size == 2) {
|
||||
image = py::array_t<uint16_t>(shape);
|
||||
} else if (item_size == 4) {
|
||||
image = py::array_t<uint32_t>(shape);
|
||||
}
|
||||
self.read_into(reinterpret_cast<std::byte *>(image.mutable_data()), n_frames);
|
||||
return image;
|
||||
});
|
||||
|
||||
py::class_<FileConfig>(m, "FileConfig")
|
||||
|
28
src/File.cpp
28
src/File.cpp
@ -7,7 +7,7 @@
|
||||
namespace aare {
|
||||
|
||||
File::File(const std::filesystem::path &fname, const std::string &mode, const FileConfig &cfg)
|
||||
: file_impl(nullptr), is_npy(true) {
|
||||
: file_impl(nullptr){
|
||||
if (mode != "r" && mode != "w" && mode != "a") {
|
||||
throw std::invalid_argument("Unsupported file mode");
|
||||
}
|
||||
@ -19,7 +19,6 @@ File::File(const std::filesystem::path &fname, const std::string &mode, const Fi
|
||||
if (fname.extension() == ".raw" || fname.extension() == ".json") {
|
||||
// aare::logger::debug("Loading raw file");
|
||||
file_impl = new RawFile(fname, mode, cfg);
|
||||
is_npy = false;
|
||||
}
|
||||
// check if extension is numpy
|
||||
else if (fname.extension() == ".npy") {
|
||||
@ -30,17 +29,10 @@ File::File(const std::filesystem::path &fname, const std::string &mode, const Fi
|
||||
}
|
||||
}
|
||||
|
||||
void File::write(Frame &frame, sls_detector_header header) {
|
||||
if (is_npy) {
|
||||
// aare::logger::info("ignoring header for npy file");
|
||||
dynamic_cast<NumpyFile *>(file_impl)->write(frame);
|
||||
} else {
|
||||
dynamic_cast<RawFile *>(file_impl)->write(frame, header);
|
||||
}
|
||||
}
|
||||
Frame File::read() { return file_impl->read(); }
|
||||
|
||||
Frame File::read_frame() { return file_impl->read_frame(); }
|
||||
size_t File::total_frames() const { return file_impl->total_frames(); }
|
||||
std::vector<Frame> File::read(size_t n_frames) { return file_impl->read(n_frames); }
|
||||
std::vector<Frame> File::read_n(size_t n_frames) { return file_impl->read_n(n_frames); }
|
||||
void File::read_into(std::byte *image_buf) { file_impl->read_into(image_buf); }
|
||||
void File::read_into(std::byte *image_buf, size_t n_frames) { file_impl->read_into(image_buf, n_frames); }
|
||||
size_t File::frame_number(size_t frame_index) { return file_impl->frame_number(frame_index); }
|
||||
@ -52,19 +44,13 @@ size_t File::rows() const { return file_impl->rows(); }
|
||||
size_t File::cols() const { return file_impl->cols(); }
|
||||
size_t File::bitdepth() const { return file_impl->bitdepth(); }
|
||||
size_t File::bytes_per_pixel() const { return file_impl->bitdepth()/8; }
|
||||
void File::set_total_frames(size_t total_frames) { return file_impl->set_total_frames(total_frames); }
|
||||
File::~File() { delete file_impl; }
|
||||
DetectorType File::detector_type() const { return file_impl->detector_type(); }
|
||||
xy File::geometry() const {
|
||||
if (is_npy) {
|
||||
return {1, 1};
|
||||
}
|
||||
return reinterpret_cast<RawFile *>(file_impl)->geometry();
|
||||
}
|
||||
|
||||
Frame File::iread(size_t frame_number) { return file_impl->iread(frame_number); }
|
||||
|
||||
File::File(File &&other) noexcept : file_impl(other.file_impl), is_npy(other.is_npy) { other.file_impl = nullptr; }
|
||||
Frame File::read_frame(size_t frame_number) { return file_impl->read_frame(frame_number); }
|
||||
|
||||
File::File(File &&other) noexcept : file_impl(other.file_impl) { other.file_impl = nullptr; }
|
||||
|
||||
// write move assignment operator
|
||||
|
||||
|
@ -75,7 +75,7 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) {
|
||||
size_t NumpyFile::pixels_per_frame() { return m_pixels_per_frame; };
|
||||
size_t NumpyFile::bytes_per_frame() { return m_bytes_per_frame; };
|
||||
|
||||
std::vector<Frame> NumpyFile::read(size_t n_frames) {
|
||||
std::vector<Frame> NumpyFile::read_n(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
|
@ -382,7 +382,7 @@ void RawFile::write(Frame &frame, sls_detector_header header) {
|
||||
this->current_frame++;
|
||||
}
|
||||
|
||||
std::vector<Frame> RawFile::read(size_t n_frames) {
|
||||
std::vector<Frame> RawFile::read_n(size_t n_frames) {
|
||||
// TODO: implement this in a more efficient way
|
||||
std::vector<Frame> frames;
|
||||
for (size_t i = 0; i < n_frames; i++) {
|
||||
|
@ -41,7 +41,7 @@ TEST_CASE("Read data from a jungfrau 500k single port raw file") {
|
||||
// we know this file has 10 frames with pixel 0,0 being: 2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102
|
||||
std::vector<uint16_t> pixel_0_0 = {2123, 2051, 2109, 2117, 2089, 2095, 2072, 2126, 2097, 2102};
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
auto frame = f.read();
|
||||
auto frame = f.read_frame();
|
||||
CHECK(frame.rows() == 512);
|
||||
CHECK(frame.cols() == 1024);
|
||||
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
|
||||
@ -75,8 +75,8 @@ TEST_CASE("Compare reading from a numpy file with a raw file") {
|
||||
CHECK(npy.total_frames() == 10);
|
||||
|
||||
for (size_t i = 0; i < 10; ++i) {
|
||||
auto raw_frame = raw.read();
|
||||
auto npy_frame = npy.read();
|
||||
auto raw_frame = raw.read_frame();
|
||||
auto npy_frame = npy.read_frame();
|
||||
CHECK((raw_frame.view<uint16_t>() == npy_frame.view<uint16_t>()));
|
||||
}
|
||||
}
|
||||
@ -95,7 +95,7 @@ TEST_CASE("Read multipart files") {
|
||||
std::vector<uint16_t> pixel_1_0 = {2748, 2614, 2665, 2629, 2618, 2630, 2631, 2634, 2577, 2598};
|
||||
|
||||
for (size_t i = 0; i < 10; i++) {
|
||||
auto frame = f.read();
|
||||
auto frame = f.read_frame();
|
||||
CHECK(frame.rows() == 512);
|
||||
CHECK(frame.cols() == 1024);
|
||||
CHECK(frame.view<uint16_t>()(0, 0) == pixel_0_0[i]);
|
||||
@ -110,5 +110,5 @@ TEST_CASE("Read file with unordered frames") {
|
||||
auto fpath = test_data_path() / "mythen" / "scan242_master_3.raw";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
File f(fpath, "r");
|
||||
REQUIRE_THROWS((f.read()));
|
||||
REQUIRE_THROWS((f.read_frame()));
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user