mirror of
https://github.com/slsdetectorgroup/aare.git
synced 2026-11-19 01:24:04 +01:00
Prototype multi threaded file reader (#343)
- Reading files from nfs shares tops out at ~1GB/s for single threaded reads. - MultiThreadedFileReader uses our File wrapper to read a generic file in parallel - Placed in aare::experimental to show that it's not production ready
This commit is contained in:
1 parent
533ecb8a4a
commit
e26db97b5c
26 files changed
+1021
-15
No files matched your search
+3
-1
@@ -127,6 +127,8 @@ cover/
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
.*
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
@@ -192,4 +194,4 @@ cython_debug/
|
||||
.ruff_cache/
|
||||
|
||||
# user defined
|
||||
wheelhouse/
|
||||
wheelhouse/
|
||||
+6
-1
@@ -396,6 +396,7 @@ set(PUBLICHEADERS
|
||||
include/aare/FileInterface.hpp
|
||||
include/aare/FilePtr.hpp
|
||||
include/aare/Frame.hpp
|
||||
include/aare/MultiThreadedFileReader.hpp
|
||||
include/aare/hist/PixelHistogram.hpp
|
||||
include/aare/hist/PixelHistogramImpl.hpp
|
||||
include/aare/hist/PedestalTrackingPixelHistogram.hpp
|
||||
@@ -414,7 +415,9 @@ set(PUBLICHEADERS
|
||||
include/aare/RawMasterFile.hpp
|
||||
include/aare/RawSubFile.hpp
|
||||
include/aare/VarClusterFinder.hpp
|
||||
include/aare/utils/task.hpp)
|
||||
include/aare/utils/task.hpp
|
||||
include/aare/utils/ifstream_helpers.hpp
|
||||
include/aare/utils/math_helpers.hpp)
|
||||
|
||||
set(SourceFiles
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/calibration.cpp
|
||||
@@ -431,6 +434,7 @@ set(SourceFiles
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolator.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/MultiThreadedFileReader.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/PixelMap.cpp
|
||||
@@ -502,6 +506,7 @@ if(AARE_TESTS)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/hist/PixelHistogramImpl.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/hist/PixelHistogram.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/MultiThreadedFileReader.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/RawFile.test.cpp
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
MultiThreadedFileReader
|
||||
=======================
|
||||
|
||||
``MultiThreadedFileReader`` reads one chunk per worker on each call. Each
|
||||
worker owns an independent :cpp:class:`aare::File`, while all workers write
|
||||
into non-overlapping regions of the destination buffer. The output is ordered
|
||||
by frame index and successive calls advance through the file.
|
||||
|
||||
.. code-block:: cpp
|
||||
|
||||
#include "aare/MultiThreadedFileReader.hpp"
|
||||
|
||||
// Four workers, chunks of 128 frames, and at most 10,000 frames.
|
||||
aare::experimental::MultiThreadedFileReader reader(path, 4, 128, 10'000);
|
||||
while (reader.remaining_frames() != 0) {
|
||||
// Contains at most 4 * 128 frames.
|
||||
auto batch = reader.read();
|
||||
process(batch);
|
||||
}
|
||||
|
||||
Omit the final argument to read every frame in the source. An explicit value
|
||||
of zero requests an empty result. The low-level ``read_into`` overload avoids
|
||||
an allocation when the caller already owns a buffer of at least
|
||||
``reader.next_read_bytes()`` bytes. Use ``read_all()`` to read every frame
|
||||
remaining from the current position, and ``seek()`` to reposition the reader.
|
||||
Call ``close()`` to release all worker file handles early.
|
||||
|
||||
.. note::
|
||||
|
||||
Multiple workers do not guarantee faster reads. Performance depends on the
|
||||
storage device and file format, so the thread count and chunk size should be
|
||||
benchmark-driven.
|
||||
|
||||
.. doxygenclass:: aare::experimental::MultiThreadedFileReader
|
||||
:members:
|
||||
:undoc-members:
|
||||
:private-members:
|
||||
@@ -28,6 +28,7 @@ AARE
|
||||
pycalibration
|
||||
python/cluster/index
|
||||
python/file/index
|
||||
python/experimental/index
|
||||
python/histogram/index
|
||||
pyFit
|
||||
|
||||
@@ -42,6 +43,7 @@ AARE
|
||||
NDView
|
||||
Frame
|
||||
File
|
||||
MultiThreadedFileReader
|
||||
Dtype
|
||||
Cluster
|
||||
ClusterFinder
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
MultiThreadedFileReader
|
||||
=======================
|
||||
|
||||
.. py:currentmodule:: aare.experimental
|
||||
|
||||
The reader returns a NumPy array with shape ``(frames, rows, cols)`` and
|
||||
preserves the source pixel dtype. Each iteration reads at most
|
||||
``n_threads * chunk_size`` frames—one chunk per worker. File I/O runs with the
|
||||
Python GIL released.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from aare.experimental import MultiThreadedFileReader
|
||||
|
||||
with MultiThreadedFileReader(
|
||||
"frames.npy", n_threads=4, chunk_size=128, total_frames=10_000
|
||||
) as reader:
|
||||
for frames in reader:
|
||||
process(frames)
|
||||
|
||||
Call ``read()`` directly for the next batch, or ``read_all()`` for all frames
|
||||
remaining from the current position. ``tell()`` and ``seek()`` expose the
|
||||
iteration position. The context manager closes all worker files on exit.
|
||||
``close()`` is also available for explicit cleanup and may be called
|
||||
repeatedly.
|
||||
|
||||
.. autoclass:: MultiThreadedFileReader
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,10 @@
|
||||
Experimental
|
||||
============
|
||||
|
||||
APIs in this module may change without notice.
|
||||
|
||||
.. toctree::
|
||||
:caption: Experimental
|
||||
:maxdepth: 1
|
||||
|
||||
MultiThreadedFileReader
|
||||
@@ -64,8 +64,9 @@ class File {
|
||||
size_t total_frames() const;
|
||||
size_t rows() const;
|
||||
size_t cols() const;
|
||||
Dtype dtype() const;
|
||||
|
||||
DetectorType detector_type() const;
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -145,10 +145,15 @@ class FileInterface {
|
||||
*/
|
||||
virtual size_t bitdepth() const = 0;
|
||||
|
||||
virtual DetectorType detector_type() const = 0;
|
||||
/**
|
||||
* @brief get the data type of the pixels
|
||||
* @return pixel data type
|
||||
*/
|
||||
virtual Dtype dtype() const {
|
||||
return Dtype::from_bitdepth(static_cast<uint8_t>(bitdepth()));
|
||||
}
|
||||
|
||||
// function to query the data type of the file
|
||||
/*virtual DataType dtype = 0; */
|
||||
virtual DetectorType detector_type() const = 0;
|
||||
|
||||
virtual ~FileInterface() = default;
|
||||
|
||||
@@ -168,4 +173,4 @@ class FileInterface {
|
||||
// size_t current_frame{};
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -49,6 +49,7 @@ class JungfrauDataFile : public FileInterface {
|
||||
size_t pixels_per_frame() override;
|
||||
size_t bytes_per_pixel() const;
|
||||
size_t bitdepth() const override;
|
||||
Dtype dtype() const override { return Dtype::UINT16; }
|
||||
void seek(size_t frame_index)
|
||||
override; //!< seek to the given frame index (note not byte offset)
|
||||
size_t tell() override; //!< get the frame index of the file pointer
|
||||
@@ -113,4 +114,4 @@ class JungfrauDataFile : public FileInterface {
|
||||
std::filesystem::path fpath(size_t frame_index) const;
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -0,0 +1,104 @@
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
#pragma once
|
||||
|
||||
#include "aare/Dtype.hpp"
|
||||
#include "aare/File.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace aare::experimental {
|
||||
|
||||
/**
|
||||
* @brief Read independent chunks of a file in parallel.
|
||||
*
|
||||
* Each worker opens its own File instance, so seeking and reading do not share
|
||||
* mutable file state. Chunks are written directly to their position in the
|
||||
* destination buffer and the resulting frame order is the same as in the file.
|
||||
*/
|
||||
class MultiThreadedFileReader {
|
||||
public:
|
||||
/**
|
||||
* @param fname path accepted by File
|
||||
* @param n_threads maximum number of worker threads
|
||||
* @param chunk_size number of frames claimed by a worker at a time
|
||||
* @param total_frames number of frames to read, or all frames when omitted
|
||||
*/
|
||||
MultiThreadedFileReader(std::filesystem::path fname, size_t n_threads,
|
||||
size_t chunk_size,
|
||||
std::optional<size_t> total_frames = std::nullopt);
|
||||
|
||||
MultiThreadedFileReader(const MultiThreadedFileReader &) = delete;
|
||||
MultiThreadedFileReader &
|
||||
operator=(const MultiThreadedFileReader &) = delete;
|
||||
MultiThreadedFileReader(MultiThreadedFileReader &&) noexcept = default;
|
||||
MultiThreadedFileReader &
|
||||
operator=(MultiThreadedFileReader &&) noexcept = default;
|
||||
|
||||
/**
|
||||
* @brief Read one chunk per active worker into a caller-owned buffer.
|
||||
*
|
||||
* The buffer must hold at least next_read_bytes() bytes. The reader's
|
||||
* position advances by the returned number of frames. At the end of the
|
||||
* configured range this function returns zero and does not access the
|
||||
* destination.
|
||||
*/
|
||||
size_t read_into(std::byte *destination);
|
||||
|
||||
/** @brief Read the next wave of chunks into an owned byte buffer. */
|
||||
std::vector<std::byte> read();
|
||||
|
||||
/** @brief Read every frame remaining from the current position. */
|
||||
std::vector<std::byte> read_all();
|
||||
|
||||
/** @brief Set the next frame index to read. The end position is valid. */
|
||||
void seek(size_t frame_index);
|
||||
|
||||
/** @brief Return the next frame index to read. */
|
||||
size_t tell() const noexcept { return m_current_frame; }
|
||||
|
||||
/** @brief Close all worker files. Safe to call more than once. */
|
||||
void close() noexcept { m_files.clear(); }
|
||||
|
||||
/** @brief Return whether the worker files are open. */
|
||||
bool is_open() const noexcept { return !m_files.empty(); }
|
||||
|
||||
size_t n_threads() const noexcept { return m_n_threads; }
|
||||
size_t chunk_size() const noexcept { return m_chunk_size; }
|
||||
size_t total_frames() const noexcept { return m_total_frames; }
|
||||
size_t source_total_frames() const noexcept {
|
||||
return m_source_total_frames;
|
||||
}
|
||||
size_t rows() const noexcept { return m_rows; }
|
||||
size_t cols() const noexcept { return m_cols; }
|
||||
size_t bitdepth() const noexcept { return m_bitdepth; }
|
||||
Dtype dtype() const noexcept { return m_dtype; }
|
||||
size_t bytes_per_frame() const noexcept { return m_bytes_per_frame; }
|
||||
size_t total_bytes() const noexcept { return m_total_bytes; }
|
||||
size_t remaining_frames() const noexcept;
|
||||
size_t next_read_frames() const noexcept;
|
||||
size_t next_read_bytes() const noexcept {
|
||||
return next_read_frames() * m_bytes_per_frame;
|
||||
}
|
||||
|
||||
private:
|
||||
std::filesystem::path m_fname;
|
||||
size_t m_n_threads;
|
||||
size_t m_chunk_size;
|
||||
size_t m_total_frames;
|
||||
size_t m_source_total_frames;
|
||||
size_t m_rows;
|
||||
size_t m_cols;
|
||||
size_t m_bitdepth;
|
||||
Dtype m_dtype;
|
||||
size_t m_bytes_per_frame;
|
||||
size_t m_total_bytes;
|
||||
size_t m_current_frame;
|
||||
std::vector<File> m_files;
|
||||
|
||||
void ensure_open() const;
|
||||
};
|
||||
|
||||
} // namespace aare::experimental
|
||||
@@ -61,7 +61,7 @@ class NumpyFile : public FileInterface {
|
||||
* @brief get the data type of the numpy file
|
||||
* @return DType
|
||||
*/
|
||||
Dtype dtype() const { return m_header.dtype; }
|
||||
Dtype dtype() const override { return m_header.dtype; }
|
||||
|
||||
/**
|
||||
* @brief get the shape of the numpy file
|
||||
@@ -129,4 +129,4 @@ class NumpyFile : public FileInterface {
|
||||
void write_impl(void *data, uint64_t size);
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -118,6 +118,7 @@ class RawFile : public FileInterface {
|
||||
*/
|
||||
size_t cols(const size_t roi_index) const;
|
||||
size_t bitdepth() const override;
|
||||
Dtype dtype() const override { return Dtype::from_bitdepth(bitdepth()); }
|
||||
size_t n_modules() const;
|
||||
|
||||
/**
|
||||
@@ -170,4 +171,4 @@ class RawFile : public FileInterface {
|
||||
void open_subfiles(const size_t roi_index);
|
||||
};
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -0,0 +1,14 @@
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
|
||||
namespace aare {
|
||||
|
||||
/**
|
||||
* @brief Compute the ceiling of the integer division of n by d.
|
||||
* @param n The numerator.
|
||||
* @param d The denominator.
|
||||
* @return The ceiling of the integer division.
|
||||
*/
|
||||
constexpr size_t ceil_div(size_t n, size_t d) { return n / d + (n % d != 0); }
|
||||
|
||||
} // namespace aare
|
||||
@@ -40,6 +40,7 @@ set(PYTHON_FILES
|
||||
aare/ClusterVector.py
|
||||
aare/Cluster.py
|
||||
aare/calibration.py
|
||||
aare/experimental.py
|
||||
aare/func.py
|
||||
aare/RawFile.py
|
||||
aare/transform.py
|
||||
|
||||
@@ -3,8 +3,14 @@
|
||||
from . import _aare
|
||||
|
||||
from . import transform
|
||||
from . import experimental
|
||||
|
||||
from ._aare import File, RawMasterFile, RawSubFile, JungfrauDataFile
|
||||
from ._aare import (
|
||||
File,
|
||||
JungfrauDataFile,
|
||||
RawMasterFile,
|
||||
RawSubFile,
|
||||
)
|
||||
from ._aare import Pedestal_d, Pedestal_f, ClusterFinder_Cluster3x3i, VarClusterFinder
|
||||
from ._aare import DetectorType, ReadoutMode
|
||||
from ._aare import hitmap
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Experimental APIs that may change without notice."""
|
||||
|
||||
from ._aare.experimental import MultiThreadedFileReader
|
||||
|
||||
__all__ = ["MultiThreadedFileReader"]
|
||||
@@ -0,0 +1,175 @@
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
#pragma once
|
||||
|
||||
#include "aare/MultiThreadedFileReader.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
#include <pybind11/numpy.h>
|
||||
#include <pybind11/pybind11.h>
|
||||
#include <pybind11/stl.h>
|
||||
#include <pybind11/stl/filesystem.h>
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
inline py::dtype multi_threaded_reader_numpy_dtype(const aare::Dtype &dtype) {
|
||||
using aare::Dtype;
|
||||
if (dtype == Dtype::INT8)
|
||||
return py::dtype::of<int8_t>();
|
||||
if (dtype == Dtype::UINT8)
|
||||
return py::dtype::of<uint8_t>();
|
||||
if (dtype == Dtype::INT16)
|
||||
return py::dtype::of<int16_t>();
|
||||
if (dtype == Dtype::UINT16)
|
||||
return py::dtype::of<uint16_t>();
|
||||
if (dtype == Dtype::INT32)
|
||||
return py::dtype::of<int32_t>();
|
||||
if (dtype == Dtype::UINT32)
|
||||
return py::dtype::of<uint32_t>();
|
||||
if (dtype == Dtype::INT64)
|
||||
return py::dtype::of<int64_t>();
|
||||
if (dtype == Dtype::UINT64)
|
||||
return py::dtype::of<uint64_t>();
|
||||
if (dtype == Dtype::FLOAT)
|
||||
return py::dtype::of<float>();
|
||||
if (dtype == Dtype::DOUBLE)
|
||||
return py::dtype::of<double>();
|
||||
throw std::runtime_error("Unsupported pixel data type");
|
||||
}
|
||||
|
||||
inline py::array
|
||||
multi_threaded_reader_read(aare::experimental::MultiThreadedFileReader &reader,
|
||||
bool read_all) {
|
||||
const size_t n_frames =
|
||||
read_all ? reader.remaining_frames() : reader.next_read_frames();
|
||||
const std::vector<py::ssize_t> shape{
|
||||
static_cast<py::ssize_t>(n_frames),
|
||||
static_cast<py::ssize_t>(reader.rows()),
|
||||
static_cast<py::ssize_t>(reader.cols())};
|
||||
|
||||
py::array image(multi_threaded_reader_numpy_dtype(reader.dtype()), shape);
|
||||
auto *destination = reinterpret_cast<std::byte *>(image.mutable_data());
|
||||
{
|
||||
py::gil_scoped_release release;
|
||||
if (read_all) {
|
||||
size_t offset = 0;
|
||||
while (reader.remaining_frames() != 0) {
|
||||
const size_t frames_read =
|
||||
reader.read_into(destination + offset);
|
||||
offset += frames_read * reader.bytes_per_frame();
|
||||
}
|
||||
} else {
|
||||
reader.read_into(destination);
|
||||
}
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
inline void define_multi_threaded_file_reader_bindings(py::module_ &m) {
|
||||
using aare::experimental::MultiThreadedFileReader;
|
||||
|
||||
auto reader =
|
||||
py::class_<MultiThreadedFileReader>(m, "MultiThreadedFileReader");
|
||||
reader.attr("__module__") = "aare.experimental";
|
||||
reader
|
||||
.def(py::init<std::filesystem::path, size_t, size_t,
|
||||
std::optional<size_t>>(),
|
||||
py::arg("fname"), py::arg("n_threads"), py::arg("chunk_size"),
|
||||
py::arg("total_frames") = py::none(),
|
||||
R"doc(
|
||||
Read chunks of detector frames concurrently.
|
||||
|
||||
Each worker opens an independent File. The returned array is
|
||||
ordered by frame index even though chunks are read in parallel.
|
||||
|
||||
Args:
|
||||
fname: Path accepted by File.
|
||||
n_threads: Maximum number of worker threads.
|
||||
chunk_size: Number of frames read per claimed chunk.
|
||||
total_frames: Optional frame limit. None reads all frames.
|
||||
)doc")
|
||||
.def(
|
||||
"read",
|
||||
[](MultiThreadedFileReader &self) {
|
||||
return multi_threaded_reader_read(self, false);
|
||||
},
|
||||
R"doc(
|
||||
Read one chunk per active worker into a NumPy array.
|
||||
|
||||
Returns:
|
||||
An array containing at most n_threads * chunk_size frames.
|
||||
An empty array is returned at the end of the configured
|
||||
frame range. The GIL is released while file data is read.
|
||||
)doc")
|
||||
.def(
|
||||
"read_all",
|
||||
[](MultiThreadedFileReader &self) {
|
||||
return multi_threaded_reader_read(self, true);
|
||||
},
|
||||
R"doc(
|
||||
Read all frames remaining from the current position.
|
||||
)doc")
|
||||
.def_property_readonly("n_threads", &MultiThreadedFileReader::n_threads)
|
||||
.def_property_readonly("chunk_size",
|
||||
&MultiThreadedFileReader::chunk_size)
|
||||
.def_property_readonly("total_frames",
|
||||
&MultiThreadedFileReader::total_frames)
|
||||
.def_property_readonly("source_total_frames",
|
||||
&MultiThreadedFileReader::source_total_frames)
|
||||
.def_property_readonly("rows", &MultiThreadedFileReader::rows)
|
||||
.def_property_readonly("cols", &MultiThreadedFileReader::cols)
|
||||
.def_property_readonly("bitdepth", &MultiThreadedFileReader::bitdepth)
|
||||
.def_property_readonly("dtype",
|
||||
[](const MultiThreadedFileReader &self) {
|
||||
return multi_threaded_reader_numpy_dtype(
|
||||
self.dtype());
|
||||
})
|
||||
.def_property_readonly("bytes_per_frame",
|
||||
&MultiThreadedFileReader::bytes_per_frame)
|
||||
.def_property_readonly("total_bytes",
|
||||
&MultiThreadedFileReader::total_bytes)
|
||||
.def_property_readonly("remaining_frames",
|
||||
&MultiThreadedFileReader::remaining_frames)
|
||||
.def_property_readonly("next_read_frames",
|
||||
&MultiThreadedFileReader::next_read_frames)
|
||||
.def_property_readonly("next_read_bytes",
|
||||
&MultiThreadedFileReader::next_read_bytes)
|
||||
.def("seek", &MultiThreadedFileReader::seek, py::arg("frame_index"))
|
||||
.def("tell", &MultiThreadedFileReader::tell)
|
||||
.def("close", &MultiThreadedFileReader::close,
|
||||
"Close all worker files. Safe to call more than once.")
|
||||
.def_property_readonly(
|
||||
"closed",
|
||||
[](const MultiThreadedFileReader &self) { return !self.is_open(); })
|
||||
.def("__len__", &MultiThreadedFileReader::total_frames)
|
||||
.def(
|
||||
"__enter__",
|
||||
[](MultiThreadedFileReader &self) -> MultiThreadedFileReader * {
|
||||
if (!self.is_open()) {
|
||||
throw std::runtime_error(
|
||||
"Cannot enter a closed MultiThreadedFileReader");
|
||||
}
|
||||
return &self;
|
||||
},
|
||||
py::return_value_policy::reference_internal)
|
||||
.def("__exit__",
|
||||
[](MultiThreadedFileReader &self, const py::object &,
|
||||
const py::object &, const py::object &) {
|
||||
self.close();
|
||||
return false;
|
||||
})
|
||||
.def(
|
||||
"__iter__", [](MultiThreadedFileReader &self) { return &self; },
|
||||
py::return_value_policy::reference_internal)
|
||||
.def("__next__", [](MultiThreadedFileReader &self) {
|
||||
if (self.remaining_frames() == 0) {
|
||||
throw py::stop_iteration();
|
||||
}
|
||||
return multi_threaded_reader_read(self, false);
|
||||
});
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "bind_Defs.hpp"
|
||||
#include "bind_Eta.hpp"
|
||||
#include "bind_Interpolator.hpp"
|
||||
#include "bind_MultiThreadedFileReader.hpp"
|
||||
#include "bind_PedestalTrackingPixelHistogram.hpp"
|
||||
#include "bind_PixelHistogram.hpp"
|
||||
#include "bind_PixelMap.hpp"
|
||||
@@ -59,7 +60,11 @@ double, 'f' for float)
|
||||
define_ClusterCollector<T, N, M, U>(m, "Cluster" #N "x" #M #TYPE_CODE);
|
||||
|
||||
PYBIND11_MODULE(_aare, m) {
|
||||
auto experimental = m.def_submodule(
|
||||
"experimental", "Experimental APIs that may change without notice");
|
||||
|
||||
define_file_io_bindings(m);
|
||||
define_multi_threaded_file_reader_bindings(experimental);
|
||||
define_raw_file_io_bindings(m);
|
||||
define_raw_sub_file_io_bindings(m);
|
||||
define_ctb_raw_file_io_bindings(m);
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import aare
|
||||
from aare.experimental import MultiThreadedFileReader
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def frame_file(tmp_path):
|
||||
data = np.arange(10 * 2 * 3, dtype=np.uint16).reshape(10, 2, 3)
|
||||
path = tmp_path / "frames.npy"
|
||||
np.save(path, data)
|
||||
return path, data
|
||||
|
||||
|
||||
def test_experimental_import_path():
|
||||
assert aare.experimental.MultiThreadedFileReader is MultiThreadedFileReader
|
||||
assert MultiThreadedFileReader.__module__ == "aare.experimental"
|
||||
assert not hasattr(aare, "MultiThreadedFileReader")
|
||||
|
||||
|
||||
def test_reads_all_frames_in_order(frame_file):
|
||||
path, expected = frame_file
|
||||
reader = MultiThreadedFileReader(path, n_threads=2, chunk_size=3)
|
||||
|
||||
first = reader.read()
|
||||
second = reader.read()
|
||||
exhausted = reader.read()
|
||||
|
||||
assert np.array_equal(first, expected[:6])
|
||||
assert np.array_equal(second, expected[6:])
|
||||
assert exhausted.shape == (0, 2, 3)
|
||||
assert first.dtype == np.uint16
|
||||
assert reader.n_threads == 2
|
||||
assert reader.chunk_size == 3
|
||||
assert reader.total_frames == 10
|
||||
assert reader.source_total_frames == 10
|
||||
assert reader.rows == 2
|
||||
assert reader.cols == 3
|
||||
assert reader.bitdepth == 16
|
||||
assert reader.dtype == np.dtype(np.uint16)
|
||||
assert reader.bytes_per_frame == 12
|
||||
assert reader.total_bytes == expected.nbytes
|
||||
assert len(reader) == 10
|
||||
assert reader.tell() == 10
|
||||
assert reader.remaining_frames == 0
|
||||
assert reader.next_read_frames == 0
|
||||
assert reader.next_read_bytes == 0
|
||||
|
||||
|
||||
def test_total_frame_limit(frame_file):
|
||||
path, expected = frame_file
|
||||
reader = MultiThreadedFileReader(
|
||||
path, n_threads=2, chunk_size=2, total_frames=7
|
||||
)
|
||||
|
||||
assert np.array_equal(reader.read(), expected[:4])
|
||||
assert np.array_equal(reader.read_all(), expected[4:7])
|
||||
|
||||
|
||||
def test_iteration_yields_one_chunk_per_thread(frame_file):
|
||||
path, expected = frame_file
|
||||
reader = MultiThreadedFileReader(path, n_threads=2, chunk_size=2)
|
||||
|
||||
batches = list(reader)
|
||||
|
||||
assert [len(batch) for batch in batches] == [4, 4, 2]
|
||||
assert np.array_equal(np.concatenate(batches), expected)
|
||||
|
||||
|
||||
def test_seek_resets_iteration_position(frame_file):
|
||||
path, expected = frame_file
|
||||
reader = MultiThreadedFileReader(path, n_threads=2, chunk_size=2)
|
||||
|
||||
reader.read()
|
||||
assert reader.tell() == 4
|
||||
|
||||
reader.seek(1)
|
||||
assert reader.tell() == 1
|
||||
assert np.array_equal(reader.read(), expected[1:5])
|
||||
|
||||
with pytest.raises(IndexError):
|
||||
reader.seek(11)
|
||||
|
||||
|
||||
def test_context_manager_closes_worker_files(frame_file):
|
||||
path, expected = frame_file
|
||||
|
||||
with MultiThreadedFileReader(path, n_threads=2, chunk_size=2) as reader:
|
||||
assert not reader.closed
|
||||
assert np.array_equal(reader.read(), expected[:4])
|
||||
|
||||
assert reader.closed
|
||||
reader.close()
|
||||
with pytest.raises(RuntimeError):
|
||||
reader.read()
|
||||
with pytest.raises(RuntimeError):
|
||||
reader.seek(0)
|
||||
with pytest.raises(RuntimeError):
|
||||
with reader:
|
||||
pass
|
||||
|
||||
|
||||
def test_explicit_zero_frame_limit(frame_file):
|
||||
path, expected = frame_file
|
||||
reader = MultiThreadedFileReader(
|
||||
path, n_threads=8, chunk_size=3, total_frames=0
|
||||
)
|
||||
|
||||
actual = reader.read()
|
||||
assert actual.shape == (0, *expected.shape[1:])
|
||||
assert actual.dtype == expected.dtype
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dtype", [np.int8, np.int32, np.uint64, np.float32, np.float64]
|
||||
)
|
||||
def test_preserves_numpy_dtype(tmp_path, dtype):
|
||||
expected = np.arange(4 * 2 * 3, dtype=dtype).reshape(4, 2, 3)
|
||||
path = tmp_path / "typed-frames.npy"
|
||||
np.save(path, expected)
|
||||
|
||||
reader = MultiThreadedFileReader(path, n_threads=2, chunk_size=3)
|
||||
actual = reader.read()
|
||||
|
||||
assert actual.dtype == expected.dtype
|
||||
assert reader.dtype == expected.dtype
|
||||
assert np.array_equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("n_threads", "chunk_size", "total_frames"),
|
||||
[(0, 1, None), (1, 0, None), (1, 1, 11)],
|
||||
)
|
||||
def test_invalid_configuration(
|
||||
frame_file, n_threads, chunk_size, total_frames
|
||||
):
|
||||
path, _ = frame_file
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
MultiThreadedFileReader(
|
||||
path,
|
||||
n_threads=n_threads,
|
||||
chunk_size=chunk_size,
|
||||
total_frames=total_frames,
|
||||
)
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from aare import PixelHistogram
|
||||
|
||||
|
||||
def _random_frames(rows, cols, n, xmin, xmax, seed=0):
|
||||
rng = np.random.default_rng(seed)
|
||||
return [rng.uniform(xmin - 0.25, xmax + 0.25, size=(rows, cols)).astype(np.float64)
|
||||
for _ in range(n)]
|
||||
|
||||
|
||||
def _reference_hdata(frames, rows, cols, n_bins, xmin, xmax):
|
||||
expected = np.zeros((rows, cols, n_bins), dtype=np.uint16)
|
||||
inv_range = n_bins / (xmax - xmin)
|
||||
for img in frames:
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
v = float(img[r, c])
|
||||
if not (xmin <= v < xmax):
|
||||
continue
|
||||
b = int((v - xmin) * inv_range)
|
||||
if b >= n_bins:
|
||||
b = n_bins - 1
|
||||
expected[r, c, b] += 1
|
||||
return expected
|
||||
|
||||
|
||||
def test_async_fill_matches_reference():
|
||||
rows, cols, n_bins = 5, 7, 8
|
||||
xmin, xmax = 0.0, 2.0
|
||||
frames = _random_frames(rows, cols, n=3, xmin=xmin, xmax=xmax, seed=1)
|
||||
|
||||
hist = PixelHistogram(rows=rows, cols=cols, n_bins=n_bins, xmin=xmin, xmax=xmax)
|
||||
for img in frames:
|
||||
hist.fill_async(img)
|
||||
|
||||
np.testing.assert_array_equal(
|
||||
hist.values(),
|
||||
_reference_hdata(frames, rows, cols, n_bins, xmin, xmax),
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
def test_fill_async_copies_buffer():
|
||||
# After fill_async returns, the caller should be free to mutate the
|
||||
# numpy array without affecting the pending fill.
|
||||
rows, cols, n_bins = 4, 4, 4
|
||||
xmin, xmax = 0.0, 1.0
|
||||
hist = PixelHistogram(rows=rows, cols=cols, n_bins=n_bins, xmin=xmin, xmax=xmax, n_threads=1, max_pending=8)
|
||||
|
||||
img = np.full((rows, cols), 0.1, dtype=np.float64) # falls in bin 0
|
||||
hist.fill_async(img)
|
||||
# Mutate the original array immediately; this must not affect the
|
||||
# value that was already enqueued.
|
||||
img[:] = 0.9 # would be bin 3
|
||||
hist.flush()
|
||||
|
||||
h = hist.values()
|
||||
assert h.shape == (rows, cols, n_bins)
|
||||
# Every pixel saw one value in bin 0, none elsewhere.
|
||||
assert (h[:, :, 0] == 1).all()
|
||||
assert (h[:, :, 1:] == 0).all()
|
||||
|
||||
|
||||
def test_fill_async_rejects_wrong_shape():
|
||||
hist = PixelHistogram(8, 8, 4, 0.0, 1.0)
|
||||
bad = np.zeros((4, 4), dtype=np.float32)
|
||||
with pytest.raises(ValueError):
|
||||
hist.fill_async(bad)
|
||||
|
||||
|
||||
def test_hdata_flushes_pending():
|
||||
# Submit several frames with a tiny queue and read hdata() without an
|
||||
# explicit flush(); hdata() must drain everything first.
|
||||
rows, cols, n_bins = 3, 3, 4
|
||||
xmin, xmax = 0.0, 1.0
|
||||
hist = PixelHistogram(rows=rows, cols=cols, n_bins=n_bins, xmin=xmin, xmax=xmax,
|
||||
n_threads=1, max_pending=1)
|
||||
frames = _random_frames(rows, cols, n=8, xmin=xmin, xmax=xmax, seed=3)
|
||||
for img in frames:
|
||||
hist.fill_async(img)
|
||||
|
||||
h = hist.values() # no explicit flush()
|
||||
np.testing.assert_array_equal(
|
||||
h, _reference_hdata(frames, rows, cols, n_bins, xmin, xmax)
|
||||
)
|
||||
|
||||
|
||||
def test_bin_centers_and_edges():
|
||||
n_bins = 5
|
||||
xmin, xmax = 0.0, 1.0
|
||||
hist = PixelHistogram(rows=2, cols=2, n_bins=n_bins, xmin=xmin, xmax=xmax)
|
||||
edges = hist.bin_edges()
|
||||
centers = hist.bin_centers()
|
||||
assert edges.shape == (n_bins + 1,)
|
||||
assert centers.shape == (n_bins,)
|
||||
np.testing.assert_allclose(edges, np.linspace(xmin, xmax, n_bins + 1), atol=1e-6)
|
||||
np.testing.assert_allclose(centers, 0.5 * (edges[:-1] + edges[1:]), atol=1e-6)
|
||||
@@ -92,6 +92,12 @@ Dtype::Dtype(Dtype::TypeIndex ti) : m_type(ti) {}
|
||||
*/
|
||||
Dtype::Dtype(std::string_view sv) {
|
||||
|
||||
// NumPy uses '|' for data types whose byte order is not applicable,
|
||||
// notably one-byte integer types.
|
||||
if (!sv.empty() && sv.front() == '|') {
|
||||
sv.remove_prefix(1);
|
||||
}
|
||||
|
||||
// Check if the file is using our native endianess
|
||||
if (auto pos = sv.find_first_of("<>"); pos != std::string_view::npos) {
|
||||
const auto endianess = [](const char c) {
|
||||
|
||||
+4
-1
@@ -12,6 +12,9 @@ TEST_CASE("Construct from typeid") {
|
||||
}
|
||||
|
||||
TEST_CASE("Construct from string") {
|
||||
REQUIRE(Dtype("|i1") == typeid(int8_t));
|
||||
REQUIRE(Dtype("|u1") == typeid(uint8_t));
|
||||
|
||||
if (endian::native == endian::little) {
|
||||
REQUIRE(Dtype("<i1") == typeid(int8_t));
|
||||
REQUIRE(Dtype("<u1") == typeid(uint8_t));
|
||||
@@ -53,4 +56,4 @@ TEST_CASE("Construct from string with endianess") {
|
||||
|
||||
TEST_CASE("Convert to string") {
|
||||
REQUIRE(Dtype(typeid(int)).to_string() == "<i4");
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -73,10 +73,11 @@ size_t File::tell() const { return file_impl->tell(); }
|
||||
size_t File::rows() const { return file_impl->rows(); }
|
||||
size_t File::cols() const { return file_impl->cols(); }
|
||||
size_t File::bitdepth() const { return file_impl->bitdepth(); }
|
||||
Dtype File::dtype() const { return file_impl->dtype(); }
|
||||
size_t File::bytes_per_pixel() const {
|
||||
return file_impl->bitdepth() / bits_per_byte;
|
||||
}
|
||||
|
||||
DetectorType File::detector_type() const { return file_impl->detector_type(); }
|
||||
|
||||
} // namespace aare
|
||||
} // namespace aare
|
||||
@@ -0,0 +1,151 @@
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
#include "aare/MultiThreadedFileReader.hpp"
|
||||
#include "aare/File.hpp"
|
||||
#include "aare/utils/math_helpers.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <future>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
|
||||
namespace aare::experimental {
|
||||
namespace {
|
||||
|
||||
size_t checked_product(size_t lhs, size_t rhs) {
|
||||
if (lhs != 0 && rhs > std::numeric_limits<size_t>::max() / lhs) {
|
||||
throw std::overflow_error(
|
||||
"MultiThreadedFileReader buffer size overflow");
|
||||
}
|
||||
return lhs * rhs;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
MultiThreadedFileReader::MultiThreadedFileReader(
|
||||
std::filesystem::path fname, size_t n_threads, size_t chunk_size,
|
||||
std::optional<size_t> total_frames)
|
||||
: m_fname(std::move(fname)), m_n_threads(n_threads),
|
||||
m_chunk_size(chunk_size), m_total_frames(0), m_source_total_frames(0),
|
||||
m_rows(0), m_cols(0), m_bitdepth(0), m_dtype(Dtype::NONE),
|
||||
m_bytes_per_frame(0), m_total_bytes(0), m_current_frame(0) {
|
||||
if (m_n_threads == 0) {
|
||||
throw std::invalid_argument(
|
||||
"MultiThreadedFileReader requires at least one thread");
|
||||
}
|
||||
if (m_chunk_size == 0) {
|
||||
throw std::invalid_argument(
|
||||
"MultiThreadedFileReader chunk size must be greater than zero");
|
||||
}
|
||||
|
||||
File file(m_fname);
|
||||
m_source_total_frames = file.total_frames();
|
||||
m_total_frames = total_frames.value_or(m_source_total_frames);
|
||||
if (m_total_frames > m_source_total_frames) {
|
||||
throw std::invalid_argument(
|
||||
"Requested frame count exceeds the number of frames in the file");
|
||||
}
|
||||
|
||||
m_rows = file.rows();
|
||||
m_cols = file.cols();
|
||||
m_bitdepth = file.bitdepth();
|
||||
m_dtype = file.dtype();
|
||||
m_bytes_per_frame = file.bytes_per_frame();
|
||||
m_total_bytes = checked_product(m_total_frames, m_bytes_per_frame);
|
||||
|
||||
m_files.reserve(m_n_threads);
|
||||
m_files.push_back(std::move(file));
|
||||
for (size_t i = 1; i < m_n_threads; ++i) {
|
||||
m_files.emplace_back(m_fname);
|
||||
}
|
||||
}
|
||||
|
||||
size_t MultiThreadedFileReader::remaining_frames() const noexcept {
|
||||
return m_total_frames - m_current_frame;
|
||||
}
|
||||
|
||||
size_t MultiThreadedFileReader::next_read_frames() const noexcept {
|
||||
const size_t remaining = remaining_frames();
|
||||
if (remaining == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const size_t chunks_remaining = ceil_div(remaining, m_chunk_size);
|
||||
if (m_n_threads >= chunks_remaining) {
|
||||
return remaining;
|
||||
}
|
||||
// This multiplication is safe: in this branch n_threads * chunk_size is
|
||||
// strictly smaller than remaining.
|
||||
return m_n_threads * m_chunk_size;
|
||||
}
|
||||
|
||||
void MultiThreadedFileReader::ensure_open() const {
|
||||
if (!is_open()) {
|
||||
throw std::runtime_error("MultiThreadedFileReader is closed");
|
||||
}
|
||||
}
|
||||
|
||||
size_t MultiThreadedFileReader::read_into(std::byte *destination) {
|
||||
ensure_open();
|
||||
const size_t frames_to_read = next_read_frames();
|
||||
if (frames_to_read == 0) {
|
||||
return 0;
|
||||
}
|
||||
if (destination == nullptr) {
|
||||
throw std::invalid_argument(
|
||||
"MultiThreadedFileReader destination must not be null");
|
||||
}
|
||||
|
||||
const size_t first_frame = m_current_frame;
|
||||
const size_t active_threads = ceil_div(frames_to_read, m_chunk_size);
|
||||
auto worker = [&](size_t worker_index) {
|
||||
File &file = m_files[worker_index];
|
||||
const size_t batch_offset = worker_index * m_chunk_size;
|
||||
const size_t begin = first_frame + batch_offset;
|
||||
const size_t count =
|
||||
std::min(m_chunk_size, frames_to_read - batch_offset);
|
||||
file.seek(begin);
|
||||
file.read_into(destination + batch_offset * m_bytes_per_frame, count);
|
||||
};
|
||||
|
||||
std::vector<std::future<void>> workers;
|
||||
workers.reserve(active_threads);
|
||||
for (size_t i = 0; i < active_threads; ++i) {
|
||||
workers.emplace_back(std::async(std::launch::async, worker, i));
|
||||
}
|
||||
|
||||
for (auto &future : workers) {
|
||||
future.get();
|
||||
}
|
||||
m_current_frame += frames_to_read;
|
||||
return frames_to_read;
|
||||
}
|
||||
|
||||
std::vector<std::byte> MultiThreadedFileReader::read() {
|
||||
std::vector<std::byte> data(next_read_bytes());
|
||||
read_into(data.data());
|
||||
return data;
|
||||
}
|
||||
|
||||
std::vector<std::byte> MultiThreadedFileReader::read_all() {
|
||||
ensure_open();
|
||||
std::vector<std::byte> data(
|
||||
checked_product(remaining_frames(), m_bytes_per_frame));
|
||||
size_t offset = 0;
|
||||
while (remaining_frames() != 0) {
|
||||
const size_t frames_read = read_into(data.data() + offset);
|
||||
offset += frames_read * m_bytes_per_frame;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
void MultiThreadedFileReader::seek(size_t frame_index) {
|
||||
ensure_open();
|
||||
if (frame_index > m_total_frames) {
|
||||
throw std::out_of_range(
|
||||
"MultiThreadedFileReader frame index is out of range");
|
||||
}
|
||||
m_current_frame = frame_index;
|
||||
}
|
||||
|
||||
} // namespace aare::experimental
|
||||
@@ -0,0 +1,189 @@
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
#include "aare/MultiThreadedFileReader.hpp"
|
||||
|
||||
#include "aare/Dtype.hpp"
|
||||
#include "aare/File.hpp"
|
||||
#include "aare/FileInterface.hpp"
|
||||
#include "aare/Frame.hpp"
|
||||
#include "aare/NumpyFile.hpp"
|
||||
|
||||
#include "test_config.hpp"
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using aare::File;
|
||||
using aare::FileConfig;
|
||||
using aare::Frame;
|
||||
using aare::NumpyFile;
|
||||
using aare::experimental::MultiThreadedFileReader;
|
||||
|
||||
namespace {
|
||||
|
||||
class TemporaryNumpyFile {
|
||||
public:
|
||||
TemporaryNumpyFile() {
|
||||
const auto unique =
|
||||
std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
m_path = std::filesystem::temp_directory_path() /
|
||||
("aare-mt-reader-" + std::to_string(unique) + ".npy");
|
||||
|
||||
FileConfig cfg;
|
||||
cfg.dtype = aare::Dtype::UINT16;
|
||||
cfg.rows = 2;
|
||||
cfg.cols = 3;
|
||||
NumpyFile file(m_path, "w", cfg);
|
||||
for (uint16_t frame_index = 0; frame_index < 10; ++frame_index) {
|
||||
Frame frame(cfg.rows, cfg.cols, cfg.dtype);
|
||||
auto image = frame.view<uint16_t>();
|
||||
for (ssize_t row = 0; row < image.shape(0); ++row) {
|
||||
for (ssize_t col = 0; col < image.shape(1); ++col) {
|
||||
image(row, col) = static_cast<uint16_t>(frame_index * 100 +
|
||||
row * 10 + col);
|
||||
}
|
||||
}
|
||||
file.write(frame);
|
||||
}
|
||||
}
|
||||
|
||||
TemporaryNumpyFile(const TemporaryNumpyFile &) = delete;
|
||||
TemporaryNumpyFile &operator=(const TemporaryNumpyFile &) = delete;
|
||||
|
||||
~TemporaryNumpyFile() { std::filesystem::remove(m_path); }
|
||||
|
||||
const std::filesystem::path &path() const { return m_path; }
|
||||
void truncate() { std::filesystem::resize_file(m_path, 0); }
|
||||
|
||||
private:
|
||||
std::filesystem::path m_path;
|
||||
};
|
||||
|
||||
std::vector<std::byte> read_reference(const std::filesystem::path &fpath,
|
||||
size_t n_frames) {
|
||||
File file(fpath);
|
||||
std::vector<std::byte> data(n_frames * file.bytes_per_frame());
|
||||
if (n_frames != 0) {
|
||||
file.read_into(data.data(), n_frames);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("Multi-threaded reader preserves numpy frame order",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
const auto &fpath = file.path();
|
||||
|
||||
MultiThreadedFileReader reader(fpath, 2, 3);
|
||||
|
||||
CHECK(reader.n_threads() == 2);
|
||||
CHECK(reader.chunk_size() == 3);
|
||||
CHECK(reader.total_frames() == 10);
|
||||
CHECK(reader.source_total_frames() == 10);
|
||||
CHECK(reader.rows() == 2);
|
||||
CHECK(reader.cols() == 3);
|
||||
CHECK(reader.bitdepth() == 16);
|
||||
CHECK(reader.total_bytes() ==
|
||||
reader.total_frames() * reader.bytes_per_frame());
|
||||
|
||||
const auto reference = read_reference(fpath, reader.total_frames());
|
||||
auto first = reader.read();
|
||||
auto second = reader.read();
|
||||
|
||||
CHECK(first == std::vector<std::byte>(reference.begin(),
|
||||
reference.begin() + 6 * 12));
|
||||
CHECK(second ==
|
||||
std::vector<std::byte>(reference.begin() + 6 * 12, reference.end()));
|
||||
CHECK(reader.read().empty());
|
||||
CHECK(reader.tell() == 10);
|
||||
CHECK(reader.remaining_frames() == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("Multi-threaded reader handles uneven raw chunks and frame limits",
|
||||
"[.with-data][MultiThreadedFileReader]") {
|
||||
const auto fpath =
|
||||
test_data_path() / "raw/jungfrau/jungfrau_single_master_0.json";
|
||||
REQUIRE(std::filesystem::exists(fpath));
|
||||
|
||||
MultiThreadedFileReader reader(fpath, 2, 3, 9);
|
||||
|
||||
CHECK(reader.total_frames() == 9);
|
||||
CHECK(reader.source_total_frames() == 10);
|
||||
CHECK(reader.read_all() == read_reference(fpath, 9));
|
||||
}
|
||||
|
||||
TEST_CASE("Multi-threaded reader can seek and reread",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
|
||||
MultiThreadedFileReader reader(file.path(), 2, 1);
|
||||
const auto first = reader.read();
|
||||
CHECK(reader.tell() == 2);
|
||||
CHECK(reader.next_read_frames() == 2);
|
||||
|
||||
reader.seek(0);
|
||||
CHECK(reader.tell() == 0);
|
||||
CHECK(reader.read() == first);
|
||||
CHECK_THROWS_AS(reader.seek(11), std::out_of_range);
|
||||
}
|
||||
|
||||
TEST_CASE("Multi-threaded reader validates its configuration",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
const auto &fpath = file.path();
|
||||
|
||||
CHECK_THROWS_AS(MultiThreadedFileReader(fpath, 0, 1),
|
||||
std::invalid_argument);
|
||||
CHECK_THROWS_AS(MultiThreadedFileReader(fpath, 1, 0),
|
||||
std::invalid_argument);
|
||||
CHECK_THROWS_AS(MultiThreadedFileReader(fpath, 1, 1, 11),
|
||||
std::invalid_argument);
|
||||
|
||||
MultiThreadedFileReader reader(fpath, 2, 2);
|
||||
CHECK_THROWS_AS(reader.read_into(nullptr), std::invalid_argument);
|
||||
}
|
||||
|
||||
TEST_CASE("An explicit zero frame limit produces an empty read",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
|
||||
MultiThreadedFileReader reader(file.path(), 8, 3, 0);
|
||||
CHECK(reader.total_frames() == 0);
|
||||
CHECK(reader.total_bytes() == 0);
|
||||
CHECK(reader.read().empty());
|
||||
CHECK_NOTHROW(reader.read_into(nullptr));
|
||||
}
|
||||
|
||||
TEST_CASE("read_into reads at most one chunk per worker",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
MultiThreadedFileReader reader(file.path(), 3, 2);
|
||||
std::vector<std::byte> data(reader.next_read_bytes());
|
||||
|
||||
CHECK(reader.next_read_frames() == 6);
|
||||
CHECK(reader.read_into(data.data()) == 6);
|
||||
CHECK(reader.tell() == 6);
|
||||
CHECK(reader.next_read_frames() == 4);
|
||||
CHECK(reader.next_read_bytes() == 4 * reader.bytes_per_frame());
|
||||
}
|
||||
|
||||
TEST_CASE("Multi-threaded reader can close its worker files",
|
||||
"[MultiThreadedFileReader]") {
|
||||
TemporaryNumpyFile file;
|
||||
MultiThreadedFileReader reader(file.path(), 2, 2);
|
||||
|
||||
CHECK(reader.is_open());
|
||||
reader.close();
|
||||
CHECK_FALSE(reader.is_open());
|
||||
CHECK_NOTHROW(reader.close());
|
||||
CHECK_THROWS_AS(reader.read(), std::runtime_error);
|
||||
CHECK_THROWS_AS(reader.read_all(), std::runtime_error);
|
||||
CHECK_THROWS_AS(reader.seek(0), std::runtime_error);
|
||||
}
|
||||
@@ -34,6 +34,9 @@ TEST_CASE("trim whitespace") {
|
||||
}
|
||||
|
||||
TEST_CASE("parse data type descriptions") {
|
||||
REQUIRE(parse_descr("|i1") == aare::Dtype::INT8);
|
||||
REQUIRE(parse_descr("|u1") == aare::Dtype::UINT8);
|
||||
|
||||
REQUIRE(parse_descr("<i1") == aare::Dtype::INT8);
|
||||
REQUIRE(parse_descr("<i2") == aare::Dtype::INT16);
|
||||
REQUIRE(parse_descr("<i4") == aare::Dtype::INT32);
|
||||
@@ -63,4 +66,4 @@ TEST_CASE("Parse numpy dict") {
|
||||
REQUIRE(map["descr"] == "'<f4'");
|
||||
REQUIRE(map["fortran_order"] == "False");
|
||||
REQUIRE(map["shape"] == "(3, 4)");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user