diff --git a/CMakeLists.txt b/CMakeLists.txt index d6a22d7..6338f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -368,6 +368,7 @@ set(PUBLICHEADERS set(SourceFiles + ${CMAKE_CURRENT_SOURCE_DIR}/src/calibration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/CtbRawFile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.cpp @@ -437,6 +438,7 @@ endif() if(AARE_TESTS) set(TestSources ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/calibration.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/defs.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp diff --git a/RELEASE.md b/RELEASE.md index 7537212..951632a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,22 @@ # Release notes -### head +### 2025.8.22 + +Features: + +- Apply calibration works in G0 if passes a 2D calibration and pedestal +- count pixels that switch +- calculate pedestal (also g0 version) +- NDArray::view() needs an lvalue to reduce issues with the view outliving the array + + +Bugfixes: + +- Now using glibc 2.17 in conda builds (was using the host) +- Fixed shifted pixels in clusters close to the edge of a frame + +### 2025.7.18 Features: @@ -15,7 +30,7 @@ Bugfixes: - Removed unused file: ClusterFile.cpp -### 2025.05.22 +### 2025.5.22 Features: diff --git a/VERSION b/VERSION index ae365e4..3469ebe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2025.5.22 \ No newline at end of file +2025.8.22 \ No newline at end of file diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 6d3d479..543bf00 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -3,3 +3,14 @@ python: - 3.12 - 3.13 +c_compiler: + - gcc # [linux] + +c_stdlib: + - sysroot # [linux] + +cxx_compiler: + - gxx # [linux] + +c_stdlib_version: # [linux] + - 2.17 # [linux] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 8fea745..7aabea1 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -16,6 +16,8 @@ build: requirements: build: + - {{ compiler('c') }} + - {{ stdlib("c") }} - {{ compiler('cxx') }} - cmake - ninja diff --git a/docs/src/pycalibration.rst b/docs/src/pycalibration.rst index 0ea3fba..6083162 100644 --- a/docs/src/pycalibration.rst +++ b/docs/src/pycalibration.rst @@ -17,8 +17,24 @@ Functions for applying calibration to data. # Apply calibration to raw data to convert from raw ADC values to keV data = aare.apply_calibration(raw_data, pd=pedestal, cal=calibration) + # If you pass a 2D pedestal and calibration only G0 will be used for the conversion + # Pixels that switched to G1 or G2 will be set to 0 + data = aare.apply_calibration(raw_data, pd=pedestal[0], cal=calibration[0]) + + + .. py:currentmodule:: aare .. autofunction:: apply_calibration .. autofunction:: load_calibration + +.. autofunction:: calculate_pedestal + +.. autofunction:: calculate_pedestal_float + +.. autofunction:: calculate_pedestal_g0 + +.. autofunction:: calculate_pedestal_g0_float + +.. autofunction:: count_switching_pixels diff --git a/include/aare/ClusterFinder.hpp b/include/aare/ClusterFinder.hpp index 069d887..4da3e44 100644 --- a/include/aare/ClusterFinder.hpp +++ b/include/aare/ClusterFinder.hpp @@ -144,9 +144,9 @@ class ClusterFinder { static_cast( m_pedestal.mean(iy + ir, ix + ic)); cluster.data[i] = - tmp; // Watch for out of bounds access - i++; + tmp; // Watch for out of bounds access } + i++; } } diff --git a/include/aare/Frame.hpp b/include/aare/Frame.hpp index 27a2a4a..16e9f44 100644 --- a/include/aare/Frame.hpp +++ b/include/aare/Frame.hpp @@ -105,7 +105,7 @@ class Frame { * @tparam T type of the pixels * @return NDView */ - template NDView view() { + template NDView view() & { std::array shape = {static_cast(m_rows), static_cast(m_cols)}; T *data = reinterpret_cast(m_data); diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index 1a501eb..731ba2f 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -25,7 +25,7 @@ template class NDArray : public ArrayExpr, Ndim> { std::array shape_; std::array strides_; - size_t size_{}; + size_t size_{}; //TODO! do we need to store size when we have shape? T *data_; public: @@ -33,7 +33,7 @@ class NDArray : public ArrayExpr, Ndim> { * @brief Default constructor. Will construct an empty NDArray. * */ - NDArray() : shape_(), strides_(c_strides(shape_)), data_(nullptr){}; + NDArray() : shape_(), strides_(c_strides(shape_)), data_(nullptr) {}; /** * @brief Construct a new NDArray object with a given shape. @@ -43,8 +43,7 @@ class NDArray : public ArrayExpr, Ndim> { */ explicit NDArray(std::array shape) : shape_(shape), strides_(c_strides(shape_)), - size_(std::accumulate(shape_.begin(), shape_.end(), 1, - std::multiplies<>())), + size_(num_elements(shape_)), data_(new T[size_]) {} /** @@ -79,6 +78,24 @@ class NDArray : public ArrayExpr, Ndim> { other.reset(); // TODO! is this necessary? } + + //Move constructor from an an array with Ndim + 1 + template > + NDArray(NDArray &&other) + : shape_(drop_first_dim(other.shape())), + strides_(c_strides(shape_)), size_(num_elements(shape_)), + data_(other.data()) { + + // For now only allow move if the size matches, to avoid unreachable data + // if the use case arises we can remove this check + if(size() != other.size()) { + data_ = nullptr; // avoid double free, other will clean up the memory in it's destructor + throw std::runtime_error(LOCATION + + "Size mismatch in move constructor of NDArray"); + } + other.reset(); + } + // Copy constructor NDArray(const NDArray &other) : shape_(other.shape_), strides_(c_strides(shape_)), @@ -380,12 +397,6 @@ NDArray NDArray::operator*(const T &value) { result *= value; return result; } -// template void NDArray::Print() { -// if (shape_[0] < 20 && shape_[1] < 20) -// Print_all(); -// else -// Print_some(); -// } template std::ostream &operator<<(std::ostream &os, const NDArray &arr) { @@ -437,4 +448,23 @@ NDArray load(const std::string &pathname, return img; } +template +NDArray safe_divide(const NDArray &numerator, + const NDArray &denominator) { + if (numerator.shape() != denominator.shape()) { + throw std::runtime_error( + "Shapes of numerator and denominator must match"); + } + NDArray result(numerator.shape()); + for (ssize_t i = 0; i < numerator.size(); ++i) { + if (denominator[i] != 0) { + result[i] = + static_cast(numerator[i]) / static_cast(denominator[i]); + } else { + result[i] = RT{0}; // or handle division by zero as needed + } + } + return result; +} + } // namespace aare \ No newline at end of file diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index 28f5371..0aa4f78 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -26,6 +26,33 @@ Shape make_shape(const std::vector &shape) { return arr; } + +/** + * @brief Helper function to drop the first dimension of a shape. + * This is useful when you want to create a 2D view from a 3D array. + * @param shape The shape to drop the first dimension from. + * @return A new shape with the first dimension dropped. + */ +template +Shape drop_first_dim(const Shape &shape) { + static_assert(Ndim > 1, "Cannot drop first dimension from a 1D shape"); + Shape new_shape; + std::copy(shape.begin() + 1, shape.end(), new_shape.begin()); + return new_shape; +} + +/** + * @brief Helper function when constructing NDArray/NDView. Calculates the number + * of elements in the resulting array from a shape. + * @param shape The shape to calculate the number of elements for. + * @return The number of elements in and NDArray/NDView of that shape. + */ +template +size_t num_elements(const Shape &shape) { + return std::accumulate(shape.begin(), shape.end(), 1, + std::multiplies()); +} + template ssize_t element_offset(const Strides & /*unused*/) { return 0; @@ -66,17 +93,28 @@ class NDView : public ArrayExpr, Ndim> { : buffer_(buffer), strides_(c_strides(shape)), shape_(shape), size_(std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>())) {} - + template std::enable_if_t operator()(Ix... index) { return buffer_[element_offset(strides_, index...)]; } template - const std::enable_if_t operator()(Ix... index) const { + std::enable_if_t 1), NDView> operator()(Ix... index) { + // return a view of the next dimension + std::array new_shape{}; + std::copy_n(shape_.begin() + 1, Ndim - 1, new_shape.begin()); + return NDView(&buffer_[element_offset(strides_, index...)], + new_shape); + + } + + template + std::enable_if_t operator()(Ix... index) const { return buffer_[element_offset(strides_, index...)]; } + ssize_t size() const { return static_cast(size_); } size_t total_bytes() const { return size_ * sizeof(T); } std::array strides() const noexcept { return strides_; } @@ -85,9 +123,19 @@ class NDView : public ArrayExpr, Ndim> { T *end() { return buffer_ + size_; } T const *begin() const { return buffer_; } T const *end() const { return buffer_ + size_; } - T &operator()(ssize_t i) { return buffer_[i]; } + + + + + + /** + * @brief Access element at index i. + */ T &operator[](ssize_t i) { return buffer_[i]; } - const T &operator()(ssize_t i) const { return buffer_[i]; } + + /** + * @brief Access element at index i. + */ const T &operator[](ssize_t i) const { return buffer_[i]; } bool operator==(const NDView &other) const { @@ -157,6 +205,22 @@ class NDView : public ArrayExpr, Ndim> { const T *data() const { return buffer_; } void print_all() const; + /** + * @brief Create a subview of a range of the first dimension. + * This is useful for splitting a batches of frames in parallel processing. + * @param first The first index of the subview (inclusive). + * @param last The last index of the subview (exclusive). + * @return A new NDView that is a subview of the current view. + * @throws std::runtime_error if the range is invalid. + */ + NDView sub_view(ssize_t first, ssize_t last) const { + if (first < 0 || last > shape_[0] || first >= last) + throw std::runtime_error(LOCATION + "Invalid sub_view range"); + auto new_shape = shape_; + new_shape[0] = last - first; + return NDView(buffer_ + first * strides_[0], new_shape); + } + private: T *buffer_{nullptr}; std::array strides_{}; diff --git a/include/aare/VarClusterFinder.hpp b/include/aare/VarClusterFinder.hpp index 3679d39..a002f9b 100644 --- a/include/aare/VarClusterFinder.hpp +++ b/include/aare/VarClusterFinder.hpp @@ -240,14 +240,14 @@ template void VarClusterFinder::first_pass() { for (ssize_t i = 0; i < original_.size(); ++i) { if (use_noise_map) - threshold_ = 5 * noiseMap(i); - binary_(i) = (original_(i) > threshold_); + threshold_ = 5 * noiseMap[i]; + binary_[i] = (original_[i] > threshold_); } for (int i = 0; i < shape_[0]; ++i) { for (int j = 0; j < shape_[1]; ++j) { - // do we have someting to process? + // do we have something to process? if (binary_(i, j)) { auto tmp = check_neighbours(i, j); if (tmp != 0) { diff --git a/include/aare/calibration.hpp b/include/aare/calibration.hpp index aab1f89..3b18da6 100644 --- a/include/aare/calibration.hpp +++ b/include/aare/calibration.hpp @@ -1,6 +1,9 @@ #pragma once +#include "aare/NDArray.hpp" +#include "aare/NDView.hpp" #include "aare/defs.hpp" +#include "aare/utils/par.hpp" #include "aare/utils/task.hpp" #include #include @@ -55,32 +58,152 @@ ALWAYS_INLINE std::pair get_value_and_gain(uint16_t raw) { template void apply_calibration_impl(NDView res, NDView raw_data, - NDView ped, NDView cal, int start, - int stop) { + NDView ped, NDView cal, int start, + int stop) { for (int frame_nr = start; frame_nr != stop; ++frame_nr) { for (int row = 0; row != raw_data.shape(1); ++row) { for (int col = 0; col != raw_data.shape(2); ++col) { - auto [value, gain] = get_value_and_gain(raw_data(frame_nr, row, col)); + auto [value, gain] = + get_value_and_gain(raw_data(frame_nr, row, col)); + + // Using multiplication does not seem to speed up the code here + // ADU/keV is the standard unit for the calibration which + // means rewriting the formula is not worth it. res(frame_nr, row, col) = - (value - ped(gain, row, col)) / cal(gain, row, col); //TODO! use multiplication + (value - ped(gain, row, col)) / cal(gain, row, col); } } } } template +void apply_calibration_impl(NDView res, NDView raw_data, + NDView ped, NDView cal, int start, + int stop) { + + for (int frame_nr = start; frame_nr != stop; ++frame_nr) { + for (int row = 0; row != raw_data.shape(1); ++row) { + for (int col = 0; col != raw_data.shape(2); ++col) { + auto [value, gain] = + get_value_and_gain(raw_data(frame_nr, row, col)); + + // Using multiplication does not seem to speed up the code here + // ADU/keV is the standard unit for the calibration which + // means rewriting the formula is not worth it. + + // Set the value to 0 if the gain is not 0 + if (gain == 0) + res(frame_nr, row, col) = + (value - ped(row, col)) / cal(row, col); + else + res(frame_nr, row, col) = 0; + } + } + } +} + +template void apply_calibration(NDView res, NDView raw_data, - NDView ped, NDView cal, + NDView ped, NDView cal, ssize_t n_threads = 4) { std::vector> futures; futures.reserve(n_threads); auto limits = split_task(0, raw_data.shape(0), n_threads); for (const auto &lim : limits) - futures.push_back(std::async(&apply_calibration_impl, res, raw_data, ped, cal, - lim.first, lim.second)); + futures.push_back(std::async( + static_cast, NDView, + NDView, NDView, int, int)>( + apply_calibration_impl), + res, raw_data, ped, cal, lim.first, lim.second)); for (auto &f : futures) f.get(); } +template +std::pair, NDArray> +sum_and_count_per_gain(NDView raw_data) { + constexpr ssize_t num_gains = only_gain0 ? 1 : 3; + NDArray accumulator( + std::array{num_gains, raw_data.shape(1), raw_data.shape(2)}, + 0); + NDArray count( + std::array{num_gains, raw_data.shape(1), raw_data.shape(2)}, + 0); + for (int frame_nr = 0; frame_nr != raw_data.shape(0); ++frame_nr) { + for (int row = 0; row != raw_data.shape(1); ++row) { + for (int col = 0; col != raw_data.shape(2); ++col) { + auto [value, gain] = + get_value_and_gain(raw_data(frame_nr, row, col)); + if (gain != 0 && only_gain0) + continue; + accumulator(gain, row, col) += value; + count(gain, row, col) += 1; + } + } + } + + return {std::move(accumulator), std::move(count)}; +} + +template +NDArray(only_gain0)> +calculate_pedestal(NDView raw_data, ssize_t n_threads) { + + constexpr ssize_t num_gains = only_gain0 ? 1 : 3; + std::vector, NDArray>>> + futures; + futures.reserve(n_threads); + + auto subviews = make_subviews(raw_data, n_threads); + + for (auto view : subviews) { + futures.push_back(std::async( + static_cast, NDArray> (*)( + NDView)>(&sum_and_count_per_gain), + view)); + } + Shape<3> shape{num_gains, raw_data.shape(1), raw_data.shape(2)}; + NDArray accumulator(shape, 0); + NDArray count(shape, 0); + + // Combine the results from the futures + for (auto &f : futures) { + auto [acc, cnt] = f.get(); + accumulator += acc; + count += cnt; + } + + + // Will move to a NDArray(only_gain0)> + // if only_gain0 is true + return safe_divide(accumulator, count); + +} + +/** + * @brief Count the number of switching pixels in the raw data. + * This function counts the number of pixels that switch between G1 and G2 gain. + * It returns an NDArray with the number of switching pixels per pixel. + * @param raw_data The NDView containing the raw data + * @return An NDArray with the number of switching pixels per pixel + */ +NDArray count_switching_pixels(NDView raw_data); + +/** + * @brief Count the number of switching pixels in the raw data. + * This function counts the number of pixels that switch between G1 and G2 gain. + * It returns an NDArray with the number of switching pixels per pixel. + * @param raw_data The NDView containing the raw data + * @param n_threads The number of threads to use for parallel processing + * @return An NDArray with the number of switching pixels per pixel + */ +NDArray count_switching_pixels(NDView raw_data, + ssize_t n_threads); + +template +auto calculate_pedestal_g0(NDView raw_data, ssize_t n_threads) { + return calculate_pedestal(raw_data, n_threads); +} + } // namespace aare \ No newline at end of file diff --git a/include/aare/utils/par.hpp b/include/aare/utils/par.hpp index e52c897..55f60f4 100644 --- a/include/aare/utils/par.hpp +++ b/include/aare/utils/par.hpp @@ -1,7 +1,10 @@ +#pragma once #include #include #include +#include "aare/utils/task.hpp" + namespace aare { template @@ -15,4 +18,17 @@ void RunInParallel(F func, const std::vector> &tasks) { thread.join(); } } + + +template +std::vector> make_subviews(NDView &data, ssize_t n_threads) { + std::vector> subviews; + subviews.reserve(n_threads); + auto limits = split_task(0, data.shape(0), n_threads); + for (const auto &lim : limits) { + subviews.push_back(data.sub_view(lim.first, lim.second)); + } + return subviews; +} + } // namespace aare \ No newline at end of file diff --git a/include/aare/utils/task.hpp b/include/aare/utils/task.hpp index a6ee142..63fe691 100644 --- a/include/aare/utils/task.hpp +++ b/include/aare/utils/task.hpp @@ -1,4 +1,4 @@ - +#pragma once #include #include diff --git a/python/aare/ClusterFinder.py b/python/aare/ClusterFinder.py index 251d938..cf5f4dd 100644 --- a/python/aare/ClusterFinder.py +++ b/python/aare/ClusterFinder.py @@ -46,14 +46,13 @@ def ClusterFinderMT(image_size, cluster_size = (3,3), dtype=np.int32, n_sigma=5, return cls(image_size, n_sigma=n_sigma, capacity=capacity, n_threads=n_threads) - -def ClusterCollector(clusterfindermt, cluster_size = (3,3), dtype=np.int32): +def ClusterCollector(clusterfindermt, dtype=np.int32): """ Factory function to create a ClusterCollector object. Provides a cleaner syntax for the templated ClusterCollector in C++. """ - - cls = _get_class("ClusterCollector", cluster_size, dtype) + + cls = _get_class("ClusterCollector", clusterfindermt.cluster_size, dtype) return cls(clusterfindermt) def ClusterFileSink(clusterfindermt, cluster_file, dtype=np.int32): diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 7a17bd9..c3b1c5a 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -32,6 +32,7 @@ from .utils import random_pixels, random_pixel, flat_list, add_colorbar from .func import * from .calibration import * -from ._aare import apply_calibration +from ._aare import apply_calibration, count_switching_pixels +from ._aare import calculate_pedestal, calculate_pedestal_float, calculate_pedestal_g0, calculate_pedestal_g0_float from ._aare import VarClusterFinder diff --git a/python/src/bind_calibration.hpp b/python/src/bind_calibration.hpp index df4b207..836a6de 100644 --- a/python/src/bind_calibration.hpp +++ b/python/src/bind_calibration.hpp @@ -17,27 +17,137 @@ py::array_t pybind_apply_calibration( calibration, int n_threads = 4) { - auto data_span = make_view_3d(data); - auto ped = make_view_3d(pedestal); - auto cal = make_view_3d(calibration); - + auto data_span = make_view_3d(data); // data is always 3D /* No pointer is passed, so NumPy will allocate the buffer */ auto result = py::array_t(data_span.shape()); auto res = make_view_3d(result); - - aare::apply_calibration(res, data_span, ped, cal, n_threads); - + if (data.ndim() == 3 && pedestal.ndim() == 3 && calibration.ndim() == 3) { + auto ped = make_view_3d(pedestal); + auto cal = make_view_3d(calibration); + aare::apply_calibration(res, data_span, ped, cal, + n_threads); + } else if (data.ndim() == 3 && pedestal.ndim() == 2 && + calibration.ndim() == 2) { + auto ped = make_view_2d(pedestal); + auto cal = make_view_2d(calibration); + aare::apply_calibration(res, data_span, ped, cal, + n_threads); + } else { + throw std::runtime_error( + "Invalid number of dimensions for data, pedestal or calibration"); + } return result; } +py::array_t pybind_count_switching_pixels( + py::array_t data, + ssize_t n_threads = 4) { + + auto data_span = make_view_3d(data); + auto arr = new NDArray{}; + *arr = aare::count_switching_pixels(data_span, n_threads); + return return_image_data(arr); +} + +template +py::array_t pybind_calculate_pedestal( + py::array_t data, + ssize_t n_threads) { + + auto data_span = make_view_3d(data); + auto arr = new NDArray{}; + *arr = aare::calculate_pedestal(data_span, n_threads); + return return_image_data(arr); +} + +template +py::array_t pybind_calculate_pedestal_g0( + py::array_t data, + ssize_t n_threads) { + + auto data_span = make_view_3d(data); + auto arr = new NDArray{}; + *arr = aare::calculate_pedestal(data_span, n_threads); + return return_image_data(arr); +} + void bind_calibration(py::module &m) { + m.def("apply_calibration", &pybind_apply_calibration, + py::arg("raw_data").noconvert(), py::kw_only(), + py::arg("pd").noconvert(), py::arg("cal").noconvert(), + py::arg("n_threads") = 4); + m.def("apply_calibration", &pybind_apply_calibration, py::arg("raw_data").noconvert(), py::kw_only(), py::arg("pd").noconvert(), py::arg("cal").noconvert(), py::arg("n_threads") = 4); - m.def("apply_calibration", &pybind_apply_calibration, + m.def("count_switching_pixels", &pybind_count_switching_pixels, + R"( + Count the number of time each pixel switches to G1 or G2. + + Parameters + ---------- + raw_data : array_like + 3D array of shape (frames, rows, cols) to count the switching pixels from. + n_threads : int + The number of threads to use for the calculation. + )", py::arg("raw_data").noconvert(), py::kw_only(), - py::arg("pd").noconvert(), py::arg("cal").noconvert(), py::arg("n_threads") = 4); + + m.def("calculate_pedestal", &pybind_calculate_pedestal, + R"( + Calculate the pedestal for all three gains and return the result as a 3D array of doubles. + + Parameters + ---------- + raw_data : array_like + 3D array of shape (frames, rows, cols) to calculate the pedestal from. + Needs to contain data for all three gains (G0, G1, G2). + n_threads : int + The number of threads to use for the calculation. + )", + py::arg("raw_data").noconvert(), py::arg("n_threads") = 4); + + m.def("calculate_pedestal_float", &pybind_calculate_pedestal, + R"( + Same as `calculate_pedestal` but returns a 3D array of floats. + + Parameters + ---------- + raw_data : array_like + 3D array of shape (frames, rows, cols) to calculate the pedestal from. + Needs to contain data for all three gains (G0, G1, G2). + n_threads : int + The number of threads to use for the calculation. + )", + py::arg("raw_data").noconvert(), py::arg("n_threads") = 4); + + m.def("calculate_pedestal_g0", &pybind_calculate_pedestal_g0, + R"( + Calculate the pedestal for G0 and return the result as a 2D array of doubles. + Pixels in G1 and G2 are ignored. + + Parameters + ---------- + raw_data : array_like + 3D array of shape (frames, rows, cols) to calculate the pedestal from. + n_threads : int + The number of threads to use for the calculation. + )", + py::arg("raw_data").noconvert(), py::arg("n_threads") = 4); + + m.def("calculate_pedestal_g0_float", &pybind_calculate_pedestal_g0, + R"( + Same as `calculate_pedestal_g0` but returns a 2D array of floats. + + Parameters + ---------- + raw_data : array_like + 3D array of shape (frames, rows, cols) to calculate the pedestal from. + n_threads : int + The number of threads to use for the calculation. + )", + py::arg("raw_data").noconvert(), py::arg("n_threads") = 4); } \ No newline at end of file diff --git a/python/tests/test_calibration.py b/python/tests/test_calibration.py index cce4344..bbd980c 100644 --- a/python/tests/test_calibration.py +++ b/python/tests/test_calibration.py @@ -1,6 +1,7 @@ import pytest import numpy as np -from aare import apply_calibration + +import aare def test_apply_calibration_small_data(): # The raw data consists of 10 4x5 images @@ -27,7 +28,7 @@ def test_apply_calibration_small_data(): - data = apply_calibration(raw, pd = pedestal, cal = calibration) + data = aare.apply_calibration(raw, pd = pedestal, cal = calibration) # The formula that is applied is: @@ -41,3 +42,94 @@ def test_apply_calibration_small_data(): assert data[2,2,2] == 0 assert data[0,1,1] == 0 assert data[1,3,0] == 0 + + +@pytest.fixture +def raw_data_3x2x2(): + raw = np.zeros((3, 2, 2), dtype=np.uint16) + raw[0, 0, 0] = 100 + raw[1,0, 0] = 200 + raw[2, 0, 0] = 300 + + raw[0, 0, 1] = (1<<14) + 100 + raw[1, 0, 1] = (1<<14) + 200 + raw[2, 0, 1] = (1<<14) + 300 + + raw[0, 1, 0] = (1<<14) + 37 + raw[1, 1, 0] = 38 + raw[2, 1, 0] = (3<<14) + 39 + + raw[0, 1, 1] = (3<<14) + 100 + raw[1, 1, 1] = (3<<14) + 200 + raw[2, 1, 1] = (3<<14) + 300 + return raw + +def test_calculate_pedestal(raw_data_3x2x2): + # Calculate the pedestal + pd = aare.calculate_pedestal(raw_data_3x2x2) + assert pd.shape == (3, 2, 2) + assert pd.dtype == np.float64 + assert pd[0, 0, 0] == 200 + assert pd[1, 0, 0] == 0 + assert pd[2, 0, 0] == 0 + + assert pd[0, 0, 1] == 0 + assert pd[1, 0, 1] == 200 + assert pd[2, 0, 1] == 0 + + assert pd[0, 1, 0] == 38 + assert pd[1, 1, 0] == 37 + assert pd[2, 1, 0] == 39 + + assert pd[0, 1, 1] == 0 + assert pd[1, 1, 1] == 0 + assert pd[2, 1, 1] == 200 + +def test_calculate_pedestal_float(raw_data_3x2x2): + #results should be the same for float + pd2 = aare.calculate_pedestal_float(raw_data_3x2x2) + assert pd2.shape == (3, 2, 2) + assert pd2.dtype == np.float32 + assert pd2[0, 0, 0] == 200 + assert pd2[1, 0, 0] == 0 + assert pd2[2, 0, 0] == 0 + + assert pd2[0, 0, 1] == 0 + assert pd2[1, 0, 1] == 200 + assert pd2[2, 0, 1] == 0 + + assert pd2[0, 1, 0] == 38 + assert pd2[1, 1, 0] == 37 + assert pd2[2, 1, 0] == 39 + + assert pd2[0, 1, 1] == 0 + assert pd2[1, 1, 1] == 0 + assert pd2[2, 1, 1] == 200 + +def test_calculate_pedestal_g0(raw_data_3x2x2): + pd = aare.calculate_pedestal_g0(raw_data_3x2x2) + assert pd.shape == (2, 2) + assert pd.dtype == np.float64 + assert pd[0, 0] == 200 + assert pd[1, 0] == 38 + assert pd[0, 1] == 0 + assert pd[1, 1] == 0 + +def test_calculate_pedestal_g0_float(raw_data_3x2x2): + pd = aare.calculate_pedestal_g0_float(raw_data_3x2x2) + assert pd.shape == (2, 2) + assert pd.dtype == np.float32 + assert pd[0, 0] == 200 + assert pd[1, 0] == 38 + assert pd[0, 1] == 0 + assert pd[1, 1] == 0 + +def test_count_switching_pixels(raw_data_3x2x2): + # Count the number of pixels that switched gain + count = aare.count_switching_pixels(raw_data_3x2x2) + assert count.shape == (2, 2) + assert count.sum() == 8 + assert count[0, 0] == 0 + assert count[1, 0] == 2 + assert count[0, 1] == 3 + assert count[1, 1] == 3 \ No newline at end of file diff --git a/src/ClusterFinderMT.test.cpp b/src/ClusterFinderMT.test.cpp index 7012705..0794687 100644 --- a/src/ClusterFinderMT.test.cpp +++ b/src/ClusterFinderMT.test.cpp @@ -57,6 +57,7 @@ class ClusterFinderMTWrapper size_t m_sink_size() const { return this->m_sink.sizeGuess(); } }; + TEST_CASE("multithreaded cluster finder", "[.with-data]") { auto fpath = test_data_path() / "raw/moench03/cu_half_speed_master_4.json"; @@ -81,7 +82,8 @@ TEST_CASE("multithreaded cluster finder", "[.with-data]") { CHECK(cf.m_input_queues_are_empty() == true); for (size_t i = 0; i < n_frames_pd; ++i) { - cf.find_clusters(file.read_frame().view()); + auto frame = file.read_frame(); + cf.find_clusters(frame.view()); } cf.stop(); diff --git a/src/NDArray.test.cpp b/src/NDArray.test.cpp index 91b5933..146426b 100644 --- a/src/NDArray.test.cpp +++ b/src/NDArray.test.cpp @@ -25,13 +25,13 @@ TEST_CASE("Construct from an NDView") { REQUIRE(image.data() != view.data()); for (uint32_t i = 0; i < image.size(); ++i) { - REQUIRE(image(i) == view(i)); + REQUIRE(image[i] == view[i]); } // Changing the image doesn't change the view image = 43; for (uint32_t i = 0; i < image.size(); ++i) { - REQUIRE(image(i) != view(i)); + REQUIRE(image[i] != view[i]); } } @@ -427,4 +427,30 @@ TEST_CASE("Construct an NDArray from an std::array") { for (uint32_t i = 0; i < a.size(); ++i) { REQUIRE(a(i) == b[i]); } -} \ No newline at end of file +} + + + +TEST_CASE("Move construct from an array with Ndim + 1") { + NDArray a({{1,2,2}}, 0); + a(0, 0, 0) = 1; + a(0, 0, 1) = 2; + a(0, 1, 0) = 3; + a(0, 1, 1) = 4; + + + NDArray b(std::move(a)); + REQUIRE(b.shape() == Shape<2>{2,2}); + REQUIRE(b.size() == 4); + REQUIRE(b(0, 0) == 1); + REQUIRE(b(0, 1) == 2); + REQUIRE(b(1, 0) == 3); + REQUIRE(b(1, 1) == 4); + +} + +TEST_CASE("Move construct from an array with Ndim + 1 throws on size mismatch") { + NDArray a({{2,2,2}}, 0); + REQUIRE_THROWS(NDArray(std::move(a))); +} + diff --git a/src/calibration.cpp b/src/calibration.cpp new file mode 100644 index 0000000..0cb3b99 --- /dev/null +++ b/src/calibration.cpp @@ -0,0 +1,44 @@ +#include "aare/calibration.hpp" + +namespace aare { + +NDArray count_switching_pixels(NDView raw_data) { + NDArray switched( + std::array{raw_data.shape(1), raw_data.shape(2)}, 0); + for (int frame_nr = 0; frame_nr != raw_data.shape(0); ++frame_nr) { + for (int row = 0; row != raw_data.shape(1); ++row) { + for (int col = 0; col != raw_data.shape(2); ++col) { + auto [value, gain] = + get_value_and_gain(raw_data(frame_nr, row, col)); + if (gain != 0) { + switched(row, col) += 1; + } + } + } + } + return switched; +} + +NDArray count_switching_pixels(NDView raw_data, + ssize_t n_threads) { + NDArray switched( + std::array{raw_data.shape(1), raw_data.shape(2)}, 0); + std::vector>> futures; + futures.reserve(n_threads); + + auto subviews = make_subviews(raw_data, n_threads); + + for (auto view : subviews) { + futures.push_back( + std::async(static_cast (*)(NDView)>( + &count_switching_pixels), + view)); + } + + for (auto &f : futures) { + switched += f.get(); + } + return switched; +} + +} // namespace aare \ No newline at end of file diff --git a/src/calibration.test.cpp b/src/calibration.test.cpp new file mode 100644 index 0000000..712dcd8 --- /dev/null +++ b/src/calibration.test.cpp @@ -0,0 +1,49 @@ +/************************************************ + * @file test-Cluster.cpp + * @short test case for generic Cluster, ClusterVector, and calculate_eta2 + ***********************************************/ + +#include "aare/calibration.hpp" + +// #include "catch.hpp" +#include +#include +#include + +using namespace aare; + +TEST_CASE("Test Pedestal Generation", "[.calibration]") { + NDArray raw(std::array{3, 2, 2}, 0); + + // gain 0 + raw(0, 0, 0) = 100; + raw(1, 0, 0) = 200; + raw(2, 0, 0) = 300; + + // gain 1 + raw(0, 0, 1) = (1 << 14) + 100; + raw(1, 0, 1) = (1 << 14) + 200; + raw(2, 0, 1) = (1 << 14) + 300; + + raw(0, 1, 0) = (1 << 14) + 37; + raw(1, 1, 0) = 38; + raw(2, 1, 0) = (3 << 14) + 39; + + // gain 2 + raw(0, 1, 1) = (3 << 14) + 100; + raw(1, 1, 1) = (3 << 14) + 200; + raw(2, 1, 1) = (3 << 14) + 300; + + auto pedestal = calculate_pedestal(raw.view(), 4); + + REQUIRE(pedestal.size() == raw.size()); + CHECK(pedestal(0, 0, 0) == 200); + CHECK(pedestal(1, 0, 0) == 0); + CHECK(pedestal(1, 0, 1) == 200); + + auto pedestal_gain0 = calculate_pedestal_g0(raw.view(), 4); + + REQUIRE(pedestal_gain0.size() == 4); + CHECK(pedestal_gain0(0, 0) == 200); + CHECK(pedestal_gain0(1, 0) == 38); +} \ No newline at end of file diff --git a/update_version.py b/update_version.py index 476895a..783eb9f 100644 --- a/update_version.py +++ b/update_version.py @@ -7,6 +7,7 @@ Script to update VERSION file with semantic versioning if provided as an argumen import sys import os import re +from datetime import datetime from packaging.version import Version, InvalidVersion @@ -26,9 +27,9 @@ def get_version(): # Check at least one argument is passed if len(sys.argv) < 2: - return "0.0.0" - - version = sys.argv[1] + version = datetime.today().strftime('%Y.%-m.%-d') + else: + version = sys.argv[1] try: v = Version(version) # normalize check if version follows PEP 440 specification @@ -54,4 +55,4 @@ def write_version_to_file(version): if __name__ == "__main__": version = get_version() - write_version_to_file(version) \ No newline at end of file + write_version_to_file(version)