Cleaned up Pedestal (#370)
Build on RHEL9 / build (push) Successful in 2m52s
Build on RHEL8 / build (push) Successful in 3m33s
Run tests using data on local RHEL8 / build (push) Successful in 4m31s
Build on local RHEL8 / build (push) Successful in 3m4s

- renamed pedestal.hpp to bind_Pedestal.hpp for python bindings
- variance is now always calculated in double and made private
- cost correctness in a few places
- removed push_fast and explicit updates of mean from Pedestal. If
performance is needed use FastPedestal
- Pedestal no longer caches std
This commit is contained in:
Erik Fröjdh
2026-09-14 16:27:53 +02:00
committed by GitHub
parent b0875106e3
commit 8551281eb7
18 changed files with 670 additions and 350 deletions
+1
View File
@@ -40,6 +40,7 @@ set(PYTHON_FILES
aare/ClusterVector.py
aare/Cluster.py
aare/FastPedestal.py
aare/Pedestal.py
aare/calibration.py
aare/factory.py
aare/experimental.py
+2 -2
View File
@@ -22,7 +22,7 @@ def FastPedestal(rows, cols, n_samples=1000, dtype=np.float64):
cols: Number of image columns.
n_samples: Initialization frame count and steady-state update-weight
denominator.
dtype: Output dtype for the mean, variance, and standard deviation.
dtype: Output dtype for the mean and standard deviation.
Supported values are ``np.float64``, ``np.float32``, and
``np.int16``.
"""
@@ -41,7 +41,7 @@ def from_file(filename, n_samples=1000, skip_first=0, dtype=np.float64):
filename: Input image file.
n_samples: Number of frames used for initialization.
skip_first: Number of leading frames to ignore.
dtype: Output dtype for the mean, variance, and standard deviation.
dtype: Output dtype for the mean and standard deviation.
Raises:
RuntimeError: If fewer than ``n_samples`` frames remain after
+26
View File
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: MPL-2.0
import numpy as np
from .factory import _get_typed_class
def Pedestal(rows, cols, n_samples=1000, dtype=np.float64):
"""Create an empty per-pixel running pedestal.
This factory hides the dtype suffix used by the templated C++ bindings.
Call ``push()`` to update the statistics and cached mean for each frame.
Statistics are available during initialization and are zero for empty pixels.
Internal moments and variance always use double precision.
Args:
rows: Number of image rows.
cols: Number of image columns.
n_samples: Number of samples accumulated before switching to
steady-state updates with weight ``1 / n_samples``.
dtype: Output dtype for the mean and standard deviation.
Supported values are ``np.float64``, ``np.float32``, and
``np.int16``.
"""
cls = _get_typed_class("Pedestal", dtype)
return cls(rows, cols, n_samples)
+1
View File
@@ -33,6 +33,7 @@ from ._aare import UDPPortPosition
from ._version import __version__
from .FastPedestal import FastPedestal
from .Pedestal import Pedestal
from .ClusterFinder import ClusterFinder, ClusterCollector, ClusterFinderMT, ClusterFileSink, ClusterFile
from .ClusterVector import ClusterVector
from .Cluster import Cluster
+1 -11
View File
@@ -15,8 +15,7 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) {
py::class_<FastPedestal<SUM_TYPE>>(
m, name.c_str(),
"Maintain a per-pixel running mean, population variance, and "
"standard deviation.",
"Maintain a per-pixel running mean and population standard deviation.",
py::buffer_protocol())
.def(py::init<uint32_t, uint32_t, uint32_t>(), py::arg("rows"),
py::arg("cols"), py::arg("n_samples"),
@@ -33,15 +32,6 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) {
return return_image_data(mean);
},
"Return a copy of the cached mean. The pedestal must be ready.")
.def(
"var",
[](FastPedestal<SUM_TYPE> &self) {
auto variance = new NDArray<SUM_TYPE, 2>{};
*variance = self.variance();
return return_image_data(variance);
},
"Return the population variance, normalized by n_samples, as a "
"NumPy array. The pedestal must be ready.")
.def(
"std",
[](FastPedestal<SUM_TYPE> &self) {
+132
View File
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: MPL-2.0
#include "aare/Pedestal.hpp"
#include "np_helper.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
template <typename SUM_TYPE>
void define_pedestal_bindings(py::module &m, const std::string &name) {
py::class_<Pedestal<SUM_TYPE>>(
m, name.c_str(),
"Maintain a per-pixel running mean and population standard deviation. "
"Statistics are available during initialization and are zero for empty "
"pixels.",
py::buffer_protocol())
.def(py::init<uint32_t, uint32_t, uint32_t>(), py::arg("rows"),
py::arg("cols"), py::arg("n_samples"),
"Construct an empty pedestal. Each pixel accumulates n_samples "
"values before switching to exponential updates.")
.def(py::init<uint32_t, uint32_t>(), py::arg("rows"), py::arg("cols"),
"Construct an empty pedestal with n_samples=1000.")
.def(
"mean",
[](Pedestal<SUM_TYPE> &self) {
auto mea = new NDArray<SUM_TYPE, 2>{};
*mea = self.mean();
return return_image_data(mea);
},
"Return a copy of the cached mean. Empty pixels return zero.")
.def(
"view",
[](py::object self_py) {
return py::module_::import("numpy").attr("asarray")(self_py);
},
"Return a non-owning, non-writable NumPy view of the cached mean.")
.def(
"std",
[](Pedestal<SUM_TYPE> &self) {
auto std = new NDArray<SUM_TYPE, 2>{};
*std = self.std();
return return_image_data(std);
},
"Return the population standard deviation as a NumPy array. "
"Empty pixels return zero.")
.def(
"__array_ufunc__",
[](py::object self, py::object ufunc, const std::string &method,
py::args inputs, py::kwargs kwargs) -> py::object {
if (method != "__call__" || inputs.size() != 2 ||
inputs[1].ptr() != self.ptr() ||
py::cast<std::string>(ufunc.attr("__name__")) !=
"subtract") {
return py::reinterpret_borrow<py::object>(
Py_NotImplemented);
}
auto mean =
py::module_::import("builtins").attr("memoryview")(self);
return ufunc(inputs[0], mean, **kwargs);
},
"Support subtracting a Pedestal from a NumPy array.")
.def("clear", py::overload_cast<>(&Pedestal<SUM_TYPE>::clear),
"Reset all statistics and per-pixel sample counts to zero.")
.def_property_readonly("rows", &Pedestal<SUM_TYPE>::rows,
"Number of image rows.")
.def_property_readonly("cols", &Pedestal<SUM_TYPE>::cols,
"Number of image columns.")
.def_property_readonly(
"n_samples", &Pedestal<SUM_TYPE>::n_samples,
"Initialization sample count per pixel and steady-state "
"update-weight denominator.")
.def(
"clone",
[&](Pedestal<SUM_TYPE> &pedestal) {
return Pedestal<SUM_TYPE>(pedestal);
},
"Return an independent copy of the pedestal and its state.")
// TODO! add push for other data types
.def(
"push",
[](Pedestal<SUM_TYPE> &pedestal,
py::array_t<uint16_t, py::array::c_style> &f) {
if (f.ndim() != 2) {
throw py::value_error("Frame must be 2-dimensional");
}
auto v = make_view_2d(f);
pedestal.push(v);
},
py::arg("frame").noconvert(),
"Accumulate or exponentially update every pixel from a "
"C-contiguous uint16 frame matching the pedestal shape. After "
"n_samples values per pixel, new values have weight 1 / n_samples.")
.def(
"push_with_threshold",
[](Pedestal<SUM_TYPE> &pedestal,
py::array_t<uint16_t, py::array::c_style> &f,
py::array_t<SUM_TYPE, py::array::c_style> &threshold) {
if (f.ndim() != 2) {
throw py::value_error("Frame must be 2-dimensional");
}
if (threshold.ndim() != 2) {
throw py::value_error("Threshold must be 2-dimensional");
}
auto frame_view = make_view_2d(f);
auto threshold_view = make_view_2d(threshold);
pedestal.push_with_threshold(frame_view, threshold_view);
},
py::arg("frame").noconvert(), py::arg("threshold").noconvert(),
"Push only pixels where abs(frame - mean) is strictly less than "
"threshold. Both arrays must be C-contiguous with the pedestal "
"shape; frame must be uint16 and threshold must use the output "
"dtype. Rejected pixels keep their statistics and sample counts.")
.def_buffer([](Pedestal<SUM_TYPE> &self) {
auto mean = self.view();
return py::buffer_info(
const_cast<SUM_TYPE *>(mean.data()), sizeof(SUM_TYPE),
py::format_descriptor<SUM_TYPE>::format(), 2,
{static_cast<py::ssize_t>(mean.shape(0)),
static_cast<py::ssize_t>(mean.shape(1))},
{static_cast<py::ssize_t>(mean.strides()[0] * sizeof(SUM_TYPE)),
static_cast<py::ssize_t>(mean.strides()[1] *
sizeof(SUM_TYPE))},
true);
});
}
+1 -1
View File
@@ -16,6 +16,7 @@
#include "bind_FastPedestal.hpp"
#include "bind_Interpolator.hpp"
#include "bind_MultiThreadedFileReader.hpp"
#include "bind_Pedestal.hpp"
#include "bind_PedestalTrackingPixelHistogram.hpp"
#include "bind_PixelHistogram.hpp"
#include "bind_PixelMap.hpp"
@@ -27,7 +28,6 @@
#include "file.hpp"
#include "fit.hpp"
#include "jungfrau_data_file.hpp"
#include "pedestal.hpp"
#include "raw_master_file.hpp"
#include "raw_sub_file.hpp"
#include "var_cluster.hpp"
-125
View File
@@ -1,125 +0,0 @@
// SPDX-License-Identifier: MPL-2.0
#include "aare/Pedestal.hpp"
#include "np_helper.hpp"
#include <cstdint>
#include <filesystem>
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
namespace py = pybind11;
template <typename SUM_TYPE>
void define_pedestal_bindings(py::module &m, const std::string &name) {
py::class_<Pedestal<SUM_TYPE>>(m, name.c_str(), py::buffer_protocol())
.def(py::init<int, int, int>())
.def(py::init<int, int>())
.def("mean",
[](Pedestal<SUM_TYPE> &self) {
auto mea = new NDArray<SUM_TYPE, 2>{};
*mea = self.mean();
return return_image_data(mea);
})
.def("view",
[](py::object self_py) {
auto &self = self_py.cast<Pedestal<SUM_TYPE> &>();
auto v = self.view();
std::array<py::ssize_t, 2> shape{
static_cast<py::ssize_t>(v.shape(0)),
static_cast<py::ssize_t>(v.shape(1))};
std::array<py::ssize_t, 2> byte_strides{
static_cast<py::ssize_t>(v.strides()[0]) *
static_cast<py::ssize_t>(sizeof(SUM_TYPE)),
static_cast<py::ssize_t>(v.strides()[1]) *
static_cast<py::ssize_t>(sizeof(SUM_TYPE))};
auto arr = py::array_t<SUM_TYPE>(shape, byte_strides, v.data(),
self_py);
arr.attr("setflags")(py::arg("write") = false);
return arr;
})
.def("variance",
[](Pedestal<SUM_TYPE> &self) {
auto var = new NDArray<SUM_TYPE, 2>{};
*var = self.variance();
return return_image_data(var);
})
.def("std",
[](Pedestal<SUM_TYPE> &self) {
auto std = new NDArray<SUM_TYPE, 2>{};
*std = self.std();
return return_image_data(std);
})
.def("cached_std",
[](Pedestal<SUM_TYPE> &self) {
auto standard_deviation = new NDArray<SUM_TYPE, 2>{};
*standard_deviation = self.cached_std();
return return_image_data(standard_deviation);
})
.def(
"__array_ufunc__",
[](py::object self, py::object ufunc, const std::string &method,
py::args inputs, py::kwargs kwargs) -> py::object {
if (method != "__call__" || inputs.size() != 2 ||
inputs[1].ptr() != self.ptr() ||
py::cast<std::string>(ufunc.attr("__name__")) !=
"subtract") {
return py::reinterpret_borrow<py::object>(
Py_NotImplemented);
}
auto mean =
py::module_::import("builtins").attr("memoryview")(self);
return ufunc(inputs[0], mean, **kwargs);
},
"Support subtracting a Pedestal from a NumPy array.")
.def("clear", py::overload_cast<>(&Pedestal<SUM_TYPE>::clear))
.def_property_readonly("rows", &Pedestal<SUM_TYPE>::rows)
.def_property_readonly("cols", &Pedestal<SUM_TYPE>::cols)
.def_property_readonly("n_samples", &Pedestal<SUM_TYPE>::n_samples)
.def_property_readonly("sum", &Pedestal<SUM_TYPE>::get_sum)
.def_property_readonly("sum2", &Pedestal<SUM_TYPE>::get_sum2)
.def("clone",
[&](Pedestal<SUM_TYPE> &pedestal) {
return Pedestal<SUM_TYPE>(pedestal);
})
// TODO! add push for other data types
.def("push",
[](Pedestal<SUM_TYPE> &pedestal, py::array_t<uint16_t> &f) {
auto v = make_view_2d(f);
pedestal.push(v);
})
.def(
"push_with_threshold",
[](Pedestal<SUM_TYPE> &pedestal,
py::array_t<uint16_t, py::array::c_style> &f,
py::array_t<SUM_TYPE, py::array::c_style> &threshold) {
auto frame_view = make_view_2d(f);
auto threshold_view = make_view_2d(threshold);
pedestal.push_with_threshold(frame_view, threshold_view);
},
py::arg("frame").noconvert(), py::arg("threshold").noconvert())
.def(
"push_no_update",
[](Pedestal<SUM_TYPE> &pedestal,
py::array_t<uint16_t, py::array::c_style> &f) {
auto v = make_view_2d(f);
pedestal.push_no_update(v);
},
py::arg().noconvert())
.def("update_mean", &Pedestal<SUM_TYPE>::update_mean)
.def_buffer([](Pedestal<SUM_TYPE> &self) {
auto mean = self.view();
return py::buffer_info(
const_cast<SUM_TYPE *>(mean.data()), sizeof(SUM_TYPE),
py::format_descriptor<SUM_TYPE>::format(), 2,
{static_cast<py::ssize_t>(mean.shape(0)),
static_cast<py::ssize_t>(mean.shape(1))},
{static_cast<py::ssize_t>(mean.strides()[0] * sizeof(SUM_TYPE)),
static_cast<py::ssize_t>(mean.strides()[1] *
sizeof(SUM_TYPE))},
true);
});
}
+14
View File
@@ -143,3 +143,17 @@ def test_fast_pedestal_rejects_wrong_shape():
with pytest.raises(RuntimeError, match="shape"):
pedestal.add_init_frame(np.zeros((2, 2), dtype=np.uint16))
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
def test_fast_pedestal_std_uses_double_variance(dtype):
pedestal = FastPedestal(1, 1, n_samples=2, dtype=dtype)
pedestal.add_init_frame(np.array([[0]], dtype=np.uint16))
pedestal.add_init_frame(np.array([[1000]], dtype=np.uint16))
assert pedestal.std().dtype == dtype
np.testing.assert_array_equal(pedestal.std(), [[500]])
pedestal.push_ema(np.array([[1000]], dtype=np.uint16))
expected = np.array([[np.sqrt(187500.0)]], dtype=dtype)
np.testing.assert_array_equal(pedestal.std(), expected)
+165 -1
View File
@@ -1,7 +1,106 @@
import numpy as np
import pytest
from aare import Pedestal_d, Pedestal_f
from aare import Pedestal, Pedestal_d, Pedestal_f, Pedestal_i16
@pytest.mark.parametrize(
("dtype", "pedestal_type"),
[(np.float64, Pedestal_d), (np.float32, Pedestal_f), (np.int16, Pedestal_i16)],
)
def test_pedestal_factory(dtype, pedestal_type):
pedestal = Pedestal(rows=2, cols=3, n_samples=4, dtype=dtype)
assert isinstance(pedestal, pedestal_type)
assert pedestal.rows == 2
assert pedestal.cols == 3
assert pedestal.n_samples == 4
def test_pedestal_factory_defaults_to_double():
pedestal = Pedestal(2, 3)
assert isinstance(pedestal, Pedestal_d)
assert pedestal.n_samples == 1000
def test_pedestal_factory_rejects_unbound_dtype():
with pytest.raises(ValueError, match="Unsupported dtype for Pedestal"):
Pedestal(2, 3, dtype=np.int32)
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
@pytest.mark.parametrize("name", ["rows", "cols", "n_samples"])
def test_pedestal_factory_rejects_negative_parameters(dtype, name):
parameters = dict(rows=1, cols=1, n_samples=10)
parameters[name] = -1
with pytest.raises(TypeError):
Pedestal(**parameters, dtype=dtype)
@pytest.mark.parametrize("pedestal_type", [Pedestal_d, Pedestal_f, Pedestal_i16])
@pytest.mark.parametrize(
"args", [(-1, 1), (1, -1), (-1, 1, 10), (1, -1, 10), (1, 1, -1)]
)
def test_pedestal_constructor_rejects_negative_parameters(pedestal_type, args):
with pytest.raises(TypeError):
pedestal_type(*args)
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
def test_pedestal_std_stays_finite_after_settling(dtype):
pedestal = Pedestal(1, 1, n_samples=10, dtype=dtype)
pedestal.push(np.array([[16382]], dtype=np.uint16))
frame = np.array([[16383]], dtype=np.uint16)
for _ in range(201):
pedestal.push(frame)
noise = pedestal.std().item()
assert np.isfinite(noise)
assert 0 <= noise < 1e-3
@pytest.mark.parametrize(
("pedestal_type", "expected_dtype"),
[(Pedestal_d, np.float64), (Pedestal_f, np.float32), (Pedestal_i16, np.int16)],
)
def test_double_precision_moments(pedestal_type, expected_dtype):
pedestal = pedestal_type(1, 1, 2)
for value in [30000, 30003, 30002]:
pedestal.push(np.array([[value]], dtype=np.uint16))
assert pedestal.mean().dtype == expected_dtype
assert pedestal.std().dtype == expected_dtype
np.testing.assert_array_equal(
pedestal.mean(), np.array([[30001.75]], dtype=expected_dtype)
)
np.testing.assert_allclose(
pedestal.std(),
np.array([[np.sqrt(1.1875)]], dtype=expected_dtype),
rtol=1e-6,
)
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
@pytest.mark.parametrize("method", ["push", "push_with_threshold"])
@pytest.mark.parametrize("shape", [(2, 2), (3, 2)])
def test_pedestal_rejects_mismatched_frame_shapes(dtype, method, shape):
pedestal = Pedestal(2, 3, dtype=dtype)
initial = np.full((2, 3), 7, dtype=np.uint16)
pedestal.push(initial)
frame = np.zeros(shape, dtype=np.uint16)
args = (
(np.full((2, 3), 10, dtype=dtype),)
if method == "push_with_threshold"
else ()
)
with pytest.raises(
RuntimeError, match="Frame shape does not match pedestal shape"
):
getattr(pedestal, method)(frame, *args)
np.testing.assert_array_equal(pedestal.mean(), initial)
@pytest.mark.parametrize(
@@ -38,3 +137,68 @@ def test_pedestal_exposes_mean_as_read_only_buffer():
np.testing.assert_array_equal(mean, pedestal.view())
assert np.shares_memory(mean, pedestal.view())
assert not mean.flags.writeable
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
@pytest.mark.parametrize("method", ["push", "push_with_threshold"])
def test_pedestal_push_accepts_contiguous_frames(dtype, method):
pedestal = Pedestal(2, 3, dtype=dtype)
frame = np.arange(6, dtype=np.uint16).reshape(2, 3)
args = (
(np.full((2, 3), 10, dtype=dtype),)
if method == "push_with_threshold"
else ()
)
getattr(pedestal, method)(frame, *args)
np.testing.assert_array_equal(pedestal.mean(), frame)
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
@pytest.mark.parametrize("input_name", ["push_frame", "threshold_frame", "threshold"])
@pytest.mark.parametrize("layout", ["transpose", "slice", "reverse"])
def test_pedestal_rejects_noncontiguous_inputs(dtype, input_name, layout):
pedestal = Pedestal(2, 3, dtype=dtype)
frame = np.zeros((2, 3), dtype=np.uint16)
threshold = np.full((2, 3), 10, dtype=dtype)
input_dtype = dtype if input_name == "threshold" else np.uint16
if layout == "transpose":
invalid = np.ones((3, 2), dtype=input_dtype).T
elif layout == "slice":
invalid = np.ones((2, 6), dtype=input_dtype)[:, ::2]
else:
invalid = np.ones((4, 6), dtype=input_dtype)[:2, :3][:, ::-1]
assert not invalid.flags.c_contiguous
with pytest.raises(TypeError):
if input_name == "push_frame":
pedestal.push(invalid)
elif input_name == "threshold_frame":
pedestal.push_with_threshold(invalid, threshold)
else:
pedestal.push_with_threshold(frame, invalid)
np.testing.assert_array_equal(pedestal.mean(), np.zeros((2, 3)))
@pytest.mark.parametrize("dtype", [np.float64, np.float32, np.int16])
@pytest.mark.parametrize("input_name", ["push_frame", "threshold_frame", "threshold"])
@pytest.mark.parametrize("shape", [(), (6,), (2, 3, 2)])
def test_pedestal_rejects_inputs_with_wrong_ndim(dtype, input_name, shape):
pedestal = Pedestal(2, 3, dtype=dtype)
frame = np.ones((2, 3), dtype=np.uint16)
threshold = np.full((2, 3), 10, dtype=dtype)
input_dtype = dtype if input_name == "threshold" else np.uint16
invalid = np.ones(shape, dtype=input_dtype)
name = "Threshold" if input_name == "threshold" else "Frame"
with pytest.raises(ValueError, match=f"{name} must be 2-dimensional"):
if input_name == "push_frame":
pedestal.push(invalid)
elif input_name == "threshold_frame":
pedestal.push_with_threshold(invalid, threshold)
else:
pedestal.push_with_threshold(frame, invalid)
np.testing.assert_array_equal(pedestal.mean(), np.zeros((2, 3)))