Merge branch 'main' into feature/cuda_clusterfinder

python/src/bind_Cluster.hpp

@@ -10,7 +10,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
 
 using namespace aare;
 

python/src/bind_ClusterCollector.hpp

@@ -6,6 +6,8 @@
 #include "aare/ClusterVector.hpp"
 #include "aare/NDView.hpp"
 #include "aare/Pedestal.hpp"
+
+#include "module_config.hpp"
 #include "np_helper.hpp"
 
 #include <cstdint>
@@ -15,7 +17,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
 
 using namespace aare;
 
@@ -30,7 +31,7 @@ void define_ClusterCollector(py::module &m, const std::string &typestr) {
     using ClusterType = Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>;
 
     py::class_<ClusterCollector<ClusterType>>(m, class_name.c_str())
-        .def(py::init<ClusterFinderMT<ClusterType, uint16_t, double> *>())
+        .def(py::init<ClusterFinderMT<ClusterType, uint16_t, pd_type> *>())
         .def("stop", &ClusterCollector<ClusterType>::stop)
         .def(
             "steal_clusters",

python/src/bind_ClusterFile.hpp

@@ -47,6 +47,8 @@ void define_ClusterFile(py::module &m, const std::string &typestr) {
              })
         .def("set_roi", &ClusterFile<ClusterType>::set_roi, py::arg("roi"))
         .def("tell", &ClusterFile<ClusterType>::tell)
+        .def("estimate_n_clusters",
+             &ClusterFile<ClusterType>::estimate_n_clusters)
         .def(
             "set_noise_map",
             [](ClusterFile<ClusterType> &self, py::array_t<int32_t> noise_map) {
@@ -82,4 +84,4 @@ void define_ClusterFile(py::module &m, const std::string &typestr) {
         });
 }
 
-#pragma GCC diagnostic pop
+#pragma GCC diagnostic pop

python/src/bind_ClusterFileSink.hpp

@@ -15,8 +15,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
-
 using namespace aare;
 
 #pragma GCC diagnostic push
@@ -29,6 +27,8 @@ void define_ClusterFileSink(py::module &m, const std::string &typestr) {
 
     using ClusterType = Cluster<T, ClusterSizeX, ClusterSizeY, CoordType>;
 
+    // TODO! adapt to set pedestal type (needs templating of ClusterFileSink)
+    // or maybe access through base class?
     py::class_<ClusterFileSink<ClusterType>>(m, class_name.c_str())
         .def(py::init<ClusterFinderMT<ClusterType, uint16_t, double> *,
                       const std::filesystem::path &>())

python/src/bind_ClusterFinder.hpp

@@ -6,6 +6,8 @@
 #include "aare/ClusterVector.hpp"
 #include "aare/NDView.hpp"
 #include "aare/Pedestal.hpp"
+
+#include "module_config.hpp"
 #include "np_helper.hpp"
 
 #include <cstdint>
@@ -15,7 +17,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
 
 using namespace aare;
 
@@ -48,6 +49,8 @@ void define_ClusterFinder(py::module &m, const std::string &typestr) {
              })
         .def("clear_pedestal",
              &ClusterFinder<ClusterType, uint16_t, pd_type>::clear_pedestal)
+        .def("update_threshold",
+             &ClusterFinder<ClusterType, uint16_t, pd_type>::update_threshold)
         .def_property_readonly(
             "pedestal",
             [](ClusterFinder<ClusterType, uint16_t, pd_type> &self) {

python/src/bind_ClusterFinderMT.hpp

@@ -6,6 +6,8 @@
 #include "aare/ClusterVector.hpp"
 #include "aare/NDView.hpp"
 #include "aare/Pedestal.hpp"
+
+#include "module_config.hpp"
 #include "np_helper.hpp"
 
 #include <cstdint>
@@ -15,7 +17,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
 
 using namespace aare;
 
@@ -31,9 +32,10 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) {
 
     py::class_<ClusterFinderMT<ClusterType, uint16_t, pd_type>>(
         m, class_name.c_str())
-        .def(py::init<Shape<2>, pd_type, size_t, size_t>(),
+        .def(py::init<Shape<2>, pd_type, size_t, size_t, size_t>(),
              py::arg("image_size"), py::arg("n_sigma") = 5.0,
-             py::arg("capacity") = 2048, py::arg("n_threads") = 3)
+             py::arg("capacity") = 2048, py::arg("n_threads") = 3,
+             py::arg("queue_depth") = 16)
         .def("push_pedestal_frame",
              [](ClusterFinderMT<ClusterType, uint16_t, pd_type> &self,
                 py::array_t<uint16_t> frame) {
@@ -46,7 +48,6 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) {
                py::array_t<uint16_t> frame, uint64_t frame_number) {
                 auto view = make_view_2d(frame);
                 self.find_clusters(view, frame_number);
-                return;
             },
             py::arg(), py::arg("frame_number") = 0)
         .def_property_readonly(
@@ -56,6 +57,8 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) {
             })
         .def("clear_pedestal",
              &ClusterFinderMT<ClusterType, uint16_t, pd_type>::clear_pedestal)
+        .def("update_threshold",
+             &ClusterFinderMT<ClusterType, uint16_t, pd_type>::update_threshold)
         .def("sync", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::sync)
         .def("stop", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::stop)
         .def("start", &ClusterFinderMT<ClusterType, uint16_t, pd_type>::start)

python/src/bind_ClusterVector.hpp

@@ -15,7 +15,6 @@
 #include <pybind11/stl_bind.h>
 
 namespace py = pybind11;
-using pd_type = double;
 
 using namespace aare;
 
@@ -72,6 +71,7 @@ void define_ClusterVector(py::module &m, const std::string &typestr) {
             R"(calculates sum of 2x2 subcluster with highest energy and index relative to cluster center 0: top_left, 1: top_right, 2: bottom_left, 3: bottom_right
             )")
         .def_property_readonly("size", &ClusterVector<ClusterType>::size)
+        .def("empty", &ClusterVector<ClusterType>::empty)
         .def("item_size", &ClusterVector<ClusterType>::item_size)
         .def_property_readonly("fmt",
                                [typestr](ClusterVector<ClusterType> &self) {
@@ -169,4 +169,4 @@ void define_3x3_reduction(py::module &m) {
         py::arg("clustervector"));
 }
 
-#pragma GCC diagnostic pop
+#pragma GCC diagnostic pop

python/src/bind_FastPedestal.hpp

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: MPL-2.0
+
+#include "aare/FastPedestal.hpp"
+#include "np_helper.hpp"
+
+#include <cstdint>
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+template <typename SUM_TYPE>
+void define_fast_pedestal_bindings(py::module &m, const std::string &name) {
+
+    py::class_<FastPedestal<SUM_TYPE>>(
+        m, name.c_str(),
+        "Maintain a per-pixel running mean, population variance, and "
+        "standard deviation.",
+        py::buffer_protocol())
+        .def(py::init<uint32_t, uint32_t, uint32_t>(), py::arg("rows"),
+             py::arg("cols"), py::arg("n_samples"),
+             "Construct an empty pedestal. It becomes ready after n_samples "
+             "calls to add_init_frame().")
+        .def(py::init<uint32_t, uint32_t>(), py::arg("rows"), py::arg("cols"),
+             "Construct an empty pedestal with n_samples=1000.")
+
+        .def(
+            "mean",
+            [](FastPedestal<SUM_TYPE> &self) {
+                auto mean = new NDArray<SUM_TYPE, 2>{};
+                *mean = self.mean();
+                return return_image_data(mean);
+            },
+            "Return a copy of the cached mean. The pedestal must be ready.")
+        .def(
+            "var",
+            [](FastPedestal<SUM_TYPE> &self) {
+                auto variance = new NDArray<SUM_TYPE, 2>{};
+                *variance = self.variance();
+                return return_image_data(variance);
+            },
+            "Return the population variance, normalized by n_samples, as a "
+            "NumPy array. The pedestal must be ready.")
+        .def(
+            "std",
+            [](FastPedestal<SUM_TYPE> &self) {
+                auto standard_deviation = new NDArray<SUM_TYPE, 2>{};
+                *standard_deviation = self.std();
+                return return_image_data(standard_deviation);
+            },
+            "Return the population standard deviation as a NumPy array. The "
+            "pedestal must be ready.")
+        .def(
+            "view",
+            [](py::object self_py) {
+                return py::module_::import("numpy").attr("asarray")(self_py);
+            },
+            "Return a non-owning, non-writable NumPy view of the cached mean. "
+            "The pedestal must be ready.")
+
+        // We need to buffer protocol to allow for numpy operations using the
+        // pedestal mean
+        .def_buffer([](FastPedestal<SUM_TYPE> &self) {
+            auto mean = self.view();
+            return py::buffer_info(
+                const_cast<SUM_TYPE *>(mean.data()), sizeof(SUM_TYPE),
+                py::format_descriptor<SUM_TYPE>::format(), 2,
+                {static_cast<py::ssize_t>(mean.shape(0)),
+                 static_cast<py::ssize_t>(mean.shape(1))},
+                {static_cast<py::ssize_t>(mean.strides()[0] * sizeof(SUM_TYPE)),
+                 static_cast<py::ssize_t>(mean.strides()[1] *
+                                          sizeof(SUM_TYPE))},
+                true);
+        })
+        // Subtracting a FastPedestal from a NumPy array
+        .def(
+            "__array_ufunc__",
+            [](py::object self, py::object ufunc, const std::string &method,
+               py::args inputs, py::kwargs kwargs) -> py::object {
+                if (method != "__call__" || inputs.size() != 2 ||
+                    inputs[1].ptr() != self.ptr() ||
+                    py::cast<std::string>(ufunc.attr("__name__")) !=
+                        "subtract") {
+                    return py::reinterpret_borrow<py::object>(
+                        Py_NotImplemented);
+                }
+
+                auto mean =
+                    py::module_::import("builtins").attr("memoryview")(self);
+                return ufunc(inputs[0], mean, **kwargs);
+            },
+            "Support subtracting a FastPedestal from a NumPy array.")
+        .def("clear", py::overload_cast<>(&FastPedestal<SUM_TYPE>::clear),
+             "Reset all statistics and initialization state to zero.")
+        .def_property_readonly("rows", &FastPedestal<SUM_TYPE>::rows,
+                               "Number of image rows.")
+        .def_property_readonly("cols", &FastPedestal<SUM_TYPE>::cols,
+                               "Number of image columns.")
+        .def_property_readonly("cur_samples",
+                               &FastPedestal<SUM_TYPE>::cur_samples,
+                               "Number of initialization frames accumulated. "
+                               "Steady-state pushes do not change it.")
+        .def_property_readonly(
+            "ready", &FastPedestal<SUM_TYPE>::ready,
+            "Whether n_samples initialization frames have been accumulated.")
+        .def_property_readonly(
+            "n_samples", &FastPedestal<SUM_TYPE>::n_samples,
+            "Initialization frame count and steady-state update-weight "
+            "denominator.")
+        .def(
+            "clone",
+            [](FastPedestal<SUM_TYPE> &pedestal) {
+                return FastPedestal<SUM_TYPE>(pedestal);
+            },
+            "Return an independent copy of the pedestal and its state.")
+        .def(
+            "push_ema",
+            [](FastPedestal<SUM_TYPE> &pedestal,
+               py::array_t<uint16_t, py::array::c_style> &frame) {
+                pedestal.push_ema(make_view_2d(frame));
+            },
+            py::arg("frame").noconvert(),
+            "Update exponential moving average. The pedstal must  "
+            "already be ready for this update.")
+        .def(
+            "add_init_frame",
+            [](FastPedestal<SUM_TYPE> &pedestal,
+               py::array_t<uint16_t, py::array::c_style> &frame) {
+                pedestal.add_init_frame(make_view_2d(frame));
+            },
+            py::arg("frame").noconvert(),
+            "Accumulate one uint16 initialization frame. Call exactly "
+            "n_samples times to make the pedestal ready.")
+        .def_static(
+            "from_file",
+            [](const std::filesystem::path &filename, uint32_t n_samples,
+               uint32_t skip_first) {
+                return FastPedestal<SUM_TYPE>::template from_file<uint16_t>(
+                    filename, n_samples, skip_first);
+            },
+            py::arg("filename"), py::arg("n_samples") = 1000,
+            py::arg("skip_first") = 0,
+            "Create a pedestal from a uint16 file. Skip skip_first frames, use "
+            "the next n_samples for initialization, then apply every remaining "
+            "frame as a steady-state update.");
+}

python/src/bind_MultiThreadedFileReader.hpp

@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: MPL-2.0
+#pragma once
+
+#include "aare/MultiThreadedFileReader.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+#include <stdexcept>
+#include <vector>
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl/filesystem.h>
+
+namespace py = pybind11;
+
+inline py::dtype multi_threaded_reader_numpy_dtype(const aare::Dtype &dtype) {
+    using aare::Dtype;
+    if (dtype == Dtype::INT8)
+        return py::dtype::of<int8_t>();
+    if (dtype == Dtype::UINT8)
+        return py::dtype::of<uint8_t>();
+    if (dtype == Dtype::INT16)
+        return py::dtype::of<int16_t>();
+    if (dtype == Dtype::UINT16)
+        return py::dtype::of<uint16_t>();
+    if (dtype == Dtype::INT32)
+        return py::dtype::of<int32_t>();
+    if (dtype == Dtype::UINT32)
+        return py::dtype::of<uint32_t>();
+    if (dtype == Dtype::INT64)
+        return py::dtype::of<int64_t>();
+    if (dtype == Dtype::UINT64)
+        return py::dtype::of<uint64_t>();
+    if (dtype == Dtype::FLOAT)
+        return py::dtype::of<float>();
+    if (dtype == Dtype::DOUBLE)
+        return py::dtype::of<double>();
+    throw std::runtime_error("Unsupported pixel data type");
+}
+
+inline py::array
+multi_threaded_reader_read(aare::experimental::MultiThreadedFileReader &reader,
+                           bool read_all) {
+    const size_t n_frames =
+        read_all ? reader.remaining_frames() : reader.next_read_frames();
+    const std::vector<py::ssize_t> shape{
+        static_cast<py::ssize_t>(n_frames),
+        static_cast<py::ssize_t>(reader.rows()),
+        static_cast<py::ssize_t>(reader.cols())};
+
+    py::array image(multi_threaded_reader_numpy_dtype(reader.dtype()), shape);
+    auto *destination = reinterpret_cast<std::byte *>(image.mutable_data());
+    {
+        py::gil_scoped_release release;
+        if (read_all) {
+            size_t offset = 0;
+            while (reader.remaining_frames() != 0) {
+                const size_t frames_read =
+                    reader.read_into(destination + offset);
+                offset += frames_read * reader.bytes_per_frame();
+            }
+        } else {
+            reader.read_into(destination);
+        }
+    }
+    return image;
+}
+
+inline void define_multi_threaded_file_reader_bindings(py::module_ &m) {
+    using aare::experimental::MultiThreadedFileReader;
+
+    auto reader =
+        py::class_<MultiThreadedFileReader>(m, "MultiThreadedFileReader");
+    reader.attr("__module__") = "aare.experimental";
+    reader
+        .def(py::init<std::filesystem::path, size_t, size_t,
+                      std::optional<size_t>>(),
+             py::arg("fname"), py::arg("n_threads"), py::arg("chunk_size"),
+             py::arg("total_frames") = py::none(),
+             R"doc(
+                 Read chunks of detector frames concurrently.
+
+                 Each worker opens an independent File. The returned array is
+                 ordered by frame index even though chunks are read in parallel.
+
+                 Args:
+                     fname: Path accepted by File.
+                     n_threads: Maximum number of worker threads.
+                     chunk_size: Number of frames read per claimed chunk.
+                     total_frames: Optional frame limit. None reads all frames.
+                 )doc")
+        .def(
+            "read",
+            [](MultiThreadedFileReader &self) {
+                return multi_threaded_reader_read(self, false);
+            },
+            R"doc(
+                 Read one chunk per active worker into a NumPy array.
+
+                 Returns:
+                     An array containing at most n_threads * chunk_size frames.
+                     An empty array is returned at the end of the configured
+                     frame range. The GIL is released while file data is read.
+                 )doc")
+        .def(
+            "read_all",
+            [](MultiThreadedFileReader &self) {
+                return multi_threaded_reader_read(self, true);
+            },
+            R"doc(
+                 Read all frames remaining from the current position.
+                 )doc")
+        .def_property_readonly("n_threads", &MultiThreadedFileReader::n_threads)
+        .def_property_readonly("chunk_size",
+                               &MultiThreadedFileReader::chunk_size)
+        .def_property_readonly("total_frames",
+                               &MultiThreadedFileReader::total_frames)
+        .def_property_readonly("source_total_frames",
+                               &MultiThreadedFileReader::source_total_frames)
+        .def_property_readonly("rows", &MultiThreadedFileReader::rows)
+        .def_property_readonly("cols", &MultiThreadedFileReader::cols)
+        .def_property_readonly("bitdepth", &MultiThreadedFileReader::bitdepth)
+        .def_property_readonly("dtype",
+                               [](const MultiThreadedFileReader &self) {
+                                   return multi_threaded_reader_numpy_dtype(
+                                       self.dtype());
+                               })
+        .def_property_readonly("bytes_per_frame",
+                               &MultiThreadedFileReader::bytes_per_frame)
+        .def_property_readonly("total_bytes",
+                               &MultiThreadedFileReader::total_bytes)
+        .def_property_readonly("remaining_frames",
+                               &MultiThreadedFileReader::remaining_frames)
+        .def_property_readonly("next_read_frames",
+                               &MultiThreadedFileReader::next_read_frames)
+        .def_property_readonly("next_read_bytes",
+                               &MultiThreadedFileReader::next_read_bytes)
+        .def("seek", &MultiThreadedFileReader::seek, py::arg("frame_index"))
+        .def("tell", &MultiThreadedFileReader::tell)
+        .def("close", &MultiThreadedFileReader::close,
+             "Close all worker files. Safe to call more than once.")
+        .def_property_readonly(
+            "closed",
+            [](const MultiThreadedFileReader &self) { return !self.is_open(); })
+        .def("__len__", &MultiThreadedFileReader::total_frames)
+        .def(
+            "__enter__",
+            [](MultiThreadedFileReader &self) -> MultiThreadedFileReader * {
+                if (!self.is_open()) {
+                    throw std::runtime_error(
+                        "Cannot enter a closed MultiThreadedFileReader");
+                }
+                return &self;
+            },
+            py::return_value_policy::reference_internal)
+        .def("__exit__",
+             [](MultiThreadedFileReader &self, const py::object &,
+                const py::object &, const py::object &) {
+                 self.close();
+                 return false;
+             })
+        .def(
+            "__iter__", [](MultiThreadedFileReader &self) { return &self; },
+            py::return_value_policy::reference_internal)
+        .def("__next__", [](MultiThreadedFileReader &self) {
+            if (self.remaining_frames() == 0) {
+                throw py::stop_iteration();
+            }
+            return multi_threaded_reader_read(self, false);
+        });
+}

python/src/bind_PedestalTrackingPixelHistogram.hpp

@@ -153,9 +153,13 @@ void define_pedestal_tracking_pixel_histogram_bindings(py::module &m) {
              Args:
                  file_path: Path to the file to fill from
                  max_frames: Maximum number of frames to fill from the file (default: -1)
+                 reader_threads: Number of parallel file reader workers (default: 2)
+                 reader_chunk_size: Frames claimed by each reader worker per batch (default: 4)
              )",
              py::call_guard<py::gil_scoped_release>(), py::arg("fname"),
-             py::arg("max_frames") = -1, py::arg("verbose") = false)
+             py::arg("max_frames") = -1, py::arg("verbose") = false,
+             py::arg("reader_threads") = std::size_t{2},
+             py::arg("reader_chunk_size") = std::size_t{4})
         .def("process_pedestal_file",
              &PedestalTrackingPixelHistogram::process_pedestal_file,
              R"(

python/src/ctb_raw_file.hpp

@@ -9,7 +9,6 @@
 
 #include "aare/decode.hpp"
 #include "aare/defs.hpp"
-// #include "aare/fClusterFileV2.hpp"
 
 #include "np_helper.hpp"
 

python/src/file.hpp

@@ -7,7 +7,6 @@
 #include "aare/RawSubFile.hpp"
 
 #include "aare/defs.hpp"
-// #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
 #include <filesystem>
@@ -195,52 +194,4 @@ void define_file_io_bindings(py::module &m) {
         });
 
 #pragma GCC diagnostic pop
-    // py::class_<ClusterHeader>(m, "ClusterHeader")
-    //     .def(py::init<>())
-    //     .def_readwrite("frame_number", &ClusterHeader::frame_number)
-    //     .def_readwrite("n_clusters", &ClusterHeader::n_clusters)
-    //     .def("__repr__", [](const ClusterHeader &a) { return "<ClusterHeader:
-    //     " + a.to_string() + ">"; });
-
-    // py::class_<ClusterV2_>(m, "ClusterV2_")
-    //     .def(py::init<>())
-    //     .def_readwrite("x", &ClusterV2_::x)
-    //     .def_readwrite("y", &ClusterV2_::y)
-    //     .def_readwrite("data", &ClusterV2_::data)
-    //     .def("__repr__", [](const ClusterV2_ &a) { return "<ClusterV2_: " +
-    //     a.to_string(false) + ">"; });
-
-    // py::class_<ClusterV2>(m, "ClusterV2")
-    //     .def(py::init<>())
-    //     .def_readwrite("cluster", &ClusterV2::cluster)
-    //     .def_readwrite("frame_number", &ClusterV2::frame_number)
-    //     .def("__repr__", [](const ClusterV2 &a) { return "<ClusterV2: " +
-    //     a.to_string() + ">"; });
-
-    // py::class_<ClusterFileV2>(m, "ClusterFileV2")
-    //     .def(py::init<const std::filesystem::path &, const std::string &>())
-    //     .def("read", py::overload_cast<>(&ClusterFileV2::read))
-    //     .def("read", py::overload_cast<int>(&ClusterFileV2::read))
-    //     .def("frame_number", &ClusterFileV2::frame_number)
-    //     .def("write", py::overload_cast<std::vector<ClusterV2> const
-    //     &>(&ClusterFileV2::write))
-
-    //     .def("close", &ClusterFileV2::close);
-
-    // m.def("to_clustV2", [](std::vector<DynamicCluster> &clusters, const int
-    // frame_number) {
-    //     std::vector<ClusterV2> clusters_;
-    //     for (auto &c : clusters) {
-    //         ClusterV2 cluster;
-    //         cluster.cluster.x = c.x;
-    //         cluster.cluster.y = c.y;
-    //         int i=0;
-    //         for(auto &d : cluster.cluster.data) {
-    //             d=c.get<double>(i++);
-    //         }
-    //         cluster.frame_number = frame_number;
-    //         clusters_.push_back(cluster);
-    //     }
-    //     return clusters_;
-    // });
-}
+}

python/src/fit.hpp

@@ -227,452 +227,6 @@ fit_dispatch(const aare::FitModel<Model> &model,
 }
 
 void define_fit_bindings(py::module &m) {
-
-    // TODO! Evaluate without converting to double
-    m.def(
-        "gaus",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
-            auto x_view = make_view_1d(x);
-            auto par_view = make_view_1d(par);
-            auto y = new NDArray<double, 1>{aare::func::gaus(x_view, par_view)};
-            return return_image_data(y);
-        },
-        R"(
-        Evaluate a 1D Gaussian function for all points in x using parameters par.
-
-        Parameters
-        ----------
-        x : array_like
-            The points at which to evaluate the Gaussian function.
-        par : array_like
-            The parameters of the Gaussian function. The first element is the amplitude, the second element is the mean, and the third element is the standard deviation.
-        )",
-        py::arg("x"), py::arg("par"));
-
-    m.def(
-        "pol1",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
-            auto x_view = make_view_1d(x);
-            auto par_view = make_view_1d(par);
-            auto y = new NDArray<double, 1>{aare::func::pol1(x_view, par_view)};
-            return return_image_data(y);
-        },
-        R"(
-        Evaluate a 1D polynomial function for all points in x using parameters par. (p0+p1*x)
-        
-        Parameters
-        ----------
-        x : array_like
-            The points at which to evaluate the polynomial function.
-        par : array_like
-            The parameters of the polynomial function. The first element is the intercept, and the second element is the slope.    
-        )",
-        py::arg("x"), py::arg("par"));
-
-    m.def(
-        "scurve",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
-            auto x_view = make_view_1d(x);
-            auto par_view = make_view_1d(par);
-            auto y =
-                new NDArray<double, 1>{aare::func::scurve(x_view, par_view)};
-            return return_image_data(y);
-        },
-        R"(
-        Evaluate a 1D scurve function for all points in x using parameters par. 
-        
-        Parameters
-        ----------
-        x : array_like
-            The points at which to evaluate the scurve function.
-        par : array_like
-            The parameters of the scurve function. The first element is the background slope, the second element is the background intercept, the third element is the mean, the fourth element is the standard deviation, the fifth element is inflexion point count number, and the sixth element is C.     
-        )",
-        py::arg("x"), py::arg("par"));
-
-    m.def(
-        "scurve2",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> par) {
-            auto x_view = make_view_1d(x);
-            auto par_view = make_view_1d(par);
-            auto y =
-                new NDArray<double, 1>{aare::func::scurve2(x_view, par_view)};
-            return return_image_data(y);
-        },
-        R"(
-        Evaluate a 1D scurve2 function for all points in x using parameters par. 
-        
-        Parameters
-        ----------
-        x : array_like
-            The points at which to evaluate the scurve function.
-        par : array_like
-            The parameters of the scurve2 function. The first element is the background slope, the second element is the background intercept, the third element is the mean, the fourth element is the standard deviation, the fifth element is inflexion point count number, and the sixth element is C.     
-        )",
-        py::arg("x"), py::arg("par"));
-
-    m.def(
-        "fit_gaus",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>{};
-                auto y_view = make_view_3d(y);
-                auto x_view = make_view_1d(x);
-                *par = aare::fit_gaus(x_view, y_view, n_threads);
-                return return_image_data(par);
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>{};
-                auto y_view = make_view_1d(y);
-                auto x_view = make_view_1d(x);
-                *par = aare::fit_gaus(x_view, y_view);
-                return return_image_data(par);
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        R"(
-        Fit a 1D Gaussian to data.
-
-        Parameters
-        ----------
-        x : array_like
-            The x values.
-        y : array_like
-            The y values.
-        n_threads : int, optional
-            The number of threads to use. Default is 4.
-        )",
-        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_gaus",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                // Allocate memory for the output
-                // Need to have pointers to allow python to manage
-                // the memory
-                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 3});
-                auto par_err =
-                    new NDArray<double, 3>({y.shape(0), y.shape(1), 3});
-                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
-
-                // Make views of the numpy arrays
-                auto y_view = make_view_3d(y);
-                auto y_view_err = make_view_3d(y_err);
-                auto x_view = make_view_1d(x);
-
-                aare::fit_gaus(x_view, y_view, y_view_err, par->view(),
-                               par_err->view(), chi2->view(), n_threads);
-
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = return_image_data(chi2),
-                                "Ndf"_a = y.shape(2) - 3);
-            } else if (y.ndim() == 1) {
-                // Allocate memory for the output
-                // Need to have pointers to allow python to manage
-                // the memory
-                auto par = new NDArray<double, 1>({3});
-                auto par_err = new NDArray<double, 1>({3});
-
-                // Decode the numpy arrays
-                auto y_view = make_view_1d(y);
-                auto y_view_err = make_view_1d(y_err);
-                auto x_view = make_view_1d(x);
-
-                double chi2 = 0;
-                aare::fit_gaus(x_view, y_view, y_view_err, par->view(),
-                               par_err->view(), chi2);
-
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = chi2, "Ndf"_a = y.size() - 3);
-
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        R"(
-        Fit a 1D Gaussian to data with error estimates.
-
-        Parameters
-        ----------
-        x : array_like
-            The x values.
-        y : array_like
-            The y values.
-        y_err : array_like
-            The error in the y values.
-        n_threads : int, optional
-            The number of threads to use. Default is 4.
-        )",
-        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_pol1",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>{};
-
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_3d(y);
-                *par = aare::fit_pol1(x_view, y_view, n_threads);
-                return return_image_data(par);
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>{};
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_1d(y);
-                *par = aare::fit_pol1(x_view, y_view);
-                return return_image_data(par);
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_pol1",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 2});
-
-                auto par_err =
-                    new NDArray<double, 3>({y.shape(0), y.shape(1), 2});
-
-                auto y_view = make_view_3d(y);
-                auto y_view_err = make_view_3d(y_err);
-                auto x_view = make_view_1d(x);
-
-                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
-
-                aare::fit_pol1(x_view, y_view, y_view_err, par->view(),
-                               par_err->view(), chi2->view(), n_threads);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = return_image_data(chi2),
-                                "Ndf"_a = y.shape(2) - 2);
-
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>({2});
-                auto par_err = new NDArray<double, 1>({2});
-
-                auto y_view = make_view_1d(y);
-                auto y_view_err = make_view_1d(y_err);
-                auto x_view = make_view_1d(x);
-
-                double chi2 = 0;
-
-                aare::fit_pol1(x_view, y_view, y_view_err, par->view(),
-                               par_err->view(), chi2);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = chi2, "Ndf"_a = y.size() - 2);
-
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        R"(
-        Fit a 1D polynomial to data with error estimates.
-
-        Parameters
-        ----------
-        x : array_like
-            The x values.
-        y : array_like
-            The y values.
-        y_err : array_like
-            The error in the y values.
-        n_threads : int, optional
-            The number of threads to use. Default is 4.
-        )",
-        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
-
-    //=========
-    m.def(
-        "fit_scurve",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>{};
-
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_3d(y);
-                *par = aare::fit_scurve(x_view, y_view, n_threads);
-                return return_image_data(par);
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>{};
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_1d(y);
-                *par = aare::fit_scurve(x_view, y_view);
-                return return_image_data(par);
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_scurve",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 6});
-
-                auto par_err =
-                    new NDArray<double, 3>({y.shape(0), y.shape(1), 6});
-
-                auto y_view = make_view_3d(y);
-                auto y_view_err = make_view_3d(y_err);
-                auto x_view = make_view_1d(x);
-
-                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
-
-                aare::fit_scurve(x_view, y_view, y_view_err, par->view(),
-                                 par_err->view(), chi2->view(), n_threads);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = return_image_data(chi2),
-                                "Ndf"_a = y.shape(2) - 2);
-
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>({2});
-                auto par_err = new NDArray<double, 1>({2});
-
-                auto y_view = make_view_1d(y);
-                auto y_view_err = make_view_1d(y_err);
-                auto x_view = make_view_1d(x);
-
-                double chi2 = 0;
-
-                aare::fit_scurve(x_view, y_view, y_view_err, par->view(),
-                                 par_err->view(), chi2);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = chi2, "Ndf"_a = y.size() - 2);
-
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        R"(
-        Fit a 1D polynomial to data with error estimates.
-
-        Parameters
-        ----------
-        x : array_like
-            The x values.
-        y : array_like
-            The y values.
-        y_err : array_like
-            The error in the y values.
-        n_threads : int, optional
-            The number of threads to use. Default is 4.
-        )",
-        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_scurve2",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>{};
-
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_3d(y);
-                *par = aare::fit_scurve2(x_view, y_view, n_threads);
-                return return_image_data(par);
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>{};
-                auto x_view = make_view_1d(x);
-                auto y_view = make_view_1d(y);
-                *par = aare::fit_scurve2(x_view, y_view);
-                return return_image_data(par);
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        py::arg("x"), py::arg("y"), py::arg("n_threads") = 4);
-
-    m.def(
-        "fit_scurve2",
-        [](py::array_t<double, py::array::c_style | py::array::forcecast> x,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y,
-           py::array_t<double, py::array::c_style | py::array::forcecast> y_err,
-           int n_threads) {
-            if (y.ndim() == 3) {
-                auto par = new NDArray<double, 3>({y.shape(0), y.shape(1), 6});
-
-                auto par_err =
-                    new NDArray<double, 3>({y.shape(0), y.shape(1), 6});
-
-                auto y_view = make_view_3d(y);
-                auto y_view_err = make_view_3d(y_err);
-                auto x_view = make_view_1d(x);
-
-                auto chi2 = new NDArray<double, 2>({y.shape(0), y.shape(1)});
-
-                aare::fit_scurve2(x_view, y_view, y_view_err, par->view(),
-                                  par_err->view(), chi2->view(), n_threads);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = return_image_data(chi2),
-                                "Ndf"_a = y.shape(2) - 2);
-
-            } else if (y.ndim() == 1) {
-                auto par = new NDArray<double, 1>({6});
-                auto par_err = new NDArray<double, 1>({6});
-
-                auto y_view = make_view_1d(y);
-                auto y_view_err = make_view_1d(y_err);
-                auto x_view = make_view_1d(x);
-
-                double chi2 = 0;
-
-                aare::fit_scurve2(x_view, y_view, y_view_err, par->view(),
-                                  par_err->view(), chi2);
-                return py::dict("par"_a = return_image_data(par),
-                                "par_err"_a = return_image_data(par_err),
-                                "chi2"_a = chi2, "Ndf"_a = y.size() - 2);
-
-            } else {
-                throw std::runtime_error("Data must be 1D or 3D");
-            }
-        },
-        R"(
-        Fit a 1D polynomial to data with error estimates.
-
-        Parameters
-        ----------
-        x : array_like
-            The x values.
-        y : array_like
-            The y values.
-        y_err : array_like
-            The error in the y values.
-        n_threads : int, optional
-            The number of threads to use. Default is 4.
-        )",
-        py::arg("x"), py::arg("y"), py::arg("y_err"), py::arg("n_threads") = 4);
-
     // ── Bind model classes ──────────────────────────────────────────
     bind_fit_model<aare::model::Gaussian>(m, "Gaussian");
     bind_fit_model<aare::model::GaussianErfcPlateau>(m, "GaussianErfcPlateau");
@@ -794,4 +348,4 @@ void define_fit_bindings(py::module &m) {
         )",
         py::arg("model"), py::arg("x"), py::arg("y"),
         py::arg("y_err") = py::none(), py::arg("n_threads") = 4);
-}
+}

python/src/module.cpp

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MPL-2.0
 // Files with bindings to the different classes
 
+#include "module_config.hpp"
+
 // New style file naming
 #include "bind_Cluster.hpp"
 #include "bind_ClusterCollector.hpp"
@@ -12,7 +14,9 @@
 #include "bind_ClusterVector.hpp"
 #include "bind_Defs.hpp"
 #include "bind_Eta.hpp"
+#include "bind_FastPedestal.hpp"
 #include "bind_Interpolator.hpp"
+#include "bind_MultiThreadedFileReader.hpp"
 #include "bind_PedestalTrackingPixelHistogram.hpp"
 #include "bind_PixelHistogram.hpp"
 #include "bind_PixelMap.hpp"
@@ -30,6 +34,7 @@
 #include "var_cluster.hpp"
 
 // Pybind stuff
+#include <cstdint>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
@@ -61,7 +66,11 @@ double, 'f' for float)
     define_ClusterCollector<T, N, M, U>(m, "Cluster" #N "x" #M #TYPE_CODE);
 
 PYBIND11_MODULE(_aare, m) {
+    auto experimental = m.def_submodule(
+        "experimental", "Experimental APIs that may change without notice");
+
     define_file_io_bindings(m);
+    define_multi_threaded_file_reader_bindings(experimental);
     define_raw_file_io_bindings(m);
     define_raw_sub_file_io_bindings(m);
     define_ctb_raw_file_io_bindings(m);
@@ -72,6 +81,10 @@ PYBIND11_MODULE(_aare, m) {
     define_pedestal_tracking_pixel_histogram_bindings(m);
     define_pedestal_bindings<double>(m, "Pedestal_d");
     define_pedestal_bindings<float>(m, "Pedestal_f");
+    define_pedestal_bindings<int16_t>(m, "Pedestal_i16");
+    define_fast_pedestal_bindings<double>(m, "FastPedestal_d");
+    define_fast_pedestal_bindings<float>(m, "FastPedestal_f");
+    define_fast_pedestal_bindings<int16_t>(m, "FastPedestal_i16");
     define_fit_bindings(m);
     define_interpolation_bindings(m);
     define_jungfrau_data_file_io_bindings(m);
@@ -103,6 +116,7 @@ PYBIND11_MODULE(_aare, m) {
     DEFINE_BINDINGS_CLUSTERFINDER(int, 3, 3, uint16_t, i);
     DEFINE_BINDINGS_CLUSTERFINDER(double, 3, 3, uint16_t, d);
     DEFINE_BINDINGS_CLUSTERFINDER(float, 3, 3, uint16_t, f);
+    DEFINE_BINDINGS_CLUSTERFINDER(int16_t, 3, 3, uint16_t, i16);
 
     DEFINE_BINDINGS_CLUSTERFINDER(int, 5, 5, uint16_t, i);
     DEFINE_BINDINGS_CLUSTERFINDER(double, 5, 5, uint16_t, d);

python/src/module_config.hpp

@@ -0,0 +1,5 @@
+#pragma once
+#include <cstdint>
+
+// Configure module wide pedestal type for cluster finding
+using pd_type = double;

python/src/pedestal.hpp

@@ -52,6 +52,12 @@ void define_pedestal_bindings(py::module &m, const std::string &name) {
                  *std = self.std();
                  return return_image_data(std);
              })
+        .def("cached_std",
+             [](Pedestal<SUM_TYPE> &self) {
+                 auto standard_deviation = new NDArray<SUM_TYPE, 2>{};
+                 *standard_deviation = self.cached_std();
+                 return return_image_data(standard_deviation);
+             })
         .def(
             "__array_ufunc__",
             [](py::object self, py::object ufunc, const std::string &method,

python/src/raw_master_file.hpp

@@ -8,7 +8,6 @@
 #include "aare/RawSubFile.hpp"
 
 #include "aare/defs.hpp"
-// #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
 #include <filesystem>

python/src/raw_sub_file.hpp

@@ -7,7 +7,6 @@
 #include "aare/RawSubFile.hpp"
 
 #include "aare/defs.hpp"
-// #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
 #include <filesystem>

python/src/var_cluster.hpp

@@ -1,17 +1,11 @@
 // SPDX-License-Identifier: MPL-2.0
 #include "aare/VarClusterFinder.hpp"
 #include "np_helper.hpp"
-// #include "aare/defs.hpp"
-// #include "aare/fClusterFileV2.hpp"
 
 #include <cstdint>
-// #include <filesystem>
 #include <pybind11/numpy.h>
-// #include <pybind11/iostream.h>
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
-// #include <pybind11/stl/filesystem.h>
-// #include <string>
 
 namespace py = pybind11;
 using namespace ::aare;