From 4c6f88c905eae63cc3215eb101bcc048b635dad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Fr=C3=B6jdh?= Date: Fri, 6 Nov 2026 12:39:06 +0100 Subject: [PATCH] Dev/build time (#413) - Make zmq opt in since nothing uses it at the moment - Turn on unity builds for Minuit2, libzmq and Catch2. Specially for Minuit2 this seems to make a difference - Split python extension into multiple translation units - Drop LTO on the python ext. Saves 22s of link time (22-->0.1s on my laptop) --- .github/workflows/build_windows.yml | 2 +- .github/workflows/build_with_docs.yml | 7 +- AGENTS.md | 42 ++++- CMakeLists.txt | 126 ++++++++----- RELEASE.md | 25 +++ benchmarks/CMakeLists.txt | 2 +- docs/src/Installation.rst | 8 + docs/src/Requirements.rst | 2 +- pyproject.toml | 3 + python/CMakeLists.txt | 83 +++++++- python/src/bind_ClusterFileSink.hpp | 1 + .../{ctb_raw_file.hpp => bind_CtbRawFile.cpp} | 2 +- ...gation.hpp => bind_DataSetAggregation.cpp} | 12 +- python/src/{bind_Defs.hpp => bind_Defs.cpp} | 5 +- python/src/bind_Eta.cpp | 31 +++ python/src/bind_Eta.hpp | 2 +- ...FastPedestal.hpp => bind_FastPedestal.cpp} | 9 +- python/src/{file.hpp => bind_File.cpp} | 4 +- python/src/{fit.hpp => bind_Fit.cpp} | 4 +- ...{bind_GapPixels.hpp => bind_GapPixels.cpp} | 2 +- ...Interpolator.hpp => bind_Interpolator.cpp} | 4 +- ...ata_file.hpp => bind_JungfrauDataFile.cpp} | 2 +- ...r.hpp => bind_MultiThreadedFileReader.cpp} | 4 +- .../{bind_Pedestal.hpp => bind_Pedestal.cpp} | 8 +- ...> bind_PedestalTrackingPixelHistogram.cpp} | 3 +- ...lHistogram.hpp => bind_PixelHistogram.cpp} | 2 +- .../{bind_PixelMap.hpp => bind_PixelMap.cpp} | 4 +- .../{bind_RawFile.hpp => bind_RawFile.cpp} | 2 +- ...master_file.hpp => bind_RawMasterFile.cpp} | 2 +- .../{raw_sub_file.hpp => bind_RawSubFile.cpp} | 2 +- ..._cluster.hpp => bind_VarClusterFinder.cpp} | 2 +- ...d_calibration.hpp => bind_calibration.cpp} | 7 +- python/src/bind_cluster_2x2.cpp | 10 + python/src/bind_cluster_3x3.cpp | 28 +++ python/src/bind_cluster_5x5.cpp | 28 +++ python/src/bind_cluster_7x7.cpp | 28 +++ python/src/bind_cluster_9x9.cpp | 28 +++ python/src/bind_cluster_common.hpp | 38 ++++ .../{bind_testing.hpp => bind_testing.cpp} | 2 +- python/src/bindings.hpp | 79 ++++++++ python/src/module.cpp | 177 +++--------------- python/src/np_helper.hpp | 6 +- tests/CMakeLists.txt | 25 +++ 43 files changed, 619 insertions(+), 244 deletions(-) rename python/src/{ctb_raw_file.hpp => bind_CtbRawFile.cpp} (99%) rename python/src/{bind_DataSetAggregation.hpp => bind_DataSetAggregation.cpp} (93%) rename python/src/{bind_Defs.hpp => bind_Defs.cpp} (98%) create mode 100644 python/src/bind_Eta.cpp rename python/src/{bind_FastPedestal.hpp => bind_FastPedestal.cpp} (95%) rename python/src/{file.hpp => bind_File.cpp} (99%) rename python/src/{fit.hpp => bind_Fit.cpp} (99%) rename python/src/{bind_GapPixels.hpp => bind_GapPixels.cpp} (99%) rename python/src/{bind_Interpolator.hpp => bind_Interpolator.cpp} (99%) rename python/src/{jungfrau_data_file.hpp => bind_JungfrauDataFile.cpp} (99%) rename python/src/{bind_MultiThreadedFileReader.hpp => bind_MultiThreadedFileReader.cpp} (98%) rename python/src/{bind_Pedestal.hpp => bind_Pedestal.cpp} (95%) rename python/src/{bind_PedestalTrackingPixelHistogram.hpp => bind_PedestalTrackingPixelHistogram.cpp} (99%) rename python/src/{bind_PixelHistogram.hpp => bind_PixelHistogram.cpp} (99%) rename python/src/{bind_PixelMap.hpp => bind_PixelMap.cpp} (98%) rename python/src/{bind_RawFile.hpp => bind_RawFile.cpp} (99%) rename python/src/{raw_master_file.hpp => bind_RawMasterFile.cpp} (99%) rename python/src/{raw_sub_file.hpp => bind_RawSubFile.cpp} (99%) rename python/src/{var_cluster.hpp => bind_VarClusterFinder.cpp} (99%) rename python/src/{bind_calibration.hpp => bind_calibration.cpp} (98%) create mode 100644 python/src/bind_cluster_2x2.cpp create mode 100644 python/src/bind_cluster_3x3.cpp create mode 100644 python/src/bind_cluster_5x5.cpp create mode 100644 python/src/bind_cluster_7x7.cpp create mode 100644 python/src/bind_cluster_9x9.cpp create mode 100644 python/src/bind_cluster_common.hpp rename python/src/{bind_testing.hpp => bind_testing.cpp} (98%) create mode 100644 python/src/bindings.hpp diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml index 1a562f39..cc8303e0 100644 --- a/.github/workflows/build_windows.yml +++ b/.github/workflows/build_windows.yml @@ -37,7 +37,7 @@ jobs: # Precompiled headers are disabled: sccache cannot cache MSVC # compilations that use one - name: Configure - run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DAARE_TESTS=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_WARNINGS_AS_ERRORS=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON + run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DAARE_TESTS=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_PYTHON_UNITY_BUILD=ON -DAARE_WARNINGS_AS_ERRORS=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON - name: Build run: cmake --build build diff --git a/.github/workflows/build_with_docs.yml b/.github/workflows/build_with_docs.yml index 9e2dc872..8dbec646 100644 --- a/.github/workflows/build_with_docs.yml +++ b/.github/workflows/build_with_docs.yml @@ -52,12 +52,15 @@ jobs: version: "v0.18.0" # Precompiled headers are disabled because sccache does not cache - # compilations that use one + # compilations that use one. The python bindings are compiled as one + # translation unit per file (AARE_PYTHON_UNITY_BUILD=OFF, the local + # default) so every unit is checked to compile on its own; the + # packaging jobs build the grouped units instead. - name: Build library and docs run: | mkdir build cd build - cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DAARE_SYSTEM_LIBRARIES=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_DOCS=ON -DAARE_TESTS=ON -DAARE_WARNINGS_AS_ERRORS=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON + cmake .. -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DAARE_SYSTEM_LIBRARIES=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_PYTHON_UNITY_BUILD=OFF -DAARE_DOCS=ON -DAARE_TESTS=ON -DAARE_WARNINGS_AS_ERRORS=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON make -j 4 make docs diff --git a/AGENTS.md b/AGENTS.md index 5646aaae..dfd5f687 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,10 +49,16 @@ cmake --build build -j4 Useful optional settings include `AARE_DOCS`, `AARE_BENCHMARKS`, `AARE_ASAN`, `AARE_WARNINGS_AS_ERRORS`, `AARE_VERBOSE` (raises the compile-time log -level), and `AARE_TUNE_LOCAL` (`-march=native`, not portable). Reconfigure an -existing build directory instead of creating alternate in-tree build layouts -unless isolation is needed; `build/CMakeCache.txt` records the options and -interpreter it was configured with. +level), `AARE_TUNE_LOCAL` (`-march=native`, not portable), `AARE_ZMQ` +(fetches or finds ZeroMQ and defines `aare::zmq`; nothing links it yet), and +`AARE_PYTHON_UNITY_BUILD` (compiles the python bindings as one translation +unit per binding area instead of one per class; less total compiler time, +less parallelism; `pip install` and the packaging CI jobs turn it on, the +`build_with_docs` job keeps it off so every unit is checked to compile on +its own). +Reconfigure an existing build directory instead of creating alternate +in-tree build layouts unless isolation is needed; `build/CMakeCache.txt` +records the options and interpreter it was configured with. `-Werror=return-type` is always enabled. New code should also compile cleanly under the project's `-Wall -Wextra -pedantic -Wshadow -Wold-style-cast @@ -169,13 +175,28 @@ agree; run them when changing any solver. ### Python layer -- `python/src/module.cpp` registers everything. Templated classes are - instantiated per value type and cluster size through the - `DEFINE_CLUSTER_BINDINGS` and `DEFINE_BINDINGS_CLUSTERFINDER` macros, +- `python/src/module.cpp` only calls the registration entry points declared + in `bindings.hpp`, in a fixed order (overload docstrings follow it). Each + `bind_.cpp` is one translation unit with the bindings of one class + or function group (`bind_RawFile.cpp`, `bind_Fit.cpp`, + `bind_calibration.cpp`, ...) and defines the entry point it declares in + `bindings.hpp`. Headers hold only templates that several units + instantiate: the `bind_Cluster*.hpp` set and `bind_Eta.hpp`, included + through `bind_cluster_common.hpp` by one `bind_cluster_NxN.cpp` per + cluster size and by `bind_Eta.cpp`. Templated classes are instantiated + per value type and cluster size through the `DEFINE_CLUSTER_BINDINGS` and + `DEFINE_BINDINGS_CLUSTERFINDER` macros in `bind_cluster_common.hpp`, producing names such as `ClusterFinder_Cluster3x3i`, `Pedestal_d`, and `FastPedestal_i16`. Supported cluster sizes are 2x2, 3x3, 5x5, 7x7, and 9x9; value types are `i`, `f`, `d`, and `i16` (3x3 only). - `module_config.hpp` fixes the module-wide pedestal type (`pd_type`). + `module_config.hpp` fixes the module-wide pedestal type (`pd_type`). A + shared header must include everything it uses itself, including the + pybind11 casters (`pybind11/stl.h`, `pybind11/stl/filesystem.h`) for the + types its bindings take or return, and non-template functions it defines + must be `inline`. `python/CMakeLists.txt` lists the units in groups + (`io`, `core`, `hist`, `pedestal`, `fit`); with `AARE_PYTHON_UNITY_BUILD` + the units of a group are concatenated into one, so file-scope names + (helpers, anonymous namespaces, macros) must not clash within a group. - `python/aare/factory.py` maps NumPy dtypes to those suffixes. The facades in `python/aare/*.py` are thin factory functions or subclasses that select the right binding and add Pythonic conveniences (iteration, context @@ -223,7 +244,10 @@ A Python-facing feature can require coordinated changes in several layers: 1. Update the C++ public API and implementation. 2. Add or update its binding in `python/src/`. -3. Register new bindings in `python/src/module.cpp`. +3. Register new bindings: put them in the matching `python/src/bind_*.cpp` + or add a new unit to its group in `python/CMakeLists.txt`, declare the + entry point in `python/src/bindings.hpp`, and call it from + `python/src/module.cpp`. 4. Update the facade or public exports in `python/aare/`. 5. If adding a Python module, add it to `PYTHON_FILES` in `python/CMakeLists.txt` so it is copied and installed. diff --git a/CMakeLists.txt b/CMakeLists.txt index 84d65e3c..8595562f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,10 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) # General options option(AARE_PYTHON_BINDINGS "Build python bindings" OFF) +option( + AARE_PYTHON_UNITY_BUILD + "Compile the python bindings as one translation unit per binding area instead of one per class: less total compiler time, fewer units to build in parallel" + OFF) option(AARE_TESTS "Build tests" OFF) option(AARE_BENCHMARKS "Build benchmarks" OFF) option(AARE_EXAMPLES "Build examples" OFF) @@ -65,6 +69,10 @@ option(AARE_FETCH_FMT "Use FetchContent to download fmt" ON) option(AARE_FETCH_PYBIND11 "Use FetchContent to download pybind11" ON) option(AARE_FETCH_CATCH "Use FetchContent to download catch2" ON) option(AARE_FETCH_JSON "Use FetchContent to download nlohmann::json" ON) +option( + AARE_ZMQ + "Fetch or find ZeroMQ. Nothing uses it yet; kept for upcoming streaming support" + OFF) option(AARE_FETCH_ZMQ "Use FetchContent to download libzmq" ON) option(AARE_FETCH_MINUIT2 "Use FetchContent to download Minuit2" ON) @@ -107,6 +115,7 @@ if(AARE_FETCH_MINUIT2) Minuit2 GIT_REPOSITORY https://github.com/GooFit/Minuit2.git GIT_TAG v6-40-02 + GIT_SHALLOW TRUE UPDATE_DISCONNECTED 1) # Disable Minuit2 extras we don't need set(minuit2_mpi @@ -124,6 +133,10 @@ if(AARE_FETCH_MINUIT2) FetchContent_MakeAvailable(Minuit2) set_property(TARGET Minuit2Math PROPERTY POSITION_INDEPENDENT_CODE ON) + # Minuit2 is ~60 small files; compiling them in batches is several times + # faster + set_target_properties(Minuit2 Minuit2Math + PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 16) set_property(TARGET Minuit2 PROPERTY POSITION_INDEPENDENT_CODE ON) add_library(aare::Minuit2 ALIAS Minuit2) @@ -144,53 +157,75 @@ else() endif() endif() -if(AARE_FETCH_ZMQ) - # Fetchcontent_Declare is deprecated need to find a way to update this for now - # setting the policy to old is enough - if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30") - cmake_policy(SET CMP0169 OLD) - endif() - set(ZMQ_PATCH_COMMAND - git apply ${CMAKE_CURRENT_SOURCE_DIR}/patches/libzmq_cmake_version.patch) - FetchContent_Declare( - libzmq - GIT_REPOSITORY https://github.com/zeromq/libzmq.git - GIT_TAG v4.3.5 - PATCH_COMMAND ${ZMQ_PATCH_COMMAND} - UPDATE_DISCONNECTED 1) - # Disable unwanted options from libzmq - set(BUILD_TESTS - OFF - CACHE BOOL "Switch off libzmq test build") - set(BUILD_SHARED - OFF - CACHE BOOL "Switch off libzmq shared libs") - set(WITH_PERF_TOOL - OFF - CACHE BOOL "") - set(ENABLE_CPACK - OFF - CACHE BOOL "") - set(ENABLE_CLANG - OFF - CACHE BOOL "") - set(ENABLE_CURVE - OFF - CACHE BOOL "") - set(ENABLE_DRAFTS - OFF - CACHE BOOL "") +# ZeroMQ is not linked anywhere at the moment, so it is only fetched and +# configured on request (saves ~8 s per configure and a 25 MB clone). Enable it +# with -DAARE_ZMQ=ON and link aare::zmq where it is needed. +if(AARE_ZMQ) + if(AARE_FETCH_ZMQ) + # Fetchcontent_Declare is deprecated need to find a way to update this for + # now setting the policy to old is enough + if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.30") + cmake_policy(SET CMP0169 OLD) + endif() + set(ZMQ_PATCH_COMMAND + git apply + ${CMAKE_CURRENT_SOURCE_DIR}/patches/libzmq_cmake_version.patch) + FetchContent_Declare( + libzmq + GIT_REPOSITORY https://github.com/zeromq/libzmq.git + GIT_TAG v4.3.5 + GIT_SHALLOW TRUE + PATCH_COMMAND ${ZMQ_PATCH_COMMAND} + UPDATE_DISCONNECTED 1) + # Disable unwanted options from libzmq + set(BUILD_TESTS + OFF + CACHE BOOL "Switch off libzmq test build") + set(BUILD_SHARED + OFF + CACHE BOOL "Switch off libzmq shared libs") + set(WITH_PERF_TOOL + OFF + CACHE BOOL "") + set(ENABLE_CPACK + OFF + CACHE BOOL "") + set(ENABLE_CLANG + OFF + CACHE BOOL "") + set(ENABLE_CURVE + OFF + CACHE BOOL "") + set(ENABLE_DRAFTS + OFF + CACHE BOOL "") - # TODO! Verify that this is what we want to do in aare Using GetProperties and - # Populate to be able to exclude zmq from install (not possible with - # FetchContent_MakeAvailable(libzmq)) - FetchContent_GetProperties(libzmq) - if(NOT libzmq_POPULATED) - FetchContent_Populate(libzmq) - add_subdirectory(${libzmq_SOURCE_DIR} ${libzmq_BINARY_DIR} EXCLUDE_FROM_ALL) + # TODO! Verify that this is what we want to do in aare Using GetProperties + # and Populate to be able to exclude zmq from install (not possible with + # FetchContent_MakeAvailable(libzmq)) + FetchContent_GetProperties(libzmq) + if(NOT libzmq_POPULATED) + FetchContent_Populate(libzmq) + add_subdirectory(${libzmq_SOURCE_DIR} ${libzmq_BINARY_DIR} + EXCLUDE_FROM_ALL) + # libzmq compiles its sources into the OBJECT library "objects" (on + # Windows directly into libzmq-static); batch them like Minuit2 + foreach(_zmq_target objects libzmq-static) + if(TARGET ${_zmq_target}) + set_target_properties( + ${_zmq_target} PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 16) + endif() + endforeach() + endif() + else() + find_package(ZeroMQ 4 REQUIRED) endif() -else() - find_package(ZeroMQ 4 REQUIRED) + if(TARGET libzmq-static) + add_library(aare::zmq ALIAS libzmq-static) + else() + add_library(aare::zmq ALIAS libzmq) + endif() + message(STATUS "ZeroMQ enabled (AARE_ZMQ=ON), link aare::zmq to use it") endif() if(AARE_FETCH_FMT) @@ -201,6 +236,7 @@ if(AARE_FETCH_FMT) fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git GIT_TAG 12.1.0 + GIT_SHALLOW TRUE GIT_PROGRESS TRUE USES_TERMINAL_DOWNLOAD TRUE) set(FMT_INSTALL diff --git a/RELEASE.md b/RELEASE.md index 74b65aa2..34e36555 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -88,6 +88,31 @@ read through ``CtbRawFile``. The transforms now flatten the frame before calling the decoders. +### Build: + +- libzmq is only fetched or searched for with ``-DAARE_ZMQ=ON`` (default + OFF) and is then available as ``aare::zmq``. Nothing in aare links it yet, + and skipping it removes about 8 s from every CMake configure. +- Minuit2, Catch2, and libzmq when enabled, are compiled as unity builds + (about 4-5x less compile time), and the fetched dependencies are shallow + clones, which makes a first configure faster and the build directory + smaller. +- The Python extension is compiled from one translation unit per bound + class or function group instead of a single file, so the bindings build + in parallel and editing one binding recompiles only its unit. + ``-DAARE_PYTHON_UNITY_BUILD=ON`` (used by ``pip install`` and the + packaging CI jobs) + combines the units of each binding area into one, which needs less total + compiler time on machines with few cores. The module's contents and + registration order are unchanged. +- The Python extension is no longer link-time optimized by default. The hot + loops are header templates compiled at ``-O3`` inside the extension and + LTO cannot reach the static ``aare_core``, so it only added a 20 s + single-threaded link and made the binary 15 % smaller. LTO follows the + standard CMake property now: + ``set_property(TARGET _aare PROPERTY INTERPROCEDURAL_OPTIMIZATION ON)`` or + ``-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON`` restores it. + ## 2026.9.30 ### New Features: diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 76732411..3b4f2f10 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -6,7 +6,7 @@ FetchContent_Declare( benchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.8.3 # Change to the latest version if needed -) + GIT_SHALLOW TRUE) # Ensure Google Benchmark is built correctly set(BENCHMARK_ENABLE_TESTING diff --git a/docs/src/Installation.rst b/docs/src/Installation.rst index 2cd8c681..eb64a216 100644 --- a/docs/src/Installation.rst +++ b/docs/src/Installation.rst @@ -109,6 +109,14 @@ Build the Python bindings. Default option is off. If you have a newer system Python compared to the one in your virtual environment, you might have to pass -DPython_FIND_VIRTUALENV=ONLY to cmake. +**AARE_PYTHON_UNITY_BUILD "Compile the python bindings as one translation unit per binding area" OFF** + +Compile the Python bindings as one translation unit per binding area (file readers, +pedestals, clusters of one size, ...) instead of one per bound class. The build then +needs less total compiler time but has fewer units to compile in parallel, which is +faster on machines with few cores. Default option is off; ``pip install`` and the packaging +CI jobs turn it on. + **AARE_TESTS "Build tests" OFF** Build unit tests. Default option is off. diff --git a/docs/src/Requirements.rst b/docs/src/Requirements.rst index 7c44dbdc..c75dfdf1 100644 --- a/docs/src/Requirements.rst +++ b/docs/src/Requirements.rst @@ -17,7 +17,7 @@ To simplify deployment we build and statically link a few libraries. - Minuit2 - nlohmann_json - pybind11 -- ZeroMQ +- ZeroMQ (only with ``-DAARE_ZMQ=ON``; nothing uses it yet) **Extra dependencies for building documentation** diff --git a/pyproject.toml b/pyproject.toml index e1176dcd..4e557c4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ install.components = ["python"] [tool.scikit-build.cmake.define] AARE_PYTHON_BINDINGS = "ON" +# A pip build is compiled once and never incrementally, so fewer, larger +# translation units only save compiler time +AARE_PYTHON_UNITY_BUILD = "ON" AARE_INSTALL_PYTHONEXT = "ON" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7f4db4c0..410564b7 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -11,22 +11,97 @@ if(AARE_FETCH_PYBIND11) FetchContent_Declare( pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v3.0.3) + GIT_TAG v3.0.3 + GIT_SHALLOW TRUE) FetchContent_MakeAvailable(pybind11) else() find_package(pybind11 3.0.3 REQUIRED) endif() -# Add the compiled python extension +# The compiled python extension. Each bind_*.cpp holds the bindings of one class +# or function group and is its own translation unit, so the units compile in +# parallel and an edit recompiles only its file. The templated cluster bindings +# are instantiated per cluster size in bind_cluster_NxN.cpp; src/module.cpp +# registers everything in a fixed order. +# +# AARE_PYTHON_UNITY_BUILD combines the units of each group below into one. That +# trades parallelism and per-file rebuilds for less total compiler time, which +# is what counts on machines with few cores; pip builds and the packaging CI +# jobs turn it on. The build_with_docs CI job keeps it off, so every unit is +# also checked to compile on its own. +# +# Without NO_EXTRAS pybind11 links pybind11::lto, which forces full LTO on the +# extension regardless of the target's INTERPROCEDURAL_OPTIMIZATION property. +# LTO cannot reach aare_core (a plain static library) and the templated hot +# paths are already compiled at -O3 inside the extension, so it only adds a long +# single-threaded link step. With NO_EXTRAS the standard target property decides +# (off by default); the other extras are re-added below. +set(_aare_py_io + src/bind_File.cpp + src/bind_RawFile.cpp + src/bind_RawSubFile.cpp + src/bind_RawMasterFile.cpp + src/bind_CtbRawFile.cpp + src/bind_JungfrauDataFile.cpp + src/bind_MultiThreadedFileReader.cpp + src/bind_DataSetAggregation.cpp) +set(_aare_py_core src/bind_Defs.cpp src/bind_GapPixels.cpp + src/bind_PixelMap.cpp src/bind_testing.cpp) +set(_aare_py_hist src/bind_PixelHistogram.cpp + src/bind_PedestalTrackingPixelHistogram.cpp) +set(_aare_py_pedestal src/bind_Pedestal.cpp src/bind_FastPedestal.cpp + src/bind_calibration.cpp) +set(_aare_py_fit src/bind_Fit.cpp src/bind_Interpolator.cpp) +set(_aare_py_groups io core hist pedestal fit) + pybind11_add_module( _aare # name of the module - src/module.cpp # source file -) + NO_EXTRAS + src/module.cpp + # the cluster units take longest, listing them first schedules them first + src/bind_cluster_3x3.cpp + src/bind_cluster_9x9.cpp + src/bind_cluster_7x7.cpp + src/bind_cluster_5x5.cpp + src/bind_cluster_2x2.cpp + ${_aare_py_io} + ${_aare_py_core} + ${_aare_py_hist} + ${_aare_py_pedestal} + ${_aare_py_fit} + src/bind_Eta.cpp + src/bind_VarClusterFinder.cpp) + +if(AARE_PYTHON_UNITY_BUILD) + if(CMAKE_VERSION VERSION_LESS 3.18) + message( + WARNING + "AARE_PYTHON_UNITY_BUILD needs CMake 3.18 or newer; compiling the python bindings as individual translation units" + ) + else() + set_target_properties(_aare PROPERTIES UNITY_BUILD ON UNITY_BUILD_MODE + GROUP) + foreach(group IN LISTS _aare_py_groups) + set_source_files_properties(${_aare_py_${group}} PROPERTIES UNITY_GROUP + ${group}) + endforeach() + endif() +endif() set_target_properties(_aare PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) target_link_libraries(_aare PRIVATE aare_core aare_compiler_flags) +# The pybind11_add_module extras that NO_EXTRAS skipped, LTO excepted +if(MSVC) + target_link_libraries(_aare PRIVATE pybind11::windows_extras) # /bigobj, /MP +elseif(CMAKE_BUILD_TYPE) + string(TOUPPER "${CMAKE_BUILD_TYPE}" _aare_build_type) + if(NOT _aare_build_type MATCHES "DEBUG|RELWITHDEBINFO|NONE") + pybind11_strip(_aare) # strip symbols from optimized builds + endif() +endif() + target_include_directories( _aare SYSTEM PRIVATE $) diff --git a/python/src/bind_ClusterFileSink.hpp b/python/src/bind_ClusterFileSink.hpp index 48820551..b6892b1c 100644 --- a/python/src/bind_ClusterFileSink.hpp +++ b/python/src/bind_ClusterFileSink.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace py = pybind11; diff --git a/python/src/ctb_raw_file.hpp b/python/src/bind_CtbRawFile.cpp similarity index 99% rename from python/src/ctb_raw_file.hpp rename to python/src/bind_CtbRawFile.cpp index 1808c01d..368f587f 100644 --- a/python/src/ctb_raw_file.hpp +++ b/python/src/bind_CtbRawFile.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/CtbRawFile.hpp" #include "aare/File.hpp" diff --git a/python/src/bind_DataSetAggregation.hpp b/python/src/bind_DataSetAggregation.cpp similarity index 93% rename from python/src/bind_DataSetAggregation.hpp rename to python/src/bind_DataSetAggregation.cpp index a12f2fb4..6e352c9c 100644 --- a/python/src/bind_DataSetAggregation.hpp +++ b/python/src/bind_DataSetAggregation.cpp @@ -1,6 +1,11 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + #include "aare/DataSetAggregation.hpp" +#include "np_helper.hpp" #include +#include #include #include @@ -75,4 +80,9 @@ void bind_read_current_source(pybind11::module_ &m) { -------- np.ndarray: A numpy array containing the mean frame stored as np.float64 for each current source with shape (rows, cols, num_sources). )pbdoc"); -} \ No newline at end of file +} + +void bind_dataset_aggregation(py::module &m) { + bind_DataSetAggregation(m); + bind_read_current_source(m); +} diff --git a/python/src/bind_Defs.hpp b/python/src/bind_Defs.cpp similarity index 98% rename from python/src/bind_Defs.hpp rename to python/src/bind_Defs.cpp index 489ca052..dcdd2800 100644 --- a/python/src/bind_Defs.hpp +++ b/python/src/bind_Defs.cpp @@ -1,11 +1,14 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include #include #include +#include "aare/ROI.hpp" #include "aare/defs.hpp" +#include + namespace py = pybind11; using namespace aare; diff --git a/python/src/bind_Eta.cpp b/python/src/bind_Eta.cpp new file mode 100644 index 00000000..707ac960 --- /dev/null +++ b/python/src/bind_Eta.cpp @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_Eta.hpp" + +#include +#include + +void bind_eta_types(py::module &m) { + using Sum_index_pair_d = Sum_index_pair; + PYBIND11_NUMPY_DTYPE(Sum_index_pair_d, sum, index); + using Sum_index_pair_f = Sum_index_pair; + PYBIND11_NUMPY_DTYPE(Sum_index_pair_f, sum, index); + using Sum_index_pair_i = Sum_index_pair; + PYBIND11_NUMPY_DTYPE(Sum_index_pair_i, sum, index); + + using eta_d = Eta2; + PYBIND11_NUMPY_DTYPE(eta_d, x, y, c, sum); + using eta_i = Eta2; + PYBIND11_NUMPY_DTYPE(eta_i, x, y, c, sum); + using eta_f = Eta2; + PYBIND11_NUMPY_DTYPE(eta_f, x, y, c, sum); + using eta_i16 = Eta2; + PYBIND11_NUMPY_DTYPE(eta_i16, x, y, c, sum); + + define_corner_enum(m); + define_eta(m, "f"); + define_eta(m, "d"); + define_eta(m, "i"); + define_eta(m, "i16"); +} diff --git a/python/src/bind_Eta.hpp b/python/src/bind_Eta.hpp index 1f2dc32b..868e7d37 100644 --- a/python/src/bind_Eta.hpp +++ b/python/src/bind_Eta.hpp @@ -29,7 +29,7 @@ void define_eta(py::module &m, const std::string &typestr) { }); } -void define_corner_enum(py::module &m) { +inline void define_corner_enum(py::module &m) { py::enum_(m, "corner", "enum.Enum") .value("cTopLeft", corner::cTopLeft) .value("cTopRight", corner::cTopRight) diff --git a/python/src/bind_FastPedestal.hpp b/python/src/bind_FastPedestal.cpp similarity index 95% rename from python/src/bind_FastPedestal.hpp rename to python/src/bind_FastPedestal.cpp index 8bba55ea..46574f9f 100644 --- a/python/src/bind_FastPedestal.hpp +++ b/python/src/bind_FastPedestal.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/FastPedestal.hpp" #include "np_helper.hpp" @@ -8,6 +8,7 @@ #include #include #include +#include namespace py = pybind11; @@ -134,3 +135,9 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) { "the next n_samples for initialization, then apply every remaining " "frame as a steady-state update."); } + +void bind_fast_pedestals(py::module &m) { + define_fast_pedestal_bindings(m, "FastPedestal_d"); + define_fast_pedestal_bindings(m, "FastPedestal_f"); + define_fast_pedestal_bindings(m, "FastPedestal_i16"); +} diff --git a/python/src/file.hpp b/python/src/bind_File.cpp similarity index 99% rename from python/src/file.hpp rename to python/src/bind_File.cpp index 96960d91..09d6de03 100644 --- a/python/src/file.hpp +++ b/python/src/bind_File.cpp @@ -1,5 +1,4 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/CtbRawFile.hpp" #include "aare/File.hpp" #include "aare/Frame.hpp" @@ -7,6 +6,7 @@ #include "aare/RawFile.hpp" #include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" +#include "bindings.hpp" #include "aare/defs.hpp" @@ -159,4 +159,4 @@ void define_file_io_bindings(py::module &m) { .def_property_readonly("start", &ScanParameters::start) .def_property_readonly("stop", &ScanParameters::stop) .def_property_readonly("step", &ScanParameters::step); -} \ No newline at end of file +} diff --git a/python/src/fit.hpp b/python/src/bind_Fit.cpp similarity index 99% rename from python/src/fit.hpp rename to python/src/bind_Fit.cpp index 4d6e3835..0e762604 100644 --- a/python/src/fit.hpp +++ b/python/src/bind_Fit.cpp @@ -1,7 +1,9 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" +#include "np_helper.hpp" #include #include +#include #include #include #include diff --git a/python/src/bind_GapPixels.hpp b/python/src/bind_GapPixels.cpp similarity index 99% rename from python/src/bind_GapPixels.hpp rename to python/src/bind_GapPixels.cpp index 41874ca5..caaa751a 100644 --- a/python/src/bind_GapPixels.hpp +++ b/python/src/bind_GapPixels.cpp @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/GapPixels.hpp" #include "aare/ROI.hpp" #include "aare/defs.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include diff --git a/python/src/bind_Interpolator.hpp b/python/src/bind_Interpolator.cpp similarity index 99% rename from python/src/bind_Interpolator.hpp rename to python/src/bind_Interpolator.cpp index de391f3b..524f1c6a 100644 --- a/python/src/bind_Interpolator.hpp +++ b/python/src/bind_Interpolator.cpp @@ -1,9 +1,9 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/CalculateEta.hpp" #include "aare/Interpolator.hpp" #include "aare/NDArray.hpp" #include "aare/NDView.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include #include @@ -209,4 +209,4 @@ void define_interpolation_bindings(py::module &m) { par : array_like The parameters of the Gaussian function. The first element is the amplitude, the second element is the mean, and the third element is the standard deviation. )"); -} \ No newline at end of file +} diff --git a/python/src/jungfrau_data_file.hpp b/python/src/bind_JungfrauDataFile.cpp similarity index 99% rename from python/src/jungfrau_data_file.hpp rename to python/src/bind_JungfrauDataFile.cpp index 84f4e6f4..41ab9743 100644 --- a/python/src/jungfrau_data_file.hpp +++ b/python/src/bind_JungfrauDataFile.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/JungfrauDataFile.hpp" #include "aare/defs.hpp" diff --git a/python/src/bind_MultiThreadedFileReader.hpp b/python/src/bind_MultiThreadedFileReader.cpp similarity index 98% rename from python/src/bind_MultiThreadedFileReader.hpp rename to python/src/bind_MultiThreadedFileReader.cpp index 0f39cb6c..2de55c7d 100644 --- a/python/src/bind_MultiThreadedFileReader.hpp +++ b/python/src/bind_MultiThreadedFileReader.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/MultiThreadedFileReader.hpp" @@ -71,7 +71,7 @@ multi_threaded_reader_read(aare::experimental::MultiThreadedFileReader &reader, return image; } -inline void define_multi_threaded_file_reader_bindings(py::module_ &m) { +void define_multi_threaded_file_reader_bindings(py::module_ &m) { using aare::experimental::MultiThreadedFileReader; auto reader = diff --git a/python/src/bind_Pedestal.hpp b/python/src/bind_Pedestal.cpp similarity index 95% rename from python/src/bind_Pedestal.hpp rename to python/src/bind_Pedestal.cpp index 02b4d32f..479ba084 100644 --- a/python/src/bind_Pedestal.hpp +++ b/python/src/bind_Pedestal.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/Pedestal.hpp" #include "np_helper.hpp" @@ -120,3 +120,9 @@ void define_pedestal_bindings(py::module &m, const std::string &name) { true); }); } + +void bind_pedestals(py::module &m) { + define_pedestal_bindings(m, "Pedestal_d"); + define_pedestal_bindings(m, "Pedestal_f"); + define_pedestal_bindings(m, "Pedestal_i16"); +} diff --git a/python/src/bind_PedestalTrackingPixelHistogram.hpp b/python/src/bind_PedestalTrackingPixelHistogram.cpp similarity index 99% rename from python/src/bind_PedestalTrackingPixelHistogram.hpp rename to python/src/bind_PedestalTrackingPixelHistogram.cpp index 1d650b26..4b8cf925 100644 --- a/python/src/bind_PedestalTrackingPixelHistogram.hpp +++ b/python/src/bind_PedestalTrackingPixelHistogram.cpp @@ -1,12 +1,13 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/hist/PedestalTrackingPixelHistogram.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include #include #include #include +#include namespace py = pybind11; using namespace ::aare; diff --git a/python/src/bind_PixelHistogram.hpp b/python/src/bind_PixelHistogram.cpp similarity index 99% rename from python/src/bind_PixelHistogram.hpp rename to python/src/bind_PixelHistogram.cpp index b0c97e35..c013cd9c 100644 --- a/python/src/bind_PixelHistogram.hpp +++ b/python/src/bind_PixelHistogram.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/hist/PixelHistogram.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include diff --git a/python/src/bind_PixelMap.hpp b/python/src/bind_PixelMap.cpp similarity index 98% rename from python/src/bind_PixelMap.hpp rename to python/src/bind_PixelMap.cpp index 6aec76c3..a4fb15c1 100644 --- a/python/src/bind_PixelMap.hpp +++ b/python/src/bind_PixelMap.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/PixelMap.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include @@ -65,4 +65,4 @@ void define_pixel_map_bindings(py::module &m) { py::arg("dynamic_range") = 16, py::arg("n_counters") = 1, R"( Generate pixel map for Matterhorn02 detector)"); -} \ No newline at end of file +} diff --git a/python/src/bind_RawFile.hpp b/python/src/bind_RawFile.cpp similarity index 99% rename from python/src/bind_RawFile.hpp rename to python/src/bind_RawFile.cpp index f3b2d672..fbdea295 100644 --- a/python/src/bind_RawFile.hpp +++ b/python/src/bind_RawFile.cpp @@ -1,5 +1,4 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/CtbRawFile.hpp" #include "aare/File.hpp" #include "aare/Frame.hpp" @@ -7,6 +6,7 @@ #include "aare/RawFile.hpp" #include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" +#include "bindings.hpp" #include "aare/defs.hpp" #include "np_helper.hpp" diff --git a/python/src/raw_master_file.hpp b/python/src/bind_RawMasterFile.cpp similarity index 99% rename from python/src/raw_master_file.hpp rename to python/src/bind_RawMasterFile.cpp index 5a0b049d..9043a300 100644 --- a/python/src/raw_master_file.hpp +++ b/python/src/bind_RawMasterFile.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once +#include "bindings.hpp" #include "aare/CtbRawFile.hpp" #include "aare/File.hpp" diff --git a/python/src/raw_sub_file.hpp b/python/src/bind_RawSubFile.cpp similarity index 99% rename from python/src/raw_sub_file.hpp rename to python/src/bind_RawSubFile.cpp index 66ec7d8b..25885bcd 100644 --- a/python/src/raw_sub_file.hpp +++ b/python/src/bind_RawSubFile.cpp @@ -1,11 +1,11 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/CtbRawFile.hpp" #include "aare/File.hpp" #include "aare/Frame.hpp" #include "aare/RawFile.hpp" #include "aare/RawMasterFile.hpp" #include "aare/RawSubFile.hpp" +#include "bindings.hpp" #include "aare/defs.hpp" diff --git a/python/src/var_cluster.hpp b/python/src/bind_VarClusterFinder.cpp similarity index 99% rename from python/src/var_cluster.hpp rename to python/src/bind_VarClusterFinder.cpp index 9aade712..980df083 100644 --- a/python/src/var_cluster.hpp +++ b/python/src/bind_VarClusterFinder.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/VarClusterFinder.hpp" +#include "bindings.hpp" #include "np_helper.hpp" #include diff --git a/python/src/bind_calibration.hpp b/python/src/bind_calibration.cpp similarity index 98% rename from python/src/bind_calibration.hpp rename to python/src/bind_calibration.cpp index 3cfc960e..c93d77f0 100644 --- a/python/src/bind_calibration.hpp +++ b/python/src/bind_calibration.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 -#pragma once #include "aare/calibration.hpp" +#include "bindings.hpp" +#include "np_helper.hpp" #include #include @@ -10,7 +11,7 @@ #include namespace py = pybind11; -using aare::ssize_t; +using namespace aare; template py::array_t @@ -164,4 +165,4 @@ void bind_calibration(py::module &m) { The number of threads to use for the calculation. )", py::arg("raw_data").noconvert(), py::arg("n_threads") = 4); -} \ No newline at end of file +} diff --git a/python/src/bind_cluster_2x2.cpp b/python/src/bind_cluster_2x2.cpp new file mode 100644 index 00000000..e7f93a3a --- /dev/null +++ b/python/src/bind_cluster_2x2.cpp @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_cluster_common.hpp" + +void bind_cluster_types_2x2(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int, 2, 2, uint16_t, i); + DEFINE_CLUSTER_BINDINGS(double, 2, 2, uint16_t, d); + DEFINE_CLUSTER_BINDINGS(float, 2, 2, uint16_t, f); +} diff --git a/python/src/bind_cluster_3x3.cpp b/python/src/bind_cluster_3x3.cpp new file mode 100644 index 00000000..d59eac31 --- /dev/null +++ b/python/src/bind_cluster_3x3.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_cluster_common.hpp" + +void bind_cluster_types_3x3(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int, 3, 3, uint16_t, i); + DEFINE_CLUSTER_BINDINGS(double, 3, 3, uint16_t, d); + DEFINE_CLUSTER_BINDINGS(float, 3, 3, uint16_t, f); +} + +void bind_cluster_types_3x3_i16(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int16_t, 3, 3, uint16_t, i16); +} + +void bind_cluster_finders_3x3(py::module &m) { + DEFINE_BINDINGS_CLUSTERFINDER(int, 3, 3, uint16_t, i); + DEFINE_BINDINGS_CLUSTERFINDER(double, 3, 3, uint16_t, d); + DEFINE_BINDINGS_CLUSTERFINDER(float, 3, 3, uint16_t, f); + DEFINE_BINDINGS_CLUSTERFINDER(int16_t, 3, 3, uint16_t, i16); +} + +void bind_eta_3x3(py::module &m) { + register_calculate_3x3eta(m); + register_calculate_3x3eta(m); + register_calculate_3x3eta(m); + register_calculate_3x3eta(m); +} diff --git a/python/src/bind_cluster_5x5.cpp b/python/src/bind_cluster_5x5.cpp new file mode 100644 index 00000000..29adec8a --- /dev/null +++ b/python/src/bind_cluster_5x5.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_cluster_common.hpp" + +void bind_cluster_types_5x5(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int, 5, 5, uint16_t, i); + DEFINE_CLUSTER_BINDINGS(double, 5, 5, uint16_t, d); + DEFINE_CLUSTER_BINDINGS(float, 5, 5, uint16_t, f); +} + +void bind_cluster_finders_5x5(py::module &m) { + DEFINE_BINDINGS_CLUSTERFINDER(int, 5, 5, uint16_t, i); + DEFINE_BINDINGS_CLUSTERFINDER(double, 5, 5, uint16_t, d); + DEFINE_BINDINGS_CLUSTERFINDER(float, 5, 5, uint16_t, f); +} + +void bind_cluster_vector_reductions_5x5(py::module &m) { + define_3x3_reduction(m); + define_3x3_reduction(m); + define_3x3_reduction(m); +} + +void bind_cluster_reductions_5x5(py::module &m) { + reduce_to_3x3(m); + reduce_to_3x3(m); + reduce_to_3x3(m); +} diff --git a/python/src/bind_cluster_7x7.cpp b/python/src/bind_cluster_7x7.cpp new file mode 100644 index 00000000..3bd2fc4b --- /dev/null +++ b/python/src/bind_cluster_7x7.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_cluster_common.hpp" + +void bind_cluster_types_7x7(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int, 7, 7, uint16_t, i); + DEFINE_CLUSTER_BINDINGS(double, 7, 7, uint16_t, d); + DEFINE_CLUSTER_BINDINGS(float, 7, 7, uint16_t, f); +} + +void bind_cluster_finders_7x7(py::module &m) { + DEFINE_BINDINGS_CLUSTERFINDER(int, 7, 7, uint16_t, i); + DEFINE_BINDINGS_CLUSTERFINDER(double, 7, 7, uint16_t, d); + DEFINE_BINDINGS_CLUSTERFINDER(float, 7, 7, uint16_t, f); +} + +void bind_cluster_vector_reductions_7x7(py::module &m) { + define_3x3_reduction(m); + define_3x3_reduction(m); + define_3x3_reduction(m); +} + +void bind_cluster_reductions_7x7(py::module &m) { + reduce_to_3x3(m); + reduce_to_3x3(m); + reduce_to_3x3(m); +} diff --git a/python/src/bind_cluster_9x9.cpp b/python/src/bind_cluster_9x9.cpp new file mode 100644 index 00000000..6ffc5b6a --- /dev/null +++ b/python/src/bind_cluster_9x9.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "bindings.hpp" + +#include "bind_cluster_common.hpp" + +void bind_cluster_types_9x9(py::module &m) { + DEFINE_CLUSTER_BINDINGS(int, 9, 9, uint16_t, i); + DEFINE_CLUSTER_BINDINGS(double, 9, 9, uint16_t, d); + DEFINE_CLUSTER_BINDINGS(float, 9, 9, uint16_t, f); +} + +void bind_cluster_finders_9x9(py::module &m) { + DEFINE_BINDINGS_CLUSTERFINDER(int, 9, 9, uint16_t, i); + DEFINE_BINDINGS_CLUSTERFINDER(double, 9, 9, uint16_t, d); + DEFINE_BINDINGS_CLUSTERFINDER(float, 9, 9, uint16_t, f); +} + +void bind_cluster_vector_reductions_9x9(py::module &m) { + define_3x3_reduction(m); + define_3x3_reduction(m); + define_3x3_reduction(m); +} + +void bind_cluster_reductions_9x9(py::module &m) { + reduce_to_3x3(m); + reduce_to_3x3(m); + reduce_to_3x3(m); +} diff --git a/python/src/bind_cluster_common.hpp b/python/src/bind_cluster_common.hpp new file mode 100644 index 00000000..809a58b8 --- /dev/null +++ b/python/src/bind_cluster_common.hpp @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MPL-2.0 +// Shared by the per-cluster-size translation units bind_cluster_NxN.cpp. +#pragma once + +#include "module_config.hpp" + +#include "bind_Cluster.hpp" +#include "bind_ClusterCollector.hpp" +#include "bind_ClusterFile.hpp" +#include "bind_ClusterFileSink.hpp" +#include "bind_ClusterFinder.hpp" +#include "bind_ClusterFinderMT.hpp" +#include "bind_ClusterVector.hpp" +#include "bind_Eta.hpp" + +/* MACRO that defines Cluster bindings for a specific size and type + +T - Storage type of the cluster data (int, float, double) +N - Number of rows in the cluster +M - Number of columns in the cluster +U - Type of the pixel data (e.g., uint16_t) +TYPE_CODE - A character representing the type code (e.g., 'i' for int, 'd' for +double, 'f' for float) + +*/ +#define DEFINE_CLUSTER_BINDINGS(T, N, M, U, TYPE_CODE) \ + define_ClusterFile(m, "Cluster" #N "x" #M #TYPE_CODE); \ + define_ClusterVector(m, "Cluster" #N "x" #M #TYPE_CODE); \ + define_Cluster(m, #N "x" #M #TYPE_CODE); \ + register_calculate_2x2eta(m); \ + define_2x2_reduction(m); \ + reduce_to_2x2(m); + +#define DEFINE_BINDINGS_CLUSTERFINDER(T, N, M, U, TYPE_CODE) \ + define_ClusterFinder(m, "Cluster" #N "x" #M #TYPE_CODE); \ + define_ClusterFinderMT(m, "Cluster" #N "x" #M #TYPE_CODE); \ + define_ClusterFileSink(m, "Cluster" #N "x" #M #TYPE_CODE); \ + define_ClusterCollector(m, "Cluster" #N "x" #M #TYPE_CODE); diff --git a/python/src/bind_testing.hpp b/python/src/bind_testing.cpp similarity index 98% rename from python/src/bind_testing.hpp rename to python/src/bind_testing.cpp index 2a4e8760..1e8199a8 100644 --- a/python/src/bind_testing.hpp +++ b/python/src/bind_testing.cpp @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 // Probes used by the python test-suite to exercise the helpers in // np_helper.hpp. Internal, no API guarantees. -#pragma once +#include "bindings.hpp" #include "aare/NDArray.hpp" #include "aare/NDView.hpp" diff --git a/python/src/bindings.hpp b/python/src/bindings.hpp new file mode 100644 index 00000000..c4cad924 --- /dev/null +++ b/python/src/bindings.hpp @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 +// Entry points of the binding translation units. Each bind_*.cpp defines the +// entry points listed under its name; module.cpp calls them in a fixed order. +// The templated cluster and eta bindings are shared headers (bind_Cluster*.hpp, +// bind_Eta.hpp) instantiated per size in bind_cluster_NxN.cpp. +#pragma once + +#include +#include + +namespace py = pybind11; + +// bind_File.cpp +void define_file_io_bindings(py::module &m); +// bind_RawFile.cpp +void define_raw_file_io_bindings(py::module &m); +// bind_RawSubFile.cpp +void define_raw_sub_file_io_bindings(py::module &m); +// bind_RawMasterFile.cpp +void define_raw_master_file_bindings(py::module &m); +// bind_CtbRawFile.cpp +void define_ctb_raw_file_io_bindings(py::module &m); +// bind_JungfrauDataFile.cpp +void define_jungfrau_data_file_io_bindings(py::module &m); +// bind_MultiThreadedFileReader.cpp +void define_multi_threaded_file_reader_bindings(py::module &m); +// bind_DataSetAggregation.cpp +void bind_dataset_aggregation(py::module &m); + +// bind_Defs.cpp +void define_defs_bindings(py::module &m); +// bind_GapPixels.cpp +void define_gap_pixels_bindings(py::module &m); +// bind_PixelMap.cpp +void define_pixel_map_bindings(py::module &m); +// bind_testing.cpp +void define_testing_bindings(py::module &m); + +// bind_PixelHistogram.cpp +void define_pixel_histogram_bindings(py::module &m); +// bind_PedestalTrackingPixelHistogram.cpp +void define_pedestal_tracking_pixel_histogram_bindings(py::module &m); + +// bind_Pedestal.cpp +void bind_pedestals(py::module &m); +// bind_FastPedestal.cpp +void bind_fast_pedestals(py::module &m); +// bind_calibration.cpp +void bind_calibration(py::module &m); + +// bind_Fit.cpp +void define_fit_bindings(py::module &m); +// bind_Interpolator.cpp +void define_interpolation_bindings(py::module &m); + +// bind_VarClusterFinder.cpp +void define_var_cluster_finder_bindings(py::module &m); + +// bind_Eta.cpp +void bind_eta_types(py::module &m); + +// bind_cluster_NxN.cpp +void bind_cluster_types_3x3(py::module &m); +void bind_cluster_types_3x3_i16(py::module &m); +void bind_cluster_types_2x2(py::module &m); +void bind_cluster_types_5x5(py::module &m); +void bind_cluster_types_7x7(py::module &m); +void bind_cluster_types_9x9(py::module &m); +void bind_cluster_finders_3x3(py::module &m); +void bind_cluster_finders_5x5(py::module &m); +void bind_cluster_finders_7x7(py::module &m); +void bind_cluster_finders_9x9(py::module &m); +void bind_cluster_vector_reductions_5x5(py::module &m); +void bind_cluster_vector_reductions_7x7(py::module &m); +void bind_cluster_vector_reductions_9x9(py::module &m); +void bind_cluster_reductions_5x5(py::module &m); +void bind_cluster_reductions_7x7(py::module &m); +void bind_cluster_reductions_9x9(py::module &m); +void bind_eta_3x3(py::module &m); diff --git a/python/src/module.cpp b/python/src/module.cpp index 9f14a72c..22d483e6 100644 --- a/python/src/module.cpp +++ b/python/src/module.cpp @@ -1,70 +1,11 @@ // SPDX-License-Identifier: MPL-2.0 -// Files with bindings to the different classes +// Module registration. Each bind_*.cpp defines the bindings of one class or +// function group and declares its entry point in bindings.hpp. The call order +// below is kept: overload docstrings follow it. -#include "module_config.hpp" +#include "bindings.hpp" -// New style file naming -#include "bind_Cluster.hpp" -#include "bind_ClusterCollector.hpp" -#include "bind_ClusterFile.hpp" -#include "bind_ClusterFileSink.hpp" -#include "bind_ClusterFinder.hpp" -#include "bind_ClusterFinderMT.hpp" -#include "bind_ClusterVector.hpp" -#include "bind_DataSetAggregation.hpp" -#include "bind_Defs.hpp" -#include "bind_Eta.hpp" -#include "bind_FastPedestal.hpp" -#include "bind_GapPixels.hpp" -#include "bind_Interpolator.hpp" -#include "bind_MultiThreadedFileReader.hpp" -#include "bind_Pedestal.hpp" -#include "bind_PedestalTrackingPixelHistogram.hpp" -#include "bind_PixelHistogram.hpp" -#include "bind_PixelMap.hpp" -#include "bind_RawFile.hpp" -#include "bind_calibration.hpp" -#include "bind_testing.hpp" - -// TODO! migrate the other names -#include "ctb_raw_file.hpp" -#include "file.hpp" -#include "fit.hpp" -#include "jungfrau_data_file.hpp" -#include "raw_master_file.hpp" -#include "raw_sub_file.hpp" -#include "var_cluster.hpp" - -// Pybind stuff -#include #include -#include - -namespace py = pybind11; - -/* MACRO that defines Cluster bindings for a specific size and type - -T - Storage type of the cluster data (int, float, double) -N - Number of rows in the cluster -M - Number of columns in the cluster -U - Type of the pixel data (e.g., uint16_t) -TYPE_CODE - A character representing the type code (e.g., 'i' for int, 'd' for -double, 'f' for float) - -*/ -#define DEFINE_CLUSTER_BINDINGS(T, N, M, U, TYPE_CODE) \ - define_ClusterFile(m, "Cluster" #N "x" #M #TYPE_CODE); \ - define_ClusterVector(m, "Cluster" #N "x" #M #TYPE_CODE); \ - define_Cluster(m, #N "x" #M #TYPE_CODE); \ - register_calculate_2x2eta(m); \ - define_2x2_reduction(m); \ - reduce_to_2x2(m); - -#define DEFINE_BINDINGS_CLUSTERFINDER(T, N, M, U, TYPE_CODE) \ - define_ClusterFinder(m, "Cluster" #N "x" #M #TYPE_CODE); \ - define_ClusterFinderMT(m, "Cluster" #N "x" #M #TYPE_CODE); \ - define_ClusterFileSink(m, "Cluster" #N "x" #M #TYPE_CODE); \ - define_ClusterCollector(m, "Cluster" #N "x" #M #TYPE_CODE); PYBIND11_MODULE(_aare, m) { auto experimental = m.def_submodule( @@ -80,12 +21,8 @@ PYBIND11_MODULE(_aare, m) { define_pixel_map_bindings(m); define_pixel_histogram_bindings(m); define_pedestal_tracking_pixel_histogram_bindings(m); - define_pedestal_bindings(m, "Pedestal_d"); - define_pedestal_bindings(m, "Pedestal_f"); - define_pedestal_bindings(m, "Pedestal_i16"); - define_fast_pedestal_bindings(m, "FastPedestal_d"); - define_fast_pedestal_bindings(m, "FastPedestal_f"); - define_fast_pedestal_bindings(m, "FastPedestal_i16"); + bind_pedestals(m); + bind_fast_pedestals(m); define_fit_bindings(m); define_interpolation_bindings(m); define_jungfrau_data_file_io_bindings(m); @@ -93,96 +30,32 @@ PYBIND11_MODULE(_aare, m) { bind_calibration(m); define_testing_bindings(m); - DEFINE_CLUSTER_BINDINGS(int, 3, 3, uint16_t, i); - DEFINE_CLUSTER_BINDINGS(double, 3, 3, uint16_t, d); - DEFINE_CLUSTER_BINDINGS(float, 3, 3, uint16_t, f); + bind_cluster_types_3x3(m); + bind_cluster_types_2x2(m); + bind_cluster_types_5x5(m); + bind_cluster_types_7x7(m); + bind_cluster_types_9x9(m); + bind_cluster_types_3x3_i16(m); - DEFINE_CLUSTER_BINDINGS(int, 2, 2, uint16_t, i); - DEFINE_CLUSTER_BINDINGS(double, 2, 2, uint16_t, d); - DEFINE_CLUSTER_BINDINGS(float, 2, 2, uint16_t, f); + bind_cluster_finders_3x3(m); + bind_cluster_finders_5x5(m); + bind_cluster_finders_7x7(m); + bind_cluster_finders_9x9(m); - DEFINE_CLUSTER_BINDINGS(int, 5, 5, uint16_t, i); - DEFINE_CLUSTER_BINDINGS(double, 5, 5, uint16_t, d); - DEFINE_CLUSTER_BINDINGS(float, 5, 5, uint16_t, f); + bind_cluster_vector_reductions_5x5(m); + bind_cluster_vector_reductions_7x7(m); + bind_cluster_vector_reductions_9x9(m); - DEFINE_CLUSTER_BINDINGS(int, 7, 7, uint16_t, i); - DEFINE_CLUSTER_BINDINGS(double, 7, 7, uint16_t, d); - DEFINE_CLUSTER_BINDINGS(float, 7, 7, uint16_t, f); + bind_cluster_reductions_5x5(m); + bind_cluster_reductions_7x7(m); + bind_cluster_reductions_9x9(m); - DEFINE_CLUSTER_BINDINGS(int, 9, 9, uint16_t, i); - DEFINE_CLUSTER_BINDINGS(double, 9, 9, uint16_t, d); - DEFINE_CLUSTER_BINDINGS(float, 9, 9, uint16_t, f); - - DEFINE_CLUSTER_BINDINGS(int16_t, 3, 3, uint16_t, i16); - - DEFINE_BINDINGS_CLUSTERFINDER(int, 3, 3, uint16_t, i); - DEFINE_BINDINGS_CLUSTERFINDER(double, 3, 3, uint16_t, d); - DEFINE_BINDINGS_CLUSTERFINDER(float, 3, 3, uint16_t, f); - DEFINE_BINDINGS_CLUSTERFINDER(int16_t, 3, 3, uint16_t, i16); - - DEFINE_BINDINGS_CLUSTERFINDER(int, 5, 5, uint16_t, i); - DEFINE_BINDINGS_CLUSTERFINDER(double, 5, 5, uint16_t, d); - DEFINE_BINDINGS_CLUSTERFINDER(float, 5, 5, uint16_t, f); - - DEFINE_BINDINGS_CLUSTERFINDER(int, 7, 7, uint16_t, i); - DEFINE_BINDINGS_CLUSTERFINDER(double, 7, 7, uint16_t, d); - DEFINE_BINDINGS_CLUSTERFINDER(float, 7, 7, uint16_t, f); - - DEFINE_BINDINGS_CLUSTERFINDER(int, 9, 9, uint16_t, i); - DEFINE_BINDINGS_CLUSTERFINDER(double, 9, 9, uint16_t, d); - DEFINE_BINDINGS_CLUSTERFINDER(float, 9, 9, uint16_t, f); - - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - define_3x3_reduction(m); - - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - reduce_to_3x3(m); - - register_calculate_3x3eta(m); - register_calculate_3x3eta(m); - register_calculate_3x3eta(m); - register_calculate_3x3eta(m); + bind_eta_3x3(m); define_defs_bindings(m); define_gap_pixels_bindings(m); - using Sum_index_pair_d = Sum_index_pair; - PYBIND11_NUMPY_DTYPE(Sum_index_pair_d, sum, index); - using Sum_index_pair_f = Sum_index_pair; - PYBIND11_NUMPY_DTYPE(Sum_index_pair_f, sum, index); - using Sum_index_pair_i = Sum_index_pair; - PYBIND11_NUMPY_DTYPE(Sum_index_pair_i, sum, index); + bind_eta_types(m); - using eta_d = Eta2; - PYBIND11_NUMPY_DTYPE(eta_d, x, y, c, sum); - using eta_i = Eta2; - PYBIND11_NUMPY_DTYPE(eta_i, x, y, c, sum); - using eta_f = Eta2; - PYBIND11_NUMPY_DTYPE(eta_f, x, y, c, sum); - using eta_i16 = Eta2; - PYBIND11_NUMPY_DTYPE(eta_i16, x, y, c, sum); - - define_corner_enum(m); - define_eta(m, "f"); - define_eta(m, "d"); - define_eta(m, "i"); - define_eta(m, "i16"); - - bind_DataSetAggregation(m); - - bind_read_current_source(m); + bind_dataset_aggregation(m); } diff --git a/python/src/np_helper.hpp b/python/src/np_helper.hpp index 83880470..6beb7318 100644 --- a/python/src/np_helper.hpp +++ b/python/src/np_helper.hpp @@ -1,11 +1,13 @@ // SPDX-License-Identifier: MPL-2.0 #pragma once +#include #include #include #include #include +#include "aare/Cluster.hpp" #include "aare/Frame.hpp" #include "aare/NDArray.hpp" #include "aare/NDView.hpp" @@ -124,8 +126,8 @@ template const std::string &fmt_format() { * Helper function to allocate image data given item size and shape * used when we want to fill a numpy array and return to python */ -py::array allocate_image_data(size_t item_size, - const std::vector &shape) { +inline py::array allocate_image_data(size_t item_size, + const std::vector &shape) { py::array image_data; if (item_size == 1) { image_data = py::array_t(shape); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eb90112c..d9cbbd25 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,9 @@ if(AARE_FETCH_CATCH) GIT_REPOSITORY https://github.com/catchorg/Catch2.git GIT_TAG v3.5.3) FetchContent_MakeAvailable(Catch2) + # Catch2 is ~100 small files; a unity build compiles them in a few batches + set_target_properties(Catch2 Catch2WithMain + PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 16) else() # Otherwise look for installed catch2 find_package(Catch2 3 REQUIRED) @@ -24,6 +27,28 @@ set_target_properties( tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} OUTPUT_NAME run_tests) +# Headers nearly every test includes, parsed once instead of per test file. Keep +# this list to the test framework and ubiquitous aare headers: anything listed +# here is available in every test without an #include. +target_precompile_headers( + tests + PRIVATE + + + + + + + + + + + "${PROJECT_SOURCE_DIR}/include/aare/NDArray.hpp" + "${PROJECT_SOURCE_DIR}/include/aare/NDView.hpp" + "${PROJECT_SOURCE_DIR}/include/aare/defs.hpp" + "${PROJECT_SOURCE_DIR}/include/aare/Frame.hpp" + "${PROJECT_SOURCE_DIR}/include/aare/Dtype.hpp") + include(CTest) include(Catch) catch_discover_tests(tests)