diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 997a19d2..bc4f6683 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest,macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml new file mode 100644 index 00000000..5078cf5e --- /dev/null +++ b/.github/workflows/build_windows.yml @@ -0,0 +1,37 @@ +name: Build and test on Windows + +on: + workflow_dispatch: + push: + pull_request: + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: python -m pip install ninja numpy matplotlib pytest pytest-check boost-histogram + + # Puts cl.exe on the path so that CMake can use the Ninja generator + - uses: ilammy/msvc-dev-cmd@v1 + + - name: Configure + run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DAARE_TESTS=ON -DAARE_PYTHON_BINDINGS=ON -DAARE_WARNINGS_AS_ERRORS=ON + + - name: Build + run: cmake --build build + + - name: C++ unit tests + run: ctest --test-dir build --output-on-failure -j4 + + - name: Python unit tests + env: + PYTHONPATH: ${{ github.workspace }}\build + run: python -m pytest python/tests diff --git a/AGENTS.md b/AGENTS.md index be6fda68..5646aaae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,8 @@ footprint. - `include/aare/` contains the public C++ API and header-defined templates. - `src/` contains C++ implementations and colocated Catch2 tests named - `*.test.cpp`. + `*.test.cpp`, grouped into `src/fit/`, `src/hist/` and `src/utils/` where + a component has several private files. - `tests/` contains the C++ test executable, configuration, and shared helpers. - `python/src/` contains pybind11 bindings and module registration. - `python/aare/` contains Python facades, convenience APIs, and public exports. @@ -47,8 +48,142 @@ cmake --build build -j4 ``` Useful optional settings include `AARE_DOCS`, `AARE_BENCHMARKS`, `AARE_ASAN`, -and `AARE_WARNINGS_AS_ERRORS`. Reconfigure an existing build directory instead -of creating alternate in-tree build layouts unless isolation is needed. +`AARE_WARNINGS_AS_ERRORS`, `AARE_VERBOSE` (raises the compile-time log +level), and `AARE_TUNE_LOCAL` (`-march=native`, not portable). Reconfigure an +existing build directory instead of creating alternate in-tree build layouts +unless isolation is needed; `build/CMakeCache.txt` records the options and +interpreter it was configured with. + +`-Werror=return-type` is always enabled. New code should also compile cleanly +under the project's `-Wall -Wextra -pedantic -Wshadow -Wold-style-cast +-Wdouble-promotion` flags, which `AARE_WARNINGS_AS_ERRORS` turns into errors. + +The build copies `python/aare/*.py` and the compiled `_aare` extension into +`build/aare/`, so `PYTHONPATH=build` makes the package importable. The +extension only imports into the interpreter CMake was configured with, which +is normally the Conda or venv Python active at configure time. Activate that +environment, or resolve the interpreter from the cache: + +```bash +PY=$(grep -E '^_?Python_EXECUTABLE' build/CMakeCache.txt | head -1 | cut -d= -f2) +PYTHONPATH="$PWD/build" "$PY" -m pytest python/tests +``` + +Do not fall back to a different system Python. + +With `AARE_BENCHMARKS=ON`, Google Benchmark programs build as +`build/run_benchmarks` and `build/fit_benchmark`. + +## Architecture + +### Core containers + +- `NDArray` owns contiguous C-order memory; `NDView` is a + non-owning view with the same shape and stride API. Both derive from + `ArrayExpr` (expression templates for element-wise math). Shapes are + `std::array`. `NDView` converts to `NDView`. +- `Frame` is a byte buffer plus rows, cols, and `Dtype`, the lowest common + denominator returned by file readers. `Dtype` is a runtime type tag built + from a bitdepth, a NumPy descriptor, or `typeid`. +- `defs.hpp` holds detector enums (`DetectorType`, `TimingMode`, + `FrameDiscardPolicy`, `ReadoutMode`), the on-disk `DetectorHeader`, chip + constants (`Moench04`, `Matterhorn02`, ...), the `LOCATION` macro used in + error messages, and `AARE_ASSERT` (active only with `AARE_CUSTOM_ASSERT`). +- `logger.hpp` provides `LOG(logLEVEL) << ...`; levels above `AARE_LOG_LEVEL` + (set by CMake) are compiled out. + +### File reading + +`File` is the generic RAII facade. It holds a `unique_ptr` and +dispatches on file extension to one of the concrete readers: + +- `RawFile` reads slsDetector `.raw` data with a `_master_N.json` (or legacy + `.raw`) master file. It owns a `RawMasterFile` (master JSON, older raw-text + masters, scan parameters, ROIs, disabled UDP ports) and a two-dimensional + table of `RawSubFile` indexed `[roi][module]`. Each `RawSubFile` handles one + module's sequence of `_dN_fM_` data files and per-frame `DetectorHeader`s. + Module placement into the assembled image goes through `DetectorGeometry`, + `ModuleGeometry`, and `ROIGeometry` (module gaps, ROIs, UDP-port halves). + Frame numbers from the detector are distinct from frame indices in the + file; the API uses both and names them accordingly. +- `NumpyFile` (`.npy`), `JungfrauDataFile` (legacy `.dat`), and `CtbRawFile` + (chip test board; raw ADC samples decoded through `decode.hpp` and the + `PixelMap` generators, exposed in Python as `aare.transform` callables). +- `experimental::MultiThreadedFileReader` opens one `File` per worker and + fills a caller-owned buffer in file order. + +Some chip-specific decoding (Moench03/05 pixel maps, ADC SAR transforms) +lives in `python/aare/transform.py` on top of C++ generators. + +### Pedestal and cluster finding + +- `Pedestal` and `FastPedestal` maintain per-pixel running mean and + standard deviation, always accumulating in `double`. `FastPedestal` must + see `min_pedestal_samples` frames before `ready()`. +- `ClusterFinder` is single-threaded: + push pedestal frames, then `find_clusters(view, frame_number)` appends to an + internal `ClusterVector` that callers take with `steal_clusters()`. + Thresholds are `n_sigma * std` and are recomputed by `update_threshold()`. +- `ClusterFinderMT` wraps N `ClusterFinder`s, one per thread, each with a + `CircularFifo` input queue backed by a preallocated `FramePool` + and a `ProducerConsumerQueue` output. A collector thread funnels all + outputs into a single sink queue. Consumers of that sink run their own + thread: `ClusterCollector` (accumulate in memory, `steal_clusters()`) or + `ClusterFileSink` (write the binary cluster file). The `stop()` and + `sync()` ordering matters; read the existing tests before changing + lifecycle code. +- `Cluster` is a POD (x, y, data[]) and + `ClusterVector>` is a frame-tagged `std::vector` of them. + `ClusterFile` reads and writes the binary format (`int32 frame_number`, + `uint32 n_clusters`, raw cluster bytes). The format carries no shape or + dtype metadata, so the caller must supply matching template arguments. + `CalculateEta.hpp` and `Interpolator` operate on `ClusterVector`s; + `algorithm.hpp` holds small 1-D search helpers. +- `hist/PixelHistogram` and `PedestalTrackingPixelHistogram` build per-pixel + histograms across a frame series, multithreaded by row bands. + +### Fitting + +`Fit.hpp`, `FitModel`, and `Models.hpp` (`Gaussian`, `Pol1`, `Pol2`, +`RisingScurve`, `FallingScurve`, `GaussianErfcPlateau`, +`GaussianChargeSharing`, ...) form the public fitting API. Each model +provides `eval`, `eval_and_grad`, `is_valid`, `estimate_par`, and +`param_info`; the separable models also declare their linear parameters +(`linear_par`, `basis_and_grad`, `basis_columns`). `FitModel` is plain data +(limits, fixed flags, start values, minimizer settings) and selects one of +four minimizers through `aare::Minimizer`: Minuit2's Migrad (default) and +Fumili, or the built-in Levenberg-Marquardt and variable projection +solvers. The private implementation lives in `src/fit/`: `Fit.cpp` holds +the `FitModel` method bodies, the per-thread `detail::PixelFitter` that +dispatches on the minimizer and falls back from VarPro to +Levenberg-Marquardt, `fit_pixel`/`fit_3d`, and the explicit +instantiations; `FitHelpers.hpp` holds the start-value precedence and +shape checks; `DampedGaussNewton.hpp` is the damped Gauss-Newton driver +shared by `LevenbergMarquardt.hpp` and `VariableProjection.hpp`, with the +small fixed-size linear algebra on `std::array` (`Vec`, `Mat`). +`FitMinuit2.cpp` (with `Chi2.hpp` and `MinuitSteps.hpp`) is the only +translation unit that includes Minuit2 headers; Minuit2 is a private, +`BUILD_INTERFACE`-only dependency of `aare_core`. The `[fit]` tests run +every configuration through all four minimizers and check that they +agree; run them when changing any solver. + +### Python layer + +- `python/src/module.cpp` registers everything. Templated classes are + instantiated per value type and cluster size through the + `DEFINE_CLUSTER_BINDINGS` and `DEFINE_BINDINGS_CLUSTERFINDER` macros, + producing names such as `ClusterFinder_Cluster3x3i`, `Pedestal_d`, and + `FastPedestal_i16`. Supported cluster sizes are 2x2, 3x3, 5x5, 7x7, and + 9x9; value types are `i`, `f`, `d`, and `i16` (3x3 only). + `module_config.hpp` fixes the module-wide pedestal type (`pd_type`). +- `python/aare/factory.py` maps NumPy dtypes to those suffixes. The facades + in `python/aare/*.py` are thin factory functions or subclasses that select + the right binding and add Pythonic conveniences (iteration, context + managers, `ScanParameters`). +- `np_helper.hpp` provides `return_image_data` (moves an `NDArray` into a + NumPy array with a capsule deleter) and `make_view_1d/2d/3d` (views into a + NumPy buffer). +- `aare.experimental` and `_aare.experimental` hold unstable APIs. ## Implementation conventions @@ -70,6 +205,14 @@ of creating alternate in-tree build layouts unless isolation is needed. namespace/module. - Start new source files with `SPDX-License-Identifier: MPL-2.0`, using the appropriate comment syntax. +- Code copied from other projects keeps its original license. Retain its + copyright and license notice, set the SPDX identifier to that license, and + list the license in `LICENSES/`, `pyproject.toml`, `conda-recipe/meta.yaml`, + and `README.md`. `etc/add_license.py` skips files with a license notice. +- When adding or bumping a fetched dependency that is compiled into the + Python extension, update its version and notice in + `THIRD-PARTY-NOTICES.txt` and the license expressions in `pyproject.toml` + and `conda-recipe/meta.yaml`. - Prefer descriptive names to comments that only restate the code. - Do not add a dependency unless the benefit justifies the packaging and deployment cost. @@ -99,15 +242,19 @@ ctest --test-dir build --output-on-failure -j4 PYTHONPATH="$PWD/build" python -m pytest python/tests ``` -For focused runs, use a Catch2 tag or an individual pytest file/test: +For focused runs, use a Catch2 tag or test name, or an individual pytest +file or test: ```bash build/run_tests "[tag]" +build/run_tests "Exact test case name" PYTHONPATH="$PWD/build" python -m pytest python/tests/test_example.py +PYTHONPATH="$PWD/build" python -m pytest python/tests/test_example.py -k name ``` -Large detector test files live outside this repository. To include data-backed -tests, set `AARE_TEST_DATA` and opt in explicitly: +Large detector test files live outside this repository in +`https://gitea.psi.ch/detectors/aare-test-data.git` (Git LFS). To include +data-backed tests, set `AARE_TEST_DATA` and opt in explicitly: ```bash export AARE_TEST_DATA=/path/to/aare-test-data @@ -116,21 +263,38 @@ PYTHONPATH="$PWD/build" python -m pytest python/tests --with-data ``` - Start bug fixes with a failing regression test when practical. -- Put C++ tests beside the relevant implementation as `src/Thing.test.cpp`. +- Put C++ tests beside the relevant implementation as `src/Thing.test.cpp` + and add them to `TestSources` in the root `CMakeLists.txt`. - Mark C++ tests requiring external files with `[.with-data]`. -- Mark Python tests requiring external files with `@pytest.mark.withdata`. +- Mark Python tests requiring external files with `@pytest.mark.withdata` and + use the `test_data_path` fixture from `python/tests/conftest.py`. +- Prefer synthesizing small input files over depending on external data. + `tests/raw_file_helpers.hpp` builds throwaway raw files in C++, and many + Python tests write small raw files into `tmp_path`. +- `tests/test_config.hpp.in` provides `test_data_path()`; + `tests/test_macros.hpp` and `tests/friend_test.hpp` (`TEST_CASE_PRIVATE` + with `FRIEND_TEST`) let a test reach private members. - Run both suites for changes that cross the C++/Python boundary. - Do not silently skip required data-backed coverage. Report when the external test data is unavailable. ## Formatting and static analysis +The project uses pre-commit. Install the git hook once per clone with +`pre-commit install` so it runs on every commit. The hooks in +`.pre-commit-config.yaml` are clang-format on C++, cmake-format on CMake +files, `typos` on `docs/` (project words are whitelisted in `typos.toml`), and +basic YAML, TOML, JSON, and large-file checks. CI runs the same hooks on pull +requests. + Before handing off a broad change, run the relevant checks: ```bash -pre-commit run --all-files -cmake --build build --target check-format -cmake --build build --target clang-tidy +pre-commit install # once per clone +pre-commit run --all-files # what CI runs +cmake --build build --target check-format # diff-based clang-format check +cmake --build build --target format-files # apply clang-format to all .cpp/.hpp +cmake --build build --target clang-tidy # uses build/compile_commands.json ``` C++ formatting follows `.clang-format` (four-space indentation and an 80-column @@ -139,7 +303,9 @@ code as part of a focused change. ## Documentation and release notes -- Update the relevant `.rst` pages for public behavior or API changes. +- Update the relevant `.rst` pages for public behavior or API changes. C++ + pages live at `docs/src/.rst`; Python pages live under + `docs/src/python//`. - Add new pages to `docs/src/index.rst` or the relevant nested toctree. - Update `RELEASE.md` under `## Next` for user-visible features, bug fixes, and API changes. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..040b659a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,8 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +All project guidance lives in `AGENTS.md`: layout, build and test commands, +architecture, conventions, formatting, documentation, and release notes. +Read it and follow it. Keep new guidance there rather than here so that every +agent and contributor sees the same instructions. diff --git a/CMakeLists.txt b/CMakeLists.txt index 17a9cfcd..94d16b95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,7 +106,7 @@ if(AARE_FETCH_MINUIT2) FetchContent_Declare( Minuit2 GIT_REPOSITORY https://github.com/GooFit/Minuit2.git - GIT_TAG master + GIT_TAG v6-40-02 UPDATE_DISCONNECTED 1) # Disable Minuit2 extras we don't need set(minuit2_mpi @@ -129,7 +129,9 @@ if(AARE_FETCH_MINUIT2) add_library(aare::Minuit2 ALIAS Minuit2) else() find_package(Minuit2 CONFIG QUIET) - if(NOT Minuit2_FOUND) + if(Minuit2_FOUND) + add_library(aare::Minuit2 ALIAS Minuit2::Minuit2) + else() message( STATUS "Stand alone Minuit2 not found. Trying to find it as a part of ROOT") @@ -153,7 +155,7 @@ if(AARE_FETCH_ZMQ) FetchContent_Declare( libzmq GIT_REPOSITORY https://github.com/zeromq/libzmq.git - GIT_TAG v4.3.4 + GIT_TAG v4.3.5 PATCH_COMMAND ${ZMQ_PATCH_COMMAND} UPDATE_DISCONNECTED 1) # Disable unwanted options from libzmq @@ -206,6 +208,11 @@ if(AARE_FETCH_FMT) CACHE BOOL "") FetchContent_MakeAvailable(fmt) set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON) + if(MSVC) + # fmt 12.1.0 calls std::isalpha only to check that ADL finds it and drops + # the result, which MSVC's standard library marks [[nodiscard]] + target_compile_options(fmt PRIVATE /wd4834) + endif() else() find_package(fmt 8 REQUIRED) endif() @@ -252,18 +259,30 @@ target_compile_features(aare_compiler_flags INTERFACE cxx_std_17) # MSVC specific # # ############################################################################## if(MSVC) - add_compile_definitions(AARE_MSVC) - if(CMAKE_BUILD_TYPE STREQUAL "Release") - message(STATUS "Release build") - target_compile_options(aare_compiler_flags INTERFACE /O2) - else() - message(STATUS "Debug build") - target_compile_options(aare_compiler_flags - INTERFACE /Od /Zi /MDd /D_ITERATOR_DEBUG_LEVEL=2) - target_link_options(aare_compiler_flags INTERFACE /DEBUG:FULL) - endif() - target_compile_options(aare_compiler_flags INTERFACE /w # disable warnings + # CMake sets the per-configuration optimisation, debug-info and + # runtime-library flags itself, so only warnings and conformance here. + target_compile_options( + aare_compiler_flags + INTERFACE /W4 # -Wall -Wextra + /permissive- # standard conformance, like CMAKE_CXX_EXTENSIONS OFF + /utf-8 # sources are UTF-8 + /we4715 # not all control paths return a value, like + # -Werror=return-type + /wd4244 # narrowing conversions: -Wconversion is not enabled + /wd4267 # on gcc/clang either + /wd4068 # unknown pragma: the GCC diagnostic pragmas + /wd4324 # structure padded due to alignas: intended ) + target_compile_definitions(aare_compiler_flags + INTERFACE _CRT_SECURE_NO_WARNINGS) + + if(AARE_WARNINGS_AS_ERRORS) + target_compile_options(aare_compiler_flags INTERFACE /WX) + endif() + + if(AARE_ASAN) + message(WARNING "AARE_ASAN is not supported with MSVC. Ignoring.") + endif() else() # ############################################################################ @@ -316,22 +335,22 @@ else() endif() endif() + if(AARE_ASAN) + message(STATUS "AddressSanitizer enabled") + target_compile_options( + aare_compiler_flags INTERFACE -fsanitize=address,undefined,pointer-compare + -fno-omit-frame-pointer) + target_link_libraries( + aare_compiler_flags INTERFACE -fsanitize=address,undefined,pointer-compare + -fno-omit-frame-pointer) + endif() + endif() # GCC/Clang specific if(AARE_PYTHON_BINDINGS) add_subdirectory(python) endif() -if(AARE_ASAN) - message(STATUS "AddressSanitizer enabled") - target_compile_options( - aare_compiler_flags INTERFACE -fsanitize=address,undefined,pointer-compare - -fno-omit-frame-pointer) - target_link_libraries( - aare_compiler_flags INTERFACE -fsanitize=address,undefined,pointer-compare - -fno-omit-frame-pointer) -endif() - if(AARE_TESTS) enable_testing() add_subdirectory(tests) @@ -404,7 +423,8 @@ set(SourceFiles ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/File.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FilePtr.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/Fit.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/fit/Fit.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/fit/FitMinuit2.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.cpp @@ -466,7 +486,7 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src/decode.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Dtype.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FilePtr.test.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/Fit.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/fit/Fit.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Frame.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/DetectorGeometry.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpolation.test.cpp @@ -479,12 +499,14 @@ if(AARE_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/src/CalculateEta.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/ClusterFinderMT.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/ProducerConsumerQueue.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Pedestal.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/FastPedestal.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/hist/PixelHistogramImpl.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/hist/PixelHistogram.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/hist/PedestalTrackingPixelHistogram.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/JungfrauDataFile.test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/logger.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/MultiThreadedFileReader.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyFile.test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/NumpyHelpers.test.cpp diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES/LGPL-2.1-or-later.txt b/LICENSES/LGPL-2.1-or-later.txt new file mode 100644 index 00000000..7f01a06c --- /dev/null +++ b/LICENSES/LGPL-2.1-or-later.txt @@ -0,0 +1,459 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt new file mode 100644 index 00000000..7dfed9f2 --- /dev/null +++ b/LICENSES/MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright 2017-2023 Leon Merten Lohse + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSES/MPL-2.0.txt b/LICENSES/MPL-2.0.txt new file mode 100644 index 00000000..f4bbcd20 --- /dev/null +++ b/LICENSES/MPL-2.0.txt @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. \ No newline at end of file diff --git a/README.md b/README.md index ca41ae10..7a9f1c8a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,20 @@ Detailed documentation including installation can be found in [Documentation](ht This project is licensed under the MPL-2.0 license. See the LICENSE file or https://www.mozilla.org/en-US/MPL/ for details. +Two files contain code from other projects and keep their original licenses: + +- `include/aare/ProducerConsumerQueue.hpp` is Apache-2.0, from + [folly](https://github.com/facebook/folly) by Meta Platforms. +- `src/NumpyHelpers.cpp` is MIT, from + [libnpy](https://github.com/llohse/libnpy) by Leon Merten Lohse. + +The full license texts are in the `LICENSES` directory. + +The Python wheels and conda packages also contain Minuit2 +(LGPL-2.1-or-later), {fmt} and nlohmann/json (MIT), and pybind11 +(BSD-3-Clause), which CMake downloads at build time. See +`THIRD-PARTY-NOTICES.txt` for their notices. + ## Build and install Prerequisites diff --git a/RELEASE.md b/RELEASE.md index 54eede05..fd23a612 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,6 +4,22 @@ ### New Features: +- Added the Python ``xy`` type with ``row`` and ``col`` fields, enabling access + to ``RawFile.geometry``, ``RawMasterFile.detector_layout``, and + ``RawMasterFile.udp_interfaces_per_module``. It unpacks as ``row, col``. +- Added Windows wheels (``win_amd64``, Python 3.12 to 3.14) to the wheel + build, so releases publish them to PyPI alongside the Linux and macOS + wheels. + +## 2026.9.30 + +### New Features: + +- Added ``pedestal_ready`` to ``ClusterFinder`` and ``ClusterFinderMT`` in C++ + and Python to check whether enough pedestal frames have been pushed to find + clusters. +- Added the Python ``FrameDiscardPolicy`` enum with ``NoDiscard``, ``Discard``, + and ``DiscardPartial``, enabling access to ``RawMasterFile.frame_discard_policy``. - Added the Python ``Pedestal`` factory with ``dtype`` selection, matching ``FastPedestal`` and defaulting to ``float64`` output. - Added ``FastPedestal`` in C++ and Python for per-pixel running mean @@ -28,9 +44,77 @@ - Added string representator in python for Cluster and Eta - Added roi slice method in python for easy slicing of numpy arrays ``array[roi.slice()]``. - added context manager for ``aare.RawMasterFile`` +- Added a configurable minimizer for the fit models, selected with the + ``minimizer`` constructor argument or property in Python, or + ``FitModel::SetMinimizer`` in C++. ``Minimizer.Migrad`` remains the default. + ``Minimizer.Fumili`` is Minuit2's Gauss-Newton minimizer, which uses the + analytic model derivatives and needs far fewer function evaluations. + ``Minimizer.LevenbergMarquardt`` is a built-in, dependency-free + Levenberg-Marquardt solver with the analytic Jacobian; it reflects steps at + parameter limits and fits data cubes without per-pixel allocations. A fit + that stalls away from a minimum, or whose width collapses below the spacing + of the scan points, is reported as failed (all zeros) rather than returned + as a result, so noisy or signal-free pixels may fail. With + ``compute_errors``, Fumili and LevenbergMarquardt report parameter errors + from their linearised covariance instead of running Hesse. Pixels for which + Fumili does not reach a valid minimum, which Minuit2's implementation cannot + once a two-sided limit becomes active, are refitted with Migrad. +- ``Minimizer.LevenbergMarquardt`` is about twice as fast for every model: it + evaluates the Jacobian at the trial point instead of re-evaluating accepted + points, accumulates the normal equations with fixed-size loops, needs a + single factorisation per iteration, and the erf-based models compute one + exponential per point instead of two. Results are unchanged, except that + ``GaussianChargeSharing`` and ``GaussianChargeSharingKb`` now evaluate the + erf from the Gaussian's exponential, a rounding-level change. One iteration + costs one model evaluation of the ``max_calls`` budget, so pixels that used + to exhaust the default budget of 100 may now converge. +- Added ``Minimizer.VarPro``, a built-in variable projection + solver. Every bundled model now declares the parameters it is linear in + (``linear_par`` and ``basis_and_grad`` in C++, checked against the model + derivatives by the tests). Each trial point solves those exactly and the + Levenberg-Marquardt iteration runs over the nonlinear parameters only, so + the fit needs fewer evaluations and does not depend on start values of the + linear parameters. When the basis functions are nearly collinear, as for a + polynomial on a narrow range far from x = 0, one more pass over the data + refines the linear solution to the precision of the other minimizers. + Limits on linear parameters, pixels that do not converge + and models without the separable structure fall back to + ``LevenbergMarquardt``. Parameter errors are the same Gauss-Newton + estimates. The Gaussian, plateau, charge-sharing and S-curve models also + provide their basis functions for all scan points at once + (``basis_columns``), a loop that compiles to vector instructions with the + new ``model::fast_exp``; ``eval`` and ``eval_and_grad``, and with them the + other minimizers, are unchanged. ``Minimizer.Migrad`` remains the + default. +- ``NDArray``/``NDView`` expressions now support scalar operands + (``2 * a + b / 4``) and can be assigned to an existing ``NDArray``, reusing + its buffer. ### API Changes: +- The C++ ``NDArray`` view constructor and ``NDArray::copy_from()`` take + ``NDView``. Mutable views still convert implicitly, so + existing callers are unaffected, and read-only views can now be copied. +- Added C++ overloads of ``adc_sar_05_06_07_08decode64to16``, + ``adc_sar_05_decode64to16`` and ``adc_sar_04_decode64to16`` that take the + packed samples as ``NDView``. Words are assembled with + ``memcpy``, so the buffer does not need to be 8-byte aligned. The existing + ``NDView`` overloads are unchanged. +- ``NDArray`` ``+ - * /`` with a scalar now returns a lazy expression instead + of an ``NDArray`` and no longer converts the scalar to the element type. +- ``FitModel`` is now plain data without a pimpl (C++ users need to rebuild); + it gained ``strategy()``, ``lower_limit()``, ``upper_limit()`` and + ``value()`` accessors and lost ``impl()``. Bad parameter indices raise + ``std::out_of_range`` (``IndexError`` in Python) and ``SetParLimits`` + rejects ``lo >= hi``. The models in ``Models.hpp`` no longer provide the + Minuit specific ``compute_steps`` and ``compute_ranges`` helpers. +- Fit behaviour common to all minimizers: user start and fixed values are + applied before the validity check and free start values are clamped into + their limits; one-sided limits are open-ended for ``LevenbergMarquardt`` + while the Minuit2 minimizers use a wide two-sided range as before; with + ``compute_errors``, fixed parameters and parameters ending on a limit report + an error of 0 (fixed parameters previously reported 1.0 with Minuit2). + - ``FastPedestal`` variance is now a private ``double`` intermediate. Removed the C++ ``variance()``/``variance_unchecked()`` APIs and Python ``var()``. Standard deviation is calculated before conversion to the output type, @@ -74,7 +158,70 @@ - ``RawMasterFile::rois()`` always returns a list of rois (no optional). Per default it returns a list of one ROI element spawing the entire detector - ``TimingMode::Auto`` changed to ``TimingMode::AUTO_TIMING``, ``TimingMode::Trigger`` changed to ``TimingMode::TRIGGER_EXPOSURE`` + ### Bugfixes: +- Fitting a 3D data cube in Python without ``y_err`` now returns ``par_err`` + when ``compute_errors`` is set, as fitting a single pixel already did. +- Samples with a zero error are ignored by every minimizer, as documented, + also when they hold NaN. They are now removed before the start values are + estimated; previously a masked NaN at a sample used by the estimate, such + as the first or last point of ``Pol1``, made even Migrad fail. A pixel with + fewer remaining samples than free parameters is reported as failed. +- Fitting a single pixel checks that ``x``, ``y`` and ``y_err`` have the same + size and raises ``RuntimeError`` otherwise. Previously a shorter ``y`` or + ``y_err`` was read past its end. +- Setting nSigma on ``ClusterFinder`` and ``ClusterFinderMT`` before the + pedestal is ready no longer raises an error. The new value is used when the + threshold is initialized. Previously ``ClusterFinderMT.set_nSigma()`` also + left the worker threads with different nSigma values. +- The Python ``Cluster`` constructors validate that the data array holds + exactly one value per pixel and raise ``ValueError`` otherwise. Previously a + longer array wrote past the end of the cluster data. +- The Python CTB decoding helpers ``adc_sar_*decode64to16``, + ``apply_custom_weights``, ``expand24to32bit``, ``expand4to8bit`` and + ``decode_my302`` validate their input through ``make_view``. They require + C-contiguous arrays of the documented rank and reject other dtypes with + ``TypeError`` instead of converting a copy. The ADC SAR decoders no longer + reinterpret the byte buffer as aligned 64-bit words; a row length that is + not a multiple of 8 bytes raises ``ValueError`` instead of dropping the + trailing bytes. Read-only arrays are accepted where the input is only read. + ``decode_my302`` raises ``ValueError`` instead of ``RuntimeError`` for a + wrong input size. +- Mismatched operators inhibited vectorization in gcc of NDArray math operators +- ``NDArray``/``NDView`` math operators and expressions were not vectorized + for ``uint8_t`` and 64 bit integers, up to 30x slower than a plain loop. +- Removed the move constructor and move assignment of + ``ProducerConsumerQueue``. They left the moved-from queue with a null + buffer, crashing its destructor if it still held elements, and leaked + the target's buffer on assignment. The queue is now non-movable, as in + the upstream folly implementation; hold it in a ``unique_ptr`` to move + it around. +- ``ProducerConsumerQueue`` rejects sizes below two with + ``std::invalid_argument`` in all build types. The previous assertion was + compiled out of Release builds, so a zero-size queue overflowed its buffer + on the first write. +- Corrected the license metadata for vendored third-party code. + ``ProducerConsumerQueue.hpp`` is tagged ``Apache-2.0`` like upstream folly + and ``NumpyHelpers.cpp`` is tagged ``MIT`` like upstream libnpy. The MPL 2.0, + Apache 2.0, and MIT texts ship in ``LICENSES/``, and both the conda package + and the Python wheel declare ``MPL-2.0 AND Apache-2.0 AND MIT`` with all + three license files. Building the wheel now requires scikit-build-core 0.11 + or newer. +- The wheel and conda package now ship ``THIRD-PARTY-NOTICES.txt`` and the + LGPL 2.1 text for the libraries compiled into the extension: Minuit2 + (LGPL-2.1-or-later), {fmt} and nlohmann/json (MIT), and pybind11 + (BSD-3-Clause). Their license expressions include these licenses. + Minuit2 is now fetched at tag ``v6-40-02`` instead of ``master``, and + libzmq at ``v4.3.5`` (MPL-2.0) instead of ``v4.3.4`` (LGPL-3.0 with a + static-linking exception). +- ``RawFile`` and ``File`` reject raw files with frame padding disabled unless + the frame discard policy is ``discardpartial``. The constructor reports the + master path before opening data subfiles. Legacy ``.raw`` master files now + parse the frame discard policy so unpadded ``discardpartial`` files remain + readable. +- Fixed ``CtbRawFile.read_frame(index)`` and reads after ``seek(index)`` at + subfile boundaries skipping a subfile, returning the wrong frame or raising + ``Subfile index out of range``. - ``Pedestal`` reports mismatched frame shapes with exceptions in all push overloads, including Debug builds, instead of aborting on assertions. - Python ``Pedestal`` constructors reject negative dimensions and sample @@ -85,6 +232,10 @@ without implicit conversion. Both ``push()`` and ``push_with_threshold()`` validate that frames and thresholds are two-dimensional before constructing views, preventing incorrect results from unsupported array layouts or ranks. +- Python ``ClusterFinder.push_pedestal_frame()`` and ``find_clusters()`` now + require C-contiguous ``uint16`` frames without implicit conversion, raising + ``TypeError`` instead of silently copying and casting the input. The frame + argument can also be passed by keyword as ``frame``. - Fixed a leaked empty ``ClusterVector`` at the end of Python ``ClusterFile`` iteration. Chunk iteration now rejects a zero chunk size. @@ -263,4 +414,3 @@ dhanya.thattil@psi.ch - diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt new file mode 100644 index 00000000..3d07d9c7 --- /dev/null +++ b/THIRD-PARTY-NOTICES.txt @@ -0,0 +1,137 @@ +Third-party notices for aare binary packages +============================================ + +The aare Python wheels and conda packages compile the following libraries into +the aare extension module. They are not part of the aare source repository; +CMake downloads them at build time. Their license notices follow below. + + Minuit2 v6-40-02 LGPL-2.1-or-later https://github.com/GooFit/Minuit2 + {fmt} 12.1.0 MIT https://github.com/fmtlib/fmt + nlohmann/json 3.11.3 MIT https://github.com/nlohmann/json + pybind11 3.0.3 BSD-3-Clause https://github.com/pybind/pybind11 + +Minuit2 is linked statically. To use aare with a modified Minuit2, build aare +from source (https://github.com/slsdetectorgroup/aare) with +-DAARE_FETCH_MINUIT2=OFF and make your Minuit2 build findable by CMake's +find_package(Minuit2). + +------------------------------------------------------------------------------ +Minuit2 +------------------------------------------------------------------------------ + +Copyright (c) 2005 LCG ROOT Math team, CERN/PH-SFT + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +See LGPL2_1.txt next to this file for the text of the LGPL version 2.1. +For questions regarding the ROOT license, please contact rootdev@cern.ch. + +The precompiled binary distributions provided as a service to our community +may feature packages released with licenses different from LGPL. + +In aare packages the text of the LGPL version 2.1 is in +LICENSES/LGPL-2.1-or-later.txt. + +------------------------------------------------------------------------------ +{fmt} +------------------------------------------------------------------------------ + +Copyright (c) 2012 - present, Victor Zverovich and {fmt} contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. + +------------------------------------------------------------------------------ +nlohmann/json +------------------------------------------------------------------------------ + +MIT License + +Copyright (c) 2013-2022 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------ +pybind11 +------------------------------------------------------------------------------ + +Copyright (c) 2016 Wenzel Jakob , All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of +external contributions to this project including patches, pull requests, etc. diff --git a/VERSION b/VERSION index 2398eb24..966e58e3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2026.7.2 \ No newline at end of file +2026.9.30 \ No newline at end of file diff --git a/benchmarks/fit_benchmark.cpp b/benchmarks/fit_benchmark.cpp index 626bde8d..2a307002 100644 --- a/benchmarks/fit_benchmark.cpp +++ b/benchmarks/fit_benchmark.cpp @@ -3,12 +3,15 @@ #include "aare/FitModel.hpp" #include "aare/Models.hpp" +#include #include #include #include #include #include +using aare::ssize_t; + struct TestCase { std::string name; double true_A; @@ -75,52 +78,31 @@ static void report_accuracy(benchmark::State &state, const TestCase &tc, // Benchmarks // ---------- -// Minuit2, analytic gradient (no Hesse) -static void BM_FitGausMinuitGrad(benchmark::State &state) { - const auto &tc = get_test_cases()[state.range(0)]; - auto data = generate_gaussian_data(tc); - auto xv = data.x.view(); - auto yv = data.y.view(); - - const auto model = aare::FitModel( - /*strategy = */ 0, - /*max_calls = */ 500, // increase for noisy signals - /*tolerance = */ 0.5, - /*compute_errors = */ false); - - aare::NDArray result; - for (auto _ : state) { - result = aare::fit_pixel(model, xv, yv); - benchmark::DoNotOptimize(result.data()); - } - - report_accuracy(state, tc, result); - state.SetLabel(tc.name); -} - -// Minuit2, analytic gradient + Hesse -static void BM_FitGausMinuitGradHesse(benchmark::State &state) { +static void run_gaussian_fit(benchmark::State &state, aare::Minimizer minimizer, + bool compute_errors) { const auto &tc = get_test_cases()[state.range(0)]; auto data = generate_gaussian_data(tc); auto xv = data.x.view(); auto yv = data.y.view(); auto ev = data.y_err.view(); - const auto model = aare::FitModel( - 0, 500, 0.5, true); // compute_errors = true -> Runs Hesse and provides - // errors on fitted params + const aare::FitModel model( + /*strategy = */ 0, + /*max_calls = */ 500, // increase for noisy signals + /*tolerance = */ 0.5, compute_errors, minimizer); aare::NDArray result; for (auto _ : state) { - result = aare::fit_pixel(model, xv, yv, ev); + result = compute_errors + ? aare::fit_pixel(model, xv, yv, ev) + : aare::fit_pixel(model, xv, yv); benchmark::DoNotOptimize(result.data()); } - // result has 6 elements: [A, mu, sig, err_A, err_mu, err_sig] report_accuracy(state, tc, result); - // Also report Hesse uncertainties - if (result.size() >= 6) { + // With errors the result is [A, mu, sig, err_A, err_mu, err_sig, chi2] + if (compute_errors && result.size() >= 6) { state.counters["errA"] = result(3); state.counters["errMu"] = result(4); state.counters["errSig"] = result(5); @@ -128,12 +110,372 @@ static void BM_FitGausMinuitGradHesse(benchmark::State &state) { state.SetLabel(tc.name); } -BENCHMARK(BM_FitGausMinuitGrad) +// Migrad, analytic gradient (no Hesse) +static void BM_FitGausMigrad(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::Migrad, false); +} + +// Migrad, analytic gradient + Hesse for the parameter errors +static void BM_FitGausMigradHesse(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::Migrad, true); +} + +// Fumili, analytic gradient and linearised Hessian +static void BM_FitGausFumili(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::Fumili, false); +} + +// Fumili, parameter errors from the linearised covariance +static void BM_FitGausFumiliErrors(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::Fumili, true); +} + +// Built-in Levenberg-Marquardt, analytic Jacobian +static void BM_FitGausLM(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::LevenbergMarquardt, false); +} + +// Built-in Levenberg-Marquardt, Gauss-Newton parameter errors +static void BM_FitGausLMErrors(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::LevenbergMarquardt, true); +} + +// Built-in variable projection: A solved exactly, iteration over mu and sigma +static void BM_FitGausVarPro(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::VarPro, false); +} + +static void BM_FitGausVarProErrors(benchmark::State &state) { + run_gaussian_fit(state, aare::Minimizer::VarPro, true); +} + +// Gaussian with sigma fixed: exercises the fixed-parameter path of the +// minimizers on the Moderate_noise case. +static void run_gaussian_fixed_sigma_fit(benchmark::State &state, + aare::Minimizer minimizer) { + const auto &tc = get_test_cases()[1]; + auto data = generate_gaussian_data(tc); + auto xv = data.x.view(); + auto yv = data.y.view(); + + aare::FitModel model(0, 500, 0.5, false, minimizer); + model.FixParameter(2, tc.true_sig); + + aare::NDArray result; + for (auto _ : state) { + result = aare::fit_pixel(model, xv, yv); + benchmark::DoNotOptimize(result.data()); + } + report_accuracy(state, tc, result); +} + +static void BM_FitGausFixedSigmaMigrad(benchmark::State &state) { + run_gaussian_fixed_sigma_fit(state, aare::Minimizer::Migrad); +} +static void BM_FitGausFixedSigmaFumili(benchmark::State &state) { + run_gaussian_fixed_sigma_fit(state, aare::Minimizer::Fumili); +} +static void BM_FitGausFixedSigmaLM(benchmark::State &state) { + run_gaussian_fixed_sigma_fit(state, aare::Minimizer::LevenbergMarquardt); +} +static void BM_FitGausFixedSigmaVarPro(benchmark::State &state) { + run_gaussian_fixed_sigma_fit(state, aare::Minimizer::VarPro); +} + +// ---------------------------------------------------------------- +// Rising S-curve (6 parameters), typical for threshold scans +// ---------------------------------------------------------------- +static constexpr ssize_t N_SCURVE_POINTS = 100; + +static void run_scurve_fit(benchmark::State &state, aare::Minimizer minimizer, + bool weighted = false) { + const std::vector truth = {5.0, 0.1, 50.0, 4.0, 200.0, 0.5}; + const double noise_sigma = 0.02 * truth[4]; + + aare::NDArray x({N_SCURVE_POINTS}); + aare::NDArray y({N_SCURVE_POINTS}); + aare::NDArray y_err({N_SCURVE_POINTS}, noise_sigma); + std::mt19937 rng(SEED); + std::normal_distribution noise(0.0, noise_sigma); + for (ssize_t i = 0; i < N_SCURVE_POINTS; ++i) { + x[i] = static_cast(i); + y[i] = aare::model::RisingScurve::eval(x[i], truth) + noise(rng); + } + auto xv = x.view(); + auto yv = y.view(); + auto ev = y_err.view(); + + const aare::FitModel model(0, 500, 0.5, false, + minimizer); + + aare::NDArray result; + for (auto _ : state) { + result = + weighted + ? aare::fit_pixel(model, xv, yv, ev) + : aare::fit_pixel(model, xv, yv); + benchmark::DoNotOptimize(result.data()); + } + + state.counters["dMu"] = result(2) - truth[2]; + state.counters["dSig"] = result(3) - truth[3]; + state.counters["dA"] = result(4) - truth[4]; + state.counters["chi2"] = result(6); +} + +static void BM_FitScurveMigrad(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::Migrad); +} + +static void BM_FitScurveFumili(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::Fumili); +} + +static void BM_FitScurveLM(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::LevenbergMarquardt); +} + +static void BM_FitScurveVarPro(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::VarPro); +} + +// Weighted fits take the per-point uncertainties into the residuals. +static void BM_FitScurveWeightedMigrad(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::Migrad, true); +} +static void BM_FitScurveWeightedFumili(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::Fumili, true); +} +static void BM_FitScurveWeightedLM(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::LevenbergMarquardt, true); +} +static void BM_FitScurveWeightedVarPro(benchmark::State &state) { + run_scurve_fit(state, aare::Minimizer::VarPro, true); +} + +// ---------------------------------------------------------------- +// Pol2 (3 parameters, cheap model): exposes the solver overhead per +// iteration rather than the model evaluation. +// ---------------------------------------------------------------- +static void run_pol2_fit(benchmark::State &state, aare::Minimizer minimizer) { + constexpr ssize_t n = 51; + const std::vector truth = {12.0, -0.8, 0.05}; + + aare::NDArray x({n}); + aare::NDArray y({n}); + std::mt19937 rng(SEED); + std::normal_distribution noise(0.0, 2.0); + for (ssize_t i = 0; i < n; ++i) { + x[i] = static_cast(i); + y[i] = aare::model::Pol2::eval(x[i], truth) + noise(rng); + } + auto xv = x.view(); + auto yv = y.view(); + + const aare::FitModel model(0, 500, 0.5, false, + minimizer); + + aare::NDArray result; + for (auto _ : state) { + result = aare::fit_pixel(model, xv, yv); + benchmark::DoNotOptimize(result.data()); + } + state.counters["dp2"] = result(2) - truth[2]; +} + +static void BM_FitPol2Migrad(benchmark::State &state) { + run_pol2_fit(state, aare::Minimizer::Migrad); +} +static void BM_FitPol2Fumili(benchmark::State &state) { + run_pol2_fit(state, aare::Minimizer::Fumili); +} +static void BM_FitPol2LM(benchmark::State &state) { + run_pol2_fit(state, aare::Minimizer::LevenbergMarquardt); +} +static void BM_FitPol2VarPro(benchmark::State &state) { + run_pol2_fit(state, aare::Minimizer::VarPro); +} + +// ---------------------------------------------------------------- +// GaussianChargeSharingKb (8 parameters, two exponentials and two erfs per +// point): the most expensive model per evaluation. +// ---------------------------------------------------------------- +static void run_charge_sharing_kb_fit(benchmark::State &state, + aare::Minimizer minimizer) { + constexpr ssize_t n = 200; + const std::vector truth = {20.0, 0.05, 110.0, 6.0, + 1500.0, 0.1, 1.1, 0.1}; + + aare::NDArray x({n}); + aare::NDArray y({n}); + std::mt19937 rng(SEED); + std::normal_distribution noise(0.0, 1.0); + for (ssize_t i = 0; i < n; ++i) { + x[i] = static_cast(i); + const double clean = + aare::model::GaussianChargeSharingKb::eval(x[i], truth); + y[i] = clean + std::sqrt(std::max(clean, 1.0)) * noise(rng); + } + auto xv = x.view(); + auto yv = y.view(); + + const aare::FitModel model( + 0, 500, 0.5, false, minimizer); + + aare::NDArray result; + for (auto _ : state) { + result = aare::fit_pixel(model, + xv, yv); + benchmark::DoNotOptimize(result.data()); + } + state.counters["dMu"] = result(2) - truth[2]; + state.counters["dN"] = result(4) - truth[4]; + state.counters["chi2"] = result(8); +} + +static void BM_FitChargeSharingKbMigrad(benchmark::State &state) { + run_charge_sharing_kb_fit(state, aare::Minimizer::Migrad); +} +static void BM_FitChargeSharingKbFumili(benchmark::State &state) { + run_charge_sharing_kb_fit(state, aare::Minimizer::Fumili); +} +static void BM_FitChargeSharingKbLM(benchmark::State &state) { + run_charge_sharing_kb_fit(state, aare::Minimizer::LevenbergMarquardt); +} +static void BM_FitChargeSharingKbVarPro(benchmark::State &state) { + run_charge_sharing_kb_fit(state, aare::Minimizer::VarPro); +} + +// ---------------------------------------------------------------- +// Data cube through fit_3d on one thread: exposes the per-pixel overhead +// of each minimizer on top of the minimisation itself. +// ---------------------------------------------------------------- +static constexpr ssize_t CUBE_ROWS = 32; +static constexpr ssize_t CUBE_COLS = 32; + +static void run_cube_fit(benchmark::State &state, aare::Minimizer minimizer, + bool compute_errors) { + const auto &tc = get_test_cases()[1]; // Moderate_noise + const double x_min = tc.true_mu - 5.0 * tc.true_sig; + const double x_max = tc.true_mu + 5.0 * tc.true_sig; + const double dx = (x_max - x_min) / (N_POINTS - 1); + const double noise_sigma = tc.noise_frac * tc.true_A; + + aare::NDArray x({N_POINTS}); + aare::NDArray y({CUBE_ROWS, CUBE_COLS, N_POINTS}); + aare::NDArray y_err({CUBE_ROWS, CUBE_COLS, N_POINTS}, + noise_sigma); + std::mt19937 rng(SEED); + std::normal_distribution noise(0.0, noise_sigma); + for (ssize_t i = 0; i < N_POINTS; ++i) + x[i] = x_min + i * dx; + for (ssize_t row = 0; row < CUBE_ROWS; ++row) { + for (ssize_t col = 0; col < CUBE_COLS; ++col) { + for (ssize_t i = 0; i < N_POINTS; ++i) { + const double clean = + tc.true_A * std::exp(-std::pow(x[i] - tc.true_mu, 2) / + (2.0 * std::pow(tc.true_sig, 2))); + y(row, col, i) = clean + noise(rng); + } + } + } + + aare::NDArray par({CUBE_ROWS, CUBE_COLS, 3}); + aare::NDArray err({CUBE_ROWS, CUBE_COLS, 3}); + aare::NDArray chi2({CUBE_ROWS, CUBE_COLS}); + const aare::FitModel model( + 0, 500, 0.5, compute_errors, minimizer); + + for (auto _ : state) { + if (compute_errors) { + aare::fit_3d(model, x.view(), y.view(), y_err.view(), par.view(), + err.view(), chi2.view(), 1); + } else { + aare::fit_3d(model, x.view(), y.view(), aare::NDView{}, + par.view(), aare::NDView{}, chi2.view(), 1); + } + benchmark::DoNotOptimize(par.data()); + } + + state.SetItemsProcessed(state.iterations() * CUBE_ROWS * CUBE_COLS); + state.counters["failed"] = + static_cast(std::count(chi2.begin(), chi2.end(), 0.0)); +} + +static void BM_FitCubeMigrad(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::Migrad, false); +} +static void BM_FitCubeFumili(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::Fumili, false); +} +static void BM_FitCubeLM(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::LevenbergMarquardt, false); +} +static void BM_FitCubeMigradErrors(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::Migrad, true); +} +static void BM_FitCubeFumiliErrors(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::Fumili, true); +} +static void BM_FitCubeLMErrors(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::LevenbergMarquardt, true); +} +static void BM_FitCubeVarPro(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::VarPro, false); +} +static void BM_FitCubeVarProErrors(benchmark::State &state) { + run_cube_fit(state, aare::Minimizer::VarPro, true); +} + +BENCHMARK(BM_FitGausMigrad)->DenseRange(0, 5)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFumili)->DenseRange(0, 5)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausLM)->DenseRange(0, 5)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausVarPro)->DenseRange(0, 5)->Unit(benchmark::kMicrosecond); + +BENCHMARK(BM_FitGausMigradHesse) + ->DenseRange(0, 5) + ->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFumiliErrors) + ->DenseRange(0, 5) + ->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausLMErrors)->DenseRange(0, 5)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausVarProErrors) ->DenseRange(0, 5) ->Unit(benchmark::kMicrosecond); -BENCHMARK(BM_FitGausMinuitGradHesse) - ->DenseRange(0, 5) - ->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFixedSigmaMigrad)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFixedSigmaFumili)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFixedSigmaLM)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitGausFixedSigmaVarPro)->Unit(benchmark::kMicrosecond); + +BENCHMARK(BM_FitScurveMigrad)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveFumili)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveLM)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveVarPro)->Unit(benchmark::kMicrosecond); + +BENCHMARK(BM_FitScurveWeightedMigrad)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveWeightedFumili)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveWeightedLM)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitScurveWeightedVarPro)->Unit(benchmark::kMicrosecond); + +BENCHMARK(BM_FitPol2Migrad)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitPol2Fumili)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitPol2LM)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitPol2VarPro)->Unit(benchmark::kMicrosecond); + +BENCHMARK(BM_FitChargeSharingKbMigrad)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitChargeSharingKbFumili)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitChargeSharingKbLM)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_FitChargeSharingKbVarPro)->Unit(benchmark::kMicrosecond); + +// fit_3d runs in worker threads, so measure wall time. +BENCHMARK(BM_FitCubeMigrad)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeFumili)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeLM)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeMigradErrors)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeFumiliErrors)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeLMErrors)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeVarPro)->Unit(benchmark::kMillisecond)->UseRealTime(); +BENCHMARK(BM_FitCubeVarProErrors)->Unit(benchmark::kMillisecond)->UseRealTime(); BENCHMARK_MAIN(); diff --git a/benchmarks/ndarray_benchmark.cpp b/benchmarks/ndarray_benchmark.cpp index 456f4d40..46e7138e 100644 --- a/benchmarks/ndarray_benchmark.cpp +++ b/benchmarks/ndarray_benchmark.cpp @@ -3,6 +3,7 @@ #include using aare::NDArray; +using aare::ssize_t; constexpr ssize_t size = 1024; class TwoArrays : public benchmark::Fixture { @@ -130,4 +131,80 @@ BENCHMARK_F(TwoArrays, MultiplyAddDivideWithIndex)(benchmark::State &st) { } } +BENCHMARK_F(TwoArrays, ScalarMultiplyAddWithOperator)(benchmark::State &st) { + for (auto _ : st) { + // This code gets timed + NDArray res = a * 2 + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_F(TwoArrays, ScalarMultiplyAddWithIndex)(benchmark::State &st) { + for (auto _ : st) { + // This code gets timed + NDArray res(a.shape()); + for (uint32_t i = 0; i < a.size(); i++) { + res(i) = a(i) * 2 + b(i); + } + benchmark::DoNotOptimize(res); + } +} + +BENCHMARK_F(TwoArrays, AddToExistingWithOperator)(benchmark::State &st) { + NDArray res(a.shape()); + for (auto _ : st) { + // This code gets timed + res = a + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_F(TwoArrays, AddToExistingWithIndex)(benchmark::State &st) { + NDArray res(a.shape()); + for (auto _ : st) { + // This code gets timed + for (uint32_t i = 0; i < a.size(); i++) { + res(i) = a(i) + b(i); + } + benchmark::DoNotOptimize(res); + } +} + +// A store to uint8_t or a 64 bit integer can alias the members of NDArray. +// Compare with a loop over raw pointers to check that this does not block +// vectorization. +template void InPlaceAddWithOperator(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + for (auto _ : st) { + a += b; + benchmark::DoNotOptimize(a); + } +} +template void InPlaceAddWithPointer(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + for (auto _ : st) { + T *dst = a.data(); + const T *src = b.data(); + for (ssize_t i = 0, n = a.size(); i < n; ++i) { + dst[i] += src[i]; + } + benchmark::DoNotOptimize(a); + } +} +template void AddToExistingWithOperator(benchmark::State &st) { + NDArray a({size, size}, 1); + NDArray b({size, size}, 2); + NDArray res({size, size}, 0); + for (auto _ : st) { + res = a + b; + benchmark::DoNotOptimize(res); + } +} +BENCHMARK_TEMPLATE(InPlaceAddWithOperator, uint8_t); +BENCHMARK_TEMPLATE(InPlaceAddWithPointer, uint8_t); +BENCHMARK_TEMPLATE(AddToExistingWithOperator, uint8_t); +BENCHMARK_TEMPLATE(InPlaceAddWithOperator, int64_t); +BENCHMARK_TEMPLATE(InPlaceAddWithPointer, int64_t); +BENCHMARK_TEMPLATE(AddToExistingWithOperator, int64_t); + BENCHMARK_MAIN(); \ No newline at end of file diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index a052e1d4..b980c93d 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: MPL-2.0 + # Look for an executable called sphinx-build find_program( SPHINX_EXECUTABLE diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 595f9530..2fe1d7cf 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 function(default_build_type val) if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "No build type selected, default to Release") diff --git a/cmake/package_config.cmake b/cmake/package_config.cmake index d018a7f0..0d9c0dcb 100644 --- a/cmake/package_config.cmake +++ b/cmake/package_config.cmake @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: MPL-2.0 + # This cmake code creates the configuration that is found and used by # find_package() of another cmake project diff --git a/cmake/project-config.cmake.in b/cmake/project-config.cmake.in index 793a373f..b3f0d569 100644 --- a/cmake/project-config.cmake.in +++ b/cmake/project-config.cmake.in @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # Config file for @PROJECT_NAME_LOWER@ # # It defines the following variables: diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 66fbe2e3..d2d39b53 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -26,7 +26,7 @@ requirements: - python - pip - numpy >=2.1 - - scikit-build-core + - scikit-build-core >=0.11 - pybind11 >=3.0.3 - matplotlib # needed in host to solve the environment for run @@ -50,5 +50,11 @@ test: - python -m pytest python/tests about: - license: SPDX-License-Identifier MPL-2.0 + license: MPL-2.0 AND Apache-2.0 AND MIT AND BSD-3-Clause AND LGPL-2.1-or-later + license_file: + - LICENSES/MPL-2.0.txt + - LICENSES/Apache-2.0.txt + - LICENSES/MIT.txt + - LICENSES/LGPL-2.1-or-later.txt + - THIRD-PARTY-NOTICES.txt summary: Data analysis library for hybrid pixel detectors from PSI diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index df589419..d83b6f29 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -36,7 +36,7 @@ add_custom_target( docs COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} COMMAND - ${SPHINX_EXECUTABLE} -a -b html + ${SPHINX_EXECUTABLE} -W -a -b html -Dbreathe_projects.aare=${CMAKE_CURRENT_BINARY_DIR}/xml -c "${SPHINX_BUILD}" ${SPHINX_BUILD}/src ${SPHINX_BUILD}/html COMMENT "Generating documentation with Sphinx") diff --git a/docs/conf.py.in b/docs/conf.py.in index ad735752..1e0b5a71 100644 --- a/docs/conf.py.in +++ b/docs/conf.py.in @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full diff --git a/docs/src/NumpyArrays.rst b/docs/src/NumpyArrays.rst new file mode 100644 index 00000000..b837771c --- /dev/null +++ b/docs/src/NumpyArrays.rst @@ -0,0 +1,147 @@ +**************** +Numpy arrays +**************** + +Most Python functions in aare take NumPy arrays as input. The bindings in +``python/src/`` receive them as ``py::array_t`` and wrap the buffer in an +``NDView`` before calling the C++ implementation. No data is copied, so the +C++ code reads and writes the memory that the caller's array owns. This page +explains the choices we make when declaring such an argument and what they +mean for the user. + +The preferred pattern +~~~~~~~~~~~~~~~~~~~~~~~~ + +Declare the argument with only the element type, take it by value, mark it +``noconvert`` in the ``py::arg`` list and let ``make_view`` check the rest: + +.. code-block:: cpp + + m.def( + "push", + [](Pedestal &self, py::array_t frame) { + auto view = make_view_2d(frame); + self.push(view); + }, + py::arg("frame").noconvert()); + +The responsibilities are split between pybind11 and ``make_view``: + +- pybind11 checks that the argument is a NumPy array with exactly the + requested dtype. Anything else fails with a ``TypeError``, and since the + argument is ``noconvert`` pybind11 never tries to build a matching array + from it. +- ``make_view`` (``python/src/np_helper.hpp``) checks that the array has the + expected number of dimensions and is C-contiguous. It raises a + ``ValueError`` with a message that tells the user what to fix, for example + ``Array is not C-contiguous. Use np.ascontiguousarray(arr)``. Writing + functions use ``make_view`` which also rejects read-only arrays. Read-only + input goes through ``make_const_view``. + +The result is a binding that either operates on the caller's memory or +refuses the call. It never silently works on a temporary copy. + +Take the array by value +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``py::array_t`` is a handle to the Python object, not the data. Passing it by +value copies the handle, which increments the reference count of the NumPy +array, and nothing else. The pixel data stays where it is. We therefore +declare the parameter as ``py::array_t frame`` rather than +``py::array_t &frame``: + +- A reference saves one reference-count increment per call, which is not + measurable next to the work done on a frame. +- A by-value parameter is a plain lvalue inside the lambda, so + ``make_view``, which calls ``mutable_data()``, works without further + qualification. A ``const py::array_t &`` does not compile with it. +- It matches the usual pybind11 style for ``py::object`` and its subclasses, + so the binding reads the same as examples in the pybind11 documentation. + +Why not let pybind11 convert +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Without ``noconvert``, pybind11 tries every overload once with conversions +disabled and then once more with conversions enabled. In the second pass a +``py::array_t`` argument accepts anything NumPy can turn into an array of +type ``T``: an array with another dtype, a non-contiguous array, or even a +Python list. pybind11 creates a new array, hands the converted copy to the +binding and drops it when the call returns. For detector data this has +several consequences: + +- The copy costs time and memory, and the user gets no indication that it + happened. For a stack of frames this can be gigabytes. +- Writes are lost. A function that fills or updates its input, for example + a pedestal subtraction in place, writes into the temporary copy. +- Values are cast without warning. ``forcecast`` allows unsafe casts, so a + ``float64`` frame is truncated to ``uint16`` instead of being rejected. +- Overloads stop working as intended. ``apply_calibration`` is registered for + both ``double`` and ``float``. With conversions allowed the first overload + would accept everything by casting and the ``float`` overload would never be + chosen. + +Argument conversion is only worth it for small inputs where convenience beats +control, for example the parameter vector passed to a fit. Even then the +function must not write into the argument. + +.. note:: + + When a binding intentionally lets pybind11 convert, say so with a comment + next to the argument. Otherwise a missing ``noconvert`` looks like an + oversight and the next person to touch the binding will add it. + + .. code-block:: cpp + + // Conversion allowed: small read-only vector, accept lists and any dtype + py::array_t par + +What the template flags mean +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``py::array_t`` takes an optional second template argument. The +flags do two things: they add requirements that the array must fulfil for the +*non-converting* check to pass, and they tell NumPy what to produce when a +conversion *is* allowed. + +``py::array::c_style`` + The array must be C-contiguous. With ``noconvert`` a non-contiguous array + is rejected by pybind11 with a ``TypeError`` listing the accepted + signatures. Without ``noconvert`` pybind11 makes a contiguous copy. + Since ``make_view`` already enforces contiguity and gives a more helpful + ``ValueError``, the flag is redundant in the preferred pattern. It is + still present in some older bindings and is harmless there. + +``py::array::f_style`` + Same as above for Fortran order. ``NDView`` assumes C order, so we do not + use it. + +``py::array::forcecast`` + Allows NumPy to perform unsafe casts, such as ``float64`` to ``uint16``, + when converting. It is the **default** value of the template argument, so + ``py::array_t`` and ``py::array_t`` + are the same type. The flag has no effect together with ``noconvert`` + because no conversion takes place. It only matters when conversions are + allowed, where it decides whether a lossy cast is accepted or the call is + rejected. + +The common spelling ``py::array_t`` +therefore means: when converting, produce a contiguous array of type ``T`` and +cast whatever you were given. Combined with ``noconvert`` it collapses to +"C-contiguous array of exactly type ``T``", which is what the plain +``py::array_t`` plus ``make_view`` already gives us. + +Summary +~~~~~~~~~~~~~~~~~~ + +- Declare the argument as ``py::array_t`` by value and add + ``.noconvert()`` to its ``py::arg``. The handle is reference counted, so + passing it by value does not copy the data. +- Create the view with ``make_view_1d/2d/3d`` or ``make_const_view_*`` and + let it validate dimensions and contiguity. Add an explicit check only when + the binding accepts several shapes, as ``apply_calibration`` does for 2D + and 3D pedestals. +- Return data to Python with ``return_image_data`` or ``return_vector`` so that + the NumPy array owns the C++ allocation. +- Only allow conversion for small, read-only inputs where accepting lists or + other dtypes is a real convenience for the user, and mark it with a comment + in the code. diff --git a/docs/src/RawFile.rst b/docs/src/RawFile.rst index bd45ad71..8ecd30e9 100644 --- a/docs/src/RawFile.rst +++ b/docs/src/RawFile.rst @@ -1,6 +1,14 @@ RawFile =============== +Reading requires frame padding to be enabled +or ``Frame Discard Policy`` to be ``discardpartial``. +The constructor checks the +parsed master metadata before opening data subfiles and throws +``std::runtime_error`` with the master path if this requirement is not met. +This applies to JSON and legacy ``.raw`` masters, including when opened through +``aare::File``. ``RawMasterFile`` can still be used to inspect their metadata. + ``total_frames()`` is the minimum actual frame count across all selected ``RawSubFile`` objects in all ROIs. Each subfile count includes its complete series of ``.raw`` files and is determined when the file is opened. Disabled diff --git a/docs/src/index.rst b/docs/src/index.rst index fc544c6e..e335a764 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -70,3 +70,4 @@ AARE Philosophy Workflow Tests + NumpyArrays diff --git a/docs/src/pyFit.rst b/docs/src/pyFit.rst index 257f7414..142b1ca6 100644 --- a/docs/src/pyFit.rst +++ b/docs/src/pyFit.rst @@ -4,11 +4,18 @@ Fitting .. py:currentmodule:: aare Aare fits one-dimensional scans and three-dimensional pixel data with -Minuit2. Create a model object and call its :meth:`fit` method:: +Minuit2's Migrad and Fumili or with the built-in Levenberg-Marquardt and +variable projection solvers. Create a model object and call its :meth:`fit` +method:: model = Gaussian(compute_errors=True) result = model.fit(x, y, y_err) +``y_err`` holds the standard deviation of each point and weights the +residuals; points with ``y_err == 0`` are left out of the fit, which skips +empty histogram bins when ``y_err = sqrt(y)``. Without ``y_err`` every point +has unit weight. + The model object is also callable, which evaluates it at the supplied points:: fitted_y = model(x, result["par"]) @@ -20,11 +27,127 @@ accepts the same model objects when a functional interface is preferred. For three-dimensional data, pass an array with shape ``(rows, columns, scan_points)`` and select the worker count with -``n_threads``:: +``n_threads`` (default 4):: result = model.fit(x, image_data, image_errors, n_threads=8) -The result dictionary contains ``par`` and ``chi2``. It also contains -``par_err`` when ``compute_errors`` is enabled. +The result dictionary contains ``par`` and ``chi2``, with shapes ``(npar,)`` +and ``(1,)`` for a scan or ``(rows, columns, npar)`` and ``(rows, columns)`` +for a cube. It also contains ``par_err`` when ``compute_errors`` is enabled. +A pixel that fails to converge returns zeros for all three. + +.. note:: + + The fit works on ``float64`` data. If ``x``, ``y`` or ``y_err`` is not + already a C-contiguous ``float64`` array, the Python bindings convert it + before fitting, which allocates a full copy. Integer or ``float32`` + input therefore works, but for a large three-dimensional ``y`` the + copy can be several times the size of the original array. Pass + ``float64`` arrays to avoid it. A ``float64`` C-contiguous array is used + in place without copying. + +Choosing the minimizer +~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: + + The built-in VarPro solver is up to ten times faster than Migrad. It and + the built-in Levenberg-Marquardt solver do not offer Minuit2's strategy + setting or Hesse errors. Try VarPro first and fall back to Migrad if you + run into problems. Minuit2's Fumili and the built-in Levenberg-Marquardt + solver are included for evaluation and may turn out to be redundant. + +Fits use Minuit2's Migrad by default. Three alternatives share the same +interface and are selected with the ``minimizer`` argument or property:: + + model = Gaussian(minimizer=Minimizer.LevenbergMarquardt) + model.minimizer = Minimizer.Fumili + +- ``Minimizer.Migrad``: Minuit2's variable-metric minimizer, driven by the + analytic gradient of the chi-squared. Parameter errors come from Hesse. +- ``Minimizer.Fumili``: Minuit2's Gauss-Newton minimizer, which controls its + steps with a trust region by default. It takes the gradient and the + linearised Hessian from the analytic derivatives of the model and typically + needs far fewer function evaluations than Migrad. The linearised Hessian at + the minimum also provides the parameter covariance, so no Hesse step runs. + + Parameter limits make Minuit2 map the bounded external parameters onto + unbounded internal ones. On an active limit the derivative of that mapping + vanishes, and Fumili's linearised Hessian, which contains only this first + derivative, loses the curvature along the bounded direction. The Hessian + can then become singular and the iteration fails to converge. Pixels for + which Fumili does not return a valid minimum are refitted with Migrad. + +- ``Minimizer.LevenbergMarquardt``: a built-in damped Gauss-Newton solver with + the analytic Jacobian of the model. It reflects steps at parameter limits + and reuses its buffers between pixels, so data cubes are fitted without + per-pixel allocations. Parameter errors are ``sqrt(diag((J^T W J)^-1))`` + with ``W`` the weights from ``y_err``. +- ``Minimizer.VarPro``: a built-in variable projection solver. + Every bundled model is linear in some of its parameters: every parameter of + a polynomial, the amplitude of a Gaussian, the amplitude and plateau of + ``GaussianErfcPlateau``, the background and amplitude of the charge-sharing + models, and the background, amplitude and charge-sharing term of the + S-curves. Each trial point solves those exactly and the Levenberg-Marquardt + iteration runs over the remaining nonlinear parameters only, so it needs + fewer evaluations than LevenbergMarquardt and is several times faster than + Migrad. Start values of the linear parameters are not used, so + ``SetParameter`` has no effect on them; ``FixParameter`` does. A pixel + whose linear solution violates a limit on a linear parameter, or that does + not converge, is refitted with LevenbergMarquardt with a fresh + ``max_calls`` budget. Parameter errors are the same Gauss-Newton estimates. + On data that barely constrains a model, VarPro can settle at a point where + an amplitude is driven to zero and the parameters attached to it lose + their meaning; such a pixel reports a valid fit with a poor ``chi2``. + +All four converge to the same minimum on well-behaved data; the ``[fit]`` +test suite checks that. The bundled ``fit_benchmark`` compares their speed on +Gaussian peaks, S-curves, polynomials, charge-sharing spectra and a data +cube. On ill-conditioned data (few points, many parameters, or parameters +that nearly trade off against each other) the minimizers stop at different +points of a flat chi-squared valley, and Migrad in particular can settle on a +parameter limit with a higher ``chi2`` than the others. Compare ``chi2`` +rather than the parameters when they disagree, fix or bound the poorly +constrained parameters, and provide start values for the rest. + +Parameters and errors +~~~~~~~~~~~~~~~~~~~~~ + +Start values are estimated from the data. ``SetParameter`` overrides the +estimate, ``FixParameter`` keeps a parameter at a given value, and +``SetParLimits`` restricts it to a range where ``-inf`` and ``inf`` leave a +side open (the Minuit2 minimizers turn an open side into a wide finite one); +free start values are clamped into their limits. The constructor arguments +control the minimizer: + +- ``max_calls`` (default 100) caps the work per pixel: Minuit2 function calls + for Migrad and Fumili, model evaluations for LevenbergMarquardt and VarPro, + which spend one evaluation per iteration (two when a step crosses a limit). + ``0`` selects Minuit's default max calls of ``200 + 100 * npar + 5 * npar**2``. + A pixel that does not converge within the budget returns zeros; the six- + and eight-parameter models on noisy data need several hundred calls, so + raise the default when such pixels fail. +- ``tolerance`` is the EDM tolerance in Minuit's convention. Migrad, + LevenbergMarquardt and VarPro stop when the estimated distance + to the minimum is below ``0.002 * tolerance``, Fumili below + ``1e-4 * tolerance``. The EDM is measured in units of ``chi2``, not + relative to the data: without ``y_err`` every sample counts with an error + of 1, so a fit of small values (amplitudes well below 1) meets the + tolerance while still far from the minimum. Pass ``y_err`` or lower + ``tolerance`` for such data. +- ``strategy`` is the Minuit2 strategy (0 is fast, 1 is Minuit2's default). + LevenbergMarquardt and VarPro ignore it. +- ``compute_errors`` adds ``par_err`` to the result. Fixed parameters and + parameters that end on a limit report an error of 0 with every minimizer. + Migrad's Hesse errors include the residual-weighted second derivatives of + the model, which the Gauss-Newton estimates of the other three omit; the + two agree when the model describes the data (``chi2`` per degree of freedom + near 1) and the Gauss-Newton errors come out smaller for a poorer fit. + +Unknown parameter names raise ``RuntimeError``, bad indices ``IndexError``, +and ``SetParLimits`` rejects a lower limit at or above the upper one. .. autofunction:: fit + +.. autoclass:: Minimizer + :members: Migrad, Fumili, LevenbergMarquardt, VarPro diff --git a/docs/src/python/file/pyCtbRawFile.rst b/docs/src/python/file/pyCtbRawFile.rst index 75d83550..f0676e6d 100644 --- a/docs/src/python/file/pyCtbRawFile.rst +++ b/docs/src/python/file/pyCtbRawFile.rst @@ -6,6 +6,10 @@ Read analog, digital and transceiver samples from a raw file containing data from the Chip Test Board. Uses :mod:`aare.transform` to decode the data into a format that the user can work with. +``read_frame(index)`` reads the frame at the zero-based index across the raw +subfiles and advances the position to ``index + 1``. A subsequent +``read_frame()`` reads the following frame, including across subfile boundaries. + .. code:: python import aare @@ -22,4 +26,4 @@ data into a format that the user can work with. :members: :undoc-members: :show-inheritance: - :inherited-members: \ No newline at end of file + :inherited-members: diff --git a/docs/src/python/file/pyRawFile.rst b/docs/src/python/file/pyRawFile.rst index deeb8f3e..a128d2fe 100644 --- a/docs/src/python/file/pyRawFile.rst +++ b/docs/src/python/file/pyRawFile.rst @@ -3,6 +3,13 @@ RawFile .. py:currentmodule:: aare +Reading requires frame padding to be enabled +or ``Frame Discard Policy`` to be ``discardpartial``. Construction raises +``RuntimeError`` with the master path before opening data subfiles if this +requirement is not met. This applies to JSON and legacy ``.raw`` masters, +including when opened through ``File``. ``RawMasterFile`` can still be used +to inspect their metadata. + ``total_frames`` and ``len(reader)`` report the minimum actual frame count across all selected subfiles and ROIs, including each subfile's complete ``.raw`` series. ``read()`` uses this count, and ``read_n()`` and diff --git a/etc/add_license.py b/etc/add_license.py index f43b6cdb..ca02ba43 100644 --- a/etc/add_license.py +++ b/etc/add_license.py @@ -4,13 +4,17 @@ import fnmatch import os from pathlib import Path -CPP_PATTERNS = ["*.h", "*.hpp", "*.cpp"] -PY_PATTERNS = ["*.py"] -CMAKE_PATTERNS = ["CMakeLists.txt"] +CPP_PATTERNS = ["*.h", "*.hpp", "*.cpp", "*.hpp.in"] +PY_PATTERNS = ["*.py", "*.py.in"] +CMAKE_PATTERNS = ["CMakeLists.txt", "*.cmake", "*.cmake.in"] FILE_PATTERNS = CPP_PATTERNS + PY_PATTERNS + CMAKE_PATTERNS LICENSE_TEXT = "SPDX-License-Identifier: MPL-2.0" +# Text found in license notices of code copied from other projects. Such files +# keep their original license and must be tagged by hand. +THIRD_PARTY_MARKERS = ["Copyright", "Permission is hereby granted", "Licensed under"] + def get_comment_prefix(filename: str) -> str | None: if any(fnmatch.fnmatch(filename, p) for p in CPP_PATTERNS): @@ -43,6 +47,10 @@ def process_file(filepath: Path) -> bool: if any("SPDX-License-Identifier" in line for line in lines): return False + if any(marker in line for line in lines for marker in THIRD_PARTY_MARKERS): + print(f"⚠️ Skipped {filepath}: contains a license notice, tag it by hand") + return False + insert_index = 0 # For Python, keep shebang on the very first line @@ -50,6 +58,11 @@ def process_file(filepath: Path) -> bool: if lines[0].startswith("#!"): insert_index = 1 + # cmake-format merges consecutive comment lines into one paragraph + if any(fnmatch.fnmatch(filename, p) for p in CMAKE_PATTERNS): + if len(lines) > insert_index and lines[insert_index].startswith("#"): + lines.insert(insert_index, "\n") + lines.insert(insert_index, license_line) try: @@ -69,7 +82,7 @@ def main() -> None: parser.add_argument( "path", help="Root directory to recursively process " - "(*.h, *.cpp, *.py, and CMakeLists.txt).", + "(C++, Python, and CMake sources, and their .in templates).", ) args = parser.parse_args() diff --git a/etc/update_version.py b/etc/update_version.py index 78eea2a3..a939d9f7 100644 --- a/etc/update_version.py +++ b/etc/update_version.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: MPL-2.0 -# Copyright (C) 2021 Contributors to the Aare Package """ Script to update VERSION file with semantic versioning if provided as an argument, or with 0.0.0 if no argument is provided. """ diff --git a/include/aare/ArrayExpr.hpp b/include/aare/ArrayExpr.hpp index 368f691f..8520208d 100644 --- a/include/aare/ArrayExpr.hpp +++ b/include/aare/ArrayExpr.hpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace aare { @@ -20,80 +22,95 @@ template class ArrayExpr { } }; -template -class ArrayAdd : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayAdd(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](int i) const { return arr1_[i] + arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArraySub : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArraySub(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](int i) const { return arr1_[i] - arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArrayMul : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayMul(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](int i) const { return arr1_[i] * arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -class ArrayDiv : public ArrayExpr, Ndim> { - const A &arr1_; - const B &arr2_; - - public: - ArrayDiv(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { - assert(arr1.size() == arr2.size()); - } - auto operator[](int i) const { return arr1_[i] / arr2_[i]; } - size_t size() const { return arr1_.size(); } - std::array shape() const { return arr1_.shape(); } -}; - -template -auto operator+(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayAdd, ArrayExpr, Ndim>(arr1, arr2); +/** + * @brief What an expression stores for one of its operands. An NDArray is + * replaced by a view, everything else is copied. The expression then holds its + * data pointers by value, so the compiler can keep them in registers even if + * the evaluation writes to a type that may alias them (e.g. uint8_t). + */ +template +auto operand(const ArrayExpr &expr) { + if constexpr (E::is_leaf) + return static_cast(expr).view(); + else + return static_cast(expr); } -template -auto operator-(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArraySub, ArrayExpr, Ndim>(arr1, arr2); +template +class ArrayBinaryOp : public ArrayExpr, Ndim> { + A arr1_; + B arr2_; + + public: + ArrayBinaryOp(const A &arr1, const B &arr2) : arr1_(arr1), arr2_(arr2) { + assert(arr1.shape() == arr2.shape()); + } + auto operator[](size_t i) const { return Op{}(arr1_[i], arr2_[i]); } + size_t size() const { return arr1_.size(); } + std::array shape() const { return arr1_.shape(); } +}; + +/** @brief Scalar operand, takes size and shape from the expression it is + * combined with. */ +template +class ArrayScalar : public ArrayExpr, Ndim> { + T value_; + size_t size_; + std::array shape_; + + public: + template + ArrayScalar(T value, const ArrayExpr &like) + : value_(value), size_(like.size()), shape_(like.shape()) {} + T operator[](size_t) const { return value_; } + size_t size() const { return size_; } + std::array shape() const { return shape_; } +}; + +// Builds the expression for expr op expr, expr op scalar and scalar op expr +template +auto binary_op(const ArrayExpr &lhs, const ArrayExpr &rhs) { + auto a = operand(lhs); + auto b = operand(rhs); + return ArrayBinaryOp(a, b); } -template -auto operator*(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayMul, ArrayExpr, Ndim>(arr1, arr2); +template >> +auto binary_op(const ArrayExpr &lhs, T rhs) { + return binary_op(lhs, ArrayScalar(rhs, lhs)); } -template -auto operator/(const ArrayExpr &arr1, const ArrayExpr &arr2) { - return ArrayDiv, ArrayExpr, Ndim>(arr1, arr2); +template >> +auto binary_op(T lhs, const ArrayExpr &rhs) { + return binary_op(ArrayScalar(lhs, rhs), rhs); +} + +// The operators only take part in overload resolution if binary_op accepts +// the operands, meaning at least one of them is an ArrayExpr +template +auto operator+(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator-(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator*(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); +} + +template +auto operator/(const L &lhs, const R &rhs) + -> decltype(binary_op>(lhs, rhs)) { + return binary_op>(lhs, rhs); } } // namespace aare \ No newline at end of file diff --git a/include/aare/Cluster.hpp b/include/aare/Cluster.hpp index b08a2878..df67884c 100755 --- a/include/aare/Cluster.hpp +++ b/include/aare/Cluster.hpp @@ -83,7 +83,7 @@ struct Cluster { data[cluster_center_index - ClusterSizeX] + data[cluster_center_index - 1 - ClusterSizeX]; // subcluster top right from center - if (ClusterSizeX > 2) { + if constexpr (ClusterSizeX > 2) { sum_2x2_subcluster[1] = data[cluster_center_index] + data[cluster_center_index + 1] + @@ -91,7 +91,7 @@ struct Cluster { data[cluster_center_index - ClusterSizeX + 1]; } // subcluster bottom left from center - if (ClusterSizeY > 2) { + if constexpr (ClusterSizeY > 2) { sum_2x2_subcluster[2] = data[cluster_center_index] + data[cluster_center_index - 1] + @@ -99,7 +99,7 @@ struct Cluster { data[cluster_center_index + ClusterSizeX - 1]; } // subcluster bottom right from center - if (ClusterSizeX > 2 && ClusterSizeY > 2) { + if constexpr (ClusterSizeX > 2 && ClusterSizeY > 2) { sum_2x2_subcluster[3] = data[cluster_center_index] + data[cluster_center_index + 1] + diff --git a/include/aare/ClusterFinder.hpp b/include/aare/ClusterFinder.hpp index 53ac4f73..ca78ad06 100644 --- a/include/aare/ClusterFinder.hpp +++ b/include/aare/ClusterFinder.hpp @@ -68,13 +68,17 @@ class ClusterFinder { } /** - * @brief Set the noise multiplier used for threshold calculation and - * recompute the threshold. + * @brief Set the noise multiplier used for threshold calculation. + * + * The threshold is recomputed immediately if the pedestal is ready, + * otherwise when the pedestal becomes ready. * @param nSigma New per-pixel noise multiplier. */ void set_nSigma(PEDESTAL_TYPE nSigma) { m_nSigma = nSigma; - update_threshold(); + if (m_pedestal.ready()) { + update_threshold(); + } } /** @brief Return the current noise multiplier used for threshold @@ -110,8 +114,23 @@ class ClusterFinder { /** @brief Clear the pedestal and mark it as not ready. */ void clear_pedestal() { m_pedestal.clear(); } - /** @brief Recompute the threshold as noise multiplied by nSigma. */ - void update_threshold() { m_threshold = m_pedestal.std() * m_nSigma; } + /** + * @brief Return whether enough pedestal frames have been pushed to find + * clusters. + */ + bool pedestal_ready() const { return m_pedestal.ready(); } + + /** + * @brief Recompute the threshold as noise multiplied by nSigma. + * @throws std::runtime_error if the pedestal is not ready. + */ + void update_threshold() { + if (!m_pedestal.ready()) { + throw std::runtime_error( + "Pedestal is not ready, cannot update threshold"); + } + m_threshold = m_pedestal.std() * m_nSigma; + } /** * @brief Move out all accumulated clusters and reset the internal vector. diff --git a/include/aare/ClusterFinderMT.hpp b/include/aare/ClusterFinderMT.hpp index b4f4fd22..f57fec88 100644 --- a/include/aare/ClusterFinderMT.hpp +++ b/include/aare/ClusterFinderMT.hpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MPL-2.0 #pragma once +#include #include #include #include @@ -131,6 +132,14 @@ class ClusterFinderMT { } } + void wait_for_input_queues() const { + for (auto &q : m_input_queues) { + while (!q->isEmpty()) { + std::this_thread::sleep_for(m_default_wait); + } + } + } + bool output_queues_are_empty() const { for (auto &q : m_output_queues) { if (!q->isEmpty()) { @@ -392,21 +401,28 @@ class ClusterFinderMT { // } /** - * @brief Set the nSigma value for all the cluster finders. + * @brief Set the nSigma value for all the cluster finders. Waits for the + * queued frames to be processed first. * @param nSigma number of sigma above the pedestal to consider a photon * during cluster finding. */ void set_nSigma(const PEDESTAL_TYPE nSigma) { - // Wait for all queues to be empty before changing the sigma - for (auto &q : m_input_queues) { - while (!q->isEmpty()) { - std::this_thread::sleep_for(m_default_wait); - } - } + wait_for_input_queues(); for (auto &cf : m_cluster_finders) { cf->set_nSigma(nSigma); } } + + /** + * @brief Return whether all cluster finders have received enough pedestal + * frames to find clusters. Waits for the queued frames to be processed + * first. + */ + bool pedestal_ready() { + wait_for_input_queues(); + return std::all_of(m_cluster_finders.begin(), m_cluster_finders.end(), + [](const auto &cf) { return cf->pedestal_ready(); }); + } }; } // namespace aare \ No newline at end of file diff --git a/include/aare/FilePtr.hpp b/include/aare/FilePtr.hpp index 2e84b87c..16ea198d 100644 --- a/include/aare/FilePtr.hpp +++ b/include/aare/FilePtr.hpp @@ -1,10 +1,29 @@ // SPDX-License-Identifier: MPL-2.0 #pragma once +#include "aare/defs.hpp" +#include #include #include namespace aare { +// MSVC's long is 32 bits, so fseek/ftell stop working past 2 GB there +inline int fseek64(FILE *fp, int64_t offset, int whence) { +#ifdef _WIN32 + return _fseeki64(fp, offset, whence); +#else + return fseek(fp, static_cast(offset), whence); +#endif +} + +inline int64_t ftell64(FILE *fp) { +#ifdef _WIN32 + return _ftelli64(fp); +#else + return ftell(fp); +#endif +} + /** * \brief RAII wrapper for FILE pointer */ @@ -22,7 +41,7 @@ class FilePtr { FILE *get(); ssize_t tell(); void seek(ssize_t offset, int whence = SEEK_SET) { - if (fseek(fp_, offset, whence) != 0) + if (fseek64(fp_, offset, whence) != 0) throw std::runtime_error("Error seeking in file"); } std::string error_msg(); diff --git a/include/aare/Fit.hpp b/include/aare/Fit.hpp index 1a2fa55f..cc7edce1 100644 --- a/include/aare/Fit.hpp +++ b/include/aare/Fit.hpp @@ -7,17 +7,24 @@ namespace aare { // --------------------------------------------------------------------------- -// Minuit2-based pixel fitting. +// Pixel fitting with the minimizer selected by the FitModel. // Template bodies and explicit instantiations live in src/Fit.cpp. // --------------------------------------------------------------------------- /** - * @brief Fit a single pixel's data using Minuit2. + * @brief Fit a single pixel's data. + * + * The minimizer (Migrad, Fumili or LevenbergMarquardt) is selected by the + * model, see aare::Minimizer. Parameter errors, when requested, come from + * MnHesse with Migrad and from the linearised covariance otherwise; fixed + * parameters and parameters ending on a limit report 0. A failed fit + * returns zeros. * * User-precedence rules: - * - Fixed parameters: untouched (value and fixed flag preserved from model). - * - User-set start: value preserved, step size auto-filled. - * - Neither: both value and step size auto-filled from data. + * - Fixed parameters: kept at their value, excluded from the fit. + * - User-set start: value used as the starting point. + * - Neither: starting value estimated from the data and clamped + * into the parameter limits. * * @tparam Model Model struct (Gaussian, RisingScurve, …). * diff --git a/include/aare/FitModel.hpp b/include/aare/FitModel.hpp index a52ea9fa..f485849e 100644 --- a/include/aare/FitModel.hpp +++ b/include/aare/FitModel.hpp @@ -3,59 +3,102 @@ #include "aare/Models.hpp" #include -#include #include #include #include namespace aare { +/** + * @brief Minimizer used by FitModel. + * + * - Migrad: Minuit2's variable-metric minimizer, driven by the analytic + * gradient of the chi-squared. Parameter errors come from MnHesse. + * - Fumili: Minuit2's Gauss-Newton minimizer with a trust region. It takes + * the gradient and a linearised Hessian from the model's analytic + * derivatives and typically needs far fewer function evaluations than + * Migrad. Parameter errors come from the covariance of that linearised + * Hessian, without a Hesse step. Minuit2's Fumili cannot converge once a + * two-sided limit becomes active; a pixel without a valid Fumili minimum + * is refitted with Migrad. + * - LevenbergMarquardt: the built-in damped Gauss-Newton solver + * (src/LevenbergMarquardt.hpp). It uses the analytic Jacobian, reflects + * steps at parameter limits and reuses its buffers between pixels, so a + * data cube is fitted without per-pixel allocations. Parameter errors are + * sqrt(diag((J^T J)^-1)) over the free parameters. + * - VarPro: the built-in variable projection solver + * (src/VariableProjection.hpp) for models that declare their linear + * parameters (model::is_separable). Every trial point solves the linear + * parameters exactly and the Levenberg-Marquardt iteration runs over the + * nonlinear ones only, which needs fewer evaluations and avoids poor + * start values of the linear parameters. Start values of free linear + * parameters are ignored. A limit on a linear parameter cannot be + * enforced by the linear solve: a pixel whose solution violates one, a + * pixel that does not converge, and every pixel of a model without the + * separable structure are fitted with LevenbergMarquardt instead. + * Parameter errors are the same Gauss-Newton estimates as with + * LevenbergMarquardt. + * + * With every minimizer, fixed parameters and parameters that end on a limit + * report an error of 0 and a failed fit returns zeros. `tolerance` is the + * EDM tolerance in Minuit's convention (Migrad, LevenbergMarquardt and + * VarPro stop below 0.002 * tolerance, Fumili below + * 1e-4 * tolerance). The EDM is in units of chi2, so an unweighted fit of + * small values meets it far from the minimum; pass errors or a lower + * tolerance for such data. `max_calls` bounds the work per pixel in the + * minimizer's own units (Minuit2 function calls, or model evaluations for + * LevenbergMarquardt and VarPro, which spend one evaluation per + * iteration and two when a step crosses a limit). `strategy` only affects + * the Minuit2 minimizers. + */ +enum class Minimizer { Migrad, Fumili, LevenbergMarquardt, VarPro }; + +/** + * @brief Fit configuration for one model: parameter limits, fixed + * parameters, user start values and minimizer settings. + * + * The object holds plain data and can be shared read-only between threads. + * Method bodies live in src/Fit.cpp, which explicitly instantiates the class + * for every supported model. + */ template class FitModel { - - // Forward declaration only — full definition lives in src/FitModelImpl.hpp - // and is never installed. Downstream code cannot dereference this pointer. - struct FitModelImpl; // stores Minuit2 user-set parameters - - std::unique_ptr impl_; - unsigned int max_calls_; - double tolerance_; - bool compute_errors_; - - std::array user_fixed_{}; - std::array user_start_{}; - - /** @brief Safely resolve a parameter name to its index. */ - unsigned int checked_index(const std::string &name) const; - public: static constexpr std::size_t npar = Model::npar; /** * @brief Construct a fit model with sensible defaults. * - * @param strategy Minuit2 strategy level (0 = fast/gradient, 1 = - * default). - * @param max_calls Maximum FCN calls per pixel minimisation. - * @param tolerance Minuit2 EDM tolerance. - * @param compute_errors If true, run MnHesse after minimisation. + * @param strategy Minuit2 strategy level (0 = fast, 1 = Minuit2's + * default). Ignored by LevenbergMarquardt. + * @param max_calls Work budget per pixel: Minuit2 function calls, + * or model evaluations for LevenbergMarquardt. + * @param tolerance EDM tolerance in Minuit's convention, see + * aare::Minimizer. + * @param compute_errors If true, also report parameter errors: from + * MnHesse with Migrad, from the linearised + * covariance with Fumili and LevenbergMarquardt. + * Fixed parameters and parameters ending on a limit + * report 0. + * @param minimizer Minimizer, see aare::Minimizer. */ explicit FitModel(unsigned int strategy = 0, unsigned int max_calls = 100, - double tolerance = 0.5, bool compute_errors = false); + double tolerance = 0.5, bool compute_errors = false, + Minimizer minimizer = Minimizer::Migrad); - // Destructor must be defined in Fit.cpp where FitModelImpl is complete. - ~FitModel(); - FitModel(const FitModel &); - FitModel &operator=(const FitModel &); - FitModel(FitModel &&) noexcept = default; - FitModel &operator=(FitModel &&) noexcept = default; - - /** @brief Set lower and upper bounds for parameter idx.*/ + /** + * @brief Set lower and upper limits for parameter idx. + * + * Infinite values leave that side open (the Minuit2 minimizers turn an + * open side into a wide finite one, at least 1e6 away from the other). + * Throws std::runtime_error when lo >= hi and std::out_of_range for a + * bad index. + */ void SetParLimits(unsigned int idx, double lo, double hi); /** * @brief Fix parameter idx at value val. * - * Excluded from minimisation. Automatic estimates will not touch it. + * Excluded from minimisation. Automatic estimates will not touch it. */ void FixParameter(unsigned int idx, double val); @@ -76,16 +119,36 @@ template class FitModel { void SetMaxCalls(unsigned int n) { max_calls_ = n; } void SetTolerance(double t) { tolerance_ = t; } void SetComputeErrors(bool b) { compute_errors_ = b; } + void SetMinimizer(Minimizer m) { minimizer_ = m; } + unsigned int strategy() const { return strategy_; } unsigned int max_calls() const { return max_calls_; } double tolerance() const { return tolerance_; } bool compute_errors() const { return compute_errors_; } + Minimizer minimizer() const { return minimizer_; } bool is_user_fixed(unsigned int idx) const { return user_fixed_[idx]; } bool is_user_start(unsigned int idx) const { return user_start_[idx]; } + /** @brief Lower limit of parameter idx, -infinity when open. */ + double lower_limit(unsigned int idx) const { return lower_[idx]; } + /** @brief Upper limit of parameter idx, +infinity when open. */ + double upper_limit(unsigned int idx) const { return upper_[idx]; } + /** @brief User start or fixed value of parameter idx, 0 when unset. */ + double value(unsigned int idx) const { return value_[idx]; } - // Returns the internal Minuit2 state. FitModelImpl is an incomplete type - // here; only callers that include src/FitModelImpl.hpp can dereference it. - FitModelImpl *impl() const { return impl_.get(); } + private: + void check_index(unsigned int idx) const; + unsigned int checked_index(const std::string &name) const; + + unsigned int strategy_; + unsigned int max_calls_; + double tolerance_; + bool compute_errors_; + Minimizer minimizer_; + std::array lower_{}; + std::array upper_{}; + std::array value_{}; + std::array user_fixed_{}; + std::array user_start_{}; }; // Suppress implicit instantiation for all supported model types. diff --git a/include/aare/Models.hpp b/include/aare/Models.hpp index eee84e2d..da908f14 100644 --- a/include/aare/Models.hpp +++ b/include/aare/Models.hpp @@ -2,10 +2,15 @@ #pragma once #include "aare/NDView.hpp" +#include "aare/defs.hpp" #include #include #include +#include +#include +#include #include +#include #include namespace aare::model { @@ -13,10 +18,14 @@ namespace aare::model { inline constexpr double inv_sqrt2 = 0.70710678118654752440; inline constexpr double inv_sqrt_2pi = 0.39894228040143267794; -inline double fast_erf(double x) { - // Abramowitz–Stegun Handbook of Mathematical Functions - // erf approximation with max error ~1.5e-7, faster than std::erf. - +/** + * @brief erf(z) given exp(-z^2), from the Abramowitz-Stegun approximation + * 7.1.26 (Handbook of Mathematical Functions, max error about 1.5e-7). + * + * Models whose derivatives need exp(-z^2) anyway pass it in, so that a + * single exponential serves both the value and the gradient. + */ +ALWAYS_INLINE double fast_erf_from_exp(double z, double exp_minus_z2) { const double a1 = 0.254829592; const double a2 = -0.284496736; const double a3 = 1.421413741; @@ -24,22 +33,69 @@ inline double fast_erf(double x) { const double a5 = 1.061405429; const double p = 0.3275911; - const int sign = x < 0 ? -1 : 1; - x = std::abs(x); - - // 7.1.26 - const double t = 1.0 / (1.0 + p * x); + const double t = 1.0 / (1.0 + p * std::abs(z)); const double y = 1.0 - - (((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t) * std::exp(-x * x); + (((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t) * exp_minus_z2; - return sign * y; + return z < 0 ? -y : y; +} + +/** @brief erf approximation, faster than std::erf; see fast_erf_from_exp. */ +ALWAYS_INLINE double fast_erf(double x) { + return fast_erf_from_exp(x, std::exp(-x * x)); +} + +/** + * @brief exp(x) without a call into libm, for loops that should vectorise. + * + * Range reduction x = k ln2 + r with the integer k taken from the mantissa + * of x * log2(e) + 1.5 * 2^52, a degree-13 Taylor polynomial for exp(r) on + * |r| <= ln2 / 2, and 2^k assembled in the exponent bits, so that a loop + * calling it compiles to vector instructions on SSE2, AVX and NEON. The + * relative error is below 4e-16 for finite x; |x| is clamped to 700. The + * bulk basis functions of the models (basis_columns) use it; eval and + * eval_and_grad keep std::exp. It is ALWAYS_INLINE because gcc otherwise + * leaves it out of line in the large fitting translation unit, and a call + * per point defeats the vectorisation of the caller. + */ +ALWAYS_INLINE double fast_exp(double x) { + constexpr double log2e = 1.4426950408889634074; + constexpr double magic = 6755399441055744.0; // 1.5 * 2^52 + constexpr double ln2_hi = 0.693145751953125; + constexpr double ln2_lo = 1.42860682030941723212e-6; + x = std::clamp(x, -700.0, 700.0); + const double t = x * log2e + magic; + const double k = t - magic; + const double r = (x - k * ln2_hi) - k * ln2_lo; + double p = 1.0 / 6227020800.0; + p = p * r + 1.0 / 479001600.0; + p = p * r + 1.0 / 39916800.0; + p = p * r + 1.0 / 3628800.0; + p = p * r + 1.0 / 362880.0; + p = p * r + 1.0 / 40320.0; + p = p * r + 1.0 / 5040.0; + p = p * r + 1.0 / 720.0; + p = p * r + 1.0 / 120.0; + p = p * r + 1.0 / 24.0; + p = p * r + 1.0 / 6.0; + p = p * r + 0.5; + p = p * r + 1.0; + p = p * r + 1.0; + std::int64_t ti = 0; + std::int64_t mi = 0; + std::memcpy(&ti, &t, sizeof(ti)); + std::memcpy(&mi, &magic, sizeof(mi)); + const std::int64_t bits = ((ti - mi) + 1023) << 52; + double scale = 0.0; + std::memcpy(&scale, &bits, sizeof(scale)); + return p * scale; } /** * @brief Per-parameter metadata: name and optional default bounds. * - * Used by FitModel to generically initialise MnUserParameters. + * Used by FitModel for parameter names and default limits. * Unbounded directions use ±no_bound as sentinels. */ struct ParamInfo { @@ -51,22 +107,89 @@ struct ParamInfo { inline constexpr double no_bound = std::numeric_limits::infinity(); /** - * @brief Compute data-range statistics used by step-size estimators. + * @brief Detects a separable model, one that declares its linear parameters + * for Minimizer::VarPro. * - * Model-independent, called once per pixel. + * A model whose function is linear in some of its parameters lists them in + * ascending order in `linear_par` and provides + * + * static void basis_and_grad(double x, const std::vector &par, + * std::array &phi, + * std::array, nnl> &dphi) + * + * with nlin = linear_par.size() and nnl = npar - nlin, such that + * + * f(x; par) = sum_j par[linear_par[j]] * phi[j] + * + * and dphi[k][j] = d phi[j] / d par[nonlinear[k]], where nonlinear lists the + * remaining parameter indices in ascending order (see separable_traits). + * basis_and_grad ignores the linear entries of par. The [fit] tests check + * phi and dphi against eval and eval_and_grad for every model. + * + * A model may additionally provide the same quantities for all scan points + * at once, see has_basis_columns: + * + * static void basis_columns(const double *__restrict x, ssize_t n, + * const std::vector &par, + * double *__restrict phi, + * double *__restrict dphi) + * + * writes phi[j * n + i] = phi_j(x_i) and dphi[(k * nlin + j) * n + i] = + * d phi_j / d par[nonlinear[k]] (x_i), one contiguous column per function. + * Written as a loop free of reductions with the parameters hoisted and + * fast_exp instead of std::exp, it compiles to vector instructions, which + * is where VarPro spends most of its time. The __restrict qualifiers are + * needed: without them gcc has to prove that the output columns do not + * overlap, exceeds its budget of run-time alias checks once a loop writes + * more than three columns, and quietly emits scalar code. The tests check + * it against basis_and_grad. */ -inline void compute_ranges(NDView x, NDView y, - double &x_range, double &y_range, - double &slope_scale) { - const auto [x_min, x_max] = std::minmax_element(x.begin(), x.end()); - const auto [y_min, y_max] = std::minmax_element(y.begin(), y.end()); +template +struct is_separable : std::false_type {}; - x_range = std::max(*x_max - *x_min, 1e-9); - y_range = std::max(*y_max - *y_min, 1e-9); +template +struct is_separable> + : std::true_type {}; - slope_scale = std::max(y_range / x_range, 1e-9); +/** @brief Detects a separable model that provides basis_columns. */ +template +struct has_basis_columns : std::false_type {}; + +template +struct has_basis_columns> + : std::true_type {}; + +/** @brief The parameter indices not listed in linear, in ascending order. */ +template +constexpr std::array +nonlinear_indices(const std::array &linear) { + std::array out{}; + std::size_t m = 0; + for (std::size_t k = 0; k < Npar; ++k) { + bool is_linear = false; + for (std::size_t j = 0; j < Nlin; ++j) + is_linear = is_linear || linear[j] == k; + if (!is_linear) + out[m++] = k; + } + return out; } +/** @brief Index bookkeeping of a separable model, see is_separable. */ +template struct separable_traits { + static constexpr std::size_t npar = Model::npar; + static constexpr std::size_t nlin = Model::linear_par.size(); + static constexpr std::size_t nnl = npar - nlin; + static constexpr std::array nonlinear_par = + nonlinear_indices(Model::linear_par); + static constexpr std::size_t linear_index(int j) { + return Model::linear_par[static_cast(j)]; + } + using Basis = std::array; + using BasisGrad = std::array, nnl>; +}; + // _____________________________________________________________________ // // Pol1 @@ -105,6 +228,17 @@ struct Pol1 { g[1] = x; // df/dp1 } + /** @brief Both parameters are linear: f = p0 * 1 + p1 * x. */ + static constexpr std::array linear_par = {0, 1}; + + static void basis_and_grad( + double x, [[maybe_unused]] const std::vector &par, + std::array &phi, + [[maybe_unused]] std::array, 0> &dphi) { + phi[0] = 1.0; + phi[1] = x; + } + static bool is_valid([[maybe_unused]] const std::vector &par) { return true; // always valid } @@ -120,14 +254,6 @@ struct Pol1 { return {intercept, slope}; } - - static void compute_steps(const std::array &start, - [[maybe_unused]] double x_range, double y_range, - double slope_scale, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.1 * slope_scale; - } }; // _____________________________________________________________________ @@ -172,6 +298,18 @@ struct Pol2 { g[2] = x * x; // df/dp2 } + /** @brief All parameters are linear: f = p0 * 1 + p1 * x + p2 * x^2. */ + static constexpr std::array linear_par = {0, 1, 2}; + + static void basis_and_grad( + double x, [[maybe_unused]] const std::vector &par, + std::array &phi, + [[maybe_unused]] std::array, 0> &dphi) { + phi[0] = 1.0; + phi[1] = x; + phi[2] = x * x; + } + static bool is_valid([[maybe_unused]] const std::vector &par) { return true; // always valid } @@ -220,15 +358,6 @@ struct Pol2 { return {p0, p1, p2}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double slope_scale, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.1 * slope_scale; - steps[2] = 0.1 * slope_scale / std::max(x_range, 1e-12); - } }; // _____________________________________________________________________ @@ -288,6 +417,44 @@ struct Gaussian { sig; // df/dsigma = A*e*(x-mu)^2/sig^3 } + /** @brief A is linear: f = A * exp(-(x - mu)^2 / (2 sigma^2)). */ + static constexpr std::array linear_par = {0}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 2> &dphi) { + const double mu = par[1]; + const double sig = par[2]; + + const double dx = x - mu; + const double inv_sig = 1.0 / sig; + const double inv_sig2 = inv_sig * inv_sig; + const double e = std::exp(-0.5 * dx * dx * inv_sig2); + + phi[0] = e; + dphi[0][0] = e * dx * inv_sig2; // d/dmu + dphi[1][0] = e * dx * dx * inv_sig2 * inv_sig; // d/dsigma + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double mu = par[1]; + const double inv_sig = 1.0 / par[2]; + const double inv_sig2 = inv_sig * inv_sig; + double *e_col = phi; + double *dmu = dphi; + double *dsig = dphi + n; + for (ssize_t i = 0; i < n; ++i) { + const double dx = x[i] - mu; + const double e = fast_exp(-0.5 * dx * dx * inv_sig2); + e_col[i] = e; + dmu[i] = e * dx * inv_sig2; + dsig[i] = e * dx * dx * inv_sig2 * inv_sig; + } + } + /** @brief Reject degenerate sigma (zero width). */ static bool is_valid(const std::vector &par) { return par[2] != 0.0; @@ -326,18 +493,6 @@ struct Gaussian { return {A, mu, sig}; } - - /** - * @brief Data-driven Minuit step sizes. - */ - static void compute_steps(const std::array &start, - double x_range, double y_range, - double /*slope_scale*/, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.05 * x_range; - steps[2] = 0.05 * x_range; - } }; // _____________________________________________________________________ @@ -393,7 +548,7 @@ struct GaussianErfcPlateau { const double z = dx * inv_sqrt2 / sig; const double e = std::exp(-z * z); - const double step = 0.5 * (1.0 - fast_erf(z)); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); return A * e + S * step; } @@ -409,7 +564,7 @@ struct GaussianErfcPlateau { const double z = dx * inv_sqrt2 / sig; const double e = std::exp(-z * z); - const double step = 0.5 * (1.0 - fast_erf(z)); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); f = A * e + S * step; @@ -426,6 +581,63 @@ struct GaussianErfcPlateau { S * inv_sqrt_2pi * e * dx * inv_sig2; // df/dsigma } + /** @brief A and S are linear: f = A * e + S * step. */ + static constexpr std::array linear_par = {0, 1}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 2> &dphi) { + const double mu = par[2]; + const double sig = par[3]; + + const double dx = x - mu; + const double inv_sig = 1.0 / sig; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + const double z = dx * inv_sqrt2 * inv_sig; + + const double e = std::exp(-z * z); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); + + phi[0] = e; + phi[1] = step; + dphi[0][0] = e * dx * inv_sig2; // de/dmu + dphi[0][1] = inv_sqrt_2pi * e * inv_sig; // dstep/dmu + dphi[1][0] = e * dx * dx * inv_sig3; // de/dsigma + dphi[1][1] = inv_sqrt_2pi * e * dx * inv_sig2; // dstep/dsigma + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double mu = par[2]; + const double inv_sig = 1.0 / par[3]; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + const double cz = inv_sqrt2 * inv_sig; + const double ce = inv_sqrt_2pi * inv_sig; + const double ce2 = inv_sqrt_2pi * inv_sig2; + double *e_col = phi; + double *step_col = phi + n; + double *de_dmu = dphi; + double *dstep_dmu = dphi + n; + double *de_dsig = dphi + 2 * n; + double *dstep_dsig = dphi + 3 * n; + for (ssize_t i = 0; i < n; ++i) { + const double dx = x[i] - mu; + const double z = dx * cz; + const double e = fast_exp(-z * z); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); + e_col[i] = e; + step_col[i] = step; + de_dmu[i] = e * dx * inv_sig2; + dstep_dmu[i] = ce * e; + de_dsig[i] = e * dx * dx * inv_sig3; + dstep_dsig[i] = ce2 * e * dx; + } + } + /** @brief Reject degenerate or negative width. */ static bool is_valid(const std::vector &par) { return par[3] > 0.0; @@ -490,16 +702,6 @@ struct GaussianErfcPlateau { return {A, S, mu, sig}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double /*slope_scale*/, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = std::max(0.1 * std::abs(start[1]), 0.1 * y_range); - steps[2] = 0.05 * x_range; - steps[3] = 0.05 * x_range; - } }; // _____________________________________________________________________ @@ -556,7 +758,7 @@ struct GaussianChargeSharing { const double u = dx / sig; const double G = std::exp(-0.5 * u * u); - const double H = 0.5 * (1.0 - fast_erf(u * inv_sqrt2)); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); return p0 - p1 * x + N * (G + C * H); } @@ -574,7 +776,7 @@ struct GaussianChargeSharing { const double u = dx / sig; const double G = std::exp(-0.5 * u * u); - const double H = 0.5 * (1.0 - fast_erf(u * inv_sqrt2)); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); f = p0 - p1 * x + N * (G + C * H); @@ -596,6 +798,84 @@ struct GaussianChargeSharing { g[5] = N * H; } + /** + * @brief p0, p1 and N are linear: f = p0 * 1 + p1 * (-x) + N * (G + C H). + * C multiplies N, so it stays a nonlinear parameter. + */ + static constexpr std::array linear_par = {0, 1, 4}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 3> &dphi) { + const double mu = par[2]; + const double sig = par[3]; + const double C = par[5]; + + const double inv_sig = 1.0 / sig; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + + const double dx = x - mu; + const double u = dx * inv_sig; + + const double G = std::exp(-0.5 * u * u); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + + const double dG_dmu = G * dx * inv_sig2; + const double dG_dsig = G * dx * dx * inv_sig3; + const double dH_dmu = inv_sqrt_2pi * G * inv_sig; + const double dH_dsig = inv_sqrt_2pi * G * dx * inv_sig2; + + phi[0] = 1.0; + phi[1] = -x; + phi[2] = G + C * H; + for (int k = 0; k < 3; ++k) { + dphi[k][0] = 0.0; + dphi[k][1] = 0.0; + } + dphi[0][2] = dG_dmu + C * dH_dmu; // d/dmu + dphi[1][2] = dG_dsig + C * dH_dsig; // d/dsigma + dphi[2][2] = H; // d/dC + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double mu = par[2]; + const double inv_sig = 1.0 / par[3]; + const double C = par[5]; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + double *one = phi; + double *minus_x = phi + n; + double *shape = phi + 2 * n; + // d phi_0 and d phi_1 vanish for every nonlinear parameter. + std::fill(dphi, dphi + 2 * n, 0.0); + std::fill(dphi + 3 * n, dphi + 5 * n, 0.0); + std::fill(dphi + 6 * n, dphi + 8 * n, 0.0); + double *d_mu = dphi + 2 * n; + double *d_sig = dphi + 5 * n; + double *d_C = dphi + 8 * n; + for (ssize_t i = 0; i < n; ++i) { + const double xi = x[i]; + const double dx = xi - mu; + const double u = dx * inv_sig; + const double G = fast_exp(-0.5 * u * u); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + const double dG_dmu = G * dx * inv_sig2; + const double dG_dsig = G * dx * dx * inv_sig3; + const double dH_dmu = inv_sqrt_2pi * G * inv_sig; + const double dH_dsig = inv_sqrt_2pi * G * dx * inv_sig2; + one[i] = 1.0; + minus_x[i] = -xi; + shape[i] = G + C * H; + d_mu[i] = dG_dmu + C * dH_dmu; + d_sig[i] = dG_dsig + C * dH_dsig; + d_C[i] = H; + } + } + static bool is_valid(const std::vector &par) { return par[3] > 0.0; } @@ -647,18 +927,6 @@ struct GaussianChargeSharing { return {p0, p1, mu, sigma, N, C}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double slope_scale, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.1 * slope_scale; - steps[2] = 0.05 * x_range; - steps[3] = 0.05 * x_range; - steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); - steps[5] = std::max(0.1 * std::abs(start[5]), 0.01); - } }; // _____________________________________________________________________ @@ -730,8 +998,8 @@ struct GaussianChargeSharingKb { const double G = std::exp(-0.5 * u * u); const double Gb = std::exp(-0.5 * ub * ub); - const double H = 0.5 * (1.0 - fast_erf(u * inv_sqrt2)); - const double Hb = 0.5 * (1.0 - fast_erf(ub * inv_sqrt2)); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + const double Hb = 0.5 * (1.0 - fast_erf_from_exp(ub * inv_sqrt2, Gb)); const double ka = G + C * H; const double kb = Gb + C * Hb; @@ -761,8 +1029,8 @@ struct GaussianChargeSharingKb { const double G = std::exp(-0.5 * u * u); const double Gb = std::exp(-0.5 * ub * ub); - const double H = 0.5 * (1.0 - fast_erf(u * inv_sqrt2)); - const double Hb = 0.5 * (1.0 - fast_erf(ub * inv_sqrt2)); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + const double Hb = 0.5 * (1.0 - fast_erf_from_exp(ub * inv_sqrt2, Gb)); const double ka = G + C * H; const double kb = Gb + C * Hb; @@ -807,6 +1075,133 @@ struct GaussianChargeSharingKb { g[7] = N * kb; } + /** + * @brief p0, p1 and N are linear: f = p0 * 1 + p1 * (-x) + N * (ka + q kb). + * C, r and q multiply N or enter the peak shapes, so they stay nonlinear. + */ + static constexpr std::array linear_par = {0, 1, 4}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 5> &dphi) { + const double mu = par[2]; + const double sig = par[3]; + const double C = par[5]; + const double r = par[6]; + const double q = par[7]; + + const double inv_sig = 1.0 / sig; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + + const double mu_b = r * mu; + + const double dx = x - mu; + const double dxb = x - mu_b; + + const double u = dx * inv_sig; + const double ub = dxb * inv_sig; + + const double G = std::exp(-0.5 * u * u); + const double Gb = std::exp(-0.5 * ub * ub); + + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + const double Hb = 0.5 * (1.0 - fast_erf_from_exp(ub * inv_sqrt2, Gb)); + + const double ka = G + C * H; + const double kb = Gb + C * Hb; + + const double dG_dmu = G * dx * inv_sig2; + const double dG_dsig = G * dx * dx * inv_sig3; + const double dH_dmu = inv_sqrt_2pi * G * inv_sig; + const double dH_dsig = inv_sqrt_2pi * G * dx * inv_sig2; + + const double dGb_dmu_b = Gb * dxb * inv_sig2; + const double dGb_dsig = Gb * dxb * dxb * inv_sig3; + const double dHb_dmu_b = inv_sqrt_2pi * Gb * inv_sig; + const double dHb_dsig = inv_sqrt_2pi * Gb * dxb * inv_sig2; + + const double dka_dmu = dG_dmu + C * dH_dmu; + const double dka_dsig = dG_dsig + C * dH_dsig; + + const double dkb_dmu_b = dGb_dmu_b + C * dHb_dmu_b; + const double dkb_dsig = dGb_dsig + C * dHb_dsig; + + phi[0] = 1.0; + phi[1] = -x; + phi[2] = ka + q * kb; + for (int k = 0; k < 5; ++k) { + dphi[k][0] = 0.0; + dphi[k][1] = 0.0; + } + dphi[0][2] = dka_dmu + q * r * dkb_dmu_b; // d/dmu, mu_b = r * mu + dphi[1][2] = dka_dsig + q * dkb_dsig; // d/dsigma + dphi[2][2] = H + q * Hb; // d/dC + dphi[3][2] = q * mu * dkb_dmu_b; // d/dr + dphi[4][2] = kb; // d/dq + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double mu = par[2]; + const double inv_sig = 1.0 / par[3]; + const double C = par[5]; + const double r = par[6]; + const double q = par[7]; + const double inv_sig2 = inv_sig * inv_sig; + const double inv_sig3 = inv_sig2 * inv_sig; + const double mu_b = r * mu; + double *one = phi; + double *minus_x = phi + n; + double *shape = phi + 2 * n; + // d phi_0 and d phi_1 vanish for every nonlinear parameter; the + // derivative of phi_2 with respect to nonlinear parameter k is + // column k * 3 + 2. + for (int k = 0; k < 5; ++k) + std::fill(dphi + (k * 3) * n, dphi + (k * 3 + 2) * n, 0.0); + double *d_mu = dphi + 2 * n; + double *d_sig = dphi + 5 * n; + double *d_C = dphi + 8 * n; + double *d_r = dphi + 11 * n; + double *d_q = dphi + 14 * n; + for (ssize_t i = 0; i < n; ++i) { + const double xi = x[i]; + const double dx = xi - mu; + const double dxb = xi - mu_b; + const double u = dx * inv_sig; + const double ub = dxb * inv_sig; + const double G = fast_exp(-0.5 * u * u); + const double Gb = fast_exp(-0.5 * ub * ub); + const double H = 0.5 * (1.0 - fast_erf_from_exp(u * inv_sqrt2, G)); + const double Hb = + 0.5 * (1.0 - fast_erf_from_exp(ub * inv_sqrt2, Gb)); + const double ka = G + C * H; + const double kb = Gb + C * Hb; + const double dG_dmu = G * dx * inv_sig2; + const double dG_dsig = G * dx * dx * inv_sig3; + const double dH_dmu = inv_sqrt_2pi * G * inv_sig; + const double dH_dsig = inv_sqrt_2pi * G * dx * inv_sig2; + const double dGb_dmu_b = Gb * dxb * inv_sig2; + const double dGb_dsig = Gb * dxb * dxb * inv_sig3; + const double dHb_dmu_b = inv_sqrt_2pi * Gb * inv_sig; + const double dHb_dsig = inv_sqrt_2pi * Gb * dxb * inv_sig2; + const double dka_dmu = dG_dmu + C * dH_dmu; + const double dka_dsig = dG_dsig + C * dH_dsig; + const double dkb_dmu_b = dGb_dmu_b + C * dHb_dmu_b; + const double dkb_dsig = dGb_dsig + C * dHb_dsig; + one[i] = 1.0; + minus_x[i] = -xi; + shape[i] = ka + q * kb; + d_mu[i] = dka_dmu + q * r * dkb_dmu_b; + d_sig[i] = dka_dsig + q * dkb_dsig; + d_C[i] = H + q * Hb; + d_r[i] = q * mu * dkb_dmu_b; + d_q[i] = kb; + } + } + static bool is_valid(const std::vector &par) { return par[3] > 0.0; } @@ -828,20 +1223,6 @@ struct GaussianChargeSharingKb { return {p0, p1, mu, sigma, N, C, kb_mean, kb_frac}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double slope_scale, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.1 * slope_scale; - steps[2] = 0.05 * x_range; - steps[3] = 0.05 * x_range; - steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); - steps[5] = std::max(0.1 * std::abs(start[5]), 0.01); - steps[6] = std::max(0.01 * std::abs(start[6]), 1e-3); - steps[7] = std::max(0.1 * std::abs(start[7]), 0.01); - } }; // _____________________________________________________________________ @@ -908,12 +1289,12 @@ struct RisingScurve { const double dx = x - p2; const double z = dx * inv_sqrt2 / p3; - const double step = 0.5 * (1.0 + fast_erf(z)); + const double e = std::exp(-z * z); + const double step = 0.5 * (1.0 + fast_erf_from_exp(z, e)); const double amp = p4 + p5 * dx; f = (p0 + p1 * x) + step * amp; - const double e = std::exp(-z * z); const double dSdp2 = -inv_sqrt_2pi * e / p3; const double dSdp3 = -inv_sqrt_2pi * e * dx / (p3 * p3); @@ -925,6 +1306,79 @@ struct RisingScurve { g[5] = step * dx; // df/dp5 } + /** + * @brief p0, p1, A and C are linear: + * f = p0 * 1 + p1 * x + A * S + C * S * (x - mu). + */ + static constexpr std::array linear_par = {0, 1, 4, 5}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 2> &dphi) { + const double p2 = par[2]; + const double p3 = par[3]; + + const double inv_p3 = 1.0 / p3; + const double dx = x - p2; + const double z = dx * inv_sqrt2 * inv_p3; + const double e = std::exp(-z * z); + const double step = 0.5 * (1.0 + fast_erf_from_exp(z, e)); + + const double dSdp2 = -inv_sqrt_2pi * e * inv_p3; + const double dSdp3 = dSdp2 * dx * inv_p3; + + phi[0] = 1.0; + phi[1] = x; + phi[2] = step; + phi[3] = step * dx; + dphi[0][0] = 0.0; + dphi[0][1] = 0.0; + dphi[0][2] = dSdp2; // dS/dmu + dphi[0][3] = dSdp2 * dx - step; // d(S dx)/dmu + dphi[1][0] = 0.0; + dphi[1][1] = 0.0; + dphi[1][2] = dSdp3; // dS/dsigma + dphi[1][3] = dSdp3 * dx; // d(S dx)/dsigma + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double p2 = par[2]; + const double inv_p3 = 1.0 / par[3]; + const double cz = inv_sqrt2 * inv_p3; + const double ce = -inv_sqrt_2pi * inv_p3; + double *one = phi; + double *x_col = phi + n; + double *step_col = phi + 2 * n; + double *step_dx = phi + 3 * n; + // d phi_0 and d phi_1 vanish for mu and sigma. + std::fill(dphi, dphi + 2 * n, 0.0); + std::fill(dphi + 4 * n, dphi + 6 * n, 0.0); + double *dmu_step = dphi + 2 * n; + double *dmu_step_dx = dphi + 3 * n; + double *dsig_step = dphi + 6 * n; + double *dsig_step_dx = dphi + 7 * n; + for (ssize_t i = 0; i < n; ++i) { + const double xi = x[i]; + const double dx = xi - p2; + const double z = dx * cz; + const double e = fast_exp(-z * z); + const double step = 0.5 * (1.0 + fast_erf_from_exp(z, e)); + const double dSdp2 = ce * e; + const double dSdp3 = dSdp2 * dx * inv_p3; + one[i] = 1.0; + x_col[i] = xi; + step_col[i] = step; + step_dx[i] = step * dx; + dmu_step[i] = dSdp2; + dmu_step_dx[i] = dSdp2 * dx - step; + dsig_step[i] = dSdp3; + dsig_step_dx[i] = dSdp3 * dx; + } + } + /** @brief Reject degenerate width (zero transition width). */ static bool is_valid(const std::vector &par) { return par[3] != 0.0; @@ -995,18 +1449,6 @@ struct RisingScurve { return {p0, p1, p2, p3, p4, p5}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double slope_scale, - std::array &steps) { - steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); - steps[1] = 0.1 * slope_scale; - steps[2] = 0.05 * x_range; - steps[3] = 0.05 * x_range; - steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); - steps[5] = 0.1 * slope_scale; - } }; // _____________________________________________________________________ @@ -1061,12 +1503,12 @@ struct FallingScurve { const double dx = x - p2; const double z = dx * inv_sqrt2 / p3; - const double step = 0.5 * (1.0 - fast_erf(z)); + const double e = std::exp(-z * z); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); const double amp = p4 + p5 * dx; f = (p0 + p1 * x) + step * amp; - const double e = std::exp(-z * z); const double dSdp2 = +inv_sqrt_2pi * e / p3; // sign flipped vs rising const double dSdp3 = +inv_sqrt_2pi * e * dx / (p3 * p3); @@ -1078,6 +1520,80 @@ struct FallingScurve { g[5] = step * dx; } + /** + * @brief p0, p1, A and C are linear: + * f = p0 * 1 + p1 * x + A * S + C * S * (x - mu). + */ + static constexpr std::array linear_par = {0, 1, 4, 5}; + + static void basis_and_grad(double x, const std::vector &par, + std::array &phi, + std::array, 2> &dphi) { + const double p2 = par[2]; + const double p3 = par[3]; + + const double inv_p3 = 1.0 / p3; + const double dx = x - p2; + const double z = dx * inv_sqrt2 * inv_p3; + const double e = std::exp(-z * z); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); + + const double dSdp2 = + inv_sqrt_2pi * e * inv_p3; // sign flipped vs rising + const double dSdp3 = dSdp2 * dx * inv_p3; + + phi[0] = 1.0; + phi[1] = x; + phi[2] = step; + phi[3] = step * dx; + dphi[0][0] = 0.0; + dphi[0][1] = 0.0; + dphi[0][2] = dSdp2; // dS/dmu + dphi[0][3] = dSdp2 * dx - step; // d(S dx)/dmu + dphi[1][0] = 0.0; + dphi[1][1] = 0.0; + dphi[1][2] = dSdp3; // dS/dsigma + dphi[1][3] = dSdp3 * dx; // d(S dx)/dsigma + } + + /** @brief basis_and_grad for all points, one column per function. */ + static void basis_columns(const double *__restrict x, ssize_t n, + const std::vector &par, + double *__restrict phi, double *__restrict dphi) { + const double p2 = par[2]; + const double inv_p3 = 1.0 / par[3]; + const double cz = inv_sqrt2 * inv_p3; + const double ce = inv_sqrt_2pi * inv_p3; // sign flipped vs rising + double *one = phi; + double *x_col = phi + n; + double *step_col = phi + 2 * n; + double *step_dx = phi + 3 * n; + // d phi_0 and d phi_1 vanish for mu and sigma. + std::fill(dphi, dphi + 2 * n, 0.0); + std::fill(dphi + 4 * n, dphi + 6 * n, 0.0); + double *dmu_step = dphi + 2 * n; + double *dmu_step_dx = dphi + 3 * n; + double *dsig_step = dphi + 6 * n; + double *dsig_step_dx = dphi + 7 * n; + for (ssize_t i = 0; i < n; ++i) { + const double xi = x[i]; + const double dx = xi - p2; + const double z = dx * cz; + const double e = fast_exp(-z * z); + const double step = 0.5 * (1.0 - fast_erf_from_exp(z, e)); + const double dSdp2 = ce * e; + const double dSdp3 = dSdp2 * dx * inv_p3; + one[i] = 1.0; + x_col[i] = xi; + step_col[i] = step; + step_dx[i] = step * dx; + dmu_step[i] = dSdp2; + dmu_step_dx[i] = dSdp2 * dx - step; + dsig_step[i] = dSdp3; + dsig_step_dx[i] = dSdp3 * dx; + } + } + /** @brief Reject degenerate width (zero transition width). */ static bool is_valid(const std::vector &par) { return par[3] != 0.0; @@ -1148,14 +1664,6 @@ struct FallingScurve { return {p0, p1, p2, p3, p4, p5}; } - - static void compute_steps(const std::array &start, - double x_range, double y_range, - double slope_scale, - std::array &steps) { - RisingScurve::compute_steps(start, x_range, y_range, slope_scale, - steps); - } }; } // namespace aare::model diff --git a/include/aare/NDArray.hpp b/include/aare/NDArray.hpp index fb6c3bb5..3a2fbda3 100644 --- a/include/aare/NDArray.hpp +++ b/include/aare/NDArray.hpp @@ -27,6 +27,8 @@ class NDArray : public ArrayExpr, Ndim> { T *data_; public: + static constexpr bool is_leaf = true; + /////////////////////////////////////////////////////////////////////////////// // Constructors // @@ -64,14 +66,20 @@ class NDArray : public ArrayExpr, Ndim> { /** * @brief Construct a new NDArray object from a NDView. - * @note The data is copied from the view to the NDArray. + * @note The data is copied from the view to the NDArray. The view is + * only read, so both NDView and NDView are + * accepted. * * @param v view of data to initialize the NDArray with */ - explicit NDArray(const NDView v) : NDArray(v.shape()) { + explicit NDArray(NDView v) : NDArray(v.shape()) { std::copy(v.begin(), v.end(), begin()); } + // Keeps an exact match for mutable views so that they do not pick the + // element-wise ArrayExpr constructor. + explicit NDArray(NDView v) : NDArray(NDView(v)) {} + /** * @brief Construct a new NDArray object from an std::array. */ @@ -135,9 +143,7 @@ class NDArray : public ArrayExpr, Ndim> { */ template NDArray(ArrayExpr &&expr) : NDArray(expr.shape()) { - for (size_t i = 0; i < size_; ++i) { - data_[i] = expr[i]; - } + *this = expr; } /** @@ -149,13 +155,14 @@ class NDArray : public ArrayExpr, Ndim> { /** * @brief Copy data from a view of matching shape into this array without * reallocating. - * @param v view to copy from, must have the same shape as this array + * @param v view to copy from, must have the same shape as this array. + * Accepts NDView and NDView. * @throws std::runtime_error if the shapes differ * * Use this instead of assigning a new NDArray when the buffer needs to be * kept, for example when the array is part of a preallocated pool. */ - void copy_from(const NDView v) { + void copy_from(NDView v) { if (v.shape() != shape_) { throw std::runtime_error(LOCATION + "Shape mismatch in NDArray::copy_from"); @@ -324,6 +331,26 @@ class NDArray : public ArrayExpr, Ndim> { return *this; } + /** + * @brief Evaluate an ArrayExpr into the NDArray. Reuses the existing + * buffer if the shape matches. + */ + template NDArray &operator=(const ArrayExpr &expr) { + if (expr.shape() != shape_) { + // Evaluate before releasing the buffer, expr might refer to it + NDArray tmp(expr.shape()); + tmp = expr; + return *this = std::move(tmp); + } + // Local copies, see elementwise() + const auto e = operand(expr); + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = e[i]; + } + return *this; + } + /////////////////////////////////////////////////////////////////////////////// // Math operators // @@ -337,10 +364,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator +=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] += other.data_[i]; - } - return *this; + return elementwise(other.data_, std::plus()); } /** @@ -351,10 +375,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator -=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] -= other.data_[i]; - } - return *this; + return elementwise(other.data_, std::minus()); } /** @@ -365,10 +386,7 @@ class NDArray : public ArrayExpr, Ndim> { throw(std::runtime_error( "Shape of NDArray must match for operator *=")); - for (size_t i = 0; i < size_; ++i) { - data_[i] *= other.data_[i]; - } - return *this; + return elementwise(other.data_, std::multiplies()); } /** @@ -379,13 +397,9 @@ class NDArray : public ArrayExpr, Ndim> { */ template NDArray &operator/=(const NDArray &other) { // check shape - if (shape_ == other.shape()) { - for (size_t i = 0; i < size_; ++i) { - data_[i] /= other(i); - } - return *this; - } - throw(std::runtime_error("Shape of NDArray must match")); + if (shape_ != other.shape()) + throw(std::runtime_error("Shape of NDArray must match")); + return elementwise(other.data(), std::divides<>()); } /** @@ -400,36 +414,28 @@ class NDArray : public ArrayExpr, Ndim> { * @brief Add a scalar value to all elements in the NDArray. */ NDArray &operator+=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] += value; - return *this; + return elementwise(value, std::plus()); } /** * @brief Subtract a scalar value to all elements in the NDArray. */ NDArray &operator-=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] -= value; - return *this; + return elementwise(value, std::minus()); } /** * @brief Multiply all elements in the NDArray with a scalar value */ NDArray &operator*=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] *= value; - return *this; + return elementwise(value, std::multiplies()); } /** * @brief Divide all elements in the NDArray with a scalar value */ NDArray &operator/=(const T &value) { - for (size_t i = 0; i < size_; ++i) - data_[i] /= value; - return *this; + return elementwise(value, std::divides()); } /** @@ -437,53 +443,7 @@ class NDArray : public ArrayExpr, Ndim> { * Used for example to mask out gain bits for Jungfrau detectors. */ NDArray &operator&=(const T &mask) { - for (auto it = begin(); it != end(); ++it) - *it &= mask; - return *this; - } - - /** - * @brief Operator + with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator+(const T &value) { - NDArray result = *this; - result += value; - return result; - } - - /** - * @brief Operator - with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator-(const T &value) { - NDArray result = *this; - result -= value; - return result; - } - - /** - * @brief Operator * with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator*(const T &value) { - NDArray result = *this; - result *= value; - return result; - } - - /** - * @brief Operator / with a scalar value. Returns a new NDArray. - * - * TODO! Expression template version of this? - */ - NDArray operator/(const T &value) { - NDArray result = *this; - result /= value; - return result; + return elementwise(mask, std::bit_and()); } /** @@ -509,19 +469,16 @@ class NDArray : public ArrayExpr, Ndim> { * @brief Compute the square root of all elements in the NDArray. */ void sqrt() { - for (size_t i = 0; i < size_; ++i) { - data_[i] = std::sqrt(data_[i]); + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = std::sqrt(data[i]); } } /* * @brief Prefix increment operator. Increments all elements by 1. */ - NDArray &operator++() { - for (size_t i = 0; i < size_; ++i) - data_[i] += T{1}; - return *this; - } + NDArray &operator++() { return elementwise(T{1}, std::plus()); } /** @brief Create a mutable view of the NDArray. */ NDView view() { return NDView{data_, shape_}; } @@ -532,6 +489,31 @@ class NDArray : public ArrayExpr, Ndim> { } private: + /** + * @brief Apply data_[i] = op(data_[i], value) to all elements. + * + * The members are copied to locals since a store to data_[i] may alias + * them for some T (uint8_t, int64_t, ...). The compiler then has to reload + * them every iteration, which blocks vectorization. + */ + template NDArray &elementwise(T value, Op op) { + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = op(data[i], value); + } + return *this; + } + + /** @brief Apply data_[i] = op(data_[i], other[i]) to all elements. */ + template + NDArray &elementwise(const V *other, Op op) { + T *data = data_; + for (size_t i = 0, n = size_; i < n; ++i) { + data[i] = op(data[i], other[i]); + } + return *this; + } + /** * @brief Reset the NDArray to an empty state. Dropping the ownership of * the data. Used internally for move operations to avoid double free or diff --git a/include/aare/NDView.hpp b/include/aare/NDView.hpp index c507b41f..715127c9 100644 --- a/include/aare/NDView.hpp +++ b/include/aare/NDView.hpp @@ -173,8 +173,7 @@ class NDView : public ArrayExpr, Ndim> { } NDView &operator=(const T val) { - for (auto it = begin(); it != end(); ++it) - *it = val; + std::fill(begin(), end(), val); return *this; } @@ -232,15 +231,19 @@ class NDView : public ArrayExpr, Ndim> { template NDView &elemenwise(T val, BinaryOperation op) { - for (uint64_t i = 0; i != size_; ++i) { - buffer_[i] = op(buffer_[i], val); + // Local copies since a store to buffer_[i] may alias the members + T *buffer = buffer_; + for (uint64_t i = 0, n = size_; i != n; ++i) { + buffer[i] = op(buffer[i], val); } return *this; } template NDView &elemenwise(const NDView &other, BinaryOperation op) { - for (uint64_t i = 0; i != size_; ++i) { - buffer_[i] = op(buffer_[i], other.buffer_[i]); + T *buffer = buffer_; + const T *other_buffer = other.buffer_; + for (uint64_t i = 0, n = size_; i != n; ++i) { + buffer[i] = op(buffer[i], other_buffer[i]); } return *this; } diff --git a/include/aare/NumpyFile.hpp b/include/aare/NumpyFile.hpp index caa68531..0d7a523d 100644 --- a/include/aare/NumpyFile.hpp +++ b/include/aare/NumpyFile.hpp @@ -2,6 +2,7 @@ #pragma once #include "aare/Dtype.hpp" #include "aare/FileInterface.hpp" +#include "aare/FilePtr.hpp" #include "aare/NumpyHelpers.hpp" #include "aare/defs.hpp" @@ -77,7 +78,7 @@ class NumpyFile : public FileInterface { */ template NDArray load() { NDArray arr(make_shape(m_header.shape)); - if (fseek(fp, static_cast(header_size), SEEK_SET)) { + if (fseek64(fp, header_size, SEEK_SET)) { throw std::runtime_error(LOCATION + "Error seeking to the start of the data"); } diff --git a/include/aare/ProducerConsumerQueue.hpp b/include/aare/ProducerConsumerQueue.hpp index b9ec9d02..d630490f 100644 --- a/include/aare/ProducerConsumerQueue.hpp +++ b/include/aare/ProducerConsumerQueue.hpp @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: Apache-2.0 /* * Copyright (c) Meta Platforms, Inc. and affiliates. * @@ -18,10 +18,18 @@ // @author Bo Hu (bhu@fb.com) // @author Jordan DeLong (delong.j@fb.com) -// Changes made by PSD Detector Group: -// Copied: Line 34 constexpr std::size_t hardware_destructive_interference_size -// = 128; from folly/lang/Align.h Changed extension to .hpp Changed namespace to -// aare +// Changes made by the CPS Detector Group at PSI relative to +// folly/ProducerConsumerQueue.h in folly v2024.03.18.00 (commit 8aceb9bf727c; +// the file is identical in v2022.01.03.00 through v2024.03.18.00): +// - Replaced the include of folly/concurrency/CacheLocality.h with a local +// constexpr hardware_destructive_interference_size = 128, the value that +// folly/lang/Align.h uses except on ARM and s390x. +// - Changed the file extension to .hpp and the namespace to aare. +// - Added a default constructor equivalent to ProducerConsumerQueue(2). +// - Declared the move constructor and move assignment deleted. The queue is +// not safe to move and an earlier attempt at move support was removed. +// - The constructor throws std::invalid_argument for size < 2 instead of +// asserting, so the check is also active in release builds. #pragma once @@ -31,12 +39,14 @@ #include #include #include +#include #include #include -constexpr std::size_t hardware_destructive_interference_size = 128; namespace aare { +constexpr std::size_t hardware_destructive_interference_size = 128; + /* * ProducerConsumerQueue is a one producer and one consumer queue * without locks. @@ -46,38 +56,18 @@ template struct ProducerConsumerQueue { ProducerConsumerQueue(const ProducerConsumerQueue &) = delete; ProducerConsumerQueue &operator=(const ProducerConsumerQueue &) = delete; - - ProducerConsumerQueue(ProducerConsumerQueue &&other) { - size_ = other.size_; - records_ = other.records_; - other.records_ = nullptr; - readIndex_ = other.readIndex_.load(std::memory_order_acquire); - writeIndex_ = other.writeIndex_.load(std::memory_order_acquire); - } - ProducerConsumerQueue &operator=(ProducerConsumerQueue &&other) { - size_ = other.size_; - records_ = other.records_; - other.records_ = nullptr; - readIndex_ = other.readIndex_.load(std::memory_order_acquire); - writeIndex_ = other.writeIndex_.load(std::memory_order_acquire); - return *this; - } + ProducerConsumerQueue(ProducerConsumerQueue &&) = delete; + ProducerConsumerQueue &operator=(ProducerConsumerQueue &&) = delete; ProducerConsumerQueue() : ProducerConsumerQueue(2) {}; - // size must be >= 2. + // size must be >= 2, otherwise std::invalid_argument is thrown. // // Also, note that the number of usable slots in the queue at any // given time is actually (size-1), so if you start with an empty queue, // isFull() will return true after size-1 insertions. explicit ProducerConsumerQueue(uint32_t size) - : size_(size), - records_(static_cast(std::malloc(sizeof(T) * size))), - readIndex_(0), writeIndex_(0) { - assert(size >= 2); - if (!records_) { - throw std::bad_alloc(); - } - } + : size_(size), records_(allocate_records(size)), readIndex_(0), + writeIndex_(0) {} ~ProducerConsumerQueue() { // We need to destruct anything that may still exist in our queue. @@ -192,11 +182,23 @@ template struct ProducerConsumerQueue { private: using AtomicIndex = std::atomic; + // Validates size before allocating, so a throw cannot leak the buffer. + static T *allocate_records(uint32_t size) { + if (size < 2) { + throw std::invalid_argument( + "ProducerConsumerQueue size must be at least 2, got " + + std::to_string(size)); + } + auto *records = static_cast(std::malloc(sizeof(T) * size)); + if (!records) { + throw std::bad_alloc(); + } + return records; + } + char pad0_[hardware_destructive_interference_size]; - // const uint32_t size_; - uint32_t size_; - // T *const records_; - T *records_; + const uint32_t size_; + T *const records_; alignas(hardware_destructive_interference_size) AtomicIndex readIndex_; alignas(hardware_destructive_interference_size) AtomicIndex writeIndex_; diff --git a/include/aare/ROI.hpp b/include/aare/ROI.hpp index 81c2a84e..36838f43 100644 --- a/include/aare/ROI.hpp +++ b/include/aare/ROI.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/defs.hpp" #include diff --git a/include/aare/ROIGeometry.hpp b/include/aare/ROIGeometry.hpp index 739b24c3..a8a90c37 100644 --- a/include/aare/ROIGeometry.hpp +++ b/include/aare/ROIGeometry.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/DetectorGeometry.hpp" #include "aare/ROI.hpp" diff --git a/include/aare/RawFile.hpp b/include/aare/RawFile.hpp index 3b27e156..4ef92962 100644 --- a/include/aare/RawFile.hpp +++ b/include/aare/RawFile.hpp @@ -39,6 +39,8 @@ class RawFile : public FileInterface { * @brief RawFile constructor * @param fname path to the master file (.json) * @param mode file mode (only "r" is supported at the moment) + * @throws std::runtime_error if frame padding is disabled and the frame + * discard policy is not DiscardPartial. */ RawFile(const std::filesystem::path &fname, const std::string &mode = "r"); virtual ~RawFile() override = default; diff --git a/include/aare/calibration.hpp b/include/aare/calibration.hpp index 28550b3a..aac09cba 100644 --- a/include/aare/calibration.hpp +++ b/include/aare/calibration.hpp @@ -11,20 +11,6 @@ namespace aare { -// Really try to convince the compile to inline this function -// TODO! Clang? -#if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) -#define STRONG_INLINE __forceinline -#else -#define STRONG_INLINE inline -#endif - -#if defined(__GNUC__) -#define ALWAYS_INLINE __attribute__((always_inline)) inline -#else -#define ALWAYS_INLINE STRONG_INLINE -#endif - /** * @brief Get the gain from the raw ADC value. In Jungfrau the gain is * encoded in the left most 2 bits of the raw value. diff --git a/include/aare/decode.hpp b/include/aare/decode.hpp index a6efd187..00433109 100644 --- a/include/aare/decode.hpp +++ b/include/aare/decode.hpp @@ -16,6 +16,23 @@ void adc_sar_05_decode64to16(NDView input, void adc_sar_04_decode64to16(NDView input, NDView output); +/** + * @brief Decode packed 64-bit ADC SAR samples stored as raw bytes. + * + * Every 8 bytes of a row form one 64-bit word in native byte order. Words + * are assembled with memcpy, so the buffer does not need to be 8-byte + * aligned. The output must have the same number of rows as the input and + * input.shape(1) / 8 columns. + * @throws std::invalid_argument if the row length is not a multiple of 8 + * bytes or the output shape does not match. + */ +void adc_sar_05_06_07_08decode64to16(NDView input, + NDView output); +void adc_sar_05_decode64to16(NDView input, + NDView output); +void adc_sar_04_decode64to16(NDView input, + NDView output); + /** * @brief Called with a 32 bit unsigned integer, shift by offset * and then return the lower 24 bits as an 32 bit integer diff --git a/include/aare/defs.hpp b/include/aare/defs.hpp index b0701912..43cae3cb 100644 --- a/include/aare/defs.hpp +++ b/include/aare/defs.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,10 @@ #include #include +#if !defined(_MSC_VER) +#include // ::ssize_t +#endif + /** * @brief LOCATION macro to get the current location in the code */ @@ -32,8 +37,40 @@ } while (0) #endif +/** + * ALWAYS_INLINE forces a function to be inlined at every call site. gcc + * refuses to inline a plain `inline` function once its translation-unit + * growth budget (--param inline-unit-growth) is used up, which happens in + * the fitting code where every model is instantiated for every minimizer; + * a per-point helper that is then called out of line also stops the + * surrounding loop from vectorising. + */ +#if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) +#define STRONG_INLINE __forceinline +#else +#define STRONG_INLINE inline +#endif + +#if defined(__GNUC__) +#define ALWAYS_INLINE __attribute__((always_inline)) inline +#else +#define ALWAYS_INLINE STRONG_INLINE +#endif + namespace aare { +/** + * Signed size and index type used throughout the library (shapes, strides, + * frame indices, ROI coordinates). On POSIX this is the platform's ssize_t, + * so nothing changes for existing code. MSVC has no ssize_t, so there it is + * std::ptrdiff_t, which is the same type as Py_ssize_t and NumPy's npy_intp. + */ +#if defined(_MSC_VER) +using ssize_t = std::ptrdiff_t; +#else +using ssize_t = ::ssize_t; +#endif + inline constexpr size_t bits_per_byte = 8; void assert_failed(const std::string &msg); diff --git a/include/aare/hist/PedestalTrackingPixelHistogram.hpp b/include/aare/hist/PedestalTrackingPixelHistogram.hpp index 88ca7f2f..2025d053 100644 --- a/include/aare/hist/PedestalTrackingPixelHistogram.hpp +++ b/include/aare/hist/PedestalTrackingPixelHistogram.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/FastPedestal.hpp" #include "aare/NDArray.hpp" diff --git a/include/aare/hist/PixelHistogram.hpp b/include/aare/hist/PixelHistogram.hpp index 05432d65..c9b199b9 100644 --- a/include/aare/hist/PixelHistogram.hpp +++ b/include/aare/hist/PixelHistogram.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include "aare/NDArray.hpp" #include "aare/NDView.hpp" diff --git a/include/aare/hist/PixelHistogramImpl.hpp b/include/aare/hist/PixelHistogramImpl.hpp index d68795c8..9af42fe1 100644 --- a/include/aare/hist/PixelHistogramImpl.hpp +++ b/include/aare/hist/PixelHistogramImpl.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once /* Basic pixel histogram class with templated axis and storage type. diff --git a/include/aare/logger.hpp b/include/aare/logger.hpp index 278d4d7d..b5504d73 100644 --- a/include/aare/logger.hpp +++ b/include/aare/logger.hpp @@ -2,9 +2,12 @@ #pragma once /*Utility to log to console*/ +#include +#include +#include #include #include -#include +#include namespace aare { @@ -55,12 +58,53 @@ enum TLogLevel { #define __AT__ \ std::string(__FILE__) + std::string("::") + std::string(__func__) + \ std::string("(): ") -#define __SHORT_FORM_OF_FILE__ \ - (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) +#define __SHORT_FORM_OF_FILE__ aare::short_file_name(__FILE__) #define __SHORT_AT__ \ std::string(__SHORT_FORM_OF_FILE__) + std::string("::") + \ std::string(__func__) + std::string("(): ") +/** + * @brief Strip the directory part of a path, accepting both '/' and '\\' as + * separators so that __FILE__ is shortened the same way on every platform. + */ +inline const char *short_file_name(const char *path) noexcept { + const char *base = path; + for (const char *p = path; *p != '\0'; ++p) { + if (*p == '/' || *p == '\\') + base = p + 1; + } + return base; +} + +/** + * @brief Local wall-clock time as HH:MM:SS.mmm (strftime "%X" plus + * milliseconds), built from std::chrono so it works on every platform. + */ +inline std::string log_timestamp() { + using namespace std::chrono; + const auto now = system_clock::now(); + const time_t t = system_clock::to_time_t(now); + const auto ms = duration_cast(now.time_since_epoch()) % 1000; + + tm local{}; +#if defined(_MSC_VER) + localtime_s(&local, &t); +#else + localtime_r(&t, &local); +#endif + + constexpr size_t buffer_len = 12; + char buffer[buffer_len]; + if (strftime(buffer, buffer_len, "%X", &local) == 0) + buffer[0] = '\0'; + + constexpr size_t result_len = 100; + char result[result_len]; + snprintf(result, result_len, "%s.%03d", buffer, + static_cast(ms.count())); + return result; +} + class Logger { std::ostringstream os; TLogLevel m_level = AARE_LOG_LEVEL; @@ -111,28 +155,12 @@ class Logger { return os; } - static std::string Timestamp() { - constexpr size_t buffer_len = 12; - char buffer[buffer_len]; - time_t t; - ::time(&t); - tm r; - strftime(buffer, buffer_len, "%X", localtime_r(&t, &r)); - buffer[buffer_len - 1] = '\0'; - struct timeval tv; - gettimeofday(&tv, nullptr); - constexpr size_t result_len = 100; - char result[result_len]; - snprintf(result, result_len, "%s.%03ld", buffer, - static_cast(tv.tv_usec) / 1000); - result[result_len - 1] = '\0'; - return result; - } + static std::string Timestamp() { return log_timestamp(); } }; // TODO! Do we need to keep the runtime option? #define LOG(level) \ - if (level > AARE_LOG_LEVEL) \ + if constexpr (level > AARE_LOG_LEVEL) \ ; \ else if (level > aare::Logger::ReportingLevel()) \ ; \ diff --git a/include/aare/to_string.hpp b/include/aare/to_string.hpp index dbb0c09f..547ebc09 100644 --- a/include/aare/to_string.hpp +++ b/include/aare/to_string.hpp @@ -19,7 +19,7 @@ T string_to(const std::string &t, const std::string &unit) { double tval{0}; try { tval = std::stod(t); - } catch (const std::invalid_argument &e) { + } catch (const std::invalid_argument &) { throw std::runtime_error("Could not convert string to time"); } diff --git a/include/aare/utils/math_helpers.hpp b/include/aare/utils/math_helpers.hpp index 9481bd8a..6d9b3c80 100644 --- a/include/aare/utils/math_helpers.hpp +++ b/include/aare/utils/math_helpers.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include diff --git a/pyproject.toml b/pyproject.toml index 8fa52bde..16636a01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ regex = '^(?P\d+(?:\.\d+)*(?:[\.\+\w]+)?)$' result = "{version}" [build-system] -requires = ["scikit-build-core>=0.10", "pybind11", "numpy"] +requires = ["scikit-build-core>=0.11", "pybind11", "numpy"] build-backend = "scikit_build_core.build" [project] @@ -16,7 +16,8 @@ dependencies = [ "numpy", "matplotlib", ] -license = { file = "LICENSE" } +license = "MPL-2.0 AND Apache-2.0 AND MIT AND BSD-3-Clause AND LGPL-2.1-or-later" +license-files = ["LICENSES/*", "THIRD-PARTY-NOTICES.txt"] @@ -27,6 +28,9 @@ build = "cp{312,313,314}-manylinux_x86_64" build = "cp{312,313,314}-macosx_*" archs = ["arm64"] +[tool.cibuildwheel.windows] +build = "cp{312,313,314}-win_amd64" + [tool.scikit-build] build.verbose = true diff --git a/python/aare/Cluster.py b/python/aare/Cluster.py index 7946567d..2da800b9 100644 --- a/python/aare/Cluster.py +++ b/python/aare/Cluster.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 from . import _aare import numpy as np from .factory import _type_to_char diff --git a/python/aare/__init__.py b/python/aare/__init__.py index 9830919d..670010d6 100644 --- a/python/aare/__init__.py +++ b/python/aare/__init__.py @@ -22,9 +22,10 @@ from ._aare import ( RawSubFile, ) from ._aare import Pedestal_d, Pedestal_f, ClusterFinder_Cluster3x3i, VarClusterFinder -from ._aare import DetectorType, ReadoutMode +from ._aare import DetectorType, FrameDiscardPolicy, ReadoutMode from ._aare import hitmap from ._aare import ROI +from ._aare import xy from ._aare import corner from ._aare import UDPPortPosition @@ -40,6 +41,7 @@ from .Cluster import Cluster from ._aare import Gaussian, RisingScurve, FallingScurve, Pol1, Pol2, GaussianErfcPlateau, GaussianChargeSharing, GaussianChargeSharingKb from ._aare import fit +from ._aare import Minimizer from ._aare import Interpolator from ._aare import calculate_eta2, calculate_eta3, calculate_cross_eta3, calculate_full_eta2 from ._aare import reduce_to_2x2, reduce_to_3x3 diff --git a/python/aare/_version.py b/python/aare/_version.py index f3530c4c..c2bab73f 100644 --- a/python/aare/_version.py +++ b/python/aare/_version.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 # This file is used to get the version of the package from the VERSION file from importlib.metadata import PackageNotFoundError, version from pathlib import Path diff --git a/python/aare/calibration.py b/python/aare/calibration.py index b7a23e74..20f8c77b 100644 --- a/python/aare/calibration.py +++ b/python/aare/calibration.py @@ -5,9 +5,12 @@ def load_calibration(fname, hg0=False): """ Load calibration data from a file. - Parameters: - fname (str): Path to the calibration file. - hg0 (bool): If True, load HG0 calibration data instead of G0. + Parameters + ---------- + fname : str + Path to the calibration file. + hg0 : bool + If True, load HG0 calibration data instead of G0. """ gains = 3 diff --git a/python/src/bind_Cluster.hpp b/python/src/bind_Cluster.hpp index 5a749bfc..49618bf2 100644 --- a/python/src/bind_Cluster.hpp +++ b/python/src/bind_Cluster.hpp @@ -13,9 +13,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_Cluster(py::module &m, const std::string &typestr) { @@ -24,14 +21,21 @@ void define_Cluster(py::module &m, const std::string &typestr) { py::class_>( m, class_name.c_str(), py::buffer_protocol()) + // conversion ok: small input, accept lists and any numeric dtype .def(py::init([](CoordType x, CoordType y, py::array_t data) { - py::buffer_info buf_info = data.request(); + constexpr py::ssize_t n_pixels = ClusterSizeX * ClusterSizeY; + if (data.ndim() != 1 || data.size() != n_pixels) { + throw py::value_error(fmt::format( + "Cluster data must be a 1D array of {} values, got {}D " + "array with {} values", + n_pixels, data.ndim(), data.size())); + } Cluster cluster; cluster.x = x; cluster.y = y; auto r = data.template unchecked<1>(); // no bounds checks - for (py::ssize_t i = 0; i < data.size(); ++i) { + for (py::ssize_t i = 0; i < n_pixels; ++i) { cluster.data[i] = r(i); } return cluster; @@ -133,5 +137,3 @@ void reduce_to_2x2(py::module &m) { row-major order. )doc"); } - -#pragma GCC diagnostic pop diff --git a/python/src/bind_ClusterCollector.hpp b/python/src/bind_ClusterCollector.hpp index 01dd03c2..fab91522 100644 --- a/python/src/bind_ClusterCollector.hpp +++ b/python/src/bind_ClusterCollector.hpp @@ -20,9 +20,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_ClusterCollector(py::module &m, const std::string &typestr) { @@ -42,5 +39,3 @@ void define_ClusterCollector(py::module &m, const std::string &typestr) { }, py::return_value_policy::take_ownership); } - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/bind_ClusterFile.hpp b/python/src/bind_ClusterFile.hpp index 09fc649b..75c5458d 100644 --- a/python/src/bind_ClusterFile.hpp +++ b/python/src/bind_ClusterFile.hpp @@ -13,11 +13,6 @@ #include #include -// Disable warnings for unused parameters, as we ignore some -// in the __exit__ method -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - namespace py = pybind11; using namespace ::aare; @@ -138,9 +133,10 @@ void define_ClusterFile(py::module &m, const std::string &typestr) { "Estimate the number of clusters from the file size. Frame " "headers can make this larger than the actual count.") .def( - "set_noise_map", + "set_noise_map", // conversions ok [](ClusterFile &self, - py::array_t noise_map) { + py::array_t + noise_map) { auto view = make_view_2d(noise_map); self.set_noise_map(view); }, @@ -149,9 +145,10 @@ void define_ClusterFile(py::module &m, const std::string &typestr) { "[y, x]. The map must cover every cluster center coordinate.") .def( - "set_gain_map", + "set_gain_map", // conversions ok [](ClusterFile &self, - py::array_t gain_map) { + py::array_t + gain_map) { auto view = make_view_2d(gain_map); self.set_gain_map(view); }, @@ -169,14 +166,12 @@ void define_ClusterFile(py::module &m, const std::string &typestr) { .def("__enter__", [](ClusterFile &self) { return &self; }) .def("__exit__", [](ClusterFile &self, - const std::optional &exc_type, - const std::optional &exc_value, - const std::optional &traceback) { - self.close(); - }) + [[maybe_unused]] const std::optional &exc_type, + [[maybe_unused]] const std::optional + &exc_value, + [[maybe_unused]] const std::optional + &traceback) { self.close(); }) .def("__iter__", [](ClusterFile &self) { return &self; }) .def("__next__", [](File &self) { return ChunkIterator(self.chunks()).next(); }); } - -#pragma GCC diagnostic pop diff --git a/python/src/bind_ClusterFileSink.hpp b/python/src/bind_ClusterFileSink.hpp index b366886a..50600fcb 100644 --- a/python/src/bind_ClusterFileSink.hpp +++ b/python/src/bind_ClusterFileSink.hpp @@ -17,9 +17,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_ClusterFileSink(py::module &m, const std::string &typestr) { @@ -34,5 +31,3 @@ void define_ClusterFileSink(py::module &m, const std::string &typestr) { const std::filesystem::path &>()) .def("stop", &ClusterFileSink::stop); } - -#pragma GCC diagnostic pop diff --git a/python/src/bind_ClusterFinder.hpp b/python/src/bind_ClusterFinder.hpp index 2e0bf8b3..8872db29 100644 --- a/python/src/bind_ClusterFinder.hpp +++ b/python/src/bind_ClusterFinder.hpp @@ -20,9 +20,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_ClusterFinder(py::module &m, const std::string &typestr) { @@ -43,16 +40,22 @@ void define_ClusterFinder(py::module &m, const std::string &typestr) { &ClusterFinder::set_nSigma, R"(number of sigma above the pedestal to consider a photon during cluster finding.)") - .def("push_pedestal_frame", - [](ClusterFinder &self, - py::array_t frame) { - auto view = make_view_2d(frame); - self.push_pedestal_frame(view); - }) + .def( + "push_pedestal_frame", + [](ClusterFinder &self, + py::array_t frame) { + auto view = make_view_2d(frame); + self.push_pedestal_frame(view); + }, + py::arg("frame").noconvert()) .def("clear_pedestal", &ClusterFinder::clear_pedestal) .def("update_threshold", &ClusterFinder::update_threshold) + .def_property_readonly( + "pedestal_ready", + &ClusterFinder::pedestal_ready, + R"(True once enough pedestal frames have been pushed to find clusters.)") .def_property_readonly( "pedestal", [](ClusterFinder &self) { @@ -84,7 +87,5 @@ void define_ClusterFinder(py::module &m, const std::string &typestr) { self.find_clusters(view, frame_number); return; }, - py::arg(), py::arg("frame_number") = 0); + py::arg("frame").noconvert(), py::arg("frame_number") = 0); } - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/bind_ClusterFinderMT.hpp b/python/src/bind_ClusterFinderMT.hpp index 37ab8d47..7e662841 100644 --- a/python/src/bind_ClusterFinderMT.hpp +++ b/python/src/bind_ClusterFinderMT.hpp @@ -20,9 +20,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_ClusterFinderMT(py::module &m, const std::string &typestr) { @@ -37,12 +34,14 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) { py::arg("capacity") = 2048, py::arg("n_threads") = 3, py::arg("queue_depth") = 16, py::arg("min_pedestal_samples") = 1000) - .def("push_pedestal_frame", - [](ClusterFinderMT &self, - py::array_t frame) { - auto view = make_view_2d(frame); - self.push_pedestal_frame(view); - }) + .def( + "push_pedestal_frame", + [](ClusterFinderMT &self, + py::array_t frame) { + auto view = make_view_2d(frame); + self.push_pedestal_frame(view); + }, + py::arg("frame").noconvert()) .def( "find_clusters", [](ClusterFinderMT &self, @@ -50,16 +49,21 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) { auto view = make_view_2d(frame); self.find_clusters(view, frame_number); }, - py::arg(), py::arg("frame_number") = 0) + py::arg("frame").noconvert(), py::arg("frame_number") = 0) .def_property_readonly( "cluster_size", - [](ClusterFinderMT &self) { + []([[maybe_unused]] ClusterFinderMT + &self) { return py::make_tuple(ClusterSizeX, ClusterSizeY); }) .def("clear_pedestal", &ClusterFinderMT::clear_pedestal) .def("update_threshold", &ClusterFinderMT::update_threshold) + .def( + "pedestal_ready", + &ClusterFinderMT::pedestal_ready, + R"(waits for the queued frames, then returns True if all cluster finders have enough pedestal frames to find clusters.)") .def("sync", &ClusterFinderMT::sync) .def("stop", &ClusterFinderMT::stop) .def("start", &ClusterFinderMT::start) @@ -87,5 +91,3 @@ void define_ClusterFinderMT(py::module &m, const std::string &typestr) { py::arg("nSigma"), R"(sets the number of sigma for all cluster finders.)"); } - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/bind_ClusterVector.hpp b/python/src/bind_ClusterVector.hpp index 095c3f8a..13ad5a6d 100644 --- a/python/src/bind_ClusterVector.hpp +++ b/python/src/bind_ClusterVector.hpp @@ -18,9 +18,6 @@ namespace py = pybind11; using namespace aare; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - template void define_ClusterVector(py::module &m, const std::string &typestr) { @@ -46,11 +43,7 @@ void define_ClusterVector(py::module &m, const std::string &typestr) { .def( "__call__", - [](ClusterVector &self, - py::array_t mask) { - if (mask.ndim() != 1) { - throw py::value_error("Mask must be one-dimensional"); - } + [](ClusterVector &self, py::array_t mask) { return self(make_view_1d(mask)); }, py::arg("mask").noconvert(), R"doc( @@ -127,8 +120,8 @@ void define_ClusterVector(py::module &m, const std::string &typestr) { "padding.") .def_property_readonly( "fmt", - [typestr](ClusterVector &self) { - return fmt_format; + [typestr]([[maybe_unused]] ClusterVector &self) { + return fmt_format(); }, "PEP 3118 format string for one stored cluster.") @@ -149,11 +142,11 @@ void define_ClusterVector(py::module &m, const std::string &typestr) { .def_buffer( [typestr](ClusterVector &self) -> py::buffer_info { return py::buffer_info( - self.data(), /* Pointer to buffer */ - self.item_size(), /* Size of one scalar */ - fmt_format, /* Format descriptor */ - 1, /* Number of dimensions */ - {self.size()}, /* Buffer dimensions */ + self.data(), /* Pointer to buffer */ + self.item_size(), /* Size of one scalar */ + fmt_format(), /* Format descriptor */ + 1, /* Number of dimensions */ + {self.size()}, /* Buffer dimensions */ {self.item_size()} /* Strides (in bytes) for each index */ ); }); @@ -232,5 +225,3 @@ void define_3x3_reduction(py::module &m) { )doc", py::arg("clustervector")); } - -#pragma GCC diagnostic pop diff --git a/python/src/bind_Defs.hpp b/python/src/bind_Defs.hpp index ba12997e..575fc53b 100644 --- a/python/src/bind_Defs.hpp +++ b/python/src/bind_Defs.hpp @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: MPL-2.0 +#include #include #include @@ -89,6 +91,21 @@ void define_defs_bindings(py::module &m) { self.ymin == other.ymin && self.ymax == other.ymax; }); + py::class_(m, "xy") + .def(py::init(), py::arg("row"), py::arg("col")) + .def_readwrite("row", &xy::row) + .def_readwrite("col", &xy::col) + .def("__repr__", + [](const xy &self) { + return fmt::format("xy(row={}, col={})", self.row, self.col); + }) + .def("__iter__", + [](const xy &self) { + return py::iter(py::make_tuple(self.row, self.col)); + }) + .def(py::self == py::self) + .def(py::self != py::self); + py::enum_(m, "DetectorType") .value("Jungfrau", DetectorType::Jungfrau) .value("Eiger", DetectorType::Eiger) @@ -111,4 +128,9 @@ void define_defs_bindings(py::module &m) { .value("GATED", TimingMode::GATED) .value("BURST_TRIGGER", TimingMode::BURST_TRIGGER) .value("TRIGGER_GATED", TimingMode::TRIGGER_GATED); + + py::enum_(m, "FrameDiscardPolicy") + .value("NoDiscard", FrameDiscardPolicy::NoDiscard) + .value("Discard", FrameDiscardPolicy::Discard) + .value("DiscardPartial", FrameDiscardPolicy::DiscardPartial); } diff --git a/python/src/bind_Eta.hpp b/python/src/bind_Eta.hpp index 716b0e72..e2b9302e 100644 --- a/python/src/bind_Eta.hpp +++ b/python/src/bind_Eta.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/CalculateEta.hpp" #include "aare/to_string.hpp" diff --git a/python/src/bind_FastPedestal.hpp b/python/src/bind_FastPedestal.hpp index b2b4fb8f..5c278490 100644 --- a/python/src/bind_FastPedestal.hpp +++ b/python/src/bind_FastPedestal.hpp @@ -106,8 +106,7 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) { "Return an independent copy of the pedestal and its state.") .def( "push_ema", - [](FastPedestal &pedestal, - py::array_t &frame) { + [](FastPedestal &pedestal, py::array_t frame) { pedestal.push_ema(make_view_2d(frame)); }, py::arg("frame").noconvert(), @@ -115,8 +114,7 @@ void define_fast_pedestal_bindings(py::module &m, const std::string &name) { "already be ready for this update.") .def( "add_init_frame", - [](FastPedestal &pedestal, - py::array_t &frame) { + [](FastPedestal &pedestal, py::array_t frame) { pedestal.add_init_frame(make_view_2d(frame)); }, py::arg("frame").noconvert(), diff --git a/python/src/bind_Interpolator.hpp b/python/src/bind_Interpolator.hpp index a98e539d..3b7a015b 100644 --- a/python/src/bind_Interpolator.hpp +++ b/python/src/bind_Interpolator.hpp @@ -101,9 +101,7 @@ void define_interpolation_bindings(py::module &m) { auto interpolator = py::class_(m, "Interpolator") .def(py::init( - [](py::array_t - etacube, + [](py::array_t etacube, py::array_t xbins, py::array_t ybins, py::array_t ebins) { return Interpolator( @@ -124,9 +122,9 @@ void define_interpolation_bindings(py::module &m) { ebins: bin edges of photon energy )doc", - py::arg("etacube"), - py::arg("xbins"), py::arg("ybins"), - py::arg("ebins")) + py::arg("etacube").noconvert(), + py::arg("xbins").noconvert(), py::arg("ybins").noconvert(), + py::arg("ebins").noconvert()) .def(py::init( [](py::array_t xbins, py::array_t ybins, @@ -146,13 +144,12 @@ void define_interpolation_bindings(py::module &m) { bin edges of etay ebins: bin edges of photon energy - )", py::arg("xbins"), - py::arg("ybins"), py::arg("ebins")) + )", py::arg("xbins").noconvert(), + py::arg("ybins").noconvert(), py::arg("ebins").noconvert()) .def( "rosenblatttransform", [](Interpolator &self, - py::array_t + py::array_t etacube) { return self.rosenblatttransform(make_view_3d(etacube)); }, @@ -162,7 +159,7 @@ void define_interpolation_bindings(py::module &m) { etacube: joint distribution of eta_x, eta_y and photon energy (**Note:** for the joint distribution first dimension is eta_x, second: eta_y, third: energy bins.) )", - py::arg("etacube")) + py::arg("etacube").noconvert()) .def("get_ietax", [](Interpolator &self) { auto *ptr = new NDArray{}; diff --git a/python/src/bind_MultiThreadedFileReader.hpp b/python/src/bind_MultiThreadedFileReader.hpp index f63f79c8..0f39cb6c 100644 --- a/python/src/bind_MultiThreadedFileReader.hpp +++ b/python/src/bind_MultiThreadedFileReader.hpp @@ -16,6 +16,7 @@ #include namespace py = pybind11; +using aare::ssize_t; inline py::dtype multi_threaded_reader_numpy_dtype(const aare::Dtype &dtype) { using aare::Dtype; diff --git a/python/src/bind_Pedestal.hpp b/python/src/bind_Pedestal.hpp index 602ee13e..9e8af455 100644 --- a/python/src/bind_Pedestal.hpp +++ b/python/src/bind_Pedestal.hpp @@ -85,11 +85,7 @@ void define_pedestal_bindings(py::module &m, const std::string &name) { // TODO! add push for other data types .def( "push", - [](Pedestal &pedestal, - py::array_t &f) { - if (f.ndim() != 2) { - throw py::value_error("Frame must be 2-dimensional"); - } + [](Pedestal &pedestal, py::array_t f) { auto v = make_view_2d(f); pedestal.push(v); }, @@ -99,15 +95,8 @@ void define_pedestal_bindings(py::module &m, const std::string &name) { "n_samples values per pixel, new values have weight 1 / n_samples.") .def( "push_with_threshold", - [](Pedestal &pedestal, - py::array_t &f, - py::array_t &threshold) { - if (f.ndim() != 2) { - throw py::value_error("Frame must be 2-dimensional"); - } - if (threshold.ndim() != 2) { - throw py::value_error("Threshold must be 2-dimensional"); - } + [](Pedestal &pedestal, py::array_t f, + py::array_t threshold) { auto frame_view = make_view_2d(f); auto threshold_view = make_view_2d(threshold); pedestal.push_with_threshold(frame_view, threshold_view); diff --git a/python/src/bind_PedestalTrackingPixelHistogram.hpp b/python/src/bind_PedestalTrackingPixelHistogram.hpp index 35b8b24a..139b58a1 100644 --- a/python/src/bind_PedestalTrackingPixelHistogram.hpp +++ b/python/src/bind_PedestalTrackingPixelHistogram.hpp @@ -50,8 +50,7 @@ void define_pedestal_tracking_pixel_histogram_bindings(py::module &m) { .def( "push_pedestal_no_update", [](PedestalTrackingPixelHistogram &self, - py::array_t - frame) { + py::array_t frame) { auto view = make_view_2d(frame); self.push_pedestal_no_update(view); }, @@ -104,8 +103,7 @@ void define_pedestal_tracking_pixel_histogram_bindings(py::module &m) { .def( "fill_async", [](PedestalTrackingPixelHistogram &self, - py::array_t - image) { + py::array_t image) { // Copy the numpy buffer into an owned NDArray while we // still hold the GIL so we don't depend on the array's // backing storage outliving this call. diff --git a/python/src/bind_PixelHistogram.hpp b/python/src/bind_PixelHistogram.hpp index d205bce2..d3a1b462 100644 --- a/python/src/bind_PixelHistogram.hpp +++ b/python/src/bind_PixelHistogram.hpp @@ -46,7 +46,7 @@ void define_pixel_histogram_binding(py::module &m, const char *class_name, .def( "fill_async", - [](Hist &self, py::array_t image) { + [](Hist &self, py::array_t image) { // Copy the numpy buffer into an owned NDArray while we // still hold the GIL so we don't depend on the array's // backing storage outliving this call. diff --git a/python/src/bind_RawFile.hpp b/python/src/bind_RawFile.hpp index 2c4e20c8..11d1d31a 100644 --- a/python/src/bind_RawFile.hpp +++ b/python/src/bind_RawFile.hpp @@ -318,7 +318,14 @@ void define_raw_file_io_bindings(py::module &m) { Cols for the given ROI. )") .def_property_readonly("bitdepth", &RawFile::bitdepth) - .def_property_readonly("geometry", &RawFile::geometry) + .def_property_readonly("geometry", &RawFile::geometry, R"( + Number of modules along each axis of the detector. + + Returns + ------- + xy + Module rows and columns. + )") .def_property_readonly("detector_type", &RawFile::detector_type) .def_property_readonly("master", &RawFile::master) .def_property_readonly("n_modules", &RawFile::n_modules) diff --git a/python/src/bind_calibration.hpp b/python/src/bind_calibration.hpp index a4a3f160..cdb16b86 100644 --- a/python/src/bind_calibration.hpp +++ b/python/src/bind_calibration.hpp @@ -9,40 +9,37 @@ #include namespace py = pybind11; +using aare::ssize_t; template -py::array_t pybind_apply_calibration( - py::array_t data, - py::array_t pedestal, - py::array_t - calibration, - int n_threads = 4) { +py::array_t +pybind_apply_calibration(py::array_t data, + py::array_t pedestal, + py::array_t calibration, int n_threads = 4) { auto data_span = make_view_3d(data); // data is always 3D /* No pointer is passed, so NumPy will allocate the buffer */ auto result = py::array_t(data_span.shape()); auto res = make_view_3d(result); - if (data.ndim() == 3 && pedestal.ndim() == 3 && calibration.ndim() == 3) { + if (pedestal.ndim() == 3 && calibration.ndim() == 3) { auto ped = make_view_3d(pedestal); auto cal = make_view_3d(calibration); aare::apply_calibration(res, data_span, ped, cal, n_threads); - } else if (data.ndim() == 3 && pedestal.ndim() == 2 && - calibration.ndim() == 2) { + } else if (pedestal.ndim() == 2 && calibration.ndim() == 2) { auto ped = make_view_2d(pedestal); auto cal = make_view_2d(calibration); aare::apply_calibration(res, data_span, ped, cal, n_threads); } else { throw std::runtime_error( - "Invalid number of dimensions for data, pedestal or calibration"); + "Invalid number of dimensions for pedestal or calibration"); } return result; } -py::array_t pybind_count_switching_pixels( - py::array_t data, - ssize_t n_threads = 4) { +py::array_t pybind_count_switching_pixels(py::array_t data, + ssize_t n_threads = 4) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; @@ -51,9 +48,8 @@ py::array_t pybind_count_switching_pixels( } template -py::array_t pybind_calculate_pedestal( - py::array_t data, - ssize_t n_threads) { +py::array_t pybind_calculate_pedestal(py::array_t data, + ssize_t n_threads) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; @@ -62,9 +58,8 @@ py::array_t pybind_calculate_pedestal( } template -py::array_t pybind_calculate_pedestal_g0( - py::array_t data, - ssize_t n_threads) { +py::array_t pybind_calculate_pedestal_g0(py::array_t data, + ssize_t n_threads) { auto data_span = make_view_3d(data); auto arr = new NDArray{}; @@ -73,10 +68,27 @@ py::array_t pybind_calculate_pedestal_g0( } void bind_calibration(py::module &m) { + py::options options; + options.disable_function_signatures(); + m.def("apply_calibration", &pybind_apply_calibration, py::arg("raw_data").noconvert(), py::kw_only(), py::arg("pd").noconvert(), py::arg("cal").noconvert(), - py::arg("n_threads") = 4); + py::arg("n_threads") = 4, + R"( + Apply the calibration to the raw data and return the result as a 3D array of doubles. + + Parameters + ---------- + raw_data : array_like + 3D array of shape ``(frames, rows, cols)``. + pd : array_like + Pedestal array. + cal : array_like + Calibration array. + n_threads : int, optional + Number of threads to use. Defaults to 4. + )"); m.def("apply_calibration", &pybind_apply_calibration, py::arg("raw_data").noconvert(), py::kw_only(), diff --git a/python/src/bind_testing.hpp b/python/src/bind_testing.hpp new file mode 100644 index 00000000..2a4e8760 --- /dev/null +++ b/python/src/bind_testing.hpp @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MPL-2.0 +// Probes used by the python test-suite to exercise the helpers in +// np_helper.hpp. Internal, no API guarantees. +#pragma once + +#include "aare/NDArray.hpp" +#include "aare/NDView.hpp" +#include "np_helper.hpp" + +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace ::aare; + +// Arrays are taken with the default flags (no c_style) so that nothing is +// made contiguous by pybind11 before it reaches make_view +template +void define_make_view_probes(py::module &m, const std::string &suffix) { + + // Copy what C++ sees through the view back to python + m.def(("view_roundtrip_" + suffix).c_str(), + [](py::array_t arr) { + auto view = make_view(arr); + return return_image_data(new NDArray(view)); + }, + py::arg("arr").noconvert()); + + // Same as above but through a read only view + m.def(("const_view_roundtrip_" + suffix).c_str(), + [](py::array_t arr) { + auto view = make_const_view(arr); + static_assert( + std::is_same_v>); + auto out = new NDArray(view.shape()); + std::copy(view.begin(), view.end(), out->begin()); + return return_image_data(out); + }, + py::arg("arr").noconvert()); + + // Write through the view + m.def(("view_fill_" + suffix).c_str(), + [](py::array_t arr, T value) { + auto view = make_view(arr); + view = value; + }, + py::arg("arr").noconvert(), py::arg("value")); +} + +void define_testing_bindings(py::module &m) { + auto testing = m.def_submodule( + "testing", "Internal helpers for the test-suite, no API guarantees"); + + // noconvert + overloads gives dispatch on dtype + define_make_view_probes(testing, "1d"); + define_make_view_probes(testing, "2d"); + define_make_view_probes(testing, "3d"); + define_make_view_probes(testing, "1d"); + define_make_view_probes(testing, "2d"); + define_make_view_probes(testing, "3d"); +} diff --git a/python/src/ctb_raw_file.hpp b/python/src/ctb_raw_file.hpp index 5a6953b7..b6338bdd 100644 --- a/python/src/ctb_raw_file.hpp +++ b/python/src/ctb_raw_file.hpp @@ -24,177 +24,119 @@ namespace py = pybind11; using namespace ::aare; +// One binding for the three ADC SAR decoders. Each is registered under +// its original Python name so existing transforms keep working. +template , NDView)> +py::array_t decode64to16(py::array_t input) { + auto bytes = make_const_view_2d(input); + constexpr ssize_t bytes_per_word = sizeof(uint64_t); + auto output = py::array_t( + {bytes.shape(0), bytes.shape(1) / bytes_per_word}); + decode(bytes, make_view_2d(output)); + return output; +} + +constexpr auto decode64to16_doc = + "Decode packed 64-bit ADC samples. Takes a two-dimensional, " + "C-contiguous uint8 array whose row length is a multiple of 8 bytes " + "and returns a uint16 array with one value per 64-bit word."; + void define_ctb_raw_file_io_bindings(py::module &m) { - m.def("adc_sar_05_06_07_08decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + m.def("adc_sar_05_06_07_08decode64to16", + &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + m.def("adc_sar_05_decode64to16", &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + m.def("adc_sar_04_decode64to16", &decode64to16, + py::arg("input").noconvert(), decode64to16_doc); - adc_sar_05_06_07_08decode64to16(input_view, output_view); + m.def( + "apply_custom_weights", + [](py::array_t input, py::array_t weights) { + auto input_view = make_view_1d(input); + auto weights_view = make_view_1d(weights); + auto output = py::array_t(input_view.size()); + apply_custom_weights(input_view, make_view_1d(output), + weights_view); + return output; + }, + py::arg("input").noconvert(), py::arg("weights").noconvert(), + "Apply per-bit weights to every value of a one-dimensional, " + "C-contiguous uint16 array and return a float64 array."); - return output; - }); + m.def( + "expand24to32bit", + [](py::array_t input, uint32_t offset) { + constexpr ssize_t bytes_per_channel = 3; // 24 bit + auto input_view = make_const_view_1d(input); + auto output = + py::array_t(input_view.size() / bytes_per_channel); + aare::expand24to32bit(input_view, make_view_1d(output), + aare::BitOffset(offset)); + return output; + }, + py::arg("input").noconvert(), py::arg("offset"), + "Expand packed 24-bit values from a one-dimensional, C-contiguous " + "uint8 array into a uint32 array."); - m.def("adc_sar_05_decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + m.def( + "expand4to8bit", + [](py::array_t input) { + auto input_view = make_const_view_1d(input); + auto output = py::array_t(input_view.size() * 2); + aare::expand4to8bit(input_view, make_view_1d(output)); + return output; + }, + py::arg("input").noconvert(), + "Expand two 4-bit values per byte of a one-dimensional, C-contiguous " + "uint8 array into a uint8 array of twice the length."); - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + m.def( + "decode_my302", + [](py::array_t input, uint32_t offset) { + // Physical layout of the chip + constexpr ssize_t channels = 64; + constexpr ssize_t counters = 3; + constexpr ssize_t bytes_per_channel = 3; // 24 bit + constexpr ssize_t n_outputs = 2; - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + auto input_view = make_const_view_1d(input); - adc_sar_05_decode64to16(input_view, output_view); + ssize_t expected_size = channels * counters * bytes_per_channel; - return output; - }); + // If we have an offset we need one extra byte per output + aare::BitOffset bitoff(offset); + if (bitoff.value()) + expected_size += n_outputs; - m.def("adc_sar_04_decode64to16", [](py::array_t input) { - if (input.ndim() != 2) { - throw std::runtime_error( - "Only 2D arrays are supported at this moment"); - } + if (input_view.size() != expected_size) { + throw py::value_error( + fmt::format("{} Expected an input size of {} bytes. Called " + "with input size of {}", + LOCATION, expected_size, input_view.size())); + } - // Create a 2D output array with the same shape as the input - std::vector shape{input.shape(0), - input.shape(1) / - static_cast(bits_per_byte)}; - py::array_t output(shape); + auto output = py::array_t(channels * counters); + auto output_view = make_view_1d(output); - // Create a view of the input and output arrays - NDView input_view( - reinterpret_cast(input.mutable_data()), - {output.shape(0), output.shape(1)}); - NDView output_view(output.mutable_data(), - {output.shape(0), output.shape(1)}); + const ssize_t in_step = input_view.size() / n_outputs; + const ssize_t out_step = output_view.size() / n_outputs; + for (ssize_t i = 0; i != n_outputs; ++i) { + NDView in_part( + input_view.data() + in_step * i, {in_step}); + NDView out_part(output_view.data() + out_step * i, + {out_step}); + aare::expand24to32bit(in_part, out_part, bitoff); + } - adc_sar_04_decode64to16(input_view, output_view); - - return output; - }); - - m.def("apply_custom_weights", - [](py::array_t - &input, - py::array_t - &weights) { - // Create new array with same shape as the input array - // (uninitialized values) - py::buffer_info buf = input.request(); - py::array_t output(buf.shape); - - // Use NDViews to call into the C++ library - auto weights_view = make_view_1d(weights); - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - apply_custom_weights(input_view, output_view, weights_view); - return output; - }); - - m.def("expand24to32bit", - [](py::array_t - &input, - uint32_t offset) { - aare::BitOffset bitoff(offset); - py::buffer_info buf = input.request(); - - constexpr uint32_t bytes_per_channel = 3; // 24 bit - py::array_t output(buf.size / bytes_per_channel); - - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - aare::expand24to32bit(input_view, output_view, bitoff); - return output; - }); - - m.def("expand4to8bit", - [](py::array_t - &input) { - py::buffer_info buf = input.request(); - - py::array_t output(buf.size * 2); - - NDView input_view(input.mutable_data(), - {input.size()}); - NDView output_view(output.mutable_data(), - {output.size()}); - - aare::expand4to8bit(input_view, output_view); - return output; - }); - - m.def("decode_my302", - [](py::array_t - &input, - uint32_t offset) { - // Physical layout of the chip - constexpr size_t channels = 64; - constexpr size_t counters = 3; - constexpr size_t bytes_per_channel = 3; // 24 bit - constexpr int n_outputs = 2; - - ssize_t expected_size = channels * counters * bytes_per_channel; - - // If whe have an offset we need one extra byte per output - aare::BitOffset bitoff(offset); - if (bitoff.value()) - expected_size += n_outputs; - - if (input.size() != expected_size) { - throw std::runtime_error(fmt::format( - "{} Expected an input size of {} bytes. Called " - "with input size of {}", - LOCATION, expected_size, input.size())); - } - - py::buffer_info buf = input.request(); - py::array_t output(channels * counters); - - for (int i = 0; i != n_outputs; ++i) { - auto step = input.size() / n_outputs; - auto out_step = output.size() / n_outputs; - NDView input_view(input.mutable_data() + step * i, - {input.size() / n_outputs}); - NDView output_view(output.mutable_data() + - out_step * i, - {output.size() / n_outputs}); - - aare::expand24to32bit(input_view, output_view, bitoff); - } - - return output; - }); + return output; + }, + py::arg("input").noconvert(), py::arg("offset"), + "Decode a Mythen 302 readout from a one-dimensional, C-contiguous " + "uint8 array into a uint32 array of 64 channels times 3 counters."); py::class_(m, "CtbRawFile") .def(py::init()) diff --git a/python/src/file.hpp b/python/src/file.hpp index a493c910..030ba181 100644 --- a/python/src/file.hpp +++ b/python/src/file.hpp @@ -21,11 +21,6 @@ namespace py = pybind11; using namespace ::aare; -// Disable warnings for unused parameters, as we ignore some -// in the __exit__ method -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - void define_file_io_bindings(py::module &m) { PYBIND11_NUMPY_DTYPE(DetectorHeader, frameNumber, expLength, packetNumber, @@ -122,9 +117,12 @@ void define_file_io_bindings(py::module &m) { }) .def("__enter__", [](File &self) { return &self; }) .def("__exit__", - [](File &self, const std::optional &exc_type, - const std::optional &exc_value, - const std::optional &traceback) { + []([[maybe_unused]] File &self, + [[maybe_unused]] const std::optional &exc_type, + [[maybe_unused]] const std::optional + &exc_value, + [[maybe_unused]] const std::optional + &traceback) { // self.close(); }) .def("__iter__", [](File &self) { return &self; }) @@ -146,7 +144,7 @@ void define_file_io_bindings(py::module &m) { self.read_into( reinterpret_cast(image.mutable_data())); return image; - } catch (std::runtime_error &e) { + } catch (std::runtime_error &) { throw py::stop_iteration(); } }); @@ -160,6 +158,4 @@ void define_file_io_bindings(py::module &m) { .def_property_readonly("start", &ScanParameters::start) .def_property_readonly("stop", &ScanParameters::stop) .def_property_readonly("step", &ScanParameters::step); - -#pragma GCC diagnostic pop } \ No newline at end of file diff --git a/python/src/fit.hpp b/python/src/fit.hpp index 82bff325..b41b1596 100644 --- a/python/src/fit.hpp +++ b/python/src/fit.hpp @@ -11,6 +11,7 @@ namespace py = pybind11; using namespace pybind11::literals; +using aare::ssize_t; template py::object @@ -22,9 +23,40 @@ fit_dispatch(const aare::FitModel &model, template void bind_fit_model(py::module &m, const char *name) { using FM = aare::FitModel; py::class_(m, name) - .def(py::init(), + .def(py::init(), py::arg("strategy") = 0, py::arg("max_calls") = 100, - py::arg("tolerance") = 0.5, py::arg("compute_errors") = false) + py::arg("tolerance") = 0.5, py::arg("compute_errors") = false, + py::arg("minimizer") = aare::Minimizer::Migrad, + R"doc( + Fit configuration for this model. + + Parameters + ---------- + strategy : int + Minuit2 strategy (0 = fast, 1 = Minuit2's default). Ignored + by ``Minimizer.LevenbergMarquardt`` and + ``Minimizer.VarPro``. + max_calls : int + Work budget per pixel: Minuit2 function calls, or model + evaluations for ``Minimizer.LevenbergMarquardt`` and + ``Minimizer.VarPro`` (one per iteration, two + when a step crosses a limit). 0 selects Minuit's default + budget. + tolerance : float + EDM tolerance in Minuit's convention. Migrad, + LevenbergMarquardt and VarPro stop below + ``0.002 * tolerance``, Fumili below ``1e-4 * tolerance``. + compute_errors : bool + Also return parameter errors: from Hesse with Migrad, from + the linearised covariance with Fumili, LevenbergMarquardt + and VarPro. Fixed parameters and parameters + ending on a limit report 0. + minimizer : Minimizer + ``Minimizer.Migrad`` (default), ``Minimizer.Fumili``, + ``Minimizer.LevenbergMarquardt`` or + ``Minimizer.VarPro``. + )doc") .def("SetParLimits", py::overload_cast(&FM::SetParLimits), py::arg("idx"), py::arg("lo"), py::arg("hi")) @@ -59,8 +91,10 @@ template void bind_fit_model(py::module &m, const char *name) { .def_property("tolerance", &FM::tolerance, &FM::SetTolerance) .def_property("compute_errors", &FM::compute_errors, &FM::SetComputeErrors) + .def_property("minimizer", &FM::minimizer, &FM::SetMinimizer) .def( - "__call__", + "__call__", // conversion ok, we want to be able to call with any + // dtype [](const FM & /*self*/, py::array_t x, py::array_t @@ -78,7 +112,7 @@ template void bind_fit_model(py::module &m, const char *name) { }, py::arg("x"), py::arg("par")) .def( - "fit", + "fit", // conversion ok [](const FM &self, py::array_t x, py::array_t y, @@ -88,6 +122,11 @@ template void bind_fit_model(py::module &m, const char *name) { R"doc( Fit this model to 1D or 3D data using Minuit2. + The minimizer is selected by the ``minimizer`` property + (``Minimizer.Migrad`` by default, ``Minimizer.Fumili``, + ``Minimizer.LevenbergMarquardt`` or + ``Minimizer.VarPro``). + Parameters ---------- x : array_like, shape (n_scan,) @@ -163,11 +202,6 @@ fit_dispatch(const aare::FitModel &model, py::array_t>( y_err_obj); - if (y_err.ndim() != 3) { - throw std::runtime_error( - "For 3D input y, y_err must also be 3D."); - } - auto err_out = new NDArray({y.shape(0), y.shape(1), npar}, 0.0); auto y_view_err = make_view_3d(y_err); @@ -186,8 +220,23 @@ fit_dispatch(const aare::FitModel &model, "chi2"_a = return_image_data(chi2_out)); } } else { - + // Unweighted fit; parameter errors are still available when + // requested, as in the 1D path. NDView dummy_err{}; + + if (model.compute_errors()) { + auto err_out = + new NDArray({y.shape(0), y.shape(1), npar}, 0.0); + + aare::fit_3d(model, x_view, y_view, dummy_err, + par_out->view(), err_out->view(), + chi2_out->view(), n_threads); + + return py::dict("par"_a = return_image_data(par_out), + "par_err"_a = return_image_data(err_out), + "chi2"_a = return_image_data(chi2_out)); + } + NDView dummy_err_out{}; aare::fit_3d(model, x_view, y_view, dummy_err, @@ -208,11 +257,6 @@ fit_dispatch(const aare::FitModel &model, py::array_t>( y_err_obj); - if (y_err.ndim() != 1) { - throw std::runtime_error( - "For 1D input y, y_err must also be 1D."); - } - auto y_view_err = make_view_1d(y_err); result = aare::fit_pixel(model, x_view, y_view, y_view_err); } else { @@ -226,7 +270,82 @@ fit_dispatch(const aare::FitModel &model, } } +// Resolve the model type behind a Python object and run the fit. +py::object dispatch_any_model( + py::object model_obj, + py::array_t x, + py::array_t y, + py::object y_err_obj, int n_threads) { + using namespace aare::model; + +#define AARE_DISPATCH_MODEL(Model) \ + if (py::isinstance>(model_obj)) { \ + const auto &mdl = model_obj.cast &>(); \ + return fit_dispatch(mdl, x, y, y_err_obj, n_threads); \ + } + + AARE_DISPATCH_MODEL(Pol1) + AARE_DISPATCH_MODEL(Pol2) + AARE_DISPATCH_MODEL(Gaussian) + AARE_DISPATCH_MODEL(GaussianErfcPlateau) + AARE_DISPATCH_MODEL(GaussianChargeSharing) + AARE_DISPATCH_MODEL(GaussianChargeSharingKb) + AARE_DISPATCH_MODEL(RisingScurve) + AARE_DISPATCH_MODEL(FallingScurve) + +#undef AARE_DISPATCH_MODEL + + throw std::runtime_error( + "Unknown model type. Expected Pol1, Pol2, Gaussian, " + "GaussianErfcPlateau, GaussianChargeSharing, GaussianChargeSharingKb, " + "RisingScurve or FallingScurve."); +} + void define_fit_bindings(py::module &m) { + // The enum must exist before it is used as a constructor default below. + py::enum_(m, "Minimizer", R"doc( + Minimizer used by the fit models. + + ``Migrad`` (default) is Minuit2's variable-metric minimizer. With + ``compute_errors`` its parameter errors come from Hesse. + + ``Fumili`` is Minuit2's Gauss-Newton minimizer with a trust region. + It takes the gradient and a linearised Hessian from the analytic + derivatives of the model and usually needs far fewer function + evaluations than Migrad. Its parameter errors come from the + covariance of the linearised Hessian, without an extra Hesse step. + Minuit2's Fumili cannot converge once a two-sided limit becomes + active; a pixel without a valid Fumili minimum is refitted with + Migrad. + + ``LevenbergMarquardt`` is the built-in damped Gauss-Newton solver. It + uses the analytic Jacobian, reflects steps at parameter limits and + reuses its buffers between pixels, so data cubes are fitted without + per-pixel allocations. Its parameter errors are + ``sqrt(diag((J^T J)^-1))``; ``max_calls`` counts model evaluations + and ``strategy`` is ignored. + + ``VarPro`` is the built-in variable projection solver. + Every bundled model is linear in some of its parameters (the + amplitude of a Gaussian, the baseline and amplitude of an S-curve, + every parameter of a polynomial); each trial point solves those + exactly and the Levenberg-Marquardt iteration runs over the + remaining nonlinear parameters only. It needs fewer evaluations than + ``LevenbergMarquardt`` and cannot be misled by poor start values of + the linear parameters, which it ignores. A pixel whose linear + solution violates a limit on a linear parameter, or that does not + converge, is refitted with ``LevenbergMarquardt``. Its parameter + errors are the same Gauss-Newton estimates; ``max_calls`` counts + evaluations and ``strategy`` is ignored. + + With every minimizer, fixed parameters and parameters that end on a + limit report an error of 0 and a failed fit returns zeros. + )doc") + .value("Migrad", aare::Minimizer::Migrad) + .value("Fumili", aare::Minimizer::Fumili) + .value("LevenbergMarquardt", aare::Minimizer::LevenbergMarquardt) + .value("VarPro", aare::Minimizer::VarPro); + // ── Bind model classes ────────────────────────────────────────── bind_fit_model(m, "Gaussian"); bind_fit_model(m, "GaussianErfcPlateau"); @@ -241,90 +360,24 @@ void define_fit_bindings(py::module &m) { m.def( "fit", - [](py::object model_obj, + [](py::object model_obj, // conversion ok, we want to be able to call + // with any dtype py::array_t x, py::array_t y, py::object y_err_obj, int n_threads) -> py::object { - using namespace aare::model; - - // ── Polynomial of degree 1 ─────── - if (py::isinstance>(model_obj)) { - const auto &mdl = - model_obj.cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, n_threads); - } - - // ── Polynomial of degree 2 ─────── - if (py::isinstance>(model_obj)) { - const auto &mdl = - model_obj.cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, n_threads); - } - - // ── Gaussian ─────── - if (py::isinstance>(model_obj)) { - const auto &mdl = - model_obj.cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, n_threads); - } - - // ── GaussianErfcPlateau ─────── - if (py::isinstance>( - model_obj)) { - const auto &mdl = - model_obj - .cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, - n_threads); - } - - // ── GaussianChargeSharing ─────── - if (py::isinstance>( - model_obj)) { - const auto &mdl = - model_obj - .cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, - n_threads); - } - - // ── GaussianChargeSharingKb ─────── - if (py::isinstance>( - model_obj)) { - const auto &mdl = model_obj.cast< - const aare::FitModel &>(); - return fit_dispatch( - mdl, x, y, y_err_obj, n_threads); - } - - // ── Rising Scurve ─────── - if (py::isinstance>(model_obj)) { - const auto &mdl = - model_obj.cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, - n_threads); - } - - // ── Falling Scurve ─────── - if (py::isinstance>(model_obj)) { - const auto &mdl = - model_obj.cast &>(); - return fit_dispatch(mdl, x, y, y_err_obj, - n_threads); - } - - throw std::runtime_error( - "Unknown model type. Expected Pol1, Pol2, Gaussian, " - "RisingScurve or FallingScurve."); + return dispatch_any_model(model_obj, x, y, y_err_obj, n_threads); }, R"( - Fit a model to 1D or 3D data using Minuit2. + Fit a model to 1D or 3D data with the minimizer selected by the + model's ``minimizer`` property. Parameters ---------- - model : Pol1, Pol2, Gaussian, RisingScurve, or FallingScurve - Configured model object. User-set limits, fixed parameters, - and start values take precedence over automatic estimates. + model : object + Configured model object: Pol1, Pol2, Gaussian, GaussianErfcPlateau, + GaussianChargeSharing, GaussianChargeSharingKb, RisingScurve or + FallingScurve. User-set limits, fixed parameters, and start values + take precedence over automatic estimates. x : array_like, shape (n_scan,) Scan points (e.g. energy or threshold values). y : array_like, shape (n_scan,) or (rows, cols, n_scan) @@ -336,15 +389,12 @@ void define_fit_bindings(py::module &m) { Returns ------- - For 1D input: - numpy array of shape (2*npar+1,) if compute_errors else (npar+1,). - Layout: [params..., (errors...,) chi2]. - - For 3D input: - dict with keys: - "par" : (rows, cols, npar) fitted parameters. - "par_err" : (rows, cols, npar) parameter errors (if compute_errors). - "chi2" : (rows, cols) chi-squared per pixel. + dict + ``par`` holds the fitted parameters, shape (npar,) for 1D input or + (rows, cols, npar) for 3D input. ``chi2`` holds the chi-squared, + shape (1,) or (rows, cols). ``par_err`` holds the parameter + errors when ``compute_errors`` is enabled; fixed parameters and + parameters on a limit report 0. A failed fit returns zeros. )", py::arg("model"), py::arg("x"), py::arg("y"), py::arg("y_err") = py::none(), py::arg("n_threads") = 4); diff --git a/python/src/jungfrau_data_file.hpp b/python/src/jungfrau_data_file.hpp index 44ecd90c..1ff4ff9d 100644 --- a/python/src/jungfrau_data_file.hpp +++ b/python/src/jungfrau_data_file.hpp @@ -15,11 +15,6 @@ namespace py = pybind11; using namespace ::aare; -// Disable warnings for unused parameters, as we ignore some -// in the __exit__ method -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - auto read_dat_frame(JungfrauDataFile &self) { py::array_t header(1); py::array_t image({self.rows(), self.cols()}); @@ -94,20 +89,20 @@ void define_jungfrau_data_file_io_bindings(py::module &m) { )") .def("__enter__", [](JungfrauDataFile &self) { return &self; }) .def("__exit__", - [](JungfrauDataFile &self, - const std::optional &exc_type, - const std::optional &exc_value, - const std::optional &traceback) { + []([[maybe_unused]] JungfrauDataFile &self, + [[maybe_unused]] const std::optional &exc_type, + [[maybe_unused]] const std::optional + &exc_value, + [[maybe_unused]] const std::optional + &traceback) { // self.close(); }) .def("__iter__", [](JungfrauDataFile &self) { return &self; }) .def("__next__", [](JungfrauDataFile &self) { try { return read_dat_frame(self); - } catch (std::runtime_error &e) { + } catch (std::runtime_error &) { throw py::stop_iteration(); } }); } - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/module.cpp b/python/src/module.cpp index e4c9706e..58dea15f 100644 --- a/python/src/module.cpp +++ b/python/src/module.cpp @@ -22,6 +22,7 @@ #include "bind_PixelMap.hpp" #include "bind_RawFile.hpp" #include "bind_calibration.hpp" +#include "bind_testing.hpp" #include "StrixelRemap/bind_InclusiveROI.hpp" #include "StrixelRemap/bind_PredefinedVariables.hpp" @@ -98,6 +99,7 @@ PYBIND11_MODULE(_aare, m) { define_jungfrau_data_file_io_bindings(m); bind_calibration(m); + define_testing_bindings(m); DEFINE_CLUSTER_BINDINGS(int, 3, 3, uint16_t, i); DEFINE_CLUSTER_BINDINGS(double, 3, 3, uint16_t, d); diff --git a/python/src/module_config.hpp b/python/src/module_config.hpp index e2f4e8a7..ad5d7ca1 100644 --- a/python/src/module_config.hpp +++ b/python/src/module_config.hpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include diff --git a/python/src/np_helper.hpp b/python/src/np_helper.hpp index c0f5c3fd..83880470 100644 --- a/python/src/np_helper.hpp +++ b/python/src/np_helper.hpp @@ -41,31 +41,60 @@ template py::array return_vector(std::vector *vec) { free_when_done); // numpy array references this parent } -// todo rewrite generic -template -auto get_shape_3d(const py::array_t &arr) { - return aare::Shape<3>{arr.shape(0), arr.shape(1), arr.shape(2)}; +// Create a NDView of a numpy array. NDView assumes C-order strides so we +// reject anything that is not Ndim dimensional and C-contiguous instead of +// silently reading the data in the wrong order. +template +auto get_checked_shape(const py::array_t &arr) { + if (arr.ndim() != Ndim) { + throw py::value_error( + fmt::format("Expected {}D array, got {}D", Ndim, arr.ndim())); + } + if (!(arr.flags() & py::array::c_style)) { + throw py::value_error( + "Array is not C-contiguous. Use np.ascontiguousarray(arr)"); + } + aare::Shape shape{}; + for (ssize_t i = 0; i < Ndim; ++i) { + shape[i] = arr.shape(i); + } + return shape; +} + +template +auto make_view(py::array_t &arr) { + auto shape = get_checked_shape(arr); + return aare::NDView(arr.mutable_data(), shape); +} + +// Read only view, also works for numpy arrays that are not writeable +template +auto make_const_view(const py::array_t &arr) { + auto shape = get_checked_shape(arr); + return aare::NDView(arr.data(), shape); } template auto make_view_3d(py::array_t &arr) { - return aare::NDView(arr.mutable_data(), get_shape_3d(arr)); + return make_view<3>(arr); } - -template -auto get_shape_2d(const py::array_t &arr) { - return aare::Shape<2>{arr.shape(0), arr.shape(1)}; -} - -template -auto get_shape_1d(const py::array_t &arr) { - return aare::Shape<1>{arr.shape(0)}; -} - template auto make_view_2d(py::array_t &arr) { - return aare::NDView(arr.mutable_data(), get_shape_2d(arr)); + return make_view<2>(arr); } template auto make_view_1d(py::array_t &arr) { - return aare::NDView(arr.mutable_data(), get_shape_1d(arr)); + return make_view<1>(arr); +} + +template +auto make_const_view_3d(const py::array_t &arr) { + return make_const_view<3>(arr); +} +template +auto make_const_view_2d(const py::array_t &arr) { + return make_const_view<2>(arr); +} +template +auto make_const_view_1d(const py::array_t &arr) { + return make_const_view<1>(arr); } template struct fmt_format_trait; // forward declaration @@ -83,8 +112,13 @@ struct fmt_format_trait> { } }; -template -auto fmt_format = fmt_format_trait::value(); +// Function-local static rather than a variable template: MSVC does not run +// the dynamic initializer of a variable template instantiated from inside +// another template, which left the format string empty. +template const std::string &fmt_format() { + static const std::string value = fmt_format_trait::value(); + return value; +} /** * Helper function to allocate image data given item size and shape diff --git a/python/src/raw_master_file.hpp b/python/src/raw_master_file.hpp index 40e942a9..d30762a4 100644 --- a/python/src/raw_master_file.hpp +++ b/python/src/raw_master_file.hpp @@ -67,9 +67,23 @@ void define_raw_master_file_bindings(py::module &m) { .def_property_readonly("total_frames_expected", &RawMasterFile::total_frames_expected) .def_property_readonly("detector_layout", - &RawMasterFile::detector_layout) + &RawMasterFile::detector_layout, R"( + Number of modules along each axis of the detector + + Returns + ---------- + xy + Module rows and columns + )") .def_property_readonly("udp_interfaces_per_module", - &RawMasterFile::udp_interfaces_per_module) + &RawMasterFile::udp_interfaces_per_module, R"( + Number of UDP interfaces along each axis of a module + + Returns + ---------- + xy + UDP interface rows and columns + )") .def_property_readonly("analog_samples", &RawMasterFile::analog_samples, R"( Number of analog samples diff --git a/python/src/raw_sub_file.hpp b/python/src/raw_sub_file.hpp index 5359c054..da858058 100644 --- a/python/src/raw_sub_file.hpp +++ b/python/src/raw_sub_file.hpp @@ -61,11 +61,6 @@ auto read_n_frames_from_RawSubFile(RawSubFile &self, size_t n_frames) { return py::make_tuple(header, image); } -// Disable warnings for unused parameters, as we ignore some -// in the __exit__ method -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-parameter" - void define_raw_sub_file_io_bindings(py::module &m) { py::class_(m, "RawSubFile") .def(py::init &exc_type, - const std::optional &exc_value, - const std::optional &traceback) {}) + []([[maybe_unused]] RawSubFile &self, + [[maybe_unused]] const std::optional &exc_type, + [[maybe_unused]] const std::optional + &exc_value, + [[maybe_unused]] const std::optional + &traceback) {}) .def("__iter__", [](RawSubFile &self) { return &self; }) .def("__next__", [](RawSubFile &self) { try { return read_frame_from_RawSubFile(self); - } catch (std::runtime_error &e) { + } catch (std::runtime_error &) { throw py::stop_iteration(); } }); } - -#pragma GCC diagnostic pop \ No newline at end of file diff --git a/python/src/var_cluster.hpp b/python/src/var_cluster.hpp index 44d70db1..1a3b674c 100644 --- a/python/src/var_cluster.hpp +++ b/python/src/var_cluster.hpp @@ -21,40 +21,42 @@ void define_var_cluster_finder_bindings(py::module &m) { auto *ptr = new NDArray(self.labeled()); return return_image_data(ptr); }) - .def("set_noiseMap", - [](VarClusterFinder &self, - py::array_t - noise_map) { - auto noise_map_span = make_view_2d(noise_map); - self.set_noiseMap(noise_map_span); - }) + .def( + "set_noiseMap", // conversions ok + [](VarClusterFinder &self, + py::array_t + noise_map) { + auto noise_map_span = make_view_2d(noise_map); + self.set_noiseMap(noise_map_span); + }, + py::arg("noise_map")) .def("set_numberOfNeighbours", &VarClusterFinder::set_numberOfNeighbours) .def("set_empty_surroundingPixels", &VarClusterFinder::set_empty_surroundingPixels) .def("set_peripheralThresholdFactor", &VarClusterFinder::set_peripheralThresholdFactor) - .def("find_clusters", - [](VarClusterFinder &self, - py::array_t - img) { - auto view = make_view_2d(img); - self.find_clusters(view); - }) - .def("find_clusters_X", - [](VarClusterFinder &self, - py::array_t - img) { - auto img_span = make_view_2d(img); - self.find_clusters_X(img_span); - }) - .def("single_pass", - [](VarClusterFinder &self, - py::array_t - img) { - auto img_span = make_view_2d(img); - self.single_pass(img_span); - }) + .def( + "find_clusters", + [](VarClusterFinder &self, py::array_t img) { + auto view = make_view_2d(img); + self.find_clusters(view); + }, + py::arg("img").noconvert()) + .def( + "find_clusters_X", + [](VarClusterFinder &self, py::array_t img) { + auto img_span = make_view_2d(img); + self.find_clusters_X(img_span); + }, + py::arg("img").noconvert()) + .def( + "single_pass", + [](VarClusterFinder &self, py::array_t img) { + auto img_span = make_view_2d(img); + self.single_pass(img_span); + }, + py::arg("img").noconvert()) .def("hits", [](VarClusterFinder &self) { auto ptr = new std::vector::Hit>( diff --git a/python/tests/test_Cluster.py b/python/tests/test_Cluster.py index 027cc203..28b90e9d 100644 --- a/python/tests/test_Cluster.py +++ b/python/tests/test_Cluster.py @@ -135,6 +135,43 @@ def test_cluster_finder(): assert clusters.size == 0 +def test_cluster_finder_set_nSigma_before_pedestal_is_ready(): + shape = [10, 10] + n_samples = 4 + cf = _aare.ClusterFinder_Cluster3x3i(shape, min_pedestal_samples=n_samples) + assert not cf.pedestal_ready + + cf.nSigma = 50 + assert cf.nSigma == 50 + + # Pedestal of 100 with a noise of 2 + for i in range(n_samples): + cf.push_pedestal_frame(np.full(shape, 98 + 4 * (i % 2), dtype=np.uint16)) + assert cf.pedestal_ready + + # 30 ADU is a cluster at the default nSigma of 5 but not at 50 + frame = np.full(shape, 100, dtype=np.uint16) + frame[5, 5] = 130 + cf.find_clusters(frame) + assert cf.steal_clusters().size == 0 + + +def test_cluster_finder_mt_set_nSigma_before_pedestal_is_ready(): + shape = [10, 10] + n_samples = 4 + cf = _aare.ClusterFinderMT_Cluster3x3i( + shape, n_threads=2, min_pedestal_samples=n_samples + ) + assert not cf.pedestal_ready() + + cf.set_nSigma(50) + + for i in range(n_samples): + cf.push_pedestal_frame(np.full(shape, 98 + 4 * (i % 2), dtype=np.uint16)) + assert cf.pedestal_ready() + cf.stop() + + def test_2x2_reduction(): """Test 2x2 Reduction""" cluster = _aare.Cluster3x3i(5,5,np.array([1, 1, 1, 2, 3, 1, 2, 2, 1], dtype=np.int32)) @@ -162,3 +199,21 @@ def test_3x3_reduction(): + + +@pytest.mark.parametrize( + "cluster_type, n_pixels", + [(_aare.Cluster3x3i, 9), (_aare.Cluster2x2i, 4)], +) +def test_cluster_constructor_requires_exact_data_length(cluster_type, n_pixels): + """Data must hold exactly one value per pixel""" + for bad in ( + np.ones(n_pixels - 1, dtype=np.int32), + np.ones(n_pixels + 1, dtype=np.int32), + np.ones((n_pixels, 1), dtype=np.int32), + ): + with pytest.raises(ValueError, match=f"{n_pixels} values"): + cluster_type(0, 0, bad) + + cluster = cluster_type(0, 0, np.arange(n_pixels, dtype=np.int32)) + assert (cluster.data.ravel() == np.arange(n_pixels)).all() diff --git a/python/tests/test_ClusterVector.py b/python/tests/test_ClusterVector.py index 8b077dc0..b0e9cffa 100644 --- a/python/tests/test_ClusterVector.py +++ b/python/tests/test_ClusterVector.py @@ -146,7 +146,7 @@ def test_masking_requires_c_contiguous_array(): mask = np.array([True, False, True, False], dtype=bool)[::2] assert not mask.flags.c_contiguous - with pytest.raises(TypeError): + with pytest.raises(ValueError, match="C-contiguous"): cv(mask) @@ -157,5 +157,5 @@ def test_masking_requires_one_dimension(): mask = np.array([[True, False]], dtype=bool) - with pytest.raises(ValueError, match="one-dimensional"): + with pytest.raises(ValueError): cv(mask) diff --git a/python/tests/test_CtbRawFile.py b/python/tests/test_CtbRawFile.py new file mode 100644 index 00000000..e23c99f2 --- /dev/null +++ b/python/tests/test_CtbRawFile.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: MPL-2.0 + + +import pytest +import numpy as np +from aare import CtbRawFile + + +@pytest.mark.withdata +def test_basic_properites(test_data_path): + f = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + assert f.total_frames == 47 + assert f.image_size_in_bytes == 640 + assert f.master.analog_samples == 10 + assert f.master.digital_samples is None + assert f.master.transceiver_samples is None + + +@pytest.mark.withdata +def test_seek_and_read(test_data_path): + f = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + f.seek(0) + header, raw_data = f.read_frame() + assert header['frameNumber'] == 48 #we know that the first frame in this file is 48 + + #Seek a multiple of frames in file + f.seek(9) + header, raw_data = f.read_frame() + assert header['frameNumber'] == 57 + +@pytest.mark.withdata +def test_read_all_frames_direct_and_iterate(test_data_path): + """ + We want to make sure that iterating and reading give the same result. + """ + f0 = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + f1 = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + + for h0, d0 in f0: + h1, d1 = f1.read_frame() + assert h0 == h1 + assert np.all(d0 == d1) + +@pytest.mark.withdata +def test_read_all_frames_with_index_and_iterate(test_data_path): + """ + We want to make sure that iterating and reading give the same result. + """ + f0 = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + f1 = CtbRawFile(test_data_path/'raw/ctb/run_master_0.json') + + frames_read = 0 + for i,(h0, d0) in enumerate(f0): + h1, d1 = f1.read_frame(i) + assert h0 == h1 + assert np.all(d0 == d1) + frames_read += 1 + + assert frames_read == f0.total_frames diff --git a/python/tests/test_FastPedestal.py b/python/tests/test_FastPedestal.py index f93560cb..1feacaea 100644 --- a/python/tests/test_FastPedestal.py +++ b/python/tests/test_FastPedestal.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import numpy as np import pytest diff --git a/python/tests/test_Fit.py b/python/tests/test_Fit.py index 3a477b27..2e876ed9 100644 --- a/python/tests/test_Fit.py +++ b/python/tests/test_Fit.py @@ -1,15 +1,401 @@ # SPDX-License-Identifier: MPL-2.0 import numpy as np +import pytest + import aare +MINIMIZERS = [ + aare.Minimizer.Migrad, + aare.Minimizer.Fumili, + aare.Minimizer.LevenbergMarquardt, + aare.Minimizer.VarPro, +] -def test_gaussian_model_evaluates_and_fits_data(): +def gaussian(x, A, mu, sigma): + return A * np.exp(-0.5 * ((x - mu) / sigma) ** 2) + + +def uses_minuit2(minimizer): + return minimizer not in ( + aare.Minimizer.LevenbergMarquardt, + aare.Minimizer.VarPro, + ) + + +@pytest.fixture(params=MINIMIZERS, ids=lambda m: m.name) +def minimizer(request): + return request.param + + +@pytest.fixture +def gaussian_data(): + x = np.linspace(-6.0, 6.0, 61) + y = gaussian(x, 120.0, 0.8, 1.3) + y_err = np.ones_like(y) + return x, y, y_err + + +def test_gaussian_model_evaluates_and_fits_data(minimizer): x = np.linspace(-5.0, 5.0, 51) expected = np.array([20.0, 0.5, 1.2]) - model = aare.Gaussian() + model = aare.Gaussian(minimizer=minimizer) y = model(x, expected) result = model.fit(x, y) np.testing.assert_allclose(result["par"], expected, atol=2e-3) + + +def test_minimizer_can_be_selected(): + assert aare.Gaussian().minimizer == aare.Minimizer.Migrad + assert set(aare.Minimizer.__members__) == { + "Migrad", + "Fumili", + "LevenbergMarquardt", + "VarPro", + } + + for minimizer in MINIMIZERS: + assert aare.Gaussian(minimizer=minimizer).minimizer == minimizer + + model = aare.Gaussian() + model.minimizer = aare.Minimizer.LevenbergMarquardt + assert model.minimizer == aare.Minimizer.LevenbergMarquardt + + +def test_fit_returns_errors_and_chi2(gaussian_data, minimizer): + x, y, y_err = gaussian_data + model = aare.Gaussian(compute_errors=True, minimizer=minimizer) + result = model.fit(x, y, y_err) + + assert set(result) == {"par", "par_err", "chi2"} + assert result["par"].shape == (3,) + assert result["par_err"].shape == (3,) + assert result["chi2"].shape == (1,) + np.testing.assert_allclose(result["par"], [120.0, 0.8, 1.3], rtol=1e-4) + np.testing.assert_allclose( + result["par_err"], [0.361, 0.00451, 0.00451], rtol=2e-2 + ) + assert result["chi2"][0] == pytest.approx(0.0, abs=1e-3) + + without_errors = aare.fit(aare.Gaussian(minimizer=minimizer), x, y) + assert set(without_errors) == {"par", "chi2"} + + +def test_pol1_weighted_errors_match_analytic_formula(minimizer): + x = np.arange(20, dtype=float) + y = 2.0 + 0.5 * x + y_err = 1.0 + 0.1 * x + w = 1.0 / y_err**2 + S, Sx, Sxx = w.sum(), (w * x).sum(), (w * x * x).sum() + delta = S * Sxx - Sx * Sx + + result = aare.Pol1(compute_errors=True, minimizer=minimizer).fit(x, y, y_err) + + np.testing.assert_allclose(result["par"], [2.0, 0.5], rtol=1e-6) + np.testing.assert_allclose( + result["par_err"], + [np.sqrt(Sxx / delta), np.sqrt(S / delta)], + rtol=1e-6, + ) + + +def test_fixed_parameter_reports_zero_error(gaussian_data, minimizer): + x, y, y_err = gaussian_data + model = aare.Gaussian(compute_errors=True, minimizer=minimizer) + model.FixParameter("sigma", 1.5) + result = model.fit(x, y, y_err) + + assert result["par"][2] == 1.5 + assert result["par_err"][2] == 0.0 + assert result["par"][0] == pytest.approx(111.15, rel=1e-3) + assert result["chi2"][0] == pytest.approx(1684.25, rel=1e-3) + + model.ReleaseParameter("sigma") + result = model.fit(x, y, y_err) + assert result["par"][2] == pytest.approx(1.3, rel=1e-4) + + +def test_active_limit_ends_on_the_limit(gaussian_data, minimizer): + x, y, y_err = gaussian_data + model = aare.Gaussian(compute_errors=True, minimizer=minimizer) + model.SetParLimits("mu", -5.0, 0.5) + result = model.fit(x, y, y_err) + + margin = 1e-6 if uses_minuit2(minimizer) else 1e-9 + assert result["par"][1] == pytest.approx(0.5, abs=margin) + assert result["par_err"][1] == 0.0 + assert result["par"][0] == pytest.approx(116.88, rel=1e-3) + assert result["chi2"][0] == pytest.approx(4301.95, rel=1e-3) + + inactive = aare.Gaussian(minimizer=minimizer) + inactive.SetParLimits(1, -5.0, 5.0) + np.testing.assert_allclose( + inactive.fit(x, y)["par"], [120.0, 0.8, 1.3], rtol=1e-4 + ) + + +def test_user_start_value(gaussian_data, minimizer): + x, y, _ = gaussian_data + model = aare.Gaussian(minimizer=minimizer) + model.SetParameter("mu", 0.0) + np.testing.assert_allclose(model.fit(x, y)["par"], [120.0, 0.8, 1.3], rtol=1e-4) + + +def test_max_calls_failure_returns_zeros(gaussian_data, minimizer): + x, y, y_err = gaussian_data + model = aare.Gaussian(max_calls=3, compute_errors=True, minimizer=minimizer) + # start far from the minimum so that no minimizer converges in 3 calls + model.SetParameter("mu", -3.0) + model.SetParameter("sigma", 3.0) + result = model.fit(x, y, y_err) + assert np.all(result["par"] == 0.0) + assert np.all(result["par_err"] == 0.0) + assert np.all(result["chi2"] == 0.0) + + +def test_strategy_is_ignored_by_levenberg_marquardt(gaussian_data): + x, y, _ = gaussian_data + lm = aare.Minimizer.LevenbergMarquardt + a = aare.Gaussian(strategy=0, minimizer=lm).fit(x, y) + b = aare.Gaussian(strategy=1, minimizer=lm).fit(x, y) + np.testing.assert_array_equal(a["par"], b["par"]) + + +def test_invalid_parameter_access_raises(): + model = aare.Gaussian() + assert model.par_names == ["A", "mu", "sigma"] + assert model.n_par == 3 + with pytest.raises(RuntimeError): + model.SetParameter("nope", 1.0) + with pytest.raises(IndexError): + model.SetParameter(3, 1.0) + with pytest.raises(RuntimeError): + model.SetParLimits("mu", 1.0, 1.0) + + +def test_mismatched_array_sizes_raise(gaussian_data, minimizer): + x, y, y_err = gaussian_data + model = aare.Gaussian(minimizer=minimizer) + with pytest.raises(RuntimeError): + model.fit(x, y[:-1]) + with pytest.raises(RuntimeError): + model.fit(x, y, y_err[:-1]) + + +@pytest.mark.parametrize("cls", [aare.RisingScurve, aare.FallingScurve]) +def test_scurve_fit(cls, minimizer): + truth = np.array([10.0, 0.1, 50.0, 3.0, 1000.0, 2.0]) + x = np.linspace(0.0, 100.0, 100) + model = cls(max_calls=500, compute_errors=True, minimizer=minimizer) + y = model(x, truth) + result = model.fit(x, y, np.ones_like(y)) + + np.testing.assert_allclose(result["par"], truth, rtol=1e-3) + assert np.all(result["par_err"] > 0.0) + + +@pytest.mark.parametrize("with_errors", [False, True]) +@pytest.mark.parametrize("weighted", [False, True]) +def test_fit_3d_matches_fit_1d(with_errors, weighted, minimizer): + x = np.linspace(-6.0, 6.0, 61) + rows, cols = 4, 3 + y = np.empty((rows, cols, x.size)) + for r in range(rows): + for c in range(cols): + y[r, c] = gaussian(x, 80.0 + 10.0 * r, -0.6 + 0.3 * c, 0.9 + 0.1 * r) + y_err = np.ones_like(y) if weighted else None + + model = aare.Gaussian(compute_errors=with_errors, minimizer=minimizer) + cube = model.fit(x, y, y_err, n_threads=3) + + expected_keys = {"par", "par_err", "chi2"} if with_errors else {"par", "chi2"} + assert set(cube) == expected_keys + assert cube["par"].shape == (rows, cols, 3) + assert cube["chi2"].shape == (rows, cols) + if with_errors: + assert cube["par_err"].shape == (rows, cols, 3) + for r in range(rows): + for c in range(cols): + single = model.fit(x, y[r, c], None if y_err is None else y_err[r, c]) + np.testing.assert_allclose(cube["par"][r, c], single["par"], rtol=1e-10) + assert cube["chi2"][r, c] == pytest.approx(single["chi2"][0], abs=1e-10) + if with_errors: + np.testing.assert_allclose( + cube["par_err"][r, c], single["par_err"], rtol=1e-10 + ) + np.testing.assert_allclose(cube["par"][3, 2], [110.0, 0.0, 1.2], rtol=1e-4, atol=1e-6) + + +def test_all_minimizers_agree_on_3d_data_with_errors(): + x = np.linspace(-5.0, 5.0, 51) + expected = np.array([20.0, 0.5, 1.2]) + single = aare.Gaussian()(x, expected) + y = np.tile(single, (2, 3, 1)) + y_err = np.ones_like(y) + + results = { + minimizer: aare.Gaussian(compute_errors=True, minimizer=minimizer).fit( + x, y, y_err, n_threads=2 + ) + for minimizer in MINIMIZERS + } + reference = results[aare.Minimizer.Migrad] + for minimizer, result in results.items(): + assert result["par"].shape == (2, 3, 3) + assert result["par_err"].shape == (2, 3, 3) + np.testing.assert_allclose(result["par"], np.broadcast_to(expected, (2, 3, 3)), atol=2e-3) + np.testing.assert_allclose(result["chi2"], 0.0, atol=1e-3) + np.testing.assert_allclose(result["par_err"], reference["par_err"], rtol=2e-2) + + +# Regressions of the built-in solvers: fits that used to be reported as valid +# although they had stopped away from the minimum or returned garbage. + + +def test_gaussian_fit_to_noise_fails_or_stays_bounded(minimizer): + # A width collapsing between two scan points let the amplitude diverge + # (up to 1e159) in a fit that was still reported as valid. + x = np.linspace(-6.0, 6.0, 61) + y = np.random.default_rng(0).normal(0.0, 1.0, (20, 20, x.size)) + result = aare.Gaussian(minimizer=minimizer).fit(x, y, n_threads=2) + + par = result["par"] + failed = np.all(par == 0.0, axis=-1) + bounded = np.all(np.isfinite(par) & (np.abs(par) < 1e6), axis=-1) + assert np.all(failed | bounded) + assert np.all(np.isfinite(result["chi2"])) + + +def test_step_reflected_onto_start_is_not_convergence(minimizer): + # The step of +2 reflects at the limit 1 back onto the start 0; the + # truncated step must still be tried. + model = aare.Pol1(minimizer=minimizer) + model.SetParameter("p0", 0.0) + model.FixParameter("p1", 0.0) + model.SetParLimits("p0", -1.0, 1.0) + result = model.fit(np.arange(4.0), np.full(4, 2.2)) + + assert result["par"][0] == pytest.approx(1.0, abs=1e-6) + assert result["chi2"][0] == pytest.approx(5.76, rel=1e-6) + + +@pytest.mark.parametrize( + "amplitude, offset", [(1e8, 0.0), (1e10, 0.0), (120.0, 1e9)] +) +def test_large_parameter_does_not_stop_the_fit(minimizer, amplitude, offset): + # One relative step norm over all parameters let a large amplitude or + # position stop the fit at the start estimate of the width. + x = np.linspace(-6.0, 6.0, 61) + y = gaussian(x, amplitude, 0.8, 1.3) + y_err = np.sqrt(y) + 1.0 + result = aare.Gaussian(minimizer=minimizer, max_calls=500).fit( + x + offset, y, y_err + ) + + par = result["par"] + assert par[0] == pytest.approx(amplitude, rel=1e-3) + assert par[1] - offset == pytest.approx(0.8, abs=1e-3) + assert par[2] == pytest.approx(1.3, rel=1e-3) + + +def test_rejected_trial_on_limit_does_not_freeze_interior_parameter(minimizer): + # A rejected trial truncated onto the lower limit froze mu at its start. + x = np.linspace(-0.1, 0.1, 11) + mu = np.sqrt(2.0 * np.log(2.0)) + y = gaussian(x, 10.0, mu, 1.0) + model = aare.Gaussian(minimizer=minimizer) + model.FixParameter("A", 10.0) + model.FixParameter("sigma", 1.0) + model.SetParameter("mu", 3.0) + model.SetParLimits("mu", 0.0, 5.0) + result = model.fit(x, y) + + assert result["par"][1] == pytest.approx(mu, abs=1e-3) + assert result["chi2"][0] < 1e-4 + + +# Samples with a zero error are ignored, also when they hold NaN (a masked +# dead frame): a zero weight does not remove a NaN from a sum. + + +@pytest.mark.parametrize("masked", [4, 0, 9]) +def test_masked_nan_sample_is_ignored(minimizer, masked): + # The start estimate of Pol1 uses the first and the last sample. + x = np.arange(10.0) + y = 2.0 + 0.5 * x + y_err = np.ones_like(y) + y[masked] = np.nan + y_err[masked] = 0.0 + result = aare.Pol1(minimizer=minimizer, compute_errors=True).fit(x, y, y_err) + + np.testing.assert_allclose(result["par"], [2.0, 0.5], atol=1e-5) + assert np.all(result["par_err"] > 0.0) + assert result["chi2"][0] == pytest.approx(0.0, abs=1e-6) + + +def test_too_few_unmasked_samples_fail(minimizer): + x = np.arange(10.0) + y = np.full_like(x, np.nan) + y_err = np.zeros_like(x) + y[0], y_err[0] = 2.0, 1.0 # one sample left for two free parameters + result = aare.Pol1(minimizer=minimizer).fit(x, y, y_err) + assert np.all(result["par"] == 0.0) + assert np.all(result["chi2"] == 0.0) + + +def test_masked_samples_in_cube_match_1d_fits(minimizer): + x = np.linspace(-6.0, 6.0, 61) + y = np.stack([gaussian(x, 80.0 + 10.0 * c, 0.8, 1.2) for c in range(4)]) + y = y.reshape(1, 4, x.size) + y_err = np.ones_like(y) + for col, masked in [(1, 10), (2, 0), (3, 34)]: + y[0, col, masked] = np.nan + y_err[0, col, masked] = 0.0 + model = aare.Gaussian(minimizer=minimizer, compute_errors=True) + cube = model.fit(x, y, y_err, n_threads=1) + + for col in range(4): + single = model.fit(x, y[0, col], y_err[0, col]) + np.testing.assert_allclose(cube["par"][0, col], single["par"], atol=1e-12) + np.testing.assert_allclose(cube["par_err"][0, col], single["par_err"], atol=1e-12) + np.testing.assert_allclose( + cube["par"][0, col], [80.0 + 10.0 * col, 0.8, 1.2], rtol=1e-3 + ) + + +def test_parameter_whose_minimum_lies_on_its_limit_reports_no_error(minimizer): + # At a minimum that lies exactly on the limit the gradient is zero. + x = np.array([-1.0, -1.0, 1.0, 1.0]) + model = aare.Pol1(minimizer=minimizer, compute_errors=True) + model.SetParLimits("p0", 0.0, 1.0) + result = model.fit(x, 1.0 + 2.0 * x) + + assert result["par"][0] == pytest.approx(1.0, abs=1e-6) + assert result["par_err"][0] == 0.0 + assert result["par_err"][1] == pytest.approx(0.5, rel=1e-6) + + +@pytest.mark.parametrize("amplitude", [0.1, 1e-4]) +def test_fit_does_not_stop_at_start_without_a_step(minimizer, amplitude): + # The EDM tolerance is absolute in chi2 units: an unweighted fit of small + # data passes it at the start estimate (sigma 1.3617) already. + x = np.linspace(-6.0, 6.0, 61) + y = gaussian(x, amplitude, 0.8, 1.3) + result = aare.Gaussian(minimizer=minimizer).fit(x, y) + assert abs(result["par"][2] - 1.3) < 0.03 # Migrad ends at 1.3205 + + one_call = aare.Gaussian(minimizer=minimizer, max_calls=1).fit(x, y) + assert np.all(one_call["par"] == 0.0) + + +@pytest.mark.parametrize("lo, span", [(8000, 1000), (20000, 500), (60000, 200)]) +def test_polynomial_far_from_zero_keeps_its_precision(minimizer, lo, span): + # Nearly collinear columns 1, x, x^2: VarPro's normal equations lost six + # digits at x = 20000. + x = np.linspace(lo, lo + span, 40) + c = np.array([1.5 - 2e-3 * lo + 3e-7 * lo**2, 2e-3 - 6e-7 * lo, 3e-7]) + y = c[0] + c[1] * x + c[2] * x**2 + result = aare.Pol2(minimizer=minimizer).fit(x, y) + np.testing.assert_allclose(result["par"], c, rtol=1e-9) diff --git a/python/tests/test_InterpolationAPI.py b/python/tests/test_InterpolationAPI.py index e110087a..48686253 100644 --- a/python/tests/test_InterpolationAPI.py +++ b/python/tests/test_InterpolationAPI.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest from aare import Interpolator, ClusterVector, Etai, Cluster diff --git a/python/tests/test_Pedestal.py b/python/tests/test_Pedestal.py index 27f9897a..d2a3ddb3 100644 --- a/python/tests/test_Pedestal.py +++ b/python/tests/test_Pedestal.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import numpy as np import pytest @@ -171,7 +172,7 @@ def test_pedestal_rejects_noncontiguous_inputs(dtype, input_name, layout): invalid = np.ones((4, 6), dtype=input_dtype)[:2, :3][:, ::-1] assert not invalid.flags.c_contiguous - with pytest.raises(TypeError): + with pytest.raises(ValueError, match="C-contiguous"): if input_name == "push_frame": pedestal.push(invalid) elif input_name == "threshold_frame": @@ -191,9 +192,8 @@ def test_pedestal_rejects_inputs_with_wrong_ndim(dtype, input_name, shape): threshold = np.full((2, 3), 10, dtype=dtype) input_dtype = dtype if input_name == "threshold" else np.uint16 invalid = np.ones(shape, dtype=input_dtype) - name = "Threshold" if input_name == "threshold" else "Frame" - with pytest.raises(ValueError, match=f"{name} must be 2-dimensional"): + with pytest.raises(ValueError): if input_name == "push_frame": pedestal.push(invalid) elif input_name == "threshold_frame": diff --git a/python/tests/test_ROI.py b/python/tests/test_ROI.py index f0b98de0..3a117016 100644 --- a/python/tests/test_ROI.py +++ b/python/tests/test_ROI.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest import numpy as np diff --git a/python/tests/test_RawFile.py b/python/tests/test_RawFile.py index 682db632..b62044fd 100644 --- a/python/tests/test_RawFile.py +++ b/python/tests/test_RawFile.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MPL-2.0 import pytest import json -from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition +from aare import File, RawFile, RawSubFile, DetectorType, ROI, UDPPortPosition, xy import numpy as np from aare import strixelremap @@ -31,6 +31,61 @@ def small_raw_file(tmp_path): return master_path +@pytest.mark.parametrize("reader_type", [RawFile, File]) +@pytest.mark.parametrize("legacy_master", [False, True]) +@pytest.mark.parametrize("padding, policy, supported", [ + (0, "nodiscard", False), + (0, "discard", False), + (0, "discardpartial", True), + (1, "nodiscard", True), + (1, "discard", True), + (1, "discardpartial", True), +]) +def test_raw_frame_policy(small_raw_file, reader_type, legacy_master, + padding, policy, supported): + master_path = small_raw_file + if legacy_master: + master_path = master_path.with_suffix(".raw") + master_path.write_text( + "Version : 6.4\n" + "Detector Type : Jungfrau\n" + "Timing Mode : auto\n" + "Geometry : [1, 1]\n" + "Image Size : 12\n" + "Pixels : [3, 2]\n" + "Dynamic Range : 16\n" + "Max Frames Per File : 1\n" + "Total Frames : 2\n" + "Frames in File : 2\n" + f"Frame Padding : {padding}\n" + f"Frame Discard Policy : {policy}\n" + ) + else: + metadata = json.loads(master_path.read_text()) + metadata["Frame Padding"] = padding + metadata["Frame Discard Policy"] = policy + master_path.write_text(json.dumps(metadata)) + + if supported: + reader = reader_type(master_path) + assert reader.total_frames == 2 + frame = reader.read_frame() + if reader_type is RawFile: + _, frame = frame + np.testing.assert_array_equal(frame, np.arange(6).reshape(2, 3)) + else: + message = "requires frame padding or discardpartial" + with pytest.raises(RuntimeError, match=message) as error: + reader_type(master_path) + assert str(master_path) in str(error.value) + + for index in range(2): + (master_path.parent / f"run_d0_f{index}_0.raw").unlink() + with pytest.raises(RuntimeError, match=message) as error: + reader_type(master_path) + assert str(master_path) in str(error.value) + + @pytest.mark.parametrize("method, args, kwargs", [ ("read_frame", (), {}), ("read_n", (1,), {}), @@ -185,6 +240,20 @@ def test_raw_frame_count_across_rois(small_raw_file, short_module): reader.read_roi(roi_index) +def test_raw_file_geometry(small_raw_file): + metadata = json.loads(small_raw_file.read_text()) + metadata["Geometry"] = {"x": 1, "y": 2} + small_raw_file.write_text(json.dumps(metadata)) + for index in range(2): + data = (small_raw_file.parent / f"run_d0_f{index}_0.raw").read_bytes() + (small_raw_file.parent / f"run_d1_f{index}_0.raw").write_bytes(data) + + reader = RawFile(small_raw_file) + assert reader.geometry == xy(row=2, col=1) + assert reader.master.detector_layout == xy(row=2, col=1) + assert reader.master.udp_interfaces_per_module == xy(row=1, col=1) + + @pytest.mark.parametrize("disabled_port", [0, 1]) def test_raw_frame_count_with_disabled_udp_port(small_raw_file, disabled_port): metadata = json.loads(small_raw_file.read_text()) diff --git a/python/tests/test_RawMasterFile.py b/python/tests/test_RawMasterFile.py index 8ca8aee1..94669d77 100644 --- a/python/tests/test_RawMasterFile.py +++ b/python/tests/test_RawMasterFile.py @@ -1,9 +1,10 @@ +# SPDX-License-Identifier: MPL-2.0 import json import pytest -from aare import RawMasterFile, ReadoutMode, DetectorType +from aare import RawMasterFile, ReadoutMode, DetectorType, FrameDiscardPolicy @pytest.mark.withdata @@ -16,7 +17,15 @@ def test_read_rawfile_quad_eiger_and_compare_to_numpy(test_data_path): assert(f.detector_type == DetectorType.Jungfrau) -def test_raw_master_file_context_manager(tmp_path): +@pytest.mark.parametrize( + "policy, expected_policy", + [ + ("nodiscard", FrameDiscardPolicy.NoDiscard), + ("discard", FrameDiscardPolicy.Discard), + ("discardpartial", FrameDiscardPolicy.DiscardPartial), + ], +) +def test_raw_master_file_context_manager(tmp_path, policy, expected_policy): file_name = tmp_path / "run_master_0.json" file_name.write_text(json.dumps({ "Version": 7.2, @@ -29,10 +38,13 @@ def test_raw_master_file_context_manager(tmp_path): "Total Frames": 2, "Frames in File": 2, "Frame Padding": 1, - "Frame Discard Policy": "nodiscard", + "Frame Discard Policy": policy, })) with RawMasterFile(file_name) as context_file: assert context_file.reading_mode == ReadoutMode.UNKNOWN assert context_file.detector_type == DetectorType.Jungfrau + frame_discard_policy = context_file.frame_discard_policy + assert isinstance(frame_discard_policy, FrameDiscardPolicy) + assert frame_discard_policy == expected_policy diff --git a/python/tests/test_Transformation.py b/python/tests/test_Transformation.py index 2b381085..0d91c10a 100644 --- a/python/tests/test_Transformation.py +++ b/python/tests/test_Transformation.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 import pytest import numpy as np from aare import CtbRawFile, transform diff --git a/python/tests/test_interpolation_simulated.py b/python/tests/test_interpolation_simulated.py index 5a3cdf3c..57a6ab63 100644 --- a/python/tests/test_interpolation_simulated.py +++ b/python/tests/test_interpolation_simulated.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: MPL-2.0 #test script to test interpolation on simulated data diff --git a/python/tests/test_make_view.py b/python/tests/test_make_view.py new file mode 100644 index 00000000..2df947ca --- /dev/null +++ b/python/tests/test_make_view.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: MPL-2.0 +"""Probe make_view in np_helper.hpp through the bindings in _aare.testing""" +import numpy as np +import pytest + +from aare._aare import testing + + + + +def test_make_view_roundtrip(): + """Send an array to C++ and check that we get the same array back""" + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + out = testing.view_roundtrip_2d(arr) + np.testing.assert_array_equal(out, arr) + + #Sanity check to make sure we got a copy an not just the same thing back + arr[3,2] = 31655 + assert arr[3,2] != out[3,2] + +def test_make_view_const_roundtrip(): + """Const works if the array is not writeable""" + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + arr.flags.writeable = False + out = testing.const_view_roundtrip_2d(arr) + np.testing.assert_array_equal(out, arr) + +def test_make_view_const_roundtrip_writable(): + """Const works also if the array is writeable""" + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + out = testing.const_view_roundtrip_2d(arr) + np.testing.assert_array_equal(out, arr) + +def test_make_view_fill(): + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + testing.view_fill_2d(arr, 7) + assert (arr == 7).all() + +def test_make_view_fill_rejects_nonwriteable(): + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + arr.flags.writeable = False + with pytest.raises(ValueError, match="not writeable"): + testing.view_fill_2d(arr, 7) + +def test_make_view_2d_rejects_1d(): + arr = np.arange(30, dtype=np.uint16) + with pytest.raises(ValueError, match="Expected 2D array, got 1D"): + testing.view_roundtrip_2d(arr) + +def test_make_view_2d_rejects_3d(): + arr = np.arange(30, dtype=np.uint16).reshape(3, 5, 2) + with pytest.raises(ValueError, match="Expected 2D array, got 3D"): + testing.view_roundtrip_2d(arr) + +def test_make_view_rejects_noncontiguous(): + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + arr = arr[::2, ::3] # non-contiguous + assert not arr.flags.c_contiguous + with pytest.raises(ValueError, match="not C-contiguous"): + testing.view_roundtrip_2d(arr) + +def test_make_view_rejects_noncontiguous_const(): + arr = np.arange(30, dtype=np.float64).reshape(5, 6) + arr = arr[::2, ::3] # non-contiguous + assert not arr.flags.c_contiguous + with pytest.raises(ValueError, match="not C-contiguous"): + testing.const_view_roundtrip_2d(arr) + +def test_make_view_rejects_fortran_order(): + arr = np.asfortranarray(np.arange(30, dtype=np.float64).reshape(5, 6)) + assert not arr.flags.c_contiguous + with pytest.raises(ValueError, match="not C-contiguous"): + testing.view_roundtrip_2d(arr) + diff --git a/python/tests/test_xy.py b/python/tests/test_xy.py new file mode 100644 index 00000000..efc9c7d2 --- /dev/null +++ b/python/tests/test_xy.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: MPL-2.0 +from aare import xy + + +def test_xy_fields(): + layout = xy(row=2, col=3) + assert layout.row == 2 + assert layout.col == 3 + + layout.row = 4 + assert layout.row == 4 + + +def test_xy_unpacks_as_row_col(): + row, col = xy(2, 3) + assert (row, col) == (2, 3) + + +def test_xy_comparison(): + assert xy(2, 3) == xy(2, 3) + assert xy(2, 3) != xy(3, 2) + assert xy(2, 3) != (2, 3) + + +def test_xy_repr(): + assert repr(xy(2, 3)) == "xy(row=2, col=3)" diff --git a/src/Chi2.hpp b/src/Chi2.hpp deleted file mode 100644 index 2aeb0b93..00000000 --- a/src/Chi2.hpp +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -#pragma once -#include "aare/Models.hpp" -#include "aare/NDView.hpp" -#include -#include -#include -#include -#include - -namespace aare { - -namespace func { - -/** - * @brief Generic chi-squared FCN with analytic gradient for a 1D model. - * - * @tparam Model A model struct that satisfies: - * - static constexpr std::size_t npar; - * - static double eval(double x, const std::vector& par); - * - static void eval_and_grad(double x, const std::vector& par, - * double& f, std::array& g); - * - static bool is_valid(const std::vector& par); - * - * Gradient: - * d(chi2)/dp_k = -2 * sum_i w_i * (y_i - f_i) * df_i/dp_k - * - * where w_i = 1/sigma_i^2 (weighted) or 1 (unweighted). - * - * By providing analytic gradients we avoid 2*npar extra function evaluations - * per Minuit step that would otherwise be spent on finite differences. - * - * @throws std::invalid_argument if par.size() != Model::npar. - * - * Invalid model parameters do not throw; they return a large penalty - * (and a zero gradient fallback) so the minimizer can remain in control. - */ -template -class Chi2Model1DGrad : public ROOT::Minuit2::FCNGradientBase { - public: - Chi2Model1DGrad(NDView x, NDView y) - : x_(x), y_(y), s_(), weighted_(false) {} - - Chi2Model1DGrad(NDView x, NDView y, - NDView y_err) - : x_(x), y_(y), s_(y_err), weighted_(true) {} - - ~Chi2Model1DGrad() override = default; - - double operator()(const std::vector &par) const override { - if (par.size() != Model::npar) { - throw std::invalid_argument( - "Chi2Model1DGrad: wrong parameter vector size."); - } - - if (!Model::is_valid(par)) - return 1e20; - - double chi2 = 0.0; - - if (weighted_) { - for (ssize_t i = 0; i < x_.size(); ++i) { - const double si = s_[i]; - if (si == 0.0) - continue; - - const double f_i = Model::eval(x_[i], par); - const double r_i = y_[i] - f_i; - chi2 += (r_i * r_i) / (si * si); - } - } else { - for (ssize_t i = 0; i < x_.size(); ++i) { - const double f_i = Model::eval(x_[i], par); - const double r_i = y_[i] - f_i; - chi2 += r_i * r_i; - } - } - - return chi2; - } - - std::vector - Gradient(const std::vector &par) const override { - if (par.size() != Model::npar) { - throw std::invalid_argument( - "Chi2Model1DGrad: wrong parameter vector size."); - } - - std::vector grad(Model::npar, 0.0); - if (!Model::is_valid(par)) - return grad; - - std::array df{}; - double f_i = 0.0; - - if (weighted_) { - for (ssize_t i = 0; i < x_.size(); ++i) { - const double si = s_[i]; - if (si == 0.0) - continue; - - Model::eval_and_grad(x_[i], par, f_i, df); - - const double r_i = y_[i] - f_i; - const double c = -2.0 * r_i / (si * si); - - for (std::size_t k = 0; k < Model::npar; ++k) { - grad[k] += c * df[k]; - } - } - } else { - for (ssize_t i = 0; i < x_.size(); ++i) { - Model::eval_and_grad(x_[i], par, f_i, df); - - const double r_i = y_[i] - f_i; - const double c = -2.0 * r_i; - - for (std::size_t k = 0; k < Model::npar; ++k) { - grad[k] += c * df[k]; - } - } - } - - return grad; - } - - /** @brief Error definition: 1.0 for chi-squared (delta_chi2 = 1 -> - * 1-sigma). */ - double Up() const override { return 1.0; } - - private: - NDView x_; - NDView y_; - NDView s_; - bool weighted_; -}; - -} // namespace func - -} // namespace aare \ No newline at end of file diff --git a/src/ClusterFinder.test.cpp b/src/ClusterFinder.test.cpp index fd7eeb37..5ce214fd 100644 --- a/src/ClusterFinder.test.cpp +++ b/src/ClusterFinder.test.cpp @@ -2,6 +2,7 @@ #include "aare/ClusterFinder.hpp" #include "aare/Pedestal.hpp" #include +#include #include #include #include @@ -45,6 +46,42 @@ TEST_CASE("Construct a cluster finder") { // REQUIRE(clusterFinder.get_nSigma() == 1); } +TEST_CASE("nSigma set before the pedestal is ready is used for the threshold") { + const Shape<2> image_size{10, 10}; + const size_t min_pedestal_samples = 4; + + // Pedestal of 100 with a noise of 2 and a 30 ADU hit: a cluster at the + // default nSigma of 5, below threshold at nSigma 50. + NDArray low(image_size, 98); + NDArray high(image_size, 102); + NDArray hit(image_size, 100); + hit(5, 5) = 130; + + const double nSigma = GENERATE(5.0, 50.0); + const size_t expected_clusters = nSigma == 5.0 ? 1 : 0; + + ClusterFinder> cf(image_size, 5.0, 100, + min_pedestal_samples); + REQUIRE_FALSE(cf.pedestal_ready()); + REQUIRE_NOTHROW(cf.set_nSigma(nSigma)); + REQUIRE(cf.get_nSigma() == nSigma); + + for (size_t i = 0; i < min_pedestal_samples / 2; ++i) { + cf.push_pedestal_frame(low.view()); + REQUIRE_FALSE(cf.pedestal_ready()); + cf.push_pedestal_frame(high.view()); + } + REQUIRE(cf.pedestal_ready()); + + cf.find_clusters(hit.view()); + REQUIRE(cf.steal_clusters().size() == expected_clusters); +} + +TEST_CASE("update_threshold throws until the pedestal is ready") { + ClusterFinder> cf({10, 10}); + REQUIRE_THROWS_AS(cf.update_threshold(), std::runtime_error); +} + // TEST_CASE("test cluster finder") { // aare::Pedestal pedestal(10, 10, 5); // NDArray frame({10, 10}); diff --git a/src/ClusterFinderMT.test.cpp b/src/ClusterFinderMT.test.cpp index ca630de4..6973d0df 100644 --- a/src/ClusterFinderMT.test.cpp +++ b/src/ClusterFinderMT.test.cpp @@ -144,6 +144,47 @@ TEST_CASE("frame buffers are recycled when pushing more frames than the pool " CHECK(cf.m_input_queues_are_empty() == true); } +TEST_CASE("nSigma set before the pedestal is ready reaches all threads") { + using ClusterType = Cluster; + + const size_t n_threads = 2; + const size_t min_pedestal_samples = 4; + const Shape<2> image_size{10, 10}; + + ClusterFinderMTWrapper cf(image_size, 5, 200, n_threads, 16, + min_pedestal_samples); + + // Pedestal of 100 with a noise of 2 and a 30 ADU hit: a cluster at the + // default nSigma of 5, below threshold at nSigma 50. + NDArray low(image_size, 98); + NDArray high(image_size, 102); + NDArray hit(image_size, 100); + hit(5, 5) = 130; + + CHECK_FALSE(cf.pedestal_ready()); + REQUIRE_NOTHROW(cf.set_nSigma(50)); + + for (size_t i = 0; i < min_pedestal_samples / 2; ++i) { + cf.push_pedestal_frame(low.view()); + cf.push_pedestal_frame(high.view()); + } + CHECK(cf.pedestal_ready()); + + for (size_t i = 0; i < n_threads; ++i) { + cf.find_clusters(hit.view(), i); + } + cf.stop(); + + ClusterCollector collector(&cf); + collector.stop(); + + auto clusters = collector.steal_clusters(); + REQUIRE(clusters.size() == n_threads); + for (const auto &frame_clusters : clusters) { + CHECK(frame_clusters.size() == 0); + } +} + TEST_CASE("cluster collector accepts finders with a matching cluster type") { using ClusterType = Cluster; using Finder = ClusterFinderMTWrapper; diff --git a/src/ClusterVector.test.cpp b/src/ClusterVector.test.cpp index 135719c1..088f0059 100644 --- a/src/ClusterVector.test.cpp +++ b/src/ClusterVector.test.cpp @@ -12,6 +12,7 @@ using aare::Cluster; using aare::ClusterVector; +using aare::ssize_t; using C1 = Cluster; TEST_CASE("A newly created ClusterVector is empty") { @@ -180,7 +181,8 @@ TEST_CASE("Summing 3x1 clusters of int64") { Cluster c3 = {11, 12, {13, 14, 15}}; cv.push_back(c3); - REQUIRE(cv.capacity() == 4); + // growth factor is implementation defined (2x libstdc++, 1.5x MSVC) + REQUIRE(cv.capacity() >= 3); REQUIRE(cv.size() == 3); /* @@ -236,7 +238,8 @@ TEST_CASE("Push back more than initial capacity") { Cluster c3 = {11, 12, {13, 14, 15, 16}}; cv.push_back(c3); REQUIRE(cv.size() == 3); - REQUIRE(cv.capacity() == 4); + // growth factor is implementation defined (2x libstdc++, 1.5x MSVC) + REQUIRE(cv.capacity() >= 3); Cluster *ptr = reinterpret_cast *>(cv.data()); diff --git a/src/CtbRawFile.cpp b/src/CtbRawFile.cpp index 260ab668..e4d0b958 100644 --- a/src/CtbRawFile.cpp +++ b/src/CtbRawFile.cpp @@ -24,9 +24,9 @@ void CtbRawFile::read_into(std::byte *image_buf, DetectorHeader *header) { throw std::runtime_error(LOCATION + " End of file reached"); } - if (m_current_frame != 0 && - m_current_frame % m_master.max_frames_per_file() == 0) { - open_data_file(m_current_subfile + 1); + const auto index = sub_file_index(m_current_frame); + if (index != m_current_subfile) { + open_data_file(index); } if (header) { @@ -79,4 +79,4 @@ void CtbRawFile::open_data_file(size_t subfile_index) { } } -} // namespace aare \ No newline at end of file +} // namespace aare diff --git a/src/Dtype.cpp b/src/Dtype.cpp index 08098874..9ad6d673 100644 --- a/src/Dtype.cpp +++ b/src/Dtype.cpp @@ -159,7 +159,7 @@ Dtype Dtype::from_bitdepth(uint8_t bitdepth) { std::string Dtype::to_string() const { char ec{}; - if (endian::native == endian::little) + if constexpr (endian::native == endian::little) ec = '<'; else ec = '>'; diff --git a/src/FilePtr.cpp b/src/FilePtr.cpp index 45e65d46..c6edfe62 100644 --- a/src/FilePtr.cpp +++ b/src/FilePtr.cpp @@ -9,10 +9,10 @@ namespace aare { FilePtr::FilePtr(const std::filesystem::path &fname, const std::string &mode = "rb") { - fp_ = fopen(fname.c_str(), mode.c_str()); + fp_ = fopen(fname.string().c_str(), mode.c_str()); if (!fp_) throw std::runtime_error( - fmt::format("Could not open: {}", fname.c_str())); + fmt::format("Could not open: {}", fname.string())); } FilePtr::FilePtr(FilePtr &&other) { std::swap(fp_, other.fp_); } @@ -27,7 +27,7 @@ FilePtr::operator bool() const noexcept { return fp_ != nullptr; } FILE *FilePtr::get() { return fp_; } ssize_t FilePtr::tell() { - auto pos = ftell(fp_); + auto pos = ftell64(fp_); if (pos == -1) throw std::runtime_error( fmt::format("Error getting file position: {}", error_msg())); diff --git a/src/Fit.cpp b/src/Fit.cpp deleted file mode 100644 index f11aa611..00000000 --- a/src/Fit.cpp +++ /dev/null @@ -1,308 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -#include "aare/Fit.hpp" -#include "Chi2.hpp" -#include "Minuit2/FunctionMinimum.h" -#include "Minuit2/MnHesse.h" -#include "Minuit2/MnMigrad.h" -#include "Minuit2/MnStrategy.h" -#include "Minuit2/MnUserParameters.h" -#include "aare/Models.hpp" -#include "aare/utils/par.hpp" -#include "aare/utils/task.hpp" -#include -#include -#include -#include -#include - -namespace aare { -// ============================================================================ -// FitModel — method definitions -// (constructor, destructor, copy, and all methods that touch Minuit2 state) -// ============================================================================ - -template struct FitModel::FitModelImpl { - ROOT::Minuit2::MnUserParameters upar; - ROOT::Minuit2::MnStrategy strategy; - - explicit FitModelImpl(unsigned int strategy_level) - : strategy(strategy_level) {} - - FitModelImpl(const FitModelImpl &) = default; - FitModelImpl &operator=(const FitModelImpl &) = default; -}; - -template -FitModel::FitModel(unsigned int strategy, unsigned int max_calls, - double tolerance, bool compute_errors) - : impl_(std::make_unique(strategy)), max_calls_(max_calls), - tolerance_(tolerance), compute_errors_(compute_errors) { - for (std::size_t i = 0; i < npar; ++i) { - const auto pi = Model::param_info[i]; - const bool has_lo = std::isfinite(pi.default_lo); - const bool has_hi = std::isfinite(pi.default_hi); - if (has_lo && has_hi) { - impl_->upar.Add(pi.name, 0.0, 1.0, pi.default_lo, pi.default_hi); - } else if (has_lo) { - impl_->upar.Add(pi.name, 0.0, 1.0, pi.default_lo, 1e6); - } else { - impl_->upar.Add(pi.name, 0.0, 1.0); - } - } -} - -template FitModel::~FitModel() = default; - -template -FitModel::FitModel(const FitModel &other) - : impl_(std::make_unique(*other.impl_)), - max_calls_(other.max_calls_), tolerance_(other.tolerance_), - compute_errors_(other.compute_errors_), user_fixed_(other.user_fixed_), - user_start_(other.user_start_) {} - -template -FitModel &FitModel::operator=(const FitModel &other) { - if (this != &other) { - impl_ = std::make_unique(*other.impl_); - max_calls_ = other.max_calls_; - tolerance_ = other.tolerance_; - compute_errors_ = other.compute_errors_; - user_fixed_ = other.user_fixed_; - user_start_ = other.user_start_; - } - return *this; -} - -template -unsigned int FitModel::checked_index(const std::string &name) const { - for (std::size_t i = 0; i < npar; ++i) { - if (impl_->upar.Name(i) == name) - return static_cast(i); - } - throw std::runtime_error("FitModel: unknown parameter name '" + name + "'"); -} - -template -void FitModel::SetParLimits(unsigned int idx, double lo, double hi) { - impl_->upar.SetLimits(idx, lo, hi); -} - -template -void FitModel::FixParameter(unsigned int idx, double val) { - SetParameter(idx, val); - impl_->upar.Fix(idx); - user_fixed_[idx] = true; -} - -template -void FitModel::ReleaseParameter(unsigned int idx) { - impl_->upar.Release(idx); - user_fixed_[idx] = false; -} - -template -void FitModel::ReleaseParameter(const std::string &name) { - ReleaseParameter(checked_index(name)); -} - -template -void FitModel::SetParameter(unsigned int idx, double val) { - impl_->upar.SetValue(idx, val); - user_start_[idx] = true; -} - -template -void FitModel::SetParameter(const std::string &name, double val) { - SetParameter(checked_index(name), val); -} - -template -void FitModel::FixParameter(const std::string &name, double val) { - FixParameter(checked_index(name), val); -} - -template -void FitModel::SetParLimits(const std::string &name, double lo, - double hi) { - SetParLimits(checked_index(name), lo, hi); -} - -template -std::string FitModel::GetParName(unsigned int idx) const { - return impl_->upar.GetName(idx); -} - -template -std::vector FitModel::GetParNames() const { - std::vector names; - for (std::size_t i = 0; i < npar; ++i) - names.push_back(GetParName(i)); - return names; -} - -// ============================================================================ -// fit_pixel / fit_3d — Minuit2 template implementations -// ============================================================================ - -template -NDArray fit_pixel(const FitModel &model, NDView x, - NDView y, NDView y_err) { - using FCN = func::Chi2Model1DGrad; - constexpr std::size_t npar = Model::npar; - const bool want_errors = model.compute_errors(); - const ssize_t result_size = want_errors ? (2 * npar + 1) : (npar + 1); - - auto start = Model::estimate_par(x, y); - - if (!Model::is_valid(std::vector(start.begin(), start.end()))) { - return NDArray({result_size}, 0.0); - } - - double x_range, y_range, slope_scale; - model::compute_ranges(x, y, x_range, y_range, slope_scale); - - std::array steps{}; - Model::compute_steps(start, x_range, y_range, slope_scale, steps); - - // thread-local copy of starting parameters - auto upar_local = model.impl()->upar; - - for (std::size_t i = 0; i < npar; ++i) { - if (model.is_user_fixed(i)) - continue; - if (!model.is_user_start(i)) - upar_local.SetValue(i, start[i]); - upar_local.SetError(i, steps[i]); - } - - auto chi2 = (y_err.size() > 0) ? FCN(x, y, y_err) : FCN(x, y); - - ROOT::Minuit2::MnMigrad migrad(chi2, upar_local, model.impl()->strategy); - ROOT::Minuit2::FunctionMinimum min = - migrad(model.max_calls(), model.tolerance()); - - if (!min.IsValid()) - return NDArray({result_size}, 0.0); - - if (want_errors) { - ROOT::Minuit2::MnHesse hesse; - hesse(chi2, min); - - const auto &values = min.UserState().Params(); - const auto &errors = min.UserState().Errors(); - - NDArray result({result_size}); - for (std::size_t k = 0; k < npar; ++k) { - result[k] = values[k]; - result[npar + k] = errors[k]; - } - result[2 * npar] = min.Fval(); - return result; - } - - const auto &values = min.UserState().Params(); - NDArray result({result_size}); - for (std::size_t k = 0; k < npar; ++k) - result[k] = values[k]; - result[npar] = min.Fval(); - return result; -} - -template -NDArray fit_pixel(const FitModel &model, NDView x, - NDView y) { - return fit_pixel(model, x, y, NDView{}); -} - -template -void fit_3d(const FitModel &model, NDView x, - NDView y, NDView y_err, - NDView par_out, NDView err_out, - NDView chi2_out, int n_threads) { - const std::size_t npar = Model::npar; - - if (x.size() != y.shape(2)) - throw std::runtime_error("fit_3d: x.size() must match y.shape(2)."); - - if (par_out.shape(0) != y.shape(0) || par_out.shape(1) != y.shape(1) || - par_out.shape(2) != npar) - throw std::runtime_error("par_out must have shape [rows, cols, npar]."); - - if (chi2_out.shape(0) != y.shape(0) || chi2_out.shape(1) != y.shape(1)) - throw std::runtime_error("chi2_out must have shape [rows, cols]."); - - const bool has_errors = (y_err.size() > 0); - const bool want_par_errors = (err_out.size() > 0) && model.compute_errors(); - - if (has_errors) { - if (y.shape(0) != y_err.shape(0) || y.shape(1) != y_err.shape(1) || - y.shape(2) != y_err.shape(2)) - throw std::runtime_error( - "fit_3d: y and y_err must have identical shape."); - - if (err_out.shape(0) != y.shape(0) || err_out.shape(1) != y.shape(1) || - err_out.shape(2) != npar) - throw std::runtime_error( - "err_out must have shape [rows, cols, npar]."); - } - - auto process = [&](ssize_t first_row, ssize_t last_row) { - for (ssize_t row = first_row; row < last_row; row++) { - for (ssize_t col = 0; col < y.shape(1); col++) { - - NDView values(&y(row, col, 0), {y.shape(2)}); - NDView errors = - has_errors ? NDView(&y_err(row, col, 0), - {y_err.shape(2)}) - : NDView{}; - - auto res = fit_pixel(model, x, values, errors); - - for (std::size_t k = 0; k < npar; ++k) - par_out(row, col, k) = res(k); - - if (want_par_errors) { - for (std::size_t k = 0; k < npar; ++k) - err_out(row, col, k) = res(npar + k); - chi2_out(row, col) = res(2 * npar); - } else { - chi2_out(row, col) = res(npar); - } - } - } - }; - - auto tasks = split_task(0, static_cast(y.shape(0)), n_threads); - RunInParallel(process, tasks); -} - -// ============================================================================ -// Explicit instantiations for all supported model types -// ============================================================================ - -// NOLINTBEGIN -#define AARE_INSTANTIATE_FIT(Model) \ - template class FitModel; \ - template NDArray fit_pixel( \ - const FitModel &, NDView, NDView, \ - NDView); \ - template NDArray fit_pixel( \ - const FitModel &, NDView, NDView); \ - template void fit_3d(const FitModel &, NDView, \ - NDView, NDView, \ - NDView, NDView, \ - NDView, int); - -AARE_INSTANTIATE_FIT(model::Gaussian) -AARE_INSTANTIATE_FIT(model::GaussianErfcPlateau) -AARE_INSTANTIATE_FIT(model::GaussianChargeSharing) -AARE_INSTANTIATE_FIT(model::GaussianChargeSharingKb) -AARE_INSTANTIATE_FIT(model::Pol1) -AARE_INSTANTIATE_FIT(model::Pol2) -AARE_INSTANTIATE_FIT(model::RisingScurve) -AARE_INSTANTIATE_FIT(model::FallingScurve) - -#undef AARE_INSTANTIATE_FIT -// NOLINTEND - -} // namespace aare diff --git a/src/Fit.test.cpp b/src/Fit.test.cpp deleted file mode 100644 index 2b0fd1bb..00000000 --- a/src/Fit.test.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -#include "aare/Fit.hpp" -#include "aare/FitModel.hpp" -#include "aare/Models.hpp" - -#include -#include -#include - -namespace { - -constexpr ssize_t n_points = 61; - -void fill_gaussian(aare::NDArray &x, aare::NDArray &y, - double amplitude, double mean, double sigma) { - for (ssize_t i = 0; i < n_points; ++i) { - x(i) = -6.0 + 0.2 * static_cast(i); - const double z = (x(i) - mean) / sigma; - y(i) = amplitude * std::exp(-0.5 * z * z); - } -} - -} // namespace - -TEST_CASE("Minuit2 fits weighted and unweighted Gaussian data", "[fit]") { - aare::NDArray x({n_points}); - aare::NDArray y({n_points}); - aare::NDArray y_err({n_points}, 1.0); - fill_gaussian(x, y, 120.0, 0.8, 1.3); - - const aare::FitModel unweighted_model; - const auto unweighted = - aare::fit_pixel(unweighted_model, x.view(), y.view()); - - REQUIRE(unweighted.size() == 4); - CHECK(unweighted(0) == Catch::Approx(120.0).epsilon(1e-5)); - CHECK(unweighted(1) == Catch::Approx(0.8).epsilon(1e-5)); - CHECK(unweighted(2) == Catch::Approx(1.3).epsilon(1e-5)); - CHECK(unweighted(3) == Catch::Approx(0.0).margin(1e-4)); - - const aare::FitModel weighted_model(0, 100, 0.5, - true); - const auto weighted = - aare::fit_pixel(weighted_model, x.view(), y.view(), y_err.view()); - - REQUIRE(weighted.size() == 7); - CHECK(weighted(0) == Catch::Approx(120.0).epsilon(1e-5)); - CHECK(weighted(1) == Catch::Approx(0.8).epsilon(1e-5)); - CHECK(weighted(2) == Catch::Approx(1.3).epsilon(1e-5)); - CHECK(weighted(6) == Catch::Approx(0.0).margin(1e-4)); -} - -TEST_CASE("Minuit2 fits a Gaussian data cube in parallel", "[fit]") { - aare::NDArray x({n_points}); - aare::NDArray values({n_points}); - fill_gaussian(x, values, 80.0, -0.6, 0.9); - - aare::NDArray y({2, 2, n_points}); - for (ssize_t row = 0; row < 2; ++row) { - for (ssize_t col = 0; col < 2; ++col) { - for (ssize_t i = 0; i < n_points; ++i) - y(row, col, i) = values(i); - } - } - - aare::NDArray par({2, 2, 3}); - aare::NDArray chi2({2, 2}); - const aare::FitModel model; - - aare::fit_3d(model, x.view(), y.view(), aare::NDView{}, - par.view(), aare::NDView{}, chi2.view(), 2); - - for (ssize_t row = 0; row < 2; ++row) { - for (ssize_t col = 0; col < 2; ++col) { - CHECK(par(row, col, 0) == Catch::Approx(80.0).epsilon(1e-5)); - CHECK(par(row, col, 1) == Catch::Approx(-0.6).epsilon(1e-5)); - CHECK(par(row, col, 2) == Catch::Approx(0.9).epsilon(1e-5)); - CHECK(chi2(row, col) == Catch::Approx(0.0).margin(1e-8)); - } - } -} diff --git a/src/Interpolation.test.cpp b/src/Interpolation.test.cpp index 41a8b70a..764de414 100644 --- a/src/Interpolation.test.cpp +++ b/src/Interpolation.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ClusterVector.hpp" #include "aare/Interpolator.hpp" #include "aare/NDArray.hpp" diff --git a/src/JungfrauDataFile.cpp b/src/JungfrauDataFile.cpp index 34820b8b..6e77a0a6 100644 --- a/src/JungfrauDataFile.cpp +++ b/src/JungfrauDataFile.cpp @@ -136,7 +136,7 @@ void JungfrauDataFile::find_frame_size(const std::filesystem::path &fname) { void JungfrauDataFile::parse_fname(const std::filesystem::path &fname) { m_path = fname.parent_path(); - m_base_name = fname.stem(); + m_base_name = fname.stem().string(); // find file index, then remove if from the base name if (auto pos = m_base_name.find_last_of('_'); pos != std::string::npos) { @@ -221,7 +221,7 @@ JungfrauDataHeader JungfrauDataFile::read_header() { throw std::runtime_error(LOCATION + "Could not read header from file" + m_fp.error_msg()); } - m_fp.seek(-header_size, SEEK_CUR); + m_fp.seek(-static_cast(header_size), SEEK_CUR); return header; } diff --git a/src/MultiThreadedFileReader.test.cpp b/src/MultiThreadedFileReader.test.cpp index 4896d788..c6fdcd90 100644 --- a/src/MultiThreadedFileReader.test.cpp +++ b/src/MultiThreadedFileReader.test.cpp @@ -22,6 +22,7 @@ using aare::File; using aare::FileConfig; using aare::Frame; using aare::NumpyFile; +using aare::ssize_t; using aare::experimental::MultiThreadedFileReader; namespace { diff --git a/src/NDArray.test.cpp b/src/NDArray.test.cpp index 3a3cafc0..56514102 100644 --- a/src/NDArray.test.cpp +++ b/src/NDArray.test.cpp @@ -9,6 +9,7 @@ using aare::NDArray; using aare::NDView; using aare::Shape; +using aare::ssize_t; TEST_CASE("Initial size is zero if no size is specified") { NDArray a; @@ -86,6 +87,40 @@ TEST_CASE("A const NDArray returns a read-only view") { REQUIRE(view(1, 2) == 1); } +TEST_CASE("NDArray can be constructed from a read-only view") { + NDArray source({2, 3}, 0); + for (ssize_t i = 0; i < source.size(); ++i) + source(i) = static_cast(i); + const auto &const_source = source; + + NDArray from_const(const_source.view()); + NDArray from_mutable(source.view()); + + REQUIRE(from_const.shape() == source.shape()); + REQUIRE(from_mutable.shape() == source.shape()); + for (ssize_t i = 0; i < source.size(); ++i) { + REQUIRE(from_const(i) == source(i)); + REQUIRE(from_mutable(i) == source(i)); + } + + // Independent copy, not a view of the source + source(0) = 42; + REQUIRE(from_const(0) == 0); +} + +TEST_CASE("NDArray::copy_from accepts a read-only view") { + NDArray source({4}, 1.5); + const auto &const_source = source; + + NDArray target({4}, 0.0); + target.copy_from(const_source.view()); + for (ssize_t i = 0; i < target.size(); ++i) + REQUIRE(target(i) == 1.5); + + NDArray wrong_shape({3}, 0.0); + REQUIRE_THROWS(wrong_shape.copy_from(const_source.view())); +} + TEST_CASE("Indexing of a 2D image") { std::array shape{{3, 7}}; NDArray img(shape, 5); @@ -412,7 +447,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 1.0; - auto C = A - v; + NDArray C = A - v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -429,7 +464,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 1.0; - auto C = A + v; + NDArray C = A + v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -446,7 +481,7 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 3.7; - auto C = A / v; + NDArray C = A / v; REQUIRE(C.data() != A.data()); // Value of C matches @@ -463,12 +498,12 @@ TEST_CASE("Elementwise operations on images") { NDArray A(shape, a_val); NDArray B(shape, b_val); double v = 3.7; - auto C = A / v; + NDArray C = A * v; REQUIRE(C.data() != A.data()); // Value of C matches for (uint32_t i = 0; i < C.size(); ++i) { - REQUIRE(C(i) == a_val / v); + REQUIRE(C(i) == a_val * v); } // Value of A is not changed @@ -478,6 +513,51 @@ TEST_CASE("Elementwise operations on images") { } } +TEST_CASE("Expressions with scalars") { + NDArray A({5, 5}, 3); + NDArray B({5, 5}, 8); + + NDArray C = 2 * A + B / 4 - 1; + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 7); + } + + // The scalar is not converted to the element type of the array + NDArray D = A * 0.5; + for (ssize_t i = 0; i < D.size(); ++i) { + REQUIRE(D(i) == 1.5); + } +} + +TEST_CASE("Assign an expression to an existing NDArray") { + NDArray A({5, 5}, 3); + NDArray B({5, 5}, 8); + + SECTION("same shape reuses the buffer") { + NDArray C({5, 5}, 0); + auto *ptr = C.data(); + C = A + B; + REQUIRE(C.data() == ptr); + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 11); + } + } + SECTION("the target can be part of the expression") { + A = A + B; + for (ssize_t i = 0; i < A.size(); ++i) { + REQUIRE(A(i) == 11); + } + } + SECTION("different shape reallocates") { + NDArray C({2, 3}, 0); + C = A + B; + REQUIRE(C.shape() == A.shape()); + for (ssize_t i = 0; i < C.size(); ++i) { + REQUIRE(C(i) == 11); + } + } +} + TEST_CASE("Assign an std::array to a 1D NDArray") { NDArray a{{5}, 0}; std::array b{1, 2, 3, 4, 5}; diff --git a/src/NDView.test.cpp b/src/NDView.test.cpp index 1a8c6a2b..59d231b3 100644 --- a/src/NDView.test.cpp +++ b/src/NDView.test.cpp @@ -11,6 +11,7 @@ using aare::NDView; using aare::num_elements; using aare::Shape; +using aare::ssize_t; static_assert(std::is_convertible_v, NDView>); static_assert(!std::is_convertible_v, NDView>); diff --git a/src/NumpyFile.cpp b/src/NumpyFile.cpp index 699fa08c..ad9cdf6f 100644 --- a/src/NumpyFile.cpp +++ b/src/NumpyFile.cpp @@ -28,8 +28,7 @@ NumpyFile::NumpyFile(const std::filesystem::path &fname, throw std::runtime_error( fmt::format("Could not open: {} for reading", fname.string())); } - initial_header_len = aare::NumpyHelpers::write_header( - std::filesystem::path(fname.c_str()), m_header); + initial_header_len = aare::NumpyHelpers::write_header(fname, m_header); } m_pixels_per_frame = std::accumulate(m_header.shape.begin() + 1, m_header.shape.end(), 1, @@ -46,7 +45,7 @@ void NumpyFile::write_impl(void *data, uint64_t size) { if (!(m_mode == "w" || m_mode == "a")) { throw std::invalid_argument("File not open for writing"); } - if (fseek(fp, 0, SEEK_END)) + if (fseek64(fp, 0, SEEK_END)) throw std::runtime_error("Could not seek to end of file"); size_t const rc = fwrite(data, size, 1, fp); if (rc != 1) { @@ -68,8 +67,8 @@ void NumpyFile::get_frame_into(size_t frame_number, std::byte *image_buf) { if (frame_number > m_header.shape[0]) { throw std::invalid_argument("Frame number out of range"); } - if (fseek(fp, header_size + frame_number * m_bytes_per_frame, - SEEK_SET)) // NOLINT + if (fseek64(fp, header_size + frame_number * m_bytes_per_frame, + SEEK_SET)) // NOLINT throw std::runtime_error("Could not seek to frame"); size_t const rc = fread(image_buf, m_bytes_per_frame, 1, fp); @@ -101,15 +100,15 @@ void NumpyFile::read_into(std::byte *image_buf, size_t n_frames) { NumpyFile::~NumpyFile() noexcept { if (m_mode == "w" || m_mode == "a") { // determine number of frames - if (fseek(fp, 0, SEEK_END)) { + if (fseek64(fp, 0, SEEK_END)) { std::cout << "Could not seek to end of file" << std::endl; } - size_t const file_size = ftell(fp); + size_t const file_size = ftell64(fp); size_t const data_size = file_size - initial_header_len; size_t const n_frames = data_size / m_bytes_per_frame; // update number of frames in header (first element of shape) m_header.shape[0] = n_frames; - if (fseek(fp, 0, SEEK_SET)) { + if (fseek64(fp, 0, SEEK_SET)) { std::cout << "Could not seek to beginning of file" << std::endl; } // create string stream to contain header diff --git a/src/NumpyHelpers.cpp b/src/NumpyHelpers.cpp index a5448338..75462953 100644 --- a/src/NumpyHelpers.cpp +++ b/src/NumpyHelpers.cpp @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: MIT /* 28-03-2024 modified by: Bechir Braham diff --git a/src/ProducerConsumerQueue.test.cpp b/src/ProducerConsumerQueue.test.cpp new file mode 100644 index 00000000..05cb4904 --- /dev/null +++ b/src/ProducerConsumerQueue.test.cpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MPL-2.0 + +#include "aare/ProducerConsumerQueue.hpp" + +#include +#include +#include +#include + +using aare::ProducerConsumerQueue; + +// The queue is a fixed-capacity SPSC ring with const size and buffer, as in +// the upstream folly implementation. Moving one is not thread safe and the +// previous move operations left the source pointing at a null buffer, so the +// type is deliberately neither copyable nor movable. Hold it by value or in a +// unique_ptr instead. +static_assert(!std::is_copy_constructible_v>); +static_assert(!std::is_copy_assignable_v>); +static_assert(!std::is_move_constructible_v>); +static_assert(!std::is_move_assignable_v>); + +TEST_CASE("ProducerConsumerQueue round trips non trivial elements", "[queue]") { + ProducerConsumerQueue q(3); // capacity 2 + REQUIRE(q.capacity() == 2); + REQUIRE(q.isEmpty()); + + REQUIRE(q.write(std::string(64, 'a'))); + REQUIRE(q.write(std::string(64, 'b'))); + CHECK(q.isFull()); + CHECK_FALSE(q.write(std::string(64, 'c'))); + + std::string out; + REQUIRE(q.read(out)); + CHECK(out == std::string(64, 'a')); + + // Wrap around the ring and leave an element behind for the destructor. + REQUIRE(q.write(std::string(64, 'd'))); + REQUIRE(q.frontPtr() != nullptr); + CHECK(*q.frontPtr() == std::string(64, 'b')); + q.popFront(); + CHECK(q.sizeGuess() == 1); +} + +TEST_CASE("ProducerConsumerQueue rejects sizes below two in every build type", + "[queue]") { + // A plain assert would be compiled out in Release builds and a zero-size + // queue would then overflow its buffer on the first write. + CHECK_THROWS_AS(ProducerConsumerQueue(0), std::invalid_argument); + CHECK_THROWS_AS(ProducerConsumerQueue(1), std::invalid_argument); + + ProducerConsumerQueue smallest(2); + CHECK(smallest.capacity() == 1); + CHECK(smallest.write(7)); + CHECK(smallest.isFull()); +} diff --git a/src/ROI.cpp b/src/ROI.cpp index 9cb37961..56882015 100644 --- a/src/ROI.cpp +++ b/src/ROI.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROI.hpp" #include "aare/DetectorGeometry.hpp" #include "aare/ROIGeometry.hpp" @@ -19,7 +20,7 @@ bool complete_ROI(const ROI &roi, const DetectorGeometry &geometry) { bool complete_ROI(const std::vector &rois, const DetectorGeometry &geometry) { - if (rois.empty() or rois.size() > 1) { + if (rois.empty() || rois.size() > 1) { return false; } else { return complete_ROI(rois[0], geometry); @@ -33,7 +34,7 @@ bool complete_ROI(const ROIGeometry &roi, const DetectorGeometry &geometry) { bool complete_ROI(const std::vector &rois, const DetectorGeometry &geometry) { - if (rois.empty() or rois.size() > 1) { + if (rois.empty() || rois.size() > 1) { return false; } else { return complete_ROI(rois[0], geometry); diff --git a/src/ROI.test.cpp b/src/ROI.test.cpp index 6cc7bb26..9bbe4dff 100644 --- a/src/ROI.test.cpp +++ b/src/ROI.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROI.hpp" #include diff --git a/src/ROIGeometry.cpp b/src/ROIGeometry.cpp index f61f9079..8008118d 100644 --- a/src/ROIGeometry.cpp +++ b/src/ROIGeometry.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/ROIGeometry.hpp" #include diff --git a/src/RawFile.cpp b/src/RawFile.cpp index 1d76b622..e416a27e 100644 --- a/src/RawFile.cpp +++ b/src/RawFile.cpp @@ -29,6 +29,14 @@ RawFile::RawFile(const std::filesystem::path &fname, const std::string &mode) m_mode = mode; if (mode == "r") { + if (m_master.frame_padding() == 0 && + m_master.frame_discard_policy() != + FrameDiscardPolicy::DiscardPartial) { + throw std::runtime_error(fmt::format( + "Cannot open '{}': RawFile requires frame padding or " + "discardpartial.", + m_master.master_fname().string())); + } m_subfiles.resize(m_master.roi_geometries().size()); // iterate over all ROIS @@ -294,7 +302,7 @@ void RawFile::open_subfiles(const size_t roi_index) { DetectorHeader RawFile::read_header(const std::filesystem::path &fname) { DetectorHeader h{}; - FILE *fp = fopen(fname.string().c_str(), "r"); + FILE *fp = fopen(fname.string().c_str(), "rb"); if (!fp) throw std::runtime_error(fmt::format( "Could not open file '{}' for frame index 0", fname.string())); diff --git a/src/RawFile.test.cpp b/src/RawFile.test.cpp index e798a610..ee0b9e24 100644 --- a/src/RawFile.test.cpp +++ b/src/RawFile.test.cpp @@ -16,6 +16,51 @@ using aare::File; using aare::RawFile; using namespace aare; +TEST_CASE("RawFile requires padding or discarding partial frames", + "[RawFile][frame-policy]") { + const auto [padding, policy, supported] = + GENERATE(table({ + {0, "nodiscard", false}, + {0, "discard", false}, + {0, "discardpartial", true}, + {1, "nodiscard", true}, + {1, "discard", true}, + {1, "discardpartial", true}, + })); + CAPTURE(padding, policy); + TemporaryRawFiles files; + nlohmann::json metadata; + std::ifstream(files.master_path()) >> metadata; + metadata["Frame Padding"] = padding; + metadata["Frame Discard Policy"] = policy; + std::ofstream(files.master_path()) << metadata; + + REQUIRE(RawMasterFile(files.master_path()).frame_padding() == padding); + if (supported) { + RawFile reader(files.master_path()); + REQUIRE(reader.total_frames() == 2); + auto frame = reader.read_frame(); + REQUIRE(frame.view()(0, 0) == 1); + File generic_reader(files.master_path()); + auto generic_frame = generic_reader.read_frame(); + REQUIRE(generic_frame.view()(0, 0) == 1); + } else { + SECTION("with data subfiles") {} + SECTION("without data subfiles") { + std::filesystem::remove(files.data_path(0, 0)); + std::filesystem::remove(files.data_path(0, 1)); + } + const auto path_matcher = + Catch::Matchers::ContainsSubstring(files.master_path().string()); + const auto policy_matcher = Catch::Matchers::ContainsSubstring( + "requires frame padding or discardpartial"); + const auto message = path_matcher && policy_matcher; + REQUIRE_THROWS_AS(RawFile(files.master_path()), std::runtime_error); + REQUIRE_THROWS_WITH(RawFile(files.master_path()), message); + REQUIRE_THROWS_WITH(File(files.master_path()), message); + } +} + TEST_CASE("RawFile read errors identify the frame index and master path", "[RawFile][read-errors]") { TemporaryRawFiles files; diff --git a/src/RawMasterFile.cpp b/src/RawMasterFile.cpp index 61999438..6e86de47 100644 --- a/src/RawMasterFile.cpp +++ b/src/RawMasterFile.cpp @@ -11,8 +11,8 @@ namespace aare { RawFileNameComponents::RawFileNameComponents( const std::filesystem::path &fname) { m_base_path = fname.parent_path(); - m_base_name = fname.stem(); - m_ext = fname.extension(); + m_base_name = fname.stem().string(); + m_ext = fname.extension().string(); if (m_ext != ".json" && m_ext != ".raw") { throw std::runtime_error(LOCATION + @@ -23,7 +23,7 @@ RawFileNameComponents::RawFileNameComponents( try { auto pos = m_base_name.rfind('_'); m_file_index = std::stoi(m_base_name.substr(pos + 1)); - } catch (const std::invalid_argument &e) { + } catch (const std::invalid_argument &) { throw std::runtime_error(LOCATION + "Could not parse file index"); } @@ -370,7 +370,7 @@ void RawMasterFile::parse_json(std::istream &is) { try { m_quad = j.at("Quad"); - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // keep the optional empty } // try{ @@ -384,7 +384,7 @@ void RawMasterFile::parse_json(std::istream &is) { if (digital_flag) { m_digital_samples = j.at("Digital Samples"); } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // keep the optional empty } try { @@ -393,7 +393,7 @@ void RawMasterFile::parse_json(std::istream &is) { if (m_transceiver_flag) { m_transceiver_samples = j.at("Transceiver Samples"); } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // keep the optional empty } try { @@ -414,7 +414,7 @@ void RawMasterFile::parse_json(std::istream &is) { m_scan_parameters .increment_stop(); // adjust for endpoint being included } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // not a scan } try { @@ -424,7 +424,7 @@ void RawMasterFile::parse_json(std::istream &is) { m_udp_port_types.value().push_back( string_to(elem)); } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // leave the optional empty } try { @@ -433,13 +433,13 @@ void RawMasterFile::parse_json(std::istream &is) { for (auto &elem : json_list_obj) { m_disabled_udp_ports.push_back(static_cast(elem)); } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { m_disabled_udp_ports .clear(); // empty list if not present in master file } try { m_udp_interfaces_per_module = {j.at("Number of UDP Interfaces"), 1}; - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { if (m_type == DetectorType::Eiger && m_quad == 1) m_udp_interfaces_per_module = {2, 1}; else if (m_type == DetectorType::Eiger) { @@ -485,7 +485,7 @@ void RawMasterFile::parse_json(std::istream &is) { } } - } catch (const json::out_of_range &e) { + } catch (const json::out_of_range &) { // fill ROI with full detector size if not present in master file m_rois.push_back( {0, m_detector_layout.col * static_cast(m_pixels_x), 0, @@ -546,9 +546,8 @@ void RawMasterFile::parse_raw(std::istream &is) { m_image_size_in_bytes = std::stoi(value); } else if (key == "Frame Padding") { m_frame_padding = std::stoi(value); - // } else if (key == "Frame Discard Policy"){ - // m_frame_discard_policy = - // StringTo(value); + } else if (key == "Frame Discard Policy") { + m_frame_discard_policy = string_to(value); // } else if (key == "Number of rows"){ // m_number_of_rows = std::stoi(value); } else if (key == "Analog Flag") { diff --git a/src/RawMasterFile.test.cpp b/src/RawMasterFile.test.cpp index bd244844..3838bfc2 100644 --- a/src/RawMasterFile.test.cpp +++ b/src/RawMasterFile.test.cpp @@ -3,11 +3,38 @@ #include "test_config.hpp" #include +#include #include #include using namespace aare; +TEST_CASE("Parse legacy master frame discard policies", + "[rawmasterfile][frame-policy]") { + const auto [policy, expected] = + GENERATE(table({ + {"nodiscard", FrameDiscardPolicy::NoDiscard}, + {"discard", FrameDiscardPolicy::Discard}, + {"discardpartial", FrameDiscardPolicy::DiscardPartial}, + {"", FrameDiscardPolicy::NoDiscard}, + })); + CAPTURE(policy); + std::ostringstream content; + content << "Version : 6.4\n" + "Detector Type : Jungfrau\n" + "Timing Mode : auto\n" + "Geometry : [1, 1]\n" + "Pixels : [3, 2]\n" + "Frame Padding : 0\n"; + if (!policy.empty()) { + content << "Frame Discard Policy : " << policy << '\n'; + } + std::istringstream input(content.str()); + RawMasterFile master(input, "run_master_0.raw"); + REQUIRE(master.frame_padding() == 0); + REQUIRE(master.frame_discard_policy() == expected); +} + TEST_CASE("Parse a master file fname") { RawFileNameComponents m("test_master_1.json"); REQUIRE(m.base_name() == "test"); diff --git a/src/RawSubFile.cpp b/src/RawSubFile.cpp index 7a42bbba..cd1a6fba 100644 --- a/src/RawSubFile.cpp +++ b/src/RawSubFile.cpp @@ -163,7 +163,7 @@ void RawSubFile::parse_fname(const std::filesystem::path &fname) { // f1 is the file index - thi is the one we need // 0 is the measurement index, will not change m_path = fname.parent_path(); - m_base_name = fname.filename(); + m_base_name = fname.filename().string(); // Regex to extract numbers after 'd' and 'f' std::regex pattern(R"(^(.*_d)(\d+)(_f)(\d+)(_\d+\.raw)$)"); diff --git a/src/algorithm.test.cpp b/src/algorithm.test.cpp index e498c3b2..026fcacb 100644 --- a/src/algorithm.test.cpp +++ b/src/algorithm.test.cpp @@ -3,6 +3,8 @@ #include #include +using aare::ssize_t; + TEST_CASE("Find the closed index in a 1D array", "[algorithm]") { aare::NDArray arr({5}); for (ssize_t i = 0; i < arr.size(); i++) { diff --git a/src/decode.cpp b/src/decode.cpp index afe261f9..f522f5c1 100644 --- a/src/decode.cpp +++ b/src/decode.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 #include "aare/decode.hpp" #include +#include #include namespace aare { @@ -102,6 +103,57 @@ void adc_sar_04_decode64to16(NDView input, } } +namespace { + +constexpr ssize_t bytes_per_word = sizeof(uint64_t); + +// Shared loop for the byte-buffer overloads. Words are copied out with +// memcpy so the input does not need to be aligned to 8 bytes. +template +void decode64to16_from_bytes(NDView input, + NDView output) { + if (input.shape(1) % bytes_per_word != 0) { + throw std::invalid_argument( + fmt::format("{} input row length of {} bytes is not a multiple " + "of {}", + LOCATION, input.shape(1), bytes_per_word)); + } + if (input.shape(0) != output.shape(0) || + input.shape(1) / bytes_per_word != output.shape(1)) { + throw std::invalid_argument(fmt::format( + "{} output shape must be ({}, {}) for an input of " + "shape ({}, {})", + LOCATION, input.shape(0), input.shape(1) / bytes_per_word, + output.shape(0), output.shape(1))); + } + + for (ssize_t i = 0; i < output.shape(0); ++i) { + const uint8_t *row = input.data() + i * input.shape(1); + for (ssize_t j = 0; j < output.shape(1); ++j) { + uint64_t word{}; + std::memcpy(&word, row + j * bytes_per_word, sizeof(word)); + output(i, j) = decode_word(word); + } + } +} + +} // namespace + +void adc_sar_05_06_07_08decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + +void adc_sar_05_decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + +void adc_sar_04_decode64to16(NDView input, + NDView output) { + decode64to16_from_bytes(input, output); +} + double apply_custom_weights(uint16_t input, const NDView weights) { if (weights.size() > 16) { throw std::invalid_argument( diff --git a/src/decode.test.cpp b/src/decode.test.cpp index bab7c38b..3fc3637f 100644 --- a/src/decode.test.cpp +++ b/src/decode.test.cpp @@ -5,9 +5,12 @@ #include #include using Catch::Matchers::WithinAbs; +#include +#include #include using aare::BitOffset; +using aare::ssize_t; TEST_CASE("test_adc_sar_05_decode64to16") { uint64_t input = 0; @@ -22,7 +25,7 @@ TEST_CASE("test_adc_sar_05_decode64to16") { // test all bits by iteratting through the bitlist std::vector bitlist = {29, 19, 28, 18, 31, 21, 27, 20, 24, 23, 25, 22}; for (size_t i = 0; i < bitlist.size(); i++) { - input = 1UL << bitlist[i]; + input = 1ULL << bitlist[i]; output = aare::adc_sar_05_decode64to16(input); CHECK(output == (1 << i)); } @@ -49,6 +52,86 @@ TEST_CASE("test_adc_sar_05_decode64to16") { CHECK(output == 3072UL); } +TEST_CASE("Decode 64 bit ADC SAR samples from a byte buffer") { + // Two rows of two words each, viewed as raw bytes + aare::NDArray words({2, 2}); + words(0, 0) = 1UL << 29; + words(0, 1) = (1UL << 25) | (1UL << 22); + words(1, 0) = 0; + words(1, 1) = (1UL << 29) | (1UL << 19) | (1UL << 28); + + aare::NDView bytes( + reinterpret_cast(words.data()), {2, 16}); + aare::NDArray out({2, 2}); + aare::adc_sar_05_decode64to16(bytes, out.view()); + + CHECK(out(0, 0) == 1); + CHECK(out(0, 1) == 3072); + CHECK(out(1, 0) == 0); + CHECK(out(1, 1) == 7); + + // Matches the overload that takes the words directly + aare::NDArray reference({2, 2}); + aare::adc_sar_05_decode64to16(words.view(), reference.view()); + for (ssize_t i = 0; i < 2; ++i) { + for (ssize_t j = 0; j < 2; ++j) { + CHECK(out(i, j) == reference(i, j)); + } + } +} + +TEST_CASE("Each byte buffer decoder maps to its own bit order") { + // First two bits of each decoder's bit list + struct Case { + void (*decode)(aare::NDView, + aare::NDView); + int bit0; + int bit1; + }; + const Case cases[] = { + {aare::adc_sar_05_06_07_08decode64to16, 29, 17}, + {aare::adc_sar_05_decode64to16, 29, 19}, + {aare::adc_sar_04_decode64to16, 15, 17}, + }; + + for (const auto &c : cases) { + uint64_t word = (1ULL << c.bit0) | (1ULL << c.bit1); + std::vector buffer(8); + std::memcpy(buffer.data(), &word, sizeof(word)); + aare::NDView bytes(buffer.data(), {1, 8}); + aare::NDArray out({1, 1}); + c.decode(bytes, out.view()); + CHECK(out(0, 0) == 3); + } +} + +TEST_CASE("Byte buffer decoding does not require an aligned buffer") { + // Place the word three bytes into the buffer + std::vector buffer(3 + 8, 0); + uint64_t word = (1UL << 29) | (1UL << 19); + std::memcpy(buffer.data() + 3, &word, sizeof(word)); + + aare::NDView bytes(buffer.data() + 3, {1, 8}); + aare::NDArray out({1, 1}); + aare::adc_sar_05_decode64to16(bytes, out.view()); + CHECK(out(0, 0) == 3); +} + +TEST_CASE("Byte buffer decoding rejects invalid shapes") { + std::vector buffer(24, 0); + aare::NDArray out({1, 1}); + + // Row length that is not a multiple of 8 + aare::NDView odd_row(buffer.data(), {2, 12}); + REQUIRE_THROWS_AS(aare::adc_sar_05_decode64to16(odd_row, out.view()), + std::invalid_argument); + + // Output shape that does not match rows x (bytes / 8) + aare::NDView two_words(buffer.data(), {1, 16}); + REQUIRE_THROWS_AS(aare::adc_sar_05_decode64to16(two_words, out.view()), + std::invalid_argument); +} + TEST_CASE("test_apply_custom_weights") { uint16_t input = 1; diff --git a/src/fit/Chi2.hpp b/src/fit/Chi2.hpp new file mode 100644 index 00000000..27fcf1af --- /dev/null +++ b/src/fit/Chi2.hpp @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once +#include "aare/Models.hpp" +#include "aare/NDView.hpp" +#include +#include +#include +#include +#include +#include + +namespace aare { + +namespace func { + +/** + * @brief Chi-squared objective of a 1D model and its analytic derivatives. + * + * Shared by the Minuit2 FCN adapters below. + * + * chi2(par) = sum_i w_i * (y_i - f(x_i; par))^2 + * + * with w_i = 1/sigma_i^2 for weighted fits (points with sigma_i == 0 are + * skipped) and w_i = 1 otherwise. + * + * @tparam Model A model struct that satisfies: + * - static constexpr std::size_t npar; + * - static double eval(double x, const std::vector& par); + * - static void eval_and_grad(double x, const std::vector& par, + * double& f, std::array& g); + * - static bool is_valid(const std::vector& par); + * + * Derivatives, with df_i/dp_k taken from Model::eval_and_grad: + * + * d(chi2)/dp_k = -2 * sum_i w_i * (y_i - f_i) * df_i/dp_k + * d2(chi2)/dp_j dp_k ~= 2 * sum_i w_i * df_i/dp_j * df_i/dp_k + * + * The second line drops the residual-weighted second derivatives of the + * model, which is the Fumili (Gauss-Newton) approximation of the Hessian. + * + * Invalid model parameters do not throw; they return a large penalty (with + * a zero gradient and unit curvature) so the minimizer can remain in control. + * + * @throws std::invalid_argument if par.size() != Model::npar. + */ +template class Chi2Data1D { + public: + static constexpr std::size_t npar = Model::npar; + /** @brief Number of elements of the packed symmetric Hessian. */ + static constexpr std::size_t nhess = npar * (npar + 1) / 2; + static constexpr double invalid_penalty = 1e20; + + /** @brief An empty y_err view selects an unweighted fit. */ + Chi2Data1D(NDView x, NDView y, + NDView y_err = NDView{}) + : x_(x), y_(y), s_(y_err), weighted_(y_err.size() > 0) {} + + double value(const std::vector &par) const { + check_size(par); + if (!Model::is_valid(par)) + return invalid_penalty; + return weighted_ ? value_impl(par) : value_impl(par); + } + + std::vector gradient(const std::vector &par) const { + check_size(par); + std::vector grad(npar, 0.0); + if (!Model::is_valid(par)) + return grad; + if (weighted_) + gradient_impl(par, grad); + else + gradient_impl(par, grad); + return grad; + } + + /** + * @brief Compute chi2, its gradient and the Gauss-Newton Hessian in a + * single pass over the data. + * + * @param grad Receives d(chi2)/dp_k, resized to npar. + * @param hess Receives the packed symmetric Hessian in the layout used by + * ROOT::Minuit2::FumiliFCNBase: element (j, k) with j <= k is + * stored at index j + k * (k + 1) / 2. Resized to nhess. + * @return chi2 at par. + */ + double evaluate_all(const std::vector &par, + std::vector &grad, + std::vector &hess) const { + check_size(par); + grad.assign(npar, 0.0); + hess.assign(nhess, 0.0); + if (!Model::is_valid(par)) { + for (std::size_t k = 0; k < npar; ++k) + hess[k + k * (k + 1) / 2] = 1.0; + return invalid_penalty; + } + return weighted_ ? evaluate_all_impl(par, grad, hess) + : evaluate_all_impl(par, grad, hess); + } + + private: + static void check_size(const std::vector &par) { + if (par.size() != npar) { + throw std::invalid_argument( + "Chi2Data1D: wrong parameter vector size."); + } + } + + template + double value_impl(const std::vector &par) const { + double chi2 = 0.0; + for (ssize_t i = 0; i < x_.size(); ++i) { + if constexpr (Weighted) { + const double si = s_[i]; + if (si == 0.0) + continue; + const double r_i = y_[i] - Model::eval(x_[i], par); + chi2 += (r_i * r_i) / (si * si); + } else { + const double r_i = y_[i] - Model::eval(x_[i], par); + chi2 += r_i * r_i; + } + } + return chi2; + } + + template + void gradient_impl(const std::vector &par, + std::vector &grad) const { + std::array df{}; + double f_i = 0.0; + for (ssize_t i = 0; i < x_.size(); ++i) { + double c = 0.0; + if constexpr (Weighted) { + const double si = s_[i]; + if (si == 0.0) + continue; + Model::eval_and_grad(x_[i], par, f_i, df); + c = -2.0 * (y_[i] - f_i) / (si * si); + } else { + Model::eval_and_grad(x_[i], par, f_i, df); + c = -2.0 * (y_[i] - f_i); + } + for (std::size_t k = 0; k < npar; ++k) + grad[k] += c * df[k]; + } + } + + template + double evaluate_all_impl(const std::vector &par, + std::vector &grad, + std::vector &hess) const { + std::array df{}; + double f_i = 0.0; + double chi2 = 0.0; + for (ssize_t i = 0; i < x_.size(); ++i) { + double w = 1.0; + if constexpr (Weighted) { + const double si = s_[i]; + if (si == 0.0) + continue; + w = 1.0 / (si * si); + } + Model::eval_and_grad(x_[i], par, f_i, df); + const double r_i = y_[i] - f_i; + chi2 += w * r_i * r_i; + const double c = -2.0 * w * r_i; + for (std::size_t k = 0; k < npar; ++k) { + grad[k] += c * df[k]; + const double h_k = 2.0 * w * df[k]; + double *column = hess.data() + k * (k + 1) / 2; + for (std::size_t j = 0; j <= k; ++j) + column[j] += h_k * df[j]; + } + } + return chi2; + } + + NDView x_; + NDView y_; + NDView s_; + bool weighted_; +}; + +/** + * @brief Chi-squared FCN with analytic gradient, used by Migrad. + * + * Providing the analytic gradient avoids the 2*npar extra function + * evaluations per Minuit step that finite differences would need. + */ +template +class Chi2Model1DGrad : public ROOT::Minuit2::FCNGradientBase { + public: + /** @brief An empty y_err view selects an unweighted fit. */ + Chi2Model1DGrad(NDView x, NDView y, + NDView y_err = NDView{}) + : data_(x, y, y_err) {} + + ~Chi2Model1DGrad() override = default; + + double operator()(const std::vector &par) const override { + return data_.value(par); + } + + std::vector + Gradient(const std::vector &par) const override { + return data_.gradient(par); + } + + /** @brief Error definition: 1.0 for chi-squared (delta_chi2 = 1 -> + * 1-sigma). */ + double Up() const override { return 1.0; } + + private: + Chi2Data1D data_; +}; + +/** + * @brief Chi-squared FCN for Fumili. + * + * EvaluateAll fills the value, gradient and Gauss-Newton Hessian cached by + * FumiliFCNBase in one pass over the data, so Fumili never needs numerical + * derivatives. + */ +template +class Chi2Model1DFumili : public ROOT::Minuit2::FumiliFCNBase { + public: + /** @brief An empty y_err view selects an unweighted fit. */ + Chi2Model1DFumili(NDView x, NDView y, + NDView y_err = NDView{}) + : ROOT::Minuit2::FumiliFCNBase(static_cast(Model::npar)), + data_(x, y, y_err) {} + + double operator()(const std::vector &par) const override { + return data_.value(par); + } + + /** @brief Error definition: 1.0 for chi-squared (delta_chi2 = 1 -> + * 1-sigma). */ + double Up() const override { return 1.0; } + + void EvaluateAll(const std::vector &par) override { + SetFCNValue(data_.evaluate_all(par, Gradient(), Hessian())); + } + + private: + Chi2Data1D data_; +}; + +} // namespace func + +} // namespace aare diff --git a/src/fit/DampedGaussNewton.hpp b/src/fit/DampedGaussNewton.hpp new file mode 100644 index 00000000..fdf1670b --- /dev/null +++ b/src/fit/DampedGaussNewton.hpp @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +#include +#include +#include +#include +#include + +namespace aare::detail { + +/** @brief Fixed-size vector and square matrix of the solvers, on the stack. */ +template using Vec = std::array; +template using Mat = std::array, N>; + +/** @brief Outcome of one of the built-in minimizers. */ +struct MinimizerResult { + bool valid = false; + int calls = 0; + int iterations = 0; + double chi2 = 0.0; +}; + +/** + * @brief Cholesky factorisation of the leading m x m block of the symmetric + * positive-definite matrix A. Writes the lower triangle of L with + * A = L L^T. Returns false when the block is not positive definite. + * + * The helpers below clamp m to N. The callers never exceed it, but with the + * bound visible gcc's range analysis no longer reports subscripts of the + * one-parameter instantiations as out of range. + */ +template bool cholesky_factor(int m, const Mat &A, Mat &L) { + m = std::min(m, N); + for (int i = 0; i < m; ++i) { + for (int j = 0; j <= i; ++j) { + double sum = A[i][j]; + for (int k = 0; k < j; ++k) + sum -= L[i][k] * L[j][k]; + if (i == j) { + if (!(sum > 0.0)) + return false; + L[i][i] = std::sqrt(sum); + } else { + L[i][j] = sum / L[j][j]; + } + } + } + return true; +} + +/** @brief Solves L L^T z = b for the leading m x m block of the factor L. */ +template +void cholesky_substitute(int m, const Mat &L, const Vec &b, Vec &z) { + m = std::min(m, N); + Vec t{}; + for (int i = 0; i < m; ++i) { + double sum = b[i]; + for (int k = 0; k < i; ++k) + sum -= L[i][k] * t[k]; + t[i] = sum / L[i][i]; + } + for (int i = m - 1; i >= 0; --i) { + double sum = t[i]; + for (int k = i + 1; k < m; ++k) + sum -= L[k][i] * z[k]; + z[i] = sum / L[i][i]; + } +} + +/** @brief Solves A z = b for the leading m x m block of the SPD matrix A. */ +template +bool cholesky_solve(int m, const Mat &A, const Vec &b, Vec &z) { + Mat L{}; + if (!cholesky_factor(m, A, L)) + return false; + cholesky_substitute(m, L, b, z); + return true; +} + +/** @brief Inverse of the leading m x m block of the SPD matrix A. */ +template bool invert_spd(int m, const Mat &A, Mat &inv) { + m = std::min(m, N); + Vec e{}; + Vec col{}; + for (int j = 0; j < m; ++j) { + for (int i = 0; i < m; ++i) + e[i] = (i == j) ? 1.0 : 0.0; + if (!cholesky_solve(m, A, e, col)) + return false; + for (int i = 0; i < m; ++i) + inv[i][j] = col[i]; + } + return true; +} + +/** + * @brief Gauss-Newton parameter errors sqrt(diag(H^-1)) over the parameters + * that are not skipped, where H is the upper triangle of the Gauss-Newton + * Hessian of chi2 / 2 over all N parameters. Skipped parameters (fixed, or + * frozen on a limit) report 0. Returns false, leaving all errors 0, when the + * block cannot be inverted. + */ +template +bool gauss_newton_errors(const Mat &H, const std::array &skip, + double *err) { + std::fill(err, err + N, 0.0); + Mat A{}; + Mat inv{}; + int m = 0; + for (int a = 0; a < N; ++a) { + if (skip[a]) + continue; + int mb = 0; + for (int c = 0; c < N; ++c) { + if (skip[c]) + continue; + A[m][mb++] = a <= c ? H[a][c] : H[c][a]; + } + ++m; + } + if (m == 0 || !invert_spd(m, A, inv)) + return false; + int i = 0; + for (int a = 0; a < N; ++a) { + if (skip[a]) + continue; + err[a] = std::sqrt(std::max(inv[i][i], 0.0)); + ++i; + } + return true; +} + +/** + * @brief Damped Gauss-Newton iteration shared by the built-in minimizers. + * + * The problem is described by an evaluator called as + * `eval(const Vec &p, Normal &out)`: at the point p (all NP parameters, + * fixed ones at their values) it fills out.F = chi2 / 2, the gradient + * out.g = dF/dp and the upper triangle of the Gauss-Newton Hessian out.H, + * both over all NP parameters, and returns false when the point is not + * admissible. LevenbergMarquardt evaluates residuals and the Jacobian of + * the full model, VariableProjection the projected problem of the nonlinear + * parameters. The driver extracts the free parameters and implements + * + * - Marquardt damping u * diag(H), starting at u = damping0, with the + * update rule of Nielsen (Madsen, Nielsen and Tingleff, Methods for + * Non-Linear Least Squares Problems, 2004). The damping also bounds the + * accuracy of the last accepted step, so a solver that converges in few + * iterations should start with a small damping. + * - Limits: a trial step that leaves the allowed box is reflected at the + * limit. The step truncated at the limit is evaluated as well and the + * better of the two is kept. A parameter sitting on a limit whose gradient + * points outward is frozen for that iteration. + * - Convergence in Minuit's convention: the iteration stops when the + * estimated distance to the minimum of chi2 (EDM, from the Gauss-Newton + * Hessian) drops below 0.002 * tolerance. A start point that passes + * already still takes one undamped step, see converged(). When the damped + * step becomes negligible for every parameter before that, the iteration + * has stalled and the fit is valid only if the EDM test passes at that + * point. + * - max_calls bounds the number of evaluations: the initial point and every + * trial point count, so an iteration costs one evaluation, or two when a + * limit is crossed. + * - With reject_degenerate, a trial point at which the model has lost its + * sensitivity to a free parameter (its Hessian diagonal collapsed by more + * than eight orders of magnitude relative to the current point or to the + * start point) is rejected like an inadmissible point, so the damping + * grows and a shorter step is tried instead. + * - With polish, a converged fit takes one more undamped Gauss-Newton step + * and keeps it when it lowers chi2, see polish_step(). + * + * After fit() the final point, its Normal and the frozen parameters remain + * available for the error computation. + */ +template class DampedGaussNewton { + public: + static_assert(NP >= 1 && NP <= 16, + "DampedGaussNewton keeps NP x NP matrices on the stack"); + + struct Normal { + double F = 0.0; // chi2 / 2 + Vec g{}; // dF/dp + Mat H{}; // Gauss-Newton Hessian, upper triangle + std::array aux{}; // evaluator payload, kept with the point + }; + + struct Options { + double tolerance = 0.5; // EDM tolerance, Minuit's convention + int max_calls = 100; // budget of evaluations + double damping0 = 0.1; // initial Marquardt damping + bool reject_degenerate = false; + bool polish = false; + }; + + template + MinimizerResult fit(Eval &eval, const Vec &start, const Vec &lower, + const Vec &upper, const std::array &fixed, + const Options &opt) { + nfree_ = 0; + accepted_ = 0; + for (int k = 0; k < NP; ++k) { + lower_[k] = lower[k]; + upper_[k] = upper[k]; + p_[k] = start[k]; + fixed_[k] = fixed[k]; + free_pos_[k] = -1; + if (!fixed[k]) { + free_pos_[k] = nfree_; + free_[nfree_++] = k; + } + } + for (int a = 0; a < nfree_; ++a) { + q_[a] = p_[free_[a]]; + active_[a] = false; + } + + MinimizerResult res; + if (!evaluate(eval, q_, cur_)) + return res; + res.calls = 1; + for (int a = 0; a < nfree_; ++a) + start_diag_[a] = cur_.H[free_[a]][free_[a]]; + const double edm_target = 0.002 * opt.tolerance; + const int max_calls = opt.max_calls; + const bool reject_degenerate = opt.reject_degenerate; + double u = opt.damping0; // damping relative to diag(H) + double nu = 2.0; + + for (;; ++res.iterations) { + extract_free(); + mark_active(); + if (gmax_ < 1e-12) { + edm(); // the undamped step for converged() + res.valid = + converged(eval, res, max_calls, reject_degenerate, false); + break; + } + // Solve for the damped step first. The solve yields + // g^T (H + u D)^-1 g, a lower bound on the EDM, so the undamped + // factorisation behind edm() is only needed when that bound + // leaves room for convergence. + const bool have_step = solve_step(u); + if (!have_step || bound_ < edm_margin * edm_target) { + const double e = edm(); + if (e < edm_target) { + // A point already deep inside the tolerance needs no + // polishing. + res.valid = + converged(eval, res, max_calls, reject_degenerate, + opt.polish && e >= 0.01 * edm_target); + break; + } + } + if (res.calls >= max_calls) + break; + if (!have_step) { + u *= nu; + nu *= 2.0; + if (u > 1e32) + break; + continue; + } + // A stalled iteration is not a minimum: repeated rejections + // shrink the damped step until it is negligible anywhere. The + // test looks at the step before it is reflected or truncated at + // a limit, so a step that a limit folds back onto the current + // point still gets its truncated alternative evaluated. + if (negligible_step()) { + res.valid = edm() < edm_target; + break; + } + bool moved = false; + bool crossed = false; + for (int a = 0; a < nfree_; ++a) { + const int k = free_[a]; + const double v = q_[a] + delta_[a]; + const double truncated = std::clamp(v, lower_[k], upper_[k]); + double reflected = v; + if (truncated != v) { + crossed = true; + if (v < lower_[k]) + reflected = 2.0 * lower_[k] - v; + if (v > upper_[k]) + reflected = 2.0 * upper_[k] - v; + reflected = std::clamp(reflected, lower_[k], upper_[k]); + } + q_new_[a] = reflected; + q_alt_[a] = truncated; + moved = moved || reflected != q_[a]; + } + // A reflection that lands on the current point needs no + // evaluation; the truncated step below is the only candidate. + bool ok = false; + if (moved) { + ++res.calls; + ok = evaluate(eval, q_new_, trial_) && + !(reject_degenerate && degenerate(trial_)); + } + if (crossed && res.calls < max_calls) { + // Also try the step truncated at the limit, keep the better. + ++res.calls; + const bool ok_alt = evaluate(eval, q_alt_, alt_) && + !(reject_degenerate && degenerate(alt_)); + if (ok_alt && (!ok || alt_.F < trial_.F)) { + q_new_ = q_alt_; + std::swap(trial_, alt_); + ok = true; + } + } + const double rho = ok ? (cur_.F - trial_.F) / predicted_ : -1.0; + if (rho > 0.0) { + // The trial becomes the current point. + q_ = q_new_; + std::swap(cur_, trial_); + for (int a = 0; a < nfree_; ++a) + p_[free_[a]] = q_[a]; + ++accepted_; + const double t = 2.0 * rho - 1.0; + u *= std::max(1.0 / 3.0, 1.0 - t * t * t); + nu = 2.0; + } else { + u *= nu; + nu *= 2.0; + if (u > 1e32) + break; + } + } + if (!res.valid) + return res; + + // p_ tracks the last evaluated point, which may be a rejected trial. + for (int a = 0; a < nfree_; ++a) + p_[free_[a]] = q_[a]; + res.chi2 = 2.0 * cur_.F; + return res; + } + + /** @brief The final point, all NP parameters. */ + const Vec &point() const { return p_; } + + /** @brief F, gradient and Hessian at the final point. */ + const Normal &normal() const { return cur_; } + + /** + * @brief True for a parameter without an error: fixed, frozen on a limit, + * or ending exactly on one. The frozen flag alone misses the last case, + * because it needs the gradient to point outward and that gradient is + * zero when the unconstrained minimum coincides with the limit. + */ + bool skipped(int k) const { + return fixed_[k] || (free_pos_[k] >= 0 && active_[free_pos_[k]]) || + p_[k] <= lower_[k] || p_[k] >= upper_[k]; + } + + /** @brief Gauss-Newton errors at the final point, see + * gauss_newton_errors(). */ + void errors(double *err_out) const { + std::array skip; + for (int k = 0; k < NP; ++k) + skip[k] = skipped(k); + gauss_newton_errors(cur_.H, skip, err_out); + } + + private: + template + bool evaluate(Eval &eval, const Vec &q, Normal &out) { + for (int a = 0; a < nfree_; ++a) + p_[free_[a]] = q[a]; + return eval(p_, out); + } + + // Gradient and Hessian of the free parameters at the current point. + void extract_free() { + gmax_ = 0.0; + for (int a = 0; a < nfree_; ++a) { + const int ka = free_[a]; + g_[a] = cur_.g[ka]; + for (int b = 0; b < nfree_; ++b) { + const int kb = free_[b]; + jtj_[a][b] = ka <= kb ? cur_.H[ka][kb] : cur_.H[kb][ka]; + } + gmax_ = std::max(gmax_, std::abs(g_[a])); + } + } + + // A parameter sitting on a limit with the descent direction pointing + // outward is frozen for this iteration. Position and gradient are both + // taken at the current point q_, so a parameter stays frozen at the end + // only where the constrained minimum lies on its limit. An earlier + // version tested the last evaluated point p_, a rejected trial, so that a + // trial that came off a limit unfroze the parameter: testing q_ had left + // more FallingScurve and GaussianChargeSharing fits stuck with the width + // on its lower limit and a higher chi2. But pairing a trial position with + // the current gradient could freeze an interior parameter and stop the + // fit there. The width no longer reaches its limit that way, because + // LevenbergMarquardt now rejects degenerate trial points as + // VariableProjection does, see degenerate(). + void mark_active() { + for (int a = 0; a < nfree_; ++a) { + const int k = free_[a]; + active_[a] = (q_[a] <= lower_[k] && g_[a] > 0.0) || + (q_[a] >= upper_[k] && g_[a] < 0.0); + } + } + + // True when the damped step is negligible for every parameter that is + // not frozen: below 1e-8 of its value plus its own scale 1/sqrt(H_aa), + // the parameter change that raises chi2 by about one. Judging each + // parameter on its own scale keeps a large parameter (an amplitude of + // 1e10, a peak position at 1e9) from hiding the steps of the others. + bool negligible_step() const { + for (int a = 0; a < nfree_; ++a) { + if (active_[a]) + continue; + const double scale = + std::abs(q_[a]) + 1.0 / std::sqrt(std::max(jtj_[a][a], 1e-300)); + if (!(std::abs(delta_[a]) <= 1e-8 * scale)) + return false; + } + return true; + } + + // A trial point where the Hessian diagonal of a free parameter collapsed + // relative to the current point or to the start point: the model no + // longer responds to that parameter there, so the iteration could stop + // at a spurious stationary point (a step function with the width driven + // to zero, for example). The start point catches a collapse spread over + // many accepted steps, each of which passes the test against its + // predecessor. + bool degenerate(const Normal &trial) const { + for (int a = 0; a < nfree_; ++a) { + if (active_[a]) + continue; + const int k = free_[a]; + const double ref = std::max(cur_.H[k][k], start_diag_[a]); + if (!(trial.H[k][k] >= 1e-8 * ref)) + return true; + } + return false; + } + + // Minuit's estimated distance to the minimum of chi2, using the + // Gauss-Newton Hessian: g^T H^-1 g with g the gradient of chi2/2. + // Also keeps the undamped Gauss-Newton step for polish_step(). + double edm() { + Mat A{}; + Vec b{}; + Vec z{}; + const int m = pack(A, b, 0.0); + for (int a = 0; a < nfree_; ++a) + gn_step_[a] = 0.0; + if (m == 0) + return 0.0; + if (!cholesky_solve(m, A, b, z)) + return std::numeric_limits::infinity(); + double e = 0.0; + int i = 0; + for (int a = 0; a < nfree_; ++a) { + if (active_[a]) + continue; + gn_step_[a] = -z[i]; + e += b[i] * z[i]; + ++i; + } + return e; + } + + // The convergence test passed. A point that passes it before any step + // was accepted may merely lie within the tolerance of chi2's scale, as + // the start estimate of an unweighted fit of data of small magnitude + // does, so it still takes the undamped step of polish_step(), as Migrad + // always iterates at least once. Without the budget for that step the + // fit has not converged. With polish, a converged fit takes the same + // step when the budget allows. Returns whether the fit is valid. + template + bool converged(Eval &eval, MinimizerResult &res, int max_calls, + bool reject_degenerate, bool polish) { + const bool first = accepted_ == 0 && nfree_ > 0; + if (!first && !polish) + return true; + if (res.calls >= max_calls) + return !first; + polish_step(eval, res, reject_degenerate); + return true; + } + + // One undamped Gauss-Newton step from a point that passed the EDM test, + // kept when it lowers F. The test stops the iteration within the + // tolerance, but a solver that converges in few iterations stops while + // the damping is still noticeable, so its last accepted step fell short + // of the minimum by about the damping factor. The extra evaluation lands + // at the minimum to the accuracy of a full Gauss-Newton step. + template + void polish_step(Eval &eval, MinimizerResult &res, bool reject_degenerate) { + for (int a = 0; a < nfree_; ++a) { + const int k = free_[a]; + q_new_[a] = std::clamp(q_[a] + gn_step_[a], lower_[k], upper_[k]); + } + ++res.calls; + const bool ok = evaluate(eval, q_new_, trial_) && + !(reject_degenerate && degenerate(trial_)) && + trial_.F < cur_.F; + if (ok) { + q_ = q_new_; + std::swap(cur_, trial_); + extract_free(); + } + for (int a = 0; a < nfree_; ++a) + p_[free_[a]] = q_[a]; + if (ok) + mark_active(); + } + + // Copy the non-frozen block of H (+ u * diag) and g into A and b. + int pack(Mat &A, Vec &b, double u) const { + double mean_diag = 0.0; + for (int a = 0; a < nfree_; ++a) + mean_diag += jtj_[a][a]; + mean_diag = std::max(mean_diag / std::max(nfree_, 1), 1e-300); + int m = 0; + for (int a = 0; a < nfree_; ++a) { + if (active_[a]) + continue; + int mb = 0; + for (int c = 0; c < nfree_; ++c) { + if (active_[c]) + continue; + A[m][mb++] = jtj_[a][c]; + } + A[m][m] += u * std::max(jtj_[a][a], 1e-10 * mean_diag); + b[m] = g_[a]; + ++m; + } + return m; + } + + // Damped Gauss-Newton step (H + u D) delta = -g for the non-frozen + // parameters, the predicted decrease of F for that step, and + // bound_ = g^T (H + u D)^-1 g, which is at most the EDM because u D is + // positive semi-definite. + bool solve_step(double u) { + Mat A{}; + Vec b{}; + Vec z{}; + const int m = pack(A, b, u); + for (int a = 0; a < nfree_; ++a) + delta_[a] = 0.0; + predicted_ = 0.0; + bound_ = 0.0; + if (m == 0) + return true; + if (!cholesky_solve(m, A, b, z)) + return false; + int i = 0; + for (int a = 0; a < nfree_; ++a) { + if (active_[a]) + continue; + delta_[a] = -z[i]; + bound_ += b[i] * z[i]; + predicted_ += + 0.5 * delta_[a] * + (u * std::max(jtj_[a][a], 1e-300) * delta_[a] - g_[a]); + ++i; + } + return predicted_ > 0.0; + } + + int nfree_ = 0; + int accepted_ = 0; // trial points accepted in this fit + std::array free_{}; + std::array free_pos_{}; + std::array fixed_{}; + std::array active_{}; + Vec lower_{}; + Vec upper_{}; + Vec p_{}; + Vec q_{}; + Vec q_new_{}; + Vec q_alt_{}; + Vec g_{}; + Vec delta_{}; + Vec gn_step_{}; + Vec start_diag_{}; // Hessian diagonal at the start point + Mat jtj_{}; + // The two solves that bound the EDM differ by rounding only when the + // damping is negligible; the margin keeps the stop decision exact. + static constexpr double edm_margin = 1.0 + 1e-6; + double gmax_ = 0.0; + double predicted_ = 0.0; + double bound_ = 0.0; + Normal cur_; + Normal trial_; + Normal alt_; +}; + +} // namespace aare::detail diff --git a/src/fit/Fit.cpp b/src/fit/Fit.cpp new file mode 100644 index 00000000..05dfa5d8 --- /dev/null +++ b/src/fit/Fit.cpp @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "aare/Fit.hpp" +#include "FitHelpers.hpp" +#include "FitMinuit2.hpp" +#include "LevenbergMarquardt.hpp" +#include "VariableProjection.hpp" +#include "aare/Models.hpp" +#include "aare/utils/par.hpp" +#include "aare/utils/task.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aare { +// ============================================================================ +// FitModel — method definitions +// ============================================================================ + +template +FitModel::FitModel(unsigned int strategy, unsigned int max_calls, + double tolerance, bool compute_errors, + Minimizer minimizer) + : strategy_(strategy), max_calls_(max_calls), tolerance_(tolerance), + compute_errors_(compute_errors), minimizer_(minimizer) { + for (std::size_t i = 0; i < npar; ++i) { + lower_[i] = Model::param_info[i].default_lo; + upper_[i] = Model::param_info[i].default_hi; + } +} + +template +void FitModel::check_index(unsigned int idx) const { + if (idx >= npar) + throw std::out_of_range("FitModel: parameter index " + + std::to_string(idx) + " out of range"); +} + +template +unsigned int FitModel::checked_index(const std::string &name) const { + for (std::size_t i = 0; i < npar; ++i) { + if (name == Model::param_info[i].name) + return static_cast(i); + } + throw std::runtime_error("FitModel: unknown parameter name '" + name + "'"); +} + +template +void FitModel::SetParLimits(unsigned int idx, double lo, double hi) { + check_index(idx); + if (!(lo < hi)) + throw std::runtime_error( + "FitModel: lower limit must be below upper limit for parameter '" + + GetParName(idx) + "'"); + lower_[idx] = lo; + upper_[idx] = hi; +} + +template +void FitModel::FixParameter(unsigned int idx, double val) { + SetParameter(idx, val); + user_fixed_[idx] = true; +} + +template +void FitModel::ReleaseParameter(unsigned int idx) { + check_index(idx); + user_fixed_[idx] = false; +} + +template +void FitModel::ReleaseParameter(const std::string &name) { + ReleaseParameter(checked_index(name)); +} + +template +void FitModel::SetParameter(unsigned int idx, double val) { + check_index(idx); + value_[idx] = val; + user_start_[idx] = true; +} + +template +void FitModel::SetParameter(const std::string &name, double val) { + SetParameter(checked_index(name), val); +} + +template +void FitModel::FixParameter(const std::string &name, double val) { + FixParameter(checked_index(name), val); +} + +template +void FitModel::SetParLimits(const std::string &name, double lo, + double hi) { + SetParLimits(checked_index(name), lo, hi); +} + +template +std::string FitModel::GetParName(unsigned int idx) const { + check_index(idx); + return Model::param_info[idx].name; +} + +template +std::vector FitModel::GetParNames() const { + std::vector names; + names.reserve(npar); + for (std::size_t i = 0; i < npar; ++i) + names.emplace_back(Model::param_info[i].name); + return names; +} + +// ============================================================================ +// fit_pixel / fit_3d +// ============================================================================ + +namespace detail { + +struct NotSeparable {}; + +/** + * @brief Fits pixels with the minimizer selected by the model. + * + * Keep one instance per thread: the buffers of the built-in minimizers are + * reused between pixels so that a data cube is fitted without per-pixel + * allocations. The Minuit2 minimizers create their state per pixel. + */ +template class PixelFitter { + public: + /** + * @brief Fit one pixel. + * + * @param par_out Receives npar fitted values; zeros on failure. + * @param err_out Receives npar errors when non-null; zeros on failure. + * @param chi2 Receives the chi-squared at the minimum; 0 on failure. + * @return true for a valid minimum. + */ + bool fit(const FitModel &model, NDView x, + NDView y, NDView y_err, double *par_out, + double *err_out, double &chi2) { + // Samples with a zero error are ignored. They are dropped here, + // before the start estimate and the minimizer: a masked sample may + // hold NaN, and a zero weight does not remove a NaN from a sum. + if (y_err.size() > 0 && + std::find(y_err.begin(), y_err.end(), 0.0) != y_err.end()) { + keep_x_.clear(); + keep_y_.clear(); + keep_err_.clear(); + for (ssize_t i = 0; i < y.size(); ++i) { + if (y_err[i] == 0.0) + continue; + keep_x_.push_back(x[i]); + keep_y_.push_back(y[i]); + keep_err_.push_back(y_err[i]); + } + const auto n = static_cast(keep_x_.size()); + x = NDView(keep_x_.data(), {n}); + y = NDView(keep_y_.data(), {n}); + y_err = NDView(keep_err_.data(), {n}); + } + // Fewer samples than free parameters leave the fit undetermined, + // and the start estimates read at least two samples. + ssize_t n_free = 0; + for (std::size_t k = 0; k < Model::npar; ++k) + n_free += model.is_user_fixed(static_cast(k)) ? 0 : 1; + if (y.size() < std::max(n_free, 2)) + return fail(par_out, err_out, chi2); + + const auto start = start_values(model, x, y); + + if (model.minimizer() == Minimizer::VarPro) { + // A model without the separable structure, a pixel that does + // not converge and a pixel whose linear solution violates a + // limit fall back to the full solver. + if constexpr (model::is_separable::value) { + const auto res = + vp_.fit(model, x, y, y_err, start, par_out, err_out); + if (res.valid && finite(par_out, err_out, res.chi2)) { + chi2 = res.chi2; + return true; + } + } + return fit_lm(model, x, y, y_err, start, par_out, err_out, chi2); + } + if (model.minimizer() == Minimizer::LevenbergMarquardt) + return fit_lm(model, x, y, y_err, start, par_out, err_out, chi2); + return fit_pixel_minuit2(model, x, y, y_err, start, par_out, err_out, + chi2); + } + + private: + // The built-in solvers do not guard every operation against NaN or + // infinity, so a result is checked before it is reported as a valid fit. + static bool finite(const double *par, const double *err, double chi2) { + bool ok = std::isfinite(chi2); + for (std::size_t k = 0; k < Model::npar; ++k) + ok = ok && std::isfinite(par[k]) && (!err || std::isfinite(err[k])); + return ok; + } + + bool fit_lm(const FitModel &model, NDView x, + NDView y, NDView y_err, + const std::array &start, double *par_out, + double *err_out, double &chi2) { + const auto res = lm_.fit(model, x, y, y_err, start, par_out, err_out); + if (res.valid && finite(par_out, err_out, res.chi2)) { + chi2 = res.chi2; + return true; + } + return fail(par_out, err_out, chi2); + } + + // A failed fit reports zeros. + static bool fail(double *par_out, double *err_out, double &chi2) { + std::fill(par_out, par_out + Model::npar, 0.0); + if (err_out) + std::fill(err_out, err_out + Model::npar, 0.0); + chi2 = 0.0; + return false; + } + + // The samples of the current pixel that have a nonzero error, see fit(). + std::vector keep_x_; + std::vector keep_y_; + std::vector keep_err_; + LevenbergMarquardt lm_; + std::conditional_t::value, + VariableProjection, NotSeparable> + vp_; +}; + +} // namespace detail + +template +NDArray fit_pixel(const FitModel &model, NDView x, + NDView y, NDView y_err) { + constexpr std::size_t npar = Model::npar; + detail::check_fit_pixel_shapes(x, y, y_err); + const bool want_errors = model.compute_errors(); + const auto result_size = + static_cast(want_errors ? (2 * npar + 1) : (npar + 1)); + NDArray result({result_size}, 0.0); + + detail::PixelFitter fitter; + double chi2 = 0.0; + double *par = result.data(); + double *err = want_errors ? result.data() + npar : nullptr; + if (!fitter.fit(model, x, y, y_err, par, err, chi2)) + return result; // all zeros + + result(result_size - 1) = chi2; + return result; +} + +template +NDArray fit_pixel(const FitModel &model, NDView x, + NDView y) { + return fit_pixel(model, x, y, NDView{}); +} + +template +void fit_3d(const FitModel &model, NDView x, + NDView y, NDView y_err, + NDView par_out, NDView err_out, + NDView chi2_out, int n_threads) { + // static so the lambda below can use it as an array size without + // capturing it, which MSVC otherwise rejects + static constexpr std::size_t npar = Model::npar; + detail::check_fit_3d_shapes(x, y, y_err, par_out, err_out, chi2_out); + + const bool has_errors = (y_err.size() > 0); + const bool want_par_errors = (err_out.size() > 0) && model.compute_errors(); + + auto process = [&](ssize_t first_row, ssize_t last_row) { + detail::PixelFitter fitter; // buffers reused within the task + std::array par{}; + std::array err{}; + for (ssize_t row = first_row; row < last_row; row++) { + for (ssize_t col = 0; col < y.shape(1); col++) { + NDView values(&y(row, col, 0), {y.shape(2)}); + NDView errors = + has_errors ? NDView(&y_err(row, col, 0), + {y_err.shape(2)}) + : NDView{}; + + double chi2 = 0.0; + fitter.fit(model, x, values, errors, par.data(), + want_par_errors ? err.data() : nullptr, chi2); + + for (std::size_t k = 0; k < npar; ++k) + par_out(row, col, static_cast(k)) = par[k]; + if (want_par_errors) { + for (std::size_t k = 0; k < npar; ++k) + err_out(row, col, static_cast(k)) = err[k]; + } + chi2_out(row, col) = chi2; + } + } + }; + + auto tasks = split_task(0, static_cast(y.shape(0)), n_threads); + RunInParallel(process, tasks); +} + +// ============================================================================ +// Explicit instantiations for all supported model types +// ============================================================================ + +// NOLINTBEGIN +#define AARE_INSTANTIATE_FIT(Model) \ + template class FitModel; \ + template NDArray fit_pixel( \ + const FitModel &, NDView, NDView, \ + NDView); \ + template NDArray fit_pixel( \ + const FitModel &, NDView, NDView); \ + template void fit_3d(const FitModel &, NDView, \ + NDView, NDView, \ + NDView, NDView, \ + NDView, int); + +AARE_INSTANTIATE_FIT(model::Gaussian) +AARE_INSTANTIATE_FIT(model::GaussianErfcPlateau) +AARE_INSTANTIATE_FIT(model::GaussianChargeSharing) +AARE_INSTANTIATE_FIT(model::GaussianChargeSharingKb) +AARE_INSTANTIATE_FIT(model::Pol1) +AARE_INSTANTIATE_FIT(model::Pol2) +AARE_INSTANTIATE_FIT(model::RisingScurve) +AARE_INSTANTIATE_FIT(model::FallingScurve) + +#undef AARE_INSTANTIATE_FIT +// NOLINTEND + +} // namespace aare diff --git a/src/fit/Fit.test.cpp b/src/fit/Fit.test.cpp new file mode 100644 index 00000000..1edadce6 --- /dev/null +++ b/src/fit/Fit.test.cpp @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "aare/Fit.hpp" +#include "aare/FitModel.hpp" +#include "aare/Models.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using aare::Minimizer; +using aare::NDArray; +using aare::NDView; +using aare::ssize_t; +using Catch::Approx; + +namespace { + +constexpr ssize_t n_points = 61; + +struct Data { + NDArray x; + NDArray y; + NDArray y_err; +}; + +void fill_gaussian(NDArray &x, NDArray &y, + double amplitude, double mean, double sigma) { + for (ssize_t i = 0; i < n_points; ++i) { + x(i) = -6.0 + 0.2 * static_cast(i); + const double z = (x(i) - mean) / sigma; + y(i) = amplitude * std::exp(-0.5 * z * z); + } +} + +// Gaussian with A = 120, mu = 0.8, sigma = 1.3 and unit errors, optionally +// with deterministic pseudo-noise. +Data gaussian_data(bool add_noise = false) { + Data d{NDArray({n_points}), NDArray({n_points}), + NDArray({n_points}, 1.0)}; + fill_gaussian(d.x, d.y, 120.0, 0.8, 1.3); + if (add_noise) { + for (ssize_t i = 0; i < n_points; ++i) + d.y(i) += 3.0 * std::sin(7.3 * static_cast(i)); + } + return d; +} + +// S-curve on 100 points in [0, 100] with unit errors, optionally with +// deterministic pseudo-noise and errors of 30. +template +Data scurve_data(const std::vector &par, bool add_noise = false) { + constexpr ssize_t n = 100; + Data d{NDArray({n}), NDArray({n}), + NDArray({n}, 1.0)}; + for (ssize_t i = 0; i < n; ++i) { + d.x(i) = 100.0 * static_cast(i) / static_cast(n - 1); + d.y(i) = Model::eval(d.x(i), par); + if (add_noise) { + d.y(i) += 30.0 * std::sin(7.3 * static_cast(i)); + d.y_err(i) = 30.0; + } + } + return d; +} + +const char *name(Minimizer m) { + switch (m) { + case Minimizer::Migrad: + return "Migrad"; + case Minimizer::Fumili: + return "Fumili"; + case Minimizer::LevenbergMarquardt: + return "LevenbergMarquardt"; + case Minimizer::VarPro: + return "VarPro"; + } + return "unknown"; +} + +bool uses_minuit2(Minimizer m) { + return m != Minimizer::LevenbergMarquardt && m != Minimizer::VarPro; +} + +template +aare::FitModel make_model(Minimizer minimizer, bool errors = false, + unsigned int max_calls = 100) { + return aare::FitModel(0, max_calls, 0.5, errors, minimizer); +} + +// Minuit2 reaches an active limit through its parameter transformation; +// the built-in solver lands on it exactly. +double limit_margin(Minimizer m) { return uses_minuit2(m) ? 1e-6 : 1e-9; } + +// Relative agreement with the true parameters: the Minuit2 minimizers stop +// at Minuit's EDM tolerance, the built-in solver lands tighter. +double truth_tolerance(Minimizer m) { return uses_minuit2(m) ? 1e-3 : 1e-4; } + +// Hesse errors at a point forced away from the free minimum differ from the +// Gauss-Newton estimates of Fumili and the built-in solver. +double misfit_error_tolerance(Minimizer m) { + return uses_minuit2(m) ? 0.1 : 5e-2; +} + +template +bool on_limit(const aare::FitModel &model, unsigned int idx, + double value) { + const double margin = 1e-6 * std::max(1.0, std::abs(value)); + return std::abs(value - model.lower_limit(idx)) <= margin || + std::abs(value - model.upper_limit(idx)) <= margin; +} + +/** + * Run one configuration through every minimizer and compare Fumili and the + * built-in solver against Migrad: parameters and chi2 within tol (relative), + * unless the candidate found a clearly lower chi2. Errors are compared for + * free parameters that do not end on a limit with tolerance err_tol. + */ +template +void check_minimizers_agree(const std::string &label, Data d, bool weighted, + bool errors, unsigned int max_calls, double tol, + Setup setup, double err_tol = 0.0) { + if (err_tol == 0.0) + err_tol = 10.0 * tol; + constexpr std::size_t npar = Model::npar; + + const auto run = [&](Minimizer minimizer) { + aare::FitModel model(0, max_calls, 0.5, errors, minimizer); + setup(model); + return weighted ? aare::fit_pixel(model, d.x.view(), d.y.view(), + d.y_err.view()) + : aare::fit_pixel(model, d.x.view(), d.y.view()); + }; + aare::FitModel reference_model(0, max_calls, 0.5, errors, + Minimizer::Migrad); + setup(reference_model); + const auto ref = run(Minimizer::Migrad); + + for (const auto candidate : + {Minimizer::Fumili, Minimizer::LevenbergMarquardt, + Minimizer::VarPro}) { + INFO(label << ": " << name(candidate) << " vs Migrad"); + const auto res = run(candidate); + REQUIRE(res.size() == ref.size()); + + const auto ichi = static_cast(errors ? 2 * npar : npar); + const double chi_ref = ref(ichi); + const double chi_res = res(ichi); + if (chi_ref > 0.0 && chi_res > 0.0 && + chi_ref - chi_res > tol * std::max(chi_ref, 1.0)) { + WARN(label + ": " + name(candidate) + " found a lower chi2 (" + + std::to_string(chi_res) + " vs " + std::to_string(chi_ref) + + ")"); + continue; + } + double scale = 0.0; + for (std::size_t k = 0; k < npar; ++k) + scale = std::max(scale, std::abs(ref(static_cast(k)))); + for (std::size_t k = 0; k < npar; ++k) { + const auto i = static_cast(k); + const double denom = + std::max({std::abs(ref(i)), 1e-2 * scale, 1e-12}); + INFO("parameter " + << reference_model.GetParName(static_cast(k))); + CHECK(std::abs(res(i) - ref(i)) / denom <= tol); + } + CHECK(std::abs(chi_res - chi_ref) / std::max(std::abs(chi_ref), 1.0) <= + tol); + if (!errors) + continue; + for (std::size_t k = 0; k < npar; ++k) { + const auto idx = static_cast(k); + const auto i = static_cast(k); + if (reference_model.is_user_fixed(idx) || + on_limit(reference_model, idx, ref(i))) + continue; + const auto ie = static_cast(npar + k); + INFO("error of " << reference_model.GetParName(idx)); + CHECK(std::abs(res(ie) - ref(ie)) / + std::max(std::abs(ref(ie)), 1e-12) <= + err_tol); + } + } +} + +} // namespace + +TEST_CASE("Fit unweighted and weighted noise-free Gaussian data", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + + const auto unweighted_model = make_model(minimizer); + const auto unweighted = + aare::fit_pixel(unweighted_model, d.x.view(), d.y.view()); + + REQUIRE(unweighted.size() == 4); + CHECK(unweighted(0) == Approx(120.0).epsilon(1e-4).margin(1e-6)); + CHECK(unweighted(1) == Approx(0.8).epsilon(1e-4).margin(1e-6)); + CHECK(unweighted(2) == Approx(1.3).epsilon(1e-4).margin(1e-6)); + CHECK(unweighted(3) == Approx(0.0).margin(1e-3)); + + const auto weighted_model = + make_model(minimizer, true); + const auto weighted = + aare::fit_pixel(weighted_model, d.x.view(), d.y.view(), d.y_err.view()); + + REQUIRE(weighted.size() == 7); + CHECK(weighted(0) == Approx(120.0).epsilon(1e-4).margin(1e-6)); + CHECK(weighted(1) == Approx(0.8).epsilon(1e-4).margin(1e-6)); + CHECK(weighted(2) == Approx(1.3).epsilon(1e-4).margin(1e-6)); + // At zero residual the Hesse, Fumili and Gauss-Newton errors coincide. + CHECK(weighted(3) == Approx(0.361).epsilon(2e-2)); + CHECK(weighted(4) == Approx(0.00451).epsilon(2e-2)); + CHECK(weighted(5) == Approx(0.00451).epsilon(2e-2)); + CHECK(weighted(6) == Approx(0.0).margin(1e-3)); +} + +TEST_CASE("Fit weighted Pol1 errors match the analytic formula", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t n = 20; + NDArray x({n}); + NDArray y({n}); + NDArray y_err({n}); + double S = 0.0, Sx = 0.0, Sxx = 0.0; + for (ssize_t i = 0; i < n; ++i) { + x(i) = static_cast(i); + y(i) = 2.0 + 0.5 * x(i); + y_err(i) = 1.0 + 0.1 * static_cast(i); + const double w = 1.0 / (y_err(i) * y_err(i)); + S += w; + Sx += w * x(i); + Sxx += w * x(i) * x(i); + } + const double delta = S * Sxx - Sx * Sx; + + const auto model = make_model(minimizer, true); + const auto res = aare::fit_pixel(model, x.view(), y.view(), y_err.view()); + + REQUIRE(res.size() == 5); + CHECK(res(0) == Approx(2.0).epsilon(1e-6)); + CHECK(res(1) == Approx(0.5).epsilon(1e-6)); + CHECK(res(2) == Approx(std::sqrt(Sxx / delta)).epsilon(1e-6)); + CHECK(res(3) == Approx(std::sqrt(S / delta)).epsilon(1e-6)); + CHECK(res(4) == Approx(0.0).margin(1e-8)); +} + +TEST_CASE("Fit with a fixed parameter", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + + SECTION("fixed at a wrong value") { + auto model = make_model(minimizer, true); + model.FixParameter("sigma", 1.5); + REQUIRE(model.is_user_fixed(2)); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + REQUIRE(res.size() == 7); + CHECK(res(2) == 1.5); + CHECK(res(0) == Approx(111.15).epsilon(1e-3)); + CHECK(res(1) == Approx(0.8).epsilon(1e-3)); + CHECK(res(3) == + Approx(0.274).epsilon(misfit_error_tolerance(minimizer))); + CHECK(res(5) == 0.0); // fixed parameters report no error + CHECK(res(6) == Approx(1684.25).epsilon(1e-3)); + } + + SECTION("fixed at the true value, then released") { + auto model = make_model(minimizer, true); + model.FixParameter(2, 1.3); + auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(0) == Approx(120.0).epsilon(1e-4).margin(1e-6)); + CHECK(res(1) == Approx(0.8).epsilon(1e-4).margin(1e-6)); + CHECK(res(2) == 1.3); + CHECK(res(5) == 0.0); + CHECK(res(6) == Approx(0.0).margin(1e-3)); + + model.ReleaseParameter("sigma"); + REQUIRE_FALSE(model.is_user_fixed(2)); + res = aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(2) == Approx(1.3).epsilon(1e-4).margin(1e-6)); + CHECK(res(5) == Approx(0.00451).epsilon(2e-2)); + } +} + +TEST_CASE("Fit with parameter limits", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + const double margin = limit_margin(minimizer); + const double err_tol = misfit_error_tolerance(minimizer); + + SECTION("active upper limit on mu") { + auto model = make_model(minimizer); + model.SetParLimits("mu", -5.0, 0.5); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + CHECK(res(1) == Approx(0.5).margin(margin)); + CHECK(res(0) == Approx(116.88).epsilon(1e-3)); + CHECK(res(2) == Approx(1.335).epsilon(1e-3)); + CHECK(res(3) == Approx(4301.95).epsilon(1e-3)); + } + + SECTION("active upper limit on sigma, with errors") { + auto model = make_model(minimizer, true); + model.SetParLimits(2, 0.5, 1.0); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(2) == Approx(1.0).margin(margin)); + CHECK(res(1) == Approx(0.8).epsilon(1e-3)); + CHECK(res(3) == Approx(0.336).epsilon(err_tol)); + CHECK(res(5) == 0.0); // on a limit: no error reported + CHECK(res(6) == Approx(5550.61).epsilon(1e-3)); + } + + SECTION("fixed amplitude and an active limit on mu") { + auto model = make_model(minimizer, true); + model.FixParameter("A", 100.0); + model.SetParLimits("mu", 0.6, 0.7); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(0) == 100.0); + CHECK(res(1) == Approx(0.7).margin(margin)); + CHECK(res(2) == Approx(1.4603).epsilon(1e-3)); + CHECK(res(3) == 0.0); + CHECK(res(4) == 0.0); + CHECK(res(5) == Approx(0.00469).epsilon(err_tol)); + CHECK(res(6) == Approx(3631.01).epsilon(1e-3)); + } + + SECTION("inactive limits do not change the result") { + const auto free_model = make_model(minimizer); + const auto reference = + aare::fit_pixel(free_model, d.x.view(), d.y.view()); + + auto model = make_model(minimizer); + model.SetParLimits("mu", -5.0, 5.0); + model.SetParLimits("sigma", 0.5, + std::numeric_limits::infinity()); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + for (ssize_t k = 0; k < 3; ++k) + CHECK(res(k) == Approx(reference(k)).epsilon(1e-4).margin(1e-6)); + } +} + +TEST_CASE("Fit honours user start values", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + + SECTION("start value for mu") { + auto model = make_model(minimizer); + model.SetParameter("mu", 0.0); + REQUIRE(model.is_user_start(1)); + CHECK(model.value(1) == 0.0); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + CHECK(res(0) == Approx(120.0).epsilon(1e-4).margin(1e-6)); + CHECK(res(1) == Approx(0.8).epsilon(1e-4).margin(1e-6)); + CHECK(res(2) == Approx(1.3).epsilon(1e-4).margin(1e-6)); + } + + SECTION("negative Gaussian with all start values given") { + Data n{NDArray({n_points}), NDArray({n_points}), + NDArray({n_points}, 1.0)}; + fill_gaussian(n.x, n.y, -120.0, 0.8, 1.3); + auto model = make_model(minimizer); + model.SetParameter("A", -100.0); + model.SetParameter("mu", 0.5); + model.SetParameter("sigma", 1.0); + const auto res = aare::fit_pixel(model, n.x.view(), n.y.view()); + CHECK(res(0) == Approx(-120.0).epsilon(1e-4).margin(1e-6)); + CHECK(res(1) == Approx(0.8).epsilon(1e-4).margin(1e-6)); + CHECK(res(2) == Approx(1.3).epsilon(1e-4).margin(1e-6)); + } +} + +TEST_CASE("Fit reports failure as zeros when max_calls is exhausted", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + + // Start far from the minimum so that no minimizer converges in 3 calls. + auto model = make_model(minimizer, false, 3); + model.SetParameter("mu", -3.0); + model.SetParameter("sigma", 3.0); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + REQUIRE(res.size() == 4); + for (ssize_t k = 0; k < res.size(); ++k) + CHECK(res(k) == 0.0); + + auto model_err = make_model(minimizer, true, 3); + model_err.SetParameter("mu", -3.0); + model_err.SetParameter("sigma", 3.0); + const auto res_err = + aare::fit_pixel(model_err, d.x.view(), d.y.view(), d.y_err.view()); + REQUIRE(res_err.size() == 7); + for (ssize_t k = 0; k < res_err.size(); ++k) + CHECK(res_err(k) == 0.0); +} + +TEST_CASE("FitModel validates parameter names, indices and limits", "[fit]") { + aare::FitModel model; + CHECK(model.GetParNames() == std::vector{"A", "mu", "sigma"}); + CHECK(model.GetParName(2) == "sigma"); + CHECK(model.GetNpar() == 3); + CHECK(model.strategy() == 0); + CHECK(model.minimizer() == Minimizer::Migrad); + CHECK(model.lower_limit(2) == 1e-12); + CHECK(std::isinf(model.upper_limit(2))); + + CHECK_THROWS_AS(model.SetParameter("nope", 1.0), std::runtime_error); + CHECK_THROWS_AS(model.SetParameter(3, 1.0), std::out_of_range); + CHECK_THROWS_AS(model.GetParName(5), std::out_of_range); + CHECK_THROWS_AS(model.SetParLimits(0, 1.0, 1.0), std::runtime_error); + CHECK_THROWS_AS(model.FixParameter("nope", 1.0), std::runtime_error); + CHECK_NOTHROW(model.SetParLimits("mu", -1.0, 1.0)); + CHECK(model.lower_limit(1) == -1.0); + CHECK(model.upper_limit(1) == 1.0); + + model.SetMinimizer(Minimizer::LevenbergMarquardt); + CHECK(model.minimizer() == Minimizer::LevenbergMarquardt); +} + +TEST_CASE("fit_pixel rejects arrays of different sizes", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + const auto model = make_model(minimizer); + NDView short_y(d.y.data(), {n_points - 1}); + NDView short_err(d.y_err.data(), {n_points - 1}); + CHECK_THROWS_AS(aare::fit_pixel(model, d.x.view(), short_y), + std::runtime_error); + CHECK_THROWS_AS(aare::fit_pixel(model, d.x.view(), d.y.view(), short_err), + std::runtime_error); +} + +TEST_CASE("Copying a FitModel keeps the minimizer", "[fit]") { + const aare::FitModel original( + 0, 100, 0.5, false, Minimizer::LevenbergMarquardt); + const aare::FitModel copied(original); + aare::FitModel assigned; + assigned = original; + + CHECK(copied.minimizer() == Minimizer::LevenbergMarquardt); + CHECK(assigned.minimizer() == Minimizer::LevenbergMarquardt); +} + +TEST_CASE("Fit rising and falling S-curves", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + const std::vector truth{10.0, 0.1, 50.0, 3.0, 1000.0, 2.0}; + + SECTION("rising, free fit with errors") { + auto d = scurve_data(truth); + const auto model = + make_model(minimizer, true, 500); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + REQUIRE(res.size() == 13); + for (std::size_t k = 0; k < truth.size(); ++k) { + CHECK(res(static_cast(k)) == + Approx(truth[k]).epsilon(1e-3)); + CHECK(res(static_cast(6 + k)) > 0.0); + } + CHECK(res(12) == Approx(0.0).margin(1e-3)); + } + + SECTION("rising, fixed slope and limited width") { + auto d = scurve_data(truth); + auto model = + make_model(minimizer, true, 500); + model.FixParameter("p1", 0.1); + model.SetParLimits("sigma", 1.0, 4.0); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(1) == 0.1); + CHECK(res(7) == 0.0); + CHECK(res(2) == Approx(50.0).epsilon(1e-3)); + CHECK(res(3) == Approx(3.0).epsilon(1e-3)); + CHECK(res(9) > 0.0); + } + + SECTION("rising, active limit on the threshold") { + auto d = scurve_data(truth); + auto model = + make_model(minimizer, false, 500); + model.SetParLimits("mu", 40.0, 48.0); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + CHECK(res(2) == Approx(48.0).margin(limit_margin(minimizer))); + CHECK(res(6) > 0.0); // chi2 of the constrained fit + } + + SECTION("falling") { + auto d = scurve_data(truth); + const auto model = + make_model(minimizer, false, 500); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + REQUIRE(res.size() == 7); + for (std::size_t k = 0; k < truth.size(); ++k) + CHECK(res(static_cast(k)) == + Approx(truth[k]).epsilon(1e-3)); + } +} + +TEST_CASE("Fit a Gaussian data cube in parallel", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + NDArray x({n_points}); + NDArray values({n_points}); + fill_gaussian(x, values, 80.0, -0.6, 0.9); + + NDArray y({2, 2, n_points}); + for (ssize_t row = 0; row < 2; ++row) { + for (ssize_t col = 0; col < 2; ++col) { + for (ssize_t i = 0; i < n_points; ++i) + y(row, col, i) = values(i); + } + } + + NDArray par({2, 2, 3}); + NDArray chi2({2, 2}); + const auto model = make_model(minimizer); + + aare::fit_3d(model, x.view(), y.view(), NDView{}, par.view(), + NDView{}, chi2.view(), 2); + + for (ssize_t row = 0; row < 2; ++row) { + for (ssize_t col = 0; col < 2; ++col) { + CHECK(par(row, col, 0) == Approx(80.0).epsilon(1e-4).margin(1e-6)); + CHECK(par(row, col, 1) == Approx(-0.6).epsilon(1e-4).margin(1e-6)); + CHECK(par(row, col, 2) == Approx(0.9).epsilon(1e-4).margin(1e-6)); + CHECK(chi2(row, col) == Approx(0.0).margin(1e-3)); + } + } + + CHECK_THROWS_AS(aare::fit_3d(model, NDView(x.data(), {10}), + y.view(), NDView{}, par.view(), + NDView{}, chi2.view(), 2), + std::runtime_error); +} + +TEST_CASE("Fit a data cube with errors matches per-pixel fits", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t rows = 4; + constexpr ssize_t cols = 3; + NDArray x({n_points}); + NDArray y({rows, cols, n_points}); + NDArray y_err({rows, cols, n_points}, 1.0); + NDArray tmp({n_points}); + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + fill_gaussian(x, tmp, 80.0 + 10.0 * static_cast(row), + -0.6 + 0.3 * static_cast(col), + 0.9 + 0.1 * static_cast(row)); + for (ssize_t i = 0; i < n_points; ++i) + y(row, col, i) = tmp(i); + } + } + + NDArray par({rows, cols, 3}); + NDArray err({rows, cols, 3}); + NDArray chi2({rows, cols}); + const auto model = make_model(minimizer, true); + + aare::fit_3d(model, x.view(), y.view(), y_err.view(), par.view(), + err.view(), chi2.view(), 3); + + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + NDView yv(&y(row, col, 0), {n_points}); + NDView ev(&y_err(row, col, 0), {n_points}); + const auto single = aare::fit_pixel(model, x.view(), yv, ev); + // reused solver buffers must not leak state between pixels + for (ssize_t k = 0; k < 3; ++k) { + CHECK(par(row, col, k) == Approx(single(k)).margin(1e-12)); + CHECK(err(row, col, k) == Approx(single(3 + k)).margin(1e-12)); + CHECK(err(row, col, k) > 0.0); + } + CHECK(chi2(row, col) == Approx(single(6)).margin(1e-12)); + const double tol = truth_tolerance(minimizer); + CHECK(par(row, col, 0) == + Approx(80.0 + 10.0 * static_cast(row)) + .epsilon(tol) + .margin(1e-6)); + CHECK(par(row, col, 1) == + Approx(-0.6 + 0.3 * static_cast(col)) + .epsilon(tol) + .margin(1e-6)); + CHECK(par(row, col, 2) == + Approx(0.9 + 0.1 * static_cast(row)) + .epsilon(tol) + .margin(1e-6)); + } + } +} + +TEST_CASE("Fitting with every parameter fixed returns the fixed values", + "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + auto d = gaussian_data(); + + auto model = make_model(minimizer, true); + model.FixParameter("A", 120.0); + model.FixParameter("mu", 0.8); + model.FixParameter("sigma", 1.3); + + const auto result = aare::fit_pixel(model, d.x.view(), d.y.view()); + + REQUIRE(result.size() == 7); + CHECK(result(0) == 120.0); + CHECK(result(1) == 0.8); + CHECK(result(2) == 1.3); + for (ssize_t k = 3; k < 6; ++k) + CHECK(result(k) == 0.0); + CHECK(result(6) == Approx(0.0).margin(1e-12)); +} + +TEST_CASE("All minimizers agree on Gaussian fits", "[fit]") { + using aare::model::Gaussian; + const auto none = [](auto &) {}; + check_minimizers_agree("exact, unweighted", gaussian_data(), + false, false, 100, 1e-4, none); + check_minimizers_agree("exact, weighted with errors", + gaussian_data(), true, true, 100, 1e-4, + none); + check_minimizers_agree( + "sigma fixed at a wrong value", gaussian_data(), true, true, 100, 1e-3, + [](auto &m) { m.FixParameter("sigma", 1.5); }, 0.2); + check_minimizers_agree( + "sigma fixed at the true value", gaussian_data(), true, true, 100, 1e-3, + [](auto &m) { m.FixParameter("sigma", 1.3); }); + check_minimizers_agree( + "active upper limit on mu", gaussian_data(), false, false, 100, 1e-3, + [](auto &m) { m.SetParLimits("mu", -5.0, 0.5); }); + check_minimizers_agree( + "inactive limits", gaussian_data(), false, false, 100, 1e-4, + [](auto &m) { m.SetParLimits("mu", -5.0, 5.0); }); + check_minimizers_agree( + "active upper limit on sigma", gaussian_data(), true, true, 100, 1e-3, + [](auto &m) { m.SetParLimits("sigma", 0.5, 1.0); }, 0.2); + check_minimizers_agree( + "user start value", gaussian_data(), false, false, 100, 1e-4, + [](auto &m) { m.SetParameter("mu", 0.0); }); + check_minimizers_agree( + "fixed amplitude and active limit on mu", gaussian_data(), true, true, + 100, 1e-3, + [](auto &m) { + m.FixParameter("A", 100.0); + m.SetParLimits("mu", 0.6, 0.7); + }, + 0.2); + check_minimizers_agree("noisy, weighted with errors", + gaussian_data(true), true, true, 500, 1e-3, + none); + check_minimizers_agree( + "noisy, fixed mu and active limit on sigma", gaussian_data(true), true, + true, 500, 1e-3, + [](auto &m) { + m.FixParameter("mu", 0.8); + m.SetParLimits("sigma", 0.5, 1.1); + }, + 0.2); +} + +TEST_CASE("All minimizers agree on S-curve fits", "[fit]") { + using aare::model::RisingScurve; + const std::vector truth{10.0, 0.1, 50.0, 3.0, 1000.0, 2.0}; + const auto none = [](auto &) {}; + check_minimizers_agree("noisy, weighted with errors", + scurve_data(truth, true), + true, true, 500, 2e-2, none); + check_minimizers_agree("noisy, fixed slope and limited width", + scurve_data(truth, true), + true, true, 500, 1e-2, [](auto &m) { + m.FixParameter("p1", 0.1); + m.SetParLimits("sigma", 1.0, 4.0); + }); + check_minimizers_agree( + "noisy, active limit on the threshold", + scurve_data(truth, true), false, false, 500, 1e-2, + [](auto &m) { m.SetParLimits("mu", 40.0, 48.0); }); + check_minimizers_agree( + "noisy, two fixed, limited amplitude, start value", + scurve_data(truth, true), true, true, 500, 1e-2, + [](auto &m) { + m.FixParameter("sigma", 3.0); + m.FixParameter("p0", 10.0); + m.SetParLimits("A", 900.0, 1100.0); + m.SetParameter("C", 0.0); + }); +} + +TEST_CASE("All minimizers agree on a noisy data cube", "[fit]") { + constexpr ssize_t rows = 3; + constexpr ssize_t cols = 2; + NDArray x({n_points}); + NDArray y({rows, cols, n_points}); + NDArray y_err({rows, cols, n_points}, 1.0); + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + const double mu = -0.6 + 0.3 * static_cast(col); + const double sigma = 0.9 + 0.1 * static_cast(row); + for (ssize_t i = 0; i < n_points; ++i) { + x(i) = -6.0 + 0.2 * static_cast(i); + const double z = (x(i) - mu) / sigma; + y(row, col, i) = 80.0 * std::exp(-0.5 * z * z) + + 2.0 * std::sin(7.3 * static_cast(i)); + } + } + } + + struct Cube { + NDArray par{{rows, cols, 3}}; + NDArray err{{rows, cols, 3}}; + NDArray chi2{{rows, cols}}; + }; + const auto fit_cube = [&](Minimizer minimizer) { + Cube c; + const auto model = + make_model(minimizer, true, 500); + aare::fit_3d(model, x.view(), y.view(), y_err.view(), c.par.view(), + c.err.view(), c.chi2.view(), 2); + return c; + }; + const auto ref = fit_cube(Minimizer::Migrad); + + for (const auto candidate : + {Minimizer::Fumili, Minimizer::LevenbergMarquardt, + Minimizer::VarPro}) { + INFO(name(candidate) << " vs Migrad"); + const auto res = fit_cube(candidate); + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + for (ssize_t k = 0; k < 3; ++k) { + CHECK(res.par(row, col, k) == + Approx(ref.par(row, col, k)).epsilon(1e-3)); + CHECK(res.err(row, col, k) == + Approx(ref.err(row, col, k)).epsilon(2e-2)); + } + CHECK(res.chi2(row, col) == + Approx(ref.chi2(row, col)).epsilon(1e-4)); + } + } + } +} + +namespace { + +// The separable structure of a model must reproduce eval and eval_and_grad: +// f = sum_j par[linear_par[j]] * phi[j], df/dpar[linear_par[j]] = phi[j] and +// df/dpar[nonlinear[k]] = sum_j par[linear_par[j]] * dphi[k][j]. +template +void check_separable(const std::vector &par, + const std::vector &xs) { + using Traits = aare::model::separable_traits; + static_assert(aare::model::is_separable::value); + typename Traits::Basis phi{}; + typename Traits::BasisGrad dphi{}; + std::array g{}; + double f = 0.0; + for (const double x : xs) { + INFO("x = " << x); + Model::basis_and_grad(x, par, phi, dphi); + Model::eval_and_grad(x, par, f, g); + double sum = 0.0; + for (std::size_t j = 0; j < Traits::nlin; ++j) { + const auto idx = Model::linear_par[j]; + sum += par[idx] * phi[j]; + CHECK(g[idx] == Approx(phi[j]).epsilon(1e-12).margin(1e-14)); + } + CHECK(Model::eval(x, par) == Approx(sum).epsilon(1e-12).margin(1e-14)); + CHECK(f == Approx(sum).epsilon(1e-12).margin(1e-14)); + for (std::size_t k = 0; k < Traits::nnl; ++k) { + double d = 0.0; + for (std::size_t j = 0; j < Traits::nlin; ++j) + d += par[Model::linear_par[j]] * dphi[k][j]; + CHECK(g[Traits::nonlinear_par[k]] == + Approx(d).epsilon(1e-12).margin(1e-14)); + } + } + // The bulk basis pass must agree with the per-point one. + if constexpr (aare::model::has_basis_columns::value) { + const auto n = static_cast(xs.size()); + std::vector cols(Traits::nlin * xs.size()); + std::vector dcols(Traits::nnl * Traits::nlin * xs.size()); + Model::basis_columns(xs.data(), n, par, cols.data(), dcols.data()); + for (ssize_t i = 0; i < n; ++i) { + INFO("x = " << xs[static_cast(i)]); + Model::basis_and_grad(xs[static_cast(i)], par, phi, + dphi); + for (std::size_t j = 0; j < Traits::nlin; ++j) + CHECK(cols[j * xs.size() + static_cast(i)] == + Approx(phi[j]).epsilon(1e-12).margin(1e-14)); + for (std::size_t k = 0; k < Traits::nnl; ++k) + for (std::size_t j = 0; j < Traits::nlin; ++j) + CHECK(dcols[(k * Traits::nlin + j) * xs.size() + + static_cast(i)] == + Approx(dphi[k][j]).epsilon(1e-12).margin(1e-14)); + } + } +} + +} // namespace + +TEST_CASE("Separable models are consistent with eval and eval_and_grad", + "[fit]") { + using namespace aare::model; + const std::vector xs = {-4.0, -1.5, 0.3, 2.0, + 7.5, 40.0, 55.0, 120.0}; + check_separable({2.0, -0.5}, xs); + check_separable({2.0, -0.5, 0.05}, xs); + check_separable({120.0, 0.8, 1.3}, xs); + check_separable({80.0, 15.0, 2.0, 1.5}, xs); + check_separable({20.0, 0.05, 50.0, 4.0, 1500.0, 0.3}, + xs); + check_separable( + {20.0, 0.05, 50.0, 4.0, 1500.0, 0.3, 1.1, 0.1}, xs); + check_separable({10.0, 0.1, 50.0, 3.0, 1000.0, 2.0}, xs); + check_separable({10.0, 0.1, 50.0, 3.0, 1000.0, 2.0}, xs); +} + +TEST_CASE("VarPro falls back to LevenbergMarquardt for a limit " + "on a linear parameter", + "[fit]") { + using aare::model::FallingScurve; + const std::vector truth{10.0, 0.1, 50.0, 3.0, 1000.0, 2.0}; + auto d = scurve_data(truth); + + auto vp = make_model(Minimizer::VarPro, true, 500); + auto lm = + make_model(Minimizer::LevenbergMarquardt, true, 500); + // The unconstrained amplitude is 1000, so the linear solve cannot + // honour this limit and the pixel is refitted with the full solver. + vp.SetParLimits("A", 0.0, 500.0); + lm.SetParLimits("A", 0.0, 500.0); + + const auto res_vp = + aare::fit_pixel(vp, d.x.view(), d.y.view(), d.y_err.view()); + const auto res_lm = + aare::fit_pixel(lm, d.x.view(), d.y.view(), d.y_err.view()); + REQUIRE(res_vp.size() == res_lm.size()); + for (ssize_t k = 0; k < res_vp.size(); ++k) + CHECK(res_vp(k) == res_lm(k)); + CHECK(res_vp(4) == Approx(500.0).margin(1e-9)); + CHECK(res_vp(12) > 0.0); +} + +TEST_CASE("fast_exp matches std::exp", "[fit]") { + double max_rel = 0.0; + for (int i = 0; i <= 14000; ++i) { + const double x = -700.0 + 1400.0 * static_cast(i) / 14000.0; + const double reference = std::exp(x); + max_rel = + std::max(max_rel, std::abs(aare::model::fast_exp(x) - reference) / + reference); + } + CHECK(max_rel < 1e-15); + CHECK(aare::model::fast_exp(0.0) == 1.0); + CHECK(aare::model::fast_exp(-1000.0) == aare::model::fast_exp(-700.0)); +} + +TEST_CASE("Fit with a one-sided limit far from the start value", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t n = 20; + NDArray x({n}); + NDArray y({n}); + for (ssize_t i = 0; i < n; ++i) { + x(i) = static_cast(i); + y(i) = 2.0 + 0.5 * x(i); + } + const double inf = std::numeric_limits::infinity(); + const bool upper = GENERATE(true, false); + auto model = make_model(minimizer); + if (upper) + model.SetParLimits(0, -inf, 1e9); + else + model.SetParLimits(0, -1e9, inf); + + const auto res = aare::fit_pixel(model, x.view(), y.view()); + CHECK(res(0) == Approx(2.0).epsilon(1e-6)); + CHECK(res(1) == Approx(0.5).epsilon(1e-6)); +} + +TEST_CASE("A parameter ending exactly on a limit reports an error of 0", + "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t n = 3; + NDArray x({n}); + NDArray y({n}, 2.0); + for (ssize_t i = 0; i < n; ++i) + x(i) = static_cast(i); + // The unconstrained minimum p0 = 2 coincides with the limit. + const bool lower = GENERATE(true, false); + auto model = make_model(minimizer, true); + model.FixParameter(1, 0.0); + if (lower) + model.SetParLimits(0, 2.0, 3.0); + else + model.SetParLimits(0, 1.0, 2.0); + + const auto res = aare::fit_pixel(model, x.view(), y.view()); + REQUIRE(res.size() == 5); + CHECK(res(0) == Approx(2.0).epsilon(1e-6)); + CHECK(res(2) == 0.0); + CHECK(res(3) == 0.0); +} + +// --------------------------------------------------------------------------- +// Regressions of the built-in solvers: fits that used to be reported as valid +// although they had stopped away from the minimum or returned garbage. +// --------------------------------------------------------------------------- + +namespace { + +// Standard normal noise from std::mt19937, whose output the standard fixes, +// and a Box-Muller transform written out here: std::normal_distribution +// differs between standard libraries. +std::vector normal_noise(std::size_t n, unsigned seed) { + std::mt19937 rng(seed); + const auto uniform = [&rng] { + return (static_cast(rng()) + 0.5) / 4294967296.0; + }; + std::vector out(n); + for (std::size_t i = 0; i < n; i += 2) { + const double r = std::sqrt(-2.0 * std::log(uniform())); + const double phi = 2.0 * std::acos(-1.0) * uniform(); + out[i] = r * std::cos(phi); + if (i + 1 < n) + out[i + 1] = r * std::sin(phi); + } + return out; +} + +} // namespace + +TEST_CASE("A Gaussian fit to pure noise fails or stays finite and bounded", + "[fit]") { + // Fitting noise, a narrow width can collapse between two scan points; + // the amplitude that fits a single point then diverges. Such a fit must + // be reported as failed, not as a valid result with |A| up to 1e159. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t rows = 20; + constexpr ssize_t cols = 20; + NDArray x({n_points}); + for (ssize_t i = 0; i < n_points; ++i) + x(i) = -6.0 + 0.2 * static_cast(i); + NDArray y({rows, cols, n_points}); + const auto noise = normal_noise(static_cast(y.size()), 1); + std::copy(noise.begin(), noise.end(), y.begin()); + + NDArray par({rows, cols, 3}); + NDArray chi2({rows, cols}); + const auto model = make_model(minimizer); + aare::fit_3d(model, x.view(), y.view(), NDView{}, par.view(), + NDView{}, chi2.view(), 1); + + int absurd = 0; + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + bool ok = std::isfinite(chi2(row, col)); + for (ssize_t k = 0; k < 3; ++k) + ok = ok && std::isfinite(par(row, col, k)) && + std::abs(par(row, col, k)) < 1e6; + absurd += ok ? 0 : 1; + } + } + CHECK(absurd == 0); +} + +TEST_CASE("A step reflected back onto the start point is not convergence", + "[fit]") { + // p0 starts at 0 with limits [-1, 1]; the damped step of +2 reflects at + // the upper limit back onto 0. The step truncated at the limit must still + // be tried: the constrained minimum is p0 = 1. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + NDArray x({4}); + NDArray y({4}, 2.2); + for (ssize_t i = 0; i < 4; ++i) + x(i) = static_cast(i); + + auto model = make_model(minimizer); + model.SetParameter("p0", 0.0); + model.FixParameter("p1", 0.0); + model.SetParLimits("p0", -1.0, 1.0); + const auto res = aare::fit_pixel(model, x.view(), y.view()); + CHECK(res(0) == Approx(1.0).margin(limit_margin(minimizer))); + CHECK(res(2) == Approx(5.76).epsilon(1e-6)); +} + +TEST_CASE("A large parameter does not hide the steps of the others", "[fit]") { + // The negligible-step test used one relative norm over all parameters, + // so an amplitude of 1e10 or a position of 1e9 stopped the fit at the + // start estimate of the width. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + const auto [amplitude, offset] = GENERATE( + std::pair{1e8, 0.0}, std::pair{1e10, 0.0}, std::pair{120.0, 1e9}); + INFO("amplitude " << amplitude << ", offset " << offset); + + Data d{NDArray({n_points}), NDArray({n_points}), + NDArray({n_points})}; + fill_gaussian(d.x, d.y, amplitude, 0.8, 1.3); + for (ssize_t i = 0; i < n_points; ++i) { + d.x(i) += offset; + d.y_err(i) = std::sqrt(d.y(i)) + 1.0; + } + const auto model = make_model(minimizer, false, 500); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + // The EDM tolerance leaves a few 1e-4 relative, far below the errors; + // the stuck fits were 0.7 % (A = 1e8) and 5 % (start estimate) off. + CHECK(res(0) == Approx(amplitude).epsilon(1e-3)); + CHECK(res(1) - offset == Approx(0.8).margin(1e-3)); + CHECK(res(2) == Approx(1.3).epsilon(1e-3)); +} + +TEST_CASE("A rejected trial on a limit does not freeze an interior parameter", + "[fit]") { + // mu starts at 3 inside [0, 5]. A trial truncated onto the lower limit is + // rejected; testing that trial's position against the gradient of the + // accepted point froze mu, and the fit stopped at the start value. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t n = 11; + const double mu = std::sqrt(2.0 * std::log(2.0)); + NDArray x({n}); + NDArray y({n}); + for (ssize_t i = 0; i < n; ++i) { + x(i) = -0.1 + 0.02 * static_cast(i); + y(i) = 10.0 * std::exp(-0.5 * (x(i) - mu) * (x(i) - mu)); + } + auto model = make_model(minimizer); + model.FixParameter("A", 10.0); + model.FixParameter("sigma", 1.0); + model.SetParameter("mu", 3.0); + model.SetParLimits("mu", 0.0, 5.0); + const auto res = aare::fit_pixel(model, x.view(), y.view()); + CHECK(res(1) == Approx(mu).margin(1e-3)); + CHECK(res(3) < 1e-4); +} + +// --------------------------------------------------------------------------- +// Samples with a zero error are ignored. They may hold NaN (a masked dead +// frame), which a zero weight does not remove from a sum, so they must not +// reach the start estimate or the minimizer at all. +// --------------------------------------------------------------------------- + +TEST_CASE("Samples with a zero error are ignored even when NaN", "[fit]") { + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + const double nan = std::numeric_limits::quiet_NaN(); + + static constexpr ssize_t n = 10; // static: used inside GENERATE's lambda + NDArray x({n}); + NDArray y({n}); + NDArray y_err({n}, 1.0); + for (ssize_t i = 0; i < n; ++i) { + x(i) = static_cast(i); + y(i) = 2.0 + 0.5 * x(i); + } + const auto line = make_model(minimizer, true); + + // The start estimate of Pol1 uses the first and the last sample. + const auto masked = GENERATE(as{}, 4, 0, n - 1); + SECTION("masked NaN on a line") { + INFO("masked sample " << masked); + y(masked) = nan; + y_err(masked) = 0.0; + const auto res = + aare::fit_pixel(line, x.view(), y.view(), y_err.view()); + CHECK(res(0) == Approx(2.0).epsilon(1e-5).margin(1e-6)); + CHECK(res(1) == Approx(0.5).epsilon(1e-5).margin(1e-6)); + CHECK(res(2) > 0.0); + CHECK(res(4) == Approx(0.0).margin(1e-6)); + } + + SECTION("masked NaN at the peak of a Gaussian") { + auto d = gaussian_data(); + constexpr ssize_t peak = 34; // x = 0.8 + d.y(peak) = nan; + d.y_err(peak) = 0.0; + const auto model = make_model(minimizer); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + const double tol = truth_tolerance(minimizer); + CHECK(res(0) == Approx(120.0).epsilon(tol)); + CHECK(res(1) == Approx(0.8).epsilon(tol)); + CHECK(res(2) == Approx(1.3).epsilon(tol)); + } + + SECTION("too few samples left fail cleanly") { + // One sample left for two free parameters, then none at all. + for (ssize_t i = 1; i < n; ++i) { + y(i) = nan; + y_err(i) = 0.0; + } + auto res = aare::fit_pixel(line, x.view(), y.view(), y_err.view()); + for (ssize_t k = 0; k < res.size(); ++k) + CHECK(res(k) == 0.0); + y_err(0) = 0.0; + res = aare::fit_pixel(line, x.view(), y.view(), y_err.view()); + for (ssize_t k = 0; k < res.size(); ++k) + CHECK(res(k) == 0.0); + } +} + +TEST_CASE("Masked samples in a data cube match per-pixel fits", "[fit]") { + // The kept samples of each pixel go through reused buffers, which must + // not carry anything over from one pixel to the next. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + constexpr ssize_t rows = 2; + constexpr ssize_t cols = 3; + // masked sample per pixel, -1 for none + constexpr ssize_t masked[rows][cols] = {{-1, 10, 0}, + {34, -1, n_points - 1}}; + NDArray x({n_points}); + NDArray tmp({n_points}); + NDArray y({rows, cols, n_points}); + NDArray y_err({rows, cols, n_points}, 1.0); + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + fill_gaussian(x, tmp, 80.0 + 10.0 * static_cast(col), 0.8, + 1.0 + 0.2 * static_cast(row)); + for (ssize_t i = 0; i < n_points; ++i) + y(row, col, i) = tmp(i); + if (masked[row][col] >= 0) { + y(row, col, masked[row][col]) = + std::numeric_limits::quiet_NaN(); + y_err(row, col, masked[row][col]) = 0.0; + } + } + } + + NDArray par({rows, cols, 3}); + NDArray err({rows, cols, 3}); + NDArray chi2({rows, cols}); + const auto model = make_model(minimizer, true); + aare::fit_3d(model, x.view(), y.view(), y_err.view(), par.view(), + err.view(), chi2.view(), 1); + + for (ssize_t row = 0; row < rows; ++row) { + for (ssize_t col = 0; col < cols; ++col) { + INFO("pixel " << row << ", " << col); + NDView yv(&y(row, col, 0), {n_points}); + NDView ev(&y_err(row, col, 0), {n_points}); + const auto single = aare::fit_pixel(model, x.view(), yv, ev); + for (ssize_t k = 0; k < 3; ++k) { + CHECK(par(row, col, k) == Approx(single(k)).margin(1e-12)); + CHECK(err(row, col, k) == Approx(single(3 + k)).margin(1e-12)); + } + CHECK(chi2(row, col) == Approx(single(6)).margin(1e-12)); + // Close to the truth within the EDM tolerance; the exact check + // is the agreement with the per-pixel fit above. + const double tol = 1e-3; + CHECK(par(row, col, 0) == + Approx(80.0 + 10.0 * static_cast(col)).epsilon(tol)); + CHECK(par(row, col, 1) == Approx(0.8).epsilon(tol)); + CHECK(par(row, col, 2) == + Approx(1.0 + 0.2 * static_cast(row)).epsilon(tol)); + } + } +} + +TEST_CASE("A parameter whose minimum lies on its limit reports no error", + "[fit]") { + // At a minimum that lies exactly on a limit the gradient is zero, so a + // test for an outward gradient does not see the parameter as bound. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + + SECTION("linear parameter") { + NDArray x({4}); + NDArray y({4}); + const double xs[4] = {-1.0, -1.0, 1.0, 1.0}; + for (ssize_t i = 0; i < 4; ++i) { + x(i) = xs[i]; + y(i) = 1.0 + 2.0 * xs[i]; + } + auto model = make_model(minimizer, true); + model.SetParLimits("p0", 0.0, 1.0); + const auto res = aare::fit_pixel(model, x.view(), y.view()); + CHECK(res(0) == Approx(1.0).margin(limit_margin(minimizer))); + CHECK(res(2) == 0.0); + CHECK(res(3) == Approx(0.5).epsilon(1e-6)); // 1 / sqrt(sum x^2) + } + + SECTION("nonlinear parameter") { + auto d = gaussian_data(); + auto model = make_model(minimizer, true); + model.SetParLimits("mu", -5.0, 0.8); + const auto res = + aare::fit_pixel(model, d.x.view(), d.y.view(), d.y_err.view()); + CHECK(res(1) == Approx(0.8).margin(1e-6)); + CHECK(res(4) == 0.0); + CHECK(res(3) > 0.0); + CHECK(res(5) > 0.0); + } +} + +TEST_CASE("A fit does not stop at its start point without taking a step", + "[fit]") { + // The EDM tolerance is absolute in chi2 units. An unweighted fit of data + // of small magnitude passes it at the start estimate already; the fit + // must still take a step, as Migrad always iterates at least once. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + const double amplitude = GENERATE(0.1, 1e-4); + INFO("amplitude " << amplitude); + Data d{NDArray({n_points}), NDArray({n_points}), + NDArray({n_points}, 1.0)}; + fill_gaussian(d.x, d.y, amplitude, 0.8, 1.3); + + // The start estimate of sigma is 1.3617; Migrad ends at 1.3205. + const auto model = make_model(minimizer); + const auto res = aare::fit_pixel(model, d.x.view(), d.y.view()); + CHECK(std::abs(res(2) - 1.3) < 0.03); + + // Without the budget for a single step the fit has not converged. + const auto one_call = + make_model(minimizer, false, 1); + const auto none = aare::fit_pixel(one_call, d.x.view(), d.y.view()); + for (ssize_t k = 0; k < none.size(); ++k) + CHECK(none(k) == 0.0); +} + +TEST_CASE("A polynomial far from x = 0 keeps its precision", "[fit]") { + // On a narrow range far from 0 the columns 1, x, x^2 are nearly + // collinear. VarPro solves them from the normal equations and lost six + // digits at x = 20000; from about x = 60000 it falls back to LM. + const auto minimizer = + GENERATE(Minimizer::Migrad, Minimizer::Fumili, + Minimizer::LevenbergMarquardt, Minimizer::VarPro); + INFO("minimizer " << name(minimizer)); + const auto [lo, span] = + GENERATE(std::pair{8000.0, 1000.0}, std::pair{20000.0, 500.0}, + std::pair{60000.0, 200.0}); + INFO("x in [" << lo << ", " << lo + span << "]"); + // 1.5 + 2e-3 (x - lo) + 3e-7 (x - lo)^2 expanded in powers of x. + const double c[3] = {1.5 - 2e-3 * lo + 3e-7 * lo * lo, 2e-3 - 6e-7 * lo, + 3e-7}; + constexpr ssize_t n = 40; + NDArray x({n}); + NDArray y({n}); + for (ssize_t i = 0; i < n; ++i) { + x(i) = lo + span * static_cast(i) / static_cast(n - 1); + y(i) = c[0] + c[1] * x(i) + c[2] * x(i) * x(i); + } + const auto model = make_model(minimizer); + const auto res = aare::fit_pixel(model, x.view(), y.view()); + for (ssize_t k = 0; k < 3; ++k) + CHECK(res(k) == Approx(c[k]).epsilon(1e-9)); +} diff --git a/src/fit/FitHelpers.hpp b/src/fit/FitHelpers.hpp new file mode 100644 index 00000000..a82599cf --- /dev/null +++ b/src/fit/FitHelpers.hpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +#include "aare/FitModel.hpp" +#include "aare/NDView.hpp" +#include +#include +#include + +// Helpers shared by the built-in solver and the optional Minuit2 backend so +// that both apply the same start-value precedence and shape checks. + +namespace aare::detail { + +/** + * @brief Starting values for one pixel. + * + * Model::estimate_par supplies data-driven estimates. User start values and + * fixed values take precedence, and free parameters are clamped into their + * limits. + */ +template +std::array start_values(const FitModel &model, + NDView x, + NDView y) { + auto start = Model::estimate_par(x, y); + for (std::size_t k = 0; k < Model::npar; ++k) { + const auto idx = static_cast(k); + if (model.is_user_fixed(idx) || model.is_user_start(idx)) + start[k] = model.value(idx); + if (!model.is_user_fixed(idx)) + start[k] = std::clamp(start[k], model.lower_limit(idx), + model.upper_limit(idx)); + } + return start; +} + +/** @brief Validate the array sizes passed to fit_pixel. */ +inline void check_fit_pixel_shapes(NDView x, NDView y, + NDView y_err) { + if (x.size() != y.size()) + throw std::runtime_error("fit_pixel: x.size() must match y.size()."); + if (y_err.size() > 0 && y_err.size() != y.size()) + throw std::runtime_error( + "fit_pixel: y and y_err must have identical size."); +} + +/** @brief Validate the array shapes passed to fit_3d. */ +template +void check_fit_3d_shapes(NDView x, NDView y, + NDView y_err, NDView par_out, + NDView err_out, + NDView chi2_out) { + constexpr auto npar = static_cast(Model::npar); + + if (x.size() != y.shape(2)) + throw std::runtime_error("fit_3d: x.size() must match y.shape(2)."); + + if (par_out.shape(0) != y.shape(0) || par_out.shape(1) != y.shape(1) || + par_out.shape(2) != npar) + throw std::runtime_error("par_out must have shape [rows, cols, npar]."); + + if (chi2_out.shape(0) != y.shape(0) || chi2_out.shape(1) != y.shape(1)) + throw std::runtime_error("chi2_out must have shape [rows, cols]."); + + const bool err_out_matches = err_out.shape(0) == y.shape(0) && + err_out.shape(1) == y.shape(1) && + err_out.shape(2) == npar; + + if (y_err.size() > 0) { + if (y.shape(0) != y_err.shape(0) || y.shape(1) != y_err.shape(1) || + y.shape(2) != y_err.shape(2)) + throw std::runtime_error( + "fit_3d: y and y_err must have identical shape."); + if (!err_out_matches) + throw std::runtime_error( + "err_out must have shape [rows, cols, npar]."); + } else if (err_out.size() > 0 && !err_out_matches) { + throw std::runtime_error("err_out must have shape [rows, cols, npar]."); + } +} + +} // namespace aare::detail diff --git a/src/fit/FitMinuit2.cpp b/src/fit/FitMinuit2.cpp new file mode 100644 index 00000000..326c4b11 --- /dev/null +++ b/src/fit/FitMinuit2.cpp @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MPL-2.0 +// Minuit2 backends of the pixel fit: Migrad (MnHesse for the errors) and +// Fumili. This is the only translation unit that includes Minuit2 headers. +#include "FitMinuit2.hpp" +#include "Chi2.hpp" +#include "Minuit2/FunctionMinimum.h" +#include "Minuit2/MnFumiliMinimize.h" +#include "Minuit2/MnHesse.h" +#include "Minuit2/MnMigrad.h" +#include "Minuit2/MnStrategy.h" +#include "Minuit2/MnUserParameterState.h" +#include "Minuit2/MnUserParameters.h" +#include "MinuitSteps.hpp" +#include "aare/Models.hpp" +#include +#include +#include +#include +#include + +namespace aare::detail { + +namespace { + +// Translate the FitModel state into Minuit2 user parameters for one pixel. +template +ROOT::Minuit2::MnUserParameters +user_parameters(const FitModel &model, NDView x, + NDView y, + const std::array &start) { + constexpr std::size_t npar = Model::npar; + + double x_range = 0.0; + double y_range = 0.0; + double slope_scale = 0.0; + minuit2::compute_ranges(x, y, x_range, y_range, slope_scale); + std::array steps{}; + minuit2::Steps::compute(start, x_range, y_range, slope_scale, steps); + + ROOT::Minuit2::MnUserParameters upar; + for (std::size_t i = 0; i < npar; ++i) { + const auto idx = static_cast(i); + upar.Add(Model::param_info[i].name, start[i], steps[i]); + const double lo = model.lower_limit(idx); + const double hi = model.upper_limit(idx); + if (std::isfinite(lo) && std::isfinite(hi)) { + upar.SetLimits(idx, lo, hi); + } else if (std::isfinite(lo) || std::isfinite(hi)) { + // Minuit2's transformation for one-sided limits converges markedly + // slower than the two-sided one, so an open side gets a wide bound. + // It is placed relative to the start value, which then always + // lies inside the range however far it is from the finite limit. + const double span = std::max(1e6, 1e3 * std::abs(start[i])); + if (std::isfinite(lo)) + upar.SetLimits(idx, lo, std::max(start[i], lo) + span); + else + upar.SetLimits(idx, std::min(start[i], hi) - span, hi); + } + if (model.is_user_fixed(idx)) + upar.Fix(idx); + } + return upar; +} + +// Minuit2 approaches an active limit asymptotically through its parameter +// transformation, so "on a limit" is decided with a small relative margin. +template +bool on_limit(const FitModel &model, unsigned int idx, double value) { + const double margin = 1e-6 * std::max(1.0, std::abs(value)); + return std::abs(value - model.lower_limit(idx)) <= margin || + std::abs(value - model.upper_limit(idx)) <= margin; +} + +// Copy the minimum into the flat result. Fixed parameters and parameters on +// a limit report an error of 0. +template +void unpack_minimum(const FitModel &model, + const ROOT::Minuit2::FunctionMinimum &min, double *par_out, + double *err_out, double &chi2) { + constexpr std::size_t npar = Model::npar; + const auto &state = min.UserState(); + const std::vector values = state.Params(); + for (std::size_t k = 0; k < npar; ++k) + par_out[k] = values[k]; + + if (err_out) { + const std::vector errors = state.Errors(); + for (std::size_t k = 0; k < npar; ++k) { + const auto idx = static_cast(k); + const bool no_error = + model.is_user_fixed(idx) || on_limit(model, idx, values[k]); + err_out[k] = no_error ? 0.0 : errors[k]; + } + } + chi2 = min.Fval(); +} + +} // namespace + +template +bool fit_pixel_minuit2(const FitModel &model, NDView x, + NDView y, NDView y_err, + const std::array &start, + double *par_out, double *err_out, double &chi2) { + constexpr std::size_t npar = Model::npar; + std::fill(par_out, par_out + npar, 0.0); + if (err_out) + std::fill(err_out, err_out + npar, 0.0); + chi2 = 0.0; + + if (!Model::is_valid(std::vector(start.begin(), start.end()))) + return false; + + const auto upar = user_parameters(model, x, y, start); + const ROOT::Minuit2::MnStrategy strategy(model.strategy()); + + if (model.minimizer() == Minimizer::Fumili) { + // Fumili takes the gradient and the Gauss-Newton Hessian from the FCN, + // so the covariance of the minimum is available without MnHesse. + func::Chi2Model1DFumili fcn(x, y, y_err); + ROOT::Minuit2::MnFumiliMinimize fumili( + fcn, ROOT::Minuit2::MnUserParameterState(upar), strategy); + const ROOT::Minuit2::FunctionMinimum min = + fumili(model.max_calls(), model.tolerance()); + if (min.IsValid()) { + unpack_minimum(model, min, par_out, err_out, chi2); + return true; + } + // Minuit2's Fumili cannot invert its linearised Hessian once a + // two-sided limit becomes active. Refit such pixels with Migrad. + } + + func::Chi2Model1DGrad fcn(x, y, y_err); + ROOT::Minuit2::MnMigrad migrad(fcn, upar, strategy); + ROOT::Minuit2::FunctionMinimum min = + migrad(model.max_calls(), model.tolerance()); + if (!min.IsValid()) + return false; + + // MnHesse cannot handle a fit in which every parameter is fixed. + if (err_out && upar.VariableParameters() > 0) { + ROOT::Minuit2::MnHesse hesse; + hesse(fcn, min); + } + unpack_minimum(model, min, par_out, err_out, chi2); + return true; +} + +// NOLINTBEGIN +#define AARE_INSTANTIATE_MINUIT2_FIT(Model) \ + template bool fit_pixel_minuit2( \ + const FitModel &, NDView, NDView, \ + NDView, const std::array &, double *, \ + double *, double &); + +AARE_INSTANTIATE_MINUIT2_FIT(model::Gaussian) +AARE_INSTANTIATE_MINUIT2_FIT(model::GaussianErfcPlateau) +AARE_INSTANTIATE_MINUIT2_FIT(model::GaussianChargeSharing) +AARE_INSTANTIATE_MINUIT2_FIT(model::GaussianChargeSharingKb) +AARE_INSTANTIATE_MINUIT2_FIT(model::Pol1) +AARE_INSTANTIATE_MINUIT2_FIT(model::Pol2) +AARE_INSTANTIATE_MINUIT2_FIT(model::RisingScurve) +AARE_INSTANTIATE_MINUIT2_FIT(model::FallingScurve) + +#undef AARE_INSTANTIATE_MINUIT2_FIT +// NOLINTEND + +} // namespace aare::detail diff --git a/src/fit/FitMinuit2.hpp b/src/fit/FitMinuit2.hpp new file mode 100644 index 00000000..c0240be8 --- /dev/null +++ b/src/fit/FitMinuit2.hpp @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +#include "aare/FitModel.hpp" +#include "aare/NDView.hpp" +#include + +// Minuit2 backends (Migrad and Fumili) of the pixel fit. FitMinuit2.cpp is +// the only translation unit that includes Minuit2 headers; it defines and +// explicitly instantiates the function below for every supported model. + +namespace aare::detail { + +/** + * @brief Fit one pixel with the Minuit2 minimizer selected by the model. + * + * @param start Starting values, see start_values() in FitHelpers.hpp. + * @param par_out Receives npar fitted values; zeros on failure. + * @param err_out Receives npar errors when non-null; zeros on failure. + * Fixed parameters and parameters on a limit report 0. + * @param chi2 Receives the chi-squared at the minimum; 0 on failure. + * @return true when Minuit2 reports a valid minimum. + */ +template +bool fit_pixel_minuit2(const FitModel &model, NDView x, + NDView y, NDView y_err, + const std::array &start, + double *par_out, double *err_out, double &chi2); + +} // namespace aare::detail diff --git a/src/fit/LevenbergMarquardt.hpp b/src/fit/LevenbergMarquardt.hpp new file mode 100644 index 00000000..8a6a7213 --- /dev/null +++ b/src/fit/LevenbergMarquardt.hpp @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +#include "DampedGaussNewton.hpp" +#include "aare/FitModel.hpp" +#include "aare/NDView.hpp" + +#include +#include +#include +#include +#include + +namespace aare::detail { + +/** + * @brief Levenberg-Marquardt least-squares minimiser for one FitModel. + * + * Minimises chi2 = sum_i ((y_i - f(x_i; p)) / s_i)^2 using the analytic + * gradient of the model (s_i = 1 for unweighted fits; points with s_i == 0 + * are ignored). Keep one instance per thread: the buffers are reused between + * pixels so that after the first fit no memory is allocated. + * + * Every evaluation computes the residuals r = (y - f) / s and the Jacobian + * J = dr/dp in one pass over the data and reduces them to the normal + * equations g = J^T r and J^T J, which DampedGaussNewton iterates on: it + * handles fixed parameters, limits, damping and the convergence test, see + * there. Rejected trials are rare with its damping rule, and a rejected one + * only wastes a single pass. Trial points at which the model loses its + * sensitivity to a free parameter are rejected. As in Minuit2, max_calls = 0 + * selects the default budget 200 + 100 * npar + 5 * npar^2. + * + * Errors are Gauss-Newton estimates, sqrt(diag((J^T J)^-1)), over the free + * parameters that do not sit on a limit. Fixed and limit-bound parameters + * report 0. + */ +template class LevenbergMarquardt { + public: + static constexpr int npar = static_cast(Model::npar); + static_assert(Model::npar >= 1 && Model::npar <= 16, + "LevenbergMarquardt keeps npar x npar matrices on the stack"); + + using Result = MinimizerResult; + using Driver = DampedGaussNewton; + using Normal = typename Driver::Normal; + + /** + * @brief Fit one pixel. + * + * @param model Limits, fixed flags and minimiser settings. + * @param x Scan points. + * @param y Measured values. + * @param s Per-point uncertainties, empty view for an unweighted fit. + * @param start Starting values, see start_values() in FitHelpers.hpp. + * @param par_out Receives npar fitted values; zeros on failure. + * @param err_out Receives npar errors when non-null; zeros on failure. + */ + Result fit(const FitModel &model, NDView x, + NDView y, NDView s, + const std::array &start, double *par_out, + double *err_out) { + x_ = x; + y_ = y; + s_ = s; + weighted_ = s.size() > 0; + const auto n = static_cast(x.size()); + Vec lower{}; + Vec upper{}; + std::array fixed{}; + for (int k = 0; k < npar; ++k) { + const auto idx = static_cast(k); + lower[k] = model.lower_limit(idx); + upper[k] = model.upper_limit(idx); + fixed[k] = model.is_user_fixed(idx); + } + r_.resize(n); + J_.resize(n * static_cast(npar)); + if (weighted_) { + // 1 / s_i once per pixel rather than once per evaluation. + w_.resize(n); + for (std::size_t i = 0; i < n; ++i) + w_[i] = s_[static_cast(i)] != 0.0 + ? 1.0 / s_[static_cast(i)] + : 0.0; + } + const int max_calls = model.max_calls() > 0 + ? static_cast(model.max_calls()) + : 200 + 100 * npar + 5 * npar * npar; + + auto eval = [this](const Vec &p, Normal &out) { + return evaluate(p, out); + }; + typename Driver::Options opt; + opt.tolerance = model.tolerance(); + opt.max_calls = max_calls; + // Start cautiously: the estimated start values of the linear + // parameters can be far off, and the first steps are then wild. + opt.damping0 = 0.1; + // A width driven onto its lower limit turns a model into a step + // function whose chi2 no longer responds to the width and jumps + // with the position; the iteration then stalls there. Such trial + // points are rejected, as in VariableProjection. + opt.reject_degenerate = true; + const Result res = driver_.fit(eval, start, lower, upper, fixed, opt); + if (!res.valid) { + std::fill(par_out, par_out + npar, 0.0); + if (err_out) + std::fill(err_out, err_out + npar, 0.0); + return res; + } + std::copy(driver_.point().begin(), driver_.point().end(), par_out); + if (err_out) + driver_.errors(err_out); + return res; + } + + private: + double weight(ssize_t i) const { + return weighted_ ? w_[static_cast(i)] : 1.0; + } + + // Residuals r = (y - f) / s and the Jacobian J = dr/dp at the point p, + // reduced to F = chi2 / 2, g = J^T r and the upper triangle of J^T J. + // The sums run over all parameters with fixed-size loops, so the + // compiler can unroll them and keep the accumulators in registers. + // Returns false when the model rejects the parameters. + bool evaluate(const Vec &p, Normal &out) { + pvec_.assign(p.begin(), p.end()); + if (!Model::is_valid(pvec_)) + return false; + const ssize_t n = x_.size(); + double F = 0.0; + std::array g{}; + double f = 0.0; + for (ssize_t i = 0; i < n; ++i) { + const double w = weight(i); + Model::eval_and_grad(x_[i], pvec_, f, g); + r_[i] = (y_[i] - f) * w; + F += 0.5 * r_[i] * r_[i]; + double *Ji = &J_[static_cast(i) * npar]; + for (int k = 0; k < npar; ++k) + Ji[k] = -g[k] * w; + } + Vec G{}; + Mat H{}; + for (ssize_t i = 0; i < n; ++i) { + const double *Ji = &J_[static_cast(i) * npar]; + const double ri = r_[i]; + for (int a = 0; a < npar; ++a) { + G[a] += Ji[a] * ri; + for (int b = a; b < npar; ++b) + H[a][b] += Ji[a] * Ji[b]; + } + } + out.F = F; + for (int a = 0; a < npar; ++a) { + out.g[a] = G[a]; + for (int b = a; b < npar; ++b) + out.H[a][b] = H[a][b]; + } + return true; + } + + NDView x_, y_, s_; + bool weighted_ = false; + Driver driver_; + std::vector r_; + std::vector J_; + std::vector w_; // 1 / s_i for weighted fits + std::vector pvec_; +}; + +} // namespace aare::detail diff --git a/src/fit/MinuitSteps.hpp b/src/fit/MinuitSteps.hpp new file mode 100644 index 00000000..d5aa8bbe --- /dev/null +++ b/src/fit/MinuitSteps.hpp @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +// Data-driven Minuit step sizes for the optional Minuit2 backend. These +// estimators used to live in aare/Models.hpp; the built-in solver does not +// need them. + +#include "aare/Models.hpp" +#include "aare/NDView.hpp" +#include +#include +#include + +namespace aare::minuit2 { + +/** + * @brief Compute data-range statistics used by the step-size estimators. + * + * Model-independent, called once per pixel. + */ +inline void compute_ranges(NDView x, NDView y, + double &x_range, double &y_range, + double &slope_scale) { + const auto [x_min, x_max] = std::minmax_element(x.begin(), x.end()); + const auto [y_min, y_max] = std::minmax_element(y.begin(), y.end()); + + x_range = std::max(*x_max - *x_min, 1e-9); + y_range = std::max(*y_max - *y_min, 1e-9); + + slope_scale = std::max(y_range / x_range, 1e-9); +} + +/** @brief Per-model Minuit step sizes, specialised for every fit model. */ +template struct Steps; + +template <> struct Steps { + static void compute(const std::array &start, + [[maybe_unused]] double x_range, double y_range, + double slope_scale, std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.1 * slope_scale; + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double slope_scale, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.1 * slope_scale; + steps[2] = 0.1 * slope_scale / std::max(x_range, 1e-12); + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double /*slope_scale*/, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.05 * x_range; + steps[2] = 0.05 * x_range; + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double /*slope_scale*/, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = std::max(0.1 * std::abs(start[1]), 0.1 * y_range); + steps[2] = 0.05 * x_range; + steps[3] = 0.05 * x_range; + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double slope_scale, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.1 * slope_scale; + steps[2] = 0.05 * x_range; + steps[3] = 0.05 * x_range; + steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); + steps[5] = std::max(0.1 * std::abs(start[5]), 0.01); + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double slope_scale, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.1 * slope_scale; + steps[2] = 0.05 * x_range; + steps[3] = 0.05 * x_range; + steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); + steps[5] = std::max(0.1 * std::abs(start[5]), 0.01); + steps[6] = std::max(0.01 * std::abs(start[6]), 1e-3); + steps[7] = std::max(0.1 * std::abs(start[7]), 0.01); + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double slope_scale, + std::array &steps) { + steps[0] = std::max(0.1 * std::abs(start[0]), 0.1 * y_range); + steps[1] = 0.1 * slope_scale; + steps[2] = 0.05 * x_range; + steps[3] = 0.05 * x_range; + steps[4] = std::max(0.1 * std::abs(start[4]), 0.1 * y_range); + steps[5] = 0.1 * slope_scale; + } +}; + +template <> struct Steps { + static void compute(const std::array &start, double x_range, + double y_range, double slope_scale, + std::array &steps) { + Steps::compute(start, x_range, y_range, + slope_scale, steps); + } +}; + +} // namespace aare::minuit2 diff --git a/src/fit/VariableProjection.hpp b/src/fit/VariableProjection.hpp new file mode 100644 index 00000000..0ea0cdbc --- /dev/null +++ b/src/fit/VariableProjection.hpp @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: MPL-2.0 +#pragma once + +#include "DampedGaussNewton.hpp" +#include "aare/FitModel.hpp" +#include "aare/Models.hpp" +#include "aare/NDView.hpp" + +#include +#include +#include +#include +#include + +namespace aare::detail { + +/** + * @brief Variable projection least-squares minimiser for a separable + * FitModel (Golub and Pereyra, 1973). + * + * The model declares the parameters it is linear in, see + * model::is_separable. For a given point of the nonlinear parameters the + * linear ones are solved exactly from their normal equations, and the + * damped Gauss-Newton iteration of DampedGaussNewton runs over the + * nonlinear parameters only, on the projected residual + * r(alpha) = y - Phi(alpha) beta(alpha). Its Jacobian uses Kaufman's + * approximation, the derivatives of the model at fixed beta projected out + * of the span of the basis functions, and the reduced Gauss-Newton Hessian + * is the Schur complement of the full one. Compared with + * LevenbergMarquardt the iteration needs fewer evaluations, cannot be misled + * by poor start values of the linear parameters, and each evaluation + * accumulates far fewer sums. The driver rejects trial points at which the + * model loses its sensitivity to a nonlinear parameter and polishes the + * converged point with one undamped step. + * + * One evaluation is a pass over the data that writes the basis functions + * and their derivatives as columns (the model's vectorisable basis_columns + * when it has one, otherwise basis_and_grad per point), a reduction to the + * normal equations of the linear parameters, the small solve, and a second + * reduction over the columns for the projected residual. + * + * Keep one instance per thread: the buffers are reused between pixels so + * that after the first fit no memory is allocated. + * + * - Fixed nonlinear parameters are left out of the iteration. A fixed + * linear parameter keeps its value and moves to the data side of the + * linear solve. + * - Limits on nonlinear parameters are handled by the driver. A limit on a + * linear parameter cannot be enforced by the linear solve: when the + * solution violates one, the fit reports failure so that the caller can + * fall back to LevenbergMarquardt. + * - Start values of free linear parameters are ignored. + * - max_calls counts evaluations of the projected problem, as in + * LevenbergMarquardt; 0 selects 200 + 100 * npar + 5 * npar^2. + * - Errors are the Gauss-Newton estimates of the full model at the + * solution, sqrt(diag((J^T J)^-1)) over the free parameters that do not + * sit on a limit, the same as LevenbergMarquardt reports. + */ +template class VariableProjection { + public: + static_assert(model::is_separable::value, + "VariableProjection needs a model with linear_par and " + "basis_and_grad"); + using Traits = model::separable_traits; + static constexpr int npar = static_cast(Model::npar); + static constexpr int nlin = static_cast(Traits::nlin); + static constexpr int nnl = static_cast(Traits::nnl); + static_assert(Model::npar >= 1 && Model::npar <= 16, + "VariableProjection keeps npar x npar matrices on the stack"); + static_assert(nlin >= 1, "a separable model has at least one linear " + "parameter"); + + using Result = MinimizerResult; + // The driver iterates over the nonlinear parameters. A fully linear + // model has none and is solved without iterating. + static constexpr int ndrv = nnl > 0 ? nnl : 1; + static constexpr int ncol = nnl * nlin > 0 ? nnl *nlin : 1; + using Driver = DampedGaussNewton; + using Normal = typename Driver::Normal; + + /** + * @brief Fit one pixel, see LevenbergMarquardt::fit for the arguments. + */ + Result fit(const FitModel &model, NDView x, + NDView y, NDView s, + const std::array &start, double *par_out, + double *err_out) { + x_ = x; + y_ = y; + const auto n = static_cast(x.size()); + phi_.resize(n * nlin); + dphi_.resize(n * nlin * nnl); + weighted_ = s.size() > 0; + if (weighted_) { + // 1 / s_i^2 once per pixel; points with s_i == 0 are ignored. + w2_.resize(n); + for (std::size_t i = 0; i < n; ++i) { + const double si = s[static_cast(i)]; + w2_[i] = si != 0.0 ? 1.0 / (si * si) : 0.0; + } + } + pvec_.assign(start.begin(), start.end()); + have_last_ = false; + have_ref_ = false; + nfree_lin_ = 0; + for (int j = 0; j < nlin; ++j) { + const auto k = static_cast(Traits::linear_index(j)); + lin_fixed_[j] = model.is_user_fixed(k); + if (!lin_fixed_[j]) + free_lin_[nfree_lin_++] = j; + } + Vec alpha{}; + Vec lower{}; + Vec upper{}; + std::array fixed{}; + for (int k = 0; k < nnl; ++k) { + const auto idx = + static_cast(Traits::nonlinear_par[k]); + alpha[k] = start[idx]; + lower[k] = model.lower_limit(idx); + upper[k] = model.upper_limit(idx); + fixed[k] = model.is_user_fixed(idx); + } + const int max_calls = model.max_calls() > 0 + ? static_cast(model.max_calls()) + : 200 + 100 * npar + 5 * npar * npar; + + Result res; + auto fail = [&]() { + res.valid = false; + res.chi2 = 0.0; + std::fill(par_out, par_out + npar, 0.0); + if (err_out) + std::fill(err_out, err_out + npar, 0.0); + return res; + }; + + if constexpr (nnl == 0) { + Normal nrm; + if (!evaluate(alpha, nrm)) + return fail(); + res.calls = 1; + res.valid = true; + res.chi2 = 2.0 * nrm.F; + for (int j = 0; j < nlin; ++j) + pvec_[Traits::linear_index(j)] = + nrm.aux[static_cast(j)]; + } else { + auto eval = [this](const Vec &a, Normal &out) { + return evaluate(a, out); + }; + typename Driver::Options opt; + opt.tolerance = model.tolerance(); + opt.max_calls = max_calls; + // The same cautious start as LevenbergMarquardt: a large first + // step can still drive a width to zero. Rejecting degenerate + // trial points guards against that, and the polish step makes + // up for the few iterations, which stop while the damping still + // shortens the steps. + opt.damping0 = 0.1; + opt.reject_degenerate = true; + opt.polish = true; + res = driver_.fit(eval, alpha, lower, upper, fixed, opt); + if (!res.valid) + return fail(); + for (int k = 0; k < nnl; ++k) + pvec_[Traits::nonlinear_par[k]] = driver_.point()[k]; + for (int j = 0; j < nlin; ++j) + pvec_[Traits::linear_index(j)] = + driver_.normal().aux[static_cast(j)]; + } + + // The linear solve knows nothing about limits. A solution beyond a + // limit fails; one within rounding of a limit is moved onto it, so + // that it counts as on the limit whichever way the rounding went. + for (int j = 0; j < nlin; ++j) { + if (lin_fixed_[j]) + continue; + const auto k = static_cast(Traits::linear_index(j)); + const double lo = model.lower_limit(k); + const double hi = model.upper_limit(k); + const double tol = 1e-12 * std::max(1.0, std::abs(pvec_[k])); + if (pvec_[k] < lo - tol || pvec_[k] > hi + tol) + return fail(); + if (std::abs(pvec_[k] - lo) <= tol) + pvec_[k] = lo; + else if (std::abs(pvec_[k] - hi) <= tol) + pvec_[k] = hi; + } + std::copy(pvec_.begin(), pvec_.end(), par_out); + if (err_out) + errors(model, err_out); + return res; + } + + private: + // F = chi2 / 2, gradient and Kaufman Hessian of the projected residual + // at the nonlinear point alpha, and the linear solution in out.aux. + // Returns false when the model rejects the point, or when the basis + // functions are numerically linearly dependent there or one of them has + // vanished on the scan points, see reduce(). + bool evaluate(const Vec &alpha, Normal &out) { + for (int k = 0; k < nnl; ++k) + pvec_[Traits::nonlinear_par[k]] = alpha[k]; + if (!Model::is_valid(pvec_)) + return false; + const ssize_t n = x_.size(); + + // Basis functions and their derivatives, one column per function: + // phi_[j * n + i] and dphi_[(k * nlin + j) * n + i]. + if constexpr (model::has_basis_columns::value) { + Model::basis_columns(x_.data(), n, pvec_, phi_.data(), + dphi_.data()); + } else { + typename Traits::Basis phi{}; + typename Traits::BasisGrad dphi{}; + double *pc = phi_.data(); + double *dc = dphi_.data(); + const auto un = static_cast(n); + for (ssize_t i = 0; i < n; ++i) { + Model::basis_and_grad(x_[i], pvec_, phi, dphi); + const auto ui = static_cast(i); + for (int j = 0; j < nlin; ++j) + pc[static_cast(j) * un + ui] = + phi[static_cast(j)]; + for (int k = 0; k < nnl; ++k) + for (int j = 0; j < nlin; ++j) + dc[static_cast(k * nlin + j) * un + ui] = + dphi[static_cast(k)] + [static_cast(j)]; + } + } + return weighted_ ? reduce(alpha, out) : reduce(alpha, out); + } + + // The reductions over the columns: normal equations and solve of the + // linear parameters, then the projected residual. All loops over the + // linear and nonlinear parameters have compile-time bounds; the free + // linear block is packed out of the full sums afterwards. + template bool reduce(const Vec &alpha, Normal &out) { + const ssize_t n = x_.size(); + const auto un = static_cast(n); + std::array pc; + for (int j = 0; j < nlin; ++j) + pc[j] = phi_.data() + static_cast(j) * un; + std::array dc; + for (int c = 0; c < nnl * nlin; ++c) + dc[c] = dphi_.data() + static_cast(c) * un; + const double *yd = y_.data(); + const double *w2 = Weighted ? w2_.data() : nullptr; + + // Pass 1: normal equations M = Phi^T W Phi, t = Phi^T W y of all + // linear parameters. + Mat M{}; + Vec t{}; + for (ssize_t i = 0; i < n; ++i) { + const double wi = Weighted ? w2[i] : 1.0; + const double yi = yd[i]; + Vec f; + for (int j = 0; j < nlin; ++j) + f[j] = pc[j][i]; + for (int a = 0; a < nlin; ++a) { + const double fa = f[a] * wi; + t[a] += fa * yi; + for (int b = a; b < nlin; ++b) + M[a][b] += fa * f[b]; + } + } + + // Free linear parameters: their block of M, with the fixed ones + // moved to the right-hand side, v_a = t_a - sum_fixed c_j M_aj. + // The lower triangle of M is filled in so that the run-time indices + // below need no branch: gcc's range analysis derives an impossible + // subscript from the branch not taken and warns about it. The loops + // run over the compile-time bound and write zeros past nfree_lin_ + // for the same reason, gcc otherwise reports v and qf as possibly + // uninitialised when they are passed to cholesky_substitute. Both + // are false positives; neither change affects the results. + for (int a = 1; a < nlin; ++a) + for (int b = 0; b < a; ++b) + M[a][b] = M[b][a]; + Mat A{}; + Vec v{}; + for (int a = 0; a < nlin; ++a) { + if (a >= nfree_lin_) { + v[a] = 0.0; + continue; + } + const int ja = free_lin_[a]; + double va = t[ja]; + for (int j = 0; j < nlin; ++j) + if (lin_fixed_[j]) + va -= pvec_[Traits::linear_index(j)] * M[ja][j]; + v[a] = va; + for (int b = 0; b < nfree_lin_; ++b) + A[a][b] = M[ja][free_lin_[b]]; + } + // A free basis function that has all but vanished on the scan points + // (a Gaussian narrower than the point spacing, centred between two + // points) leaves its coefficient undetermined, and the solve returns + // an absurd one that fits a single point. Its squared norm is compared + // with that at the pixel's first evaluation, the start point. + if (!have_ref_) { + for (int j = 0; j < nlin; ++j) + ref_norm2_[j] = M[j][j]; + have_ref_ = true; + } + for (int a = 0; a < nfree_lin_; ++a) { + const int j = free_lin_[a]; + if (!(M[j][j] >= vanished_basis * ref_norm2_[j])) + return false; + } + Mat L{}; + if (!cholesky_factor(nfree_lin_, A, L)) + return false; + // Nearly collinear basis functions: a pivot of the factorisation far + // below its diagonal element means that the linear solution loses + // about log10(diagonal / pivot) digits, beyond which it is noise. + bool refine = false; + for (int a = 0; a < nfree_lin_; ++a) { + if (!(L[a][a] * L[a][a] >= collinear_basis * A[a][a])) + return false; + refine = refine || L[a][a] * L[a][a] < refine_basis * A[a][a]; + } + Vec beta_free{}; + cholesky_substitute(nfree_lin_, L, v, beta_free); + for (int a = 0; a < nfree_lin_; ++a) + pvec_[Traits::linear_index(free_lin_[a])] = beta_free[a]; + // Short of that, the digits lost to the normal equations are + // recovered by one step of iterative refinement on the data (the + // corrected semi-normal equations): the correction solves the same + // equations for Phi^T W r of the residual of the first solution. A + // polynomial fitted far from x = 0 needs it; x in [20000, 20500] + // otherwise costs a Pol2 six digits. + if (refine) { + Vec beta; + for (int j = 0; j < nlin; ++j) + beta[j] = pvec_[Traits::linear_index(j)]; + Vec s{}; + for (ssize_t i = 0; i < n; ++i) { + double r = yd[i]; + for (int j = 0; j < nlin; ++j) + r -= beta[j] * pc[j][i]; + const double rw = Weighted ? r * w2[i] : r; + for (int j = 0; j < nlin; ++j) + s[j] += pc[j][i] * rw; + } + Vec sf{}; + for (int a = 0; a < nlin; ++a) + sf[a] = a < nfree_lin_ ? s[free_lin_[a]] : 0.0; + Vec delta{}; + cholesky_substitute(nfree_lin_, L, sf, delta); + for (int a = 0; a < nfree_lin_; ++a) + pvec_[Traits::linear_index(free_lin_[a])] += delta[a]; + } + Vec beta; + for (int j = 0; j < nlin; ++j) { + beta[j] = pvec_[Traits::linear_index(j)]; + out.aux[static_cast(j)] = beta[j]; + } + + // Pass 2: residual r = y - Phi beta and u_k = df/dalpha_k at fixed + // beta; F, b = U^T W r, N = U^T W U and Q = Phi^T W U. + double F = 0.0; + Vec b{}; + Mat N{}; + std::array, ndrv> Q{}; + for (ssize_t i = 0; i < n; ++i) { + const double wi = Weighted ? w2[i] : 1.0; + Vec f; + for (int j = 0; j < nlin; ++j) + f[j] = pc[j][i]; + double r = yd[i]; + for (int j = 0; j < nlin; ++j) + r -= beta[j] * f[j]; + Vec u{}; + for (int k = 0; k < nnl; ++k) + for (int j = 0; j < nlin; ++j) + u[k] += beta[j] * dc[k * nlin + j][i]; + const double rw = r * wi; + F += 0.5 * r * rw; + for (int k = 0; k < nnl; ++k) { + const double uw = u[k] * wi; + b[k] += uw * r; + for (int l = k; l < nnl; ++l) + N[k][l] += uw * u[l]; + for (int j = 0; j < nlin; ++j) + Q[k][j] += uw * f[j]; + } + } + + // The full Gauss-Newton Hessian of the model at this point is + // [[M, Q^T], [Q, N]]; kept for the errors of the final point. + for (int k = 0; k < nnl; ++k) + last_alpha_[k] = alpha[k]; + for (int a = 0; a < nlin; ++a) + for (int c = a; c < nlin; ++c) + last_M_[a][c] = M[a][c]; + for (int k = 0; k < nnl; ++k) { + for (int j = 0; j < nlin; ++j) + last_Q_[k][j] = Q[k][j]; + for (int l = k; l < nnl; ++l) + last_N_[k][l] = N[k][l]; + } + have_last_ = true; + + // Kaufman's Jacobian: U projected out of the span of the free basis, + // H = N - Q_free^T A^-1 Q_free. + std::array, ndrv> qf{}; + std::array, ndrv> z{}; + for (int k = 0; k < nnl; ++k) { + for (int a = 0; a < nlin; ++a) + qf[k][a] = a < nfree_lin_ ? Q[k][free_lin_[a]] : 0.0; + cholesky_substitute(nfree_lin_, L, qf[k], z[k]); + } + out.F = F; + for (int k = 0; k < nnl; ++k) { + out.g[k] = -b[k]; + for (int l = k; l < nnl; ++l) { + double h = N[k][l]; + for (int a = 0; a < nfree_lin_; ++a) + h -= qf[k][a] * z[l][a]; + out.H[k][l] = h; + } + } + return true; + } + + // Gauss-Newton errors from the Jacobian of the full model at the + // solution, the estimate LevenbergMarquardt reports as well. The + // Jacobian's normal matrix is assembled from the sums of the last + // evaluation when that was the final point, which it is unless the last + // trial was rejected; otherwise one pass over the data computes it. + void errors(const FitModel &model, double *err_out) { + Mat H{}; + bool at_solution = have_last_; + for (int k = 0; k < nnl; ++k) + at_solution = at_solution && + pvec_[Traits::nonlinear_par[k]] == last_alpha_[k]; + if (at_solution) { + for (int a = 0; a < nlin; ++a) + for (int c = a; c < nlin; ++c) + H[Traits::linear_index(a)][Traits::linear_index(c)] = + last_M_[a][c]; + for (int k = 0; k < nnl; ++k) { + const auto m = static_cast(Traits::nonlinear_par[k]); + for (int j = 0; j < nlin; ++j) { + const auto i = static_cast(Traits::linear_index(j)); + H[std::min(i, m)][std::max(i, m)] = last_Q_[k][j]; + } + for (int l = k; l < nnl; ++l) + H[m][Traits::nonlinear_par[l]] = last_N_[k][l]; + } + } else { + const ssize_t n = x_.size(); + std::array g{}; + double f = 0.0; + for (ssize_t i = 0; i < n; ++i) { + Model::eval_and_grad(x_[i], pvec_, f, g); + const double wi = + weighted_ ? w2_[static_cast(i)] : 1.0; + for (int a = 0; a < npar; ++a) + for (int c = a; c < npar; ++c) + H[a][c] += g[a] * g[c] * wi; + } + } + // Fixed parameters and parameters that end on a limit have no error. + // The driver knows the nonlinear ones; the linear solve can land a + // linear parameter exactly on a limit without violating it. + std::array skip; + for (int k = 0; k < npar; ++k) { + const auto idx = static_cast(k); + const double v = pvec_[static_cast(k)]; + skip[k] = model.is_user_fixed(idx) || v <= model.lower_limit(idx) || + v >= model.upper_limit(idx); + } + if constexpr (nnl > 0) { + for (int k = 0; k < nnl; ++k) + skip[Traits::nonlinear_par[k]] = + skip[Traits::nonlinear_par[k]] || driver_.skipped(k); + } + gauss_newton_errors(H, skip, err_out); + } + + NDView x_, y_; + Driver driver_; + bool weighted_ = false; + // Sums of the last evaluation, see errors(). + bool have_last_ = false; + Vec last_alpha_{}; + Mat last_M_{}; + std::array, ndrv> last_Q_{}; + Mat last_N_{}; + // Squared norms of the basis functions at the start point, see reduce(). + bool have_ref_ = false; + Vec ref_norm2_{}; + static constexpr double vanished_basis = 1e-8; + static constexpr double collinear_basis = 1e-10; + // Pivot ratio below which the linear solution is refined, about four + // lost digits. + static constexpr double refine_basis = 1e-4; + int nfree_lin_ = 0; + std::array free_lin_{}; + std::array lin_fixed_{}; + std::vector phi_; // nlin columns of n basis values + std::vector dphi_; // nnl * nlin columns of their derivatives + std::vector w2_; // 1 / s_i^2 for weighted fits + std::vector pvec_; +}; + +} // namespace aare::detail diff --git a/src/hist/PedestalTrackingPixelHistogram.cpp b/src/hist/PedestalTrackingPixelHistogram.cpp index f02004ab..b97b5ad9 100644 --- a/src/hist/PedestalTrackingPixelHistogram.cpp +++ b/src/hist/PedestalTrackingPixelHistogram.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/hist/PedestalTrackingPixelHistogram.hpp" #include "aare/File.hpp" #include "aare/MultiThreadedFileReader.hpp" diff --git a/src/hist/PedestalTrackingPixelHistogram.test.cpp b/src/hist/PedestalTrackingPixelHistogram.test.cpp index 2c9f1590..b8d81df4 100644 --- a/src/hist/PedestalTrackingPixelHistogram.test.cpp +++ b/src/hist/PedestalTrackingPixelHistogram.test.cpp @@ -20,6 +20,7 @@ using aare::FileConfig; using aare::Frame; using aare::NumpyFile; using aare::PedestalTrackingPixelHistogram; +using aare::ssize_t; namespace { diff --git a/src/hist/PixelHistogram.test.cpp b/src/hist/PixelHistogram.test.cpp index d681c789..25767257 100644 --- a/src/hist/PixelHistogram.test.cpp +++ b/src/hist/PixelHistogram.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include #include @@ -17,6 +18,7 @@ using aare::NDArray; using aare::NDView; using aare::PixelHistogram; +using aare::ssize_t; namespace { // The synchronous fill() has been removed; fill_async() is the only entry diff --git a/src/hist/PixelHistogramImpl.test.cpp b/src/hist/PixelHistogramImpl.test.cpp index f3526c17..f9456efa 100644 --- a/src/hist/PixelHistogramImpl.test.cpp +++ b/src/hist/PixelHistogramImpl.test.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include #include diff --git a/src/logger.test.cpp b/src/logger.test.cpp new file mode 100644 index 00000000..17218d96 --- /dev/null +++ b/src/logger.test.cpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MPL-2.0 +#include "aare/logger.hpp" + +#include +#include + +#include +#include +#include + +using Catch::Matchers::EndsWith; +using Catch::Matchers::Matches; +using Catch::Matchers::StartsWith; + +TEST_CASE("Log timestamp is HH:MM:SS.mmm", "[logger]") { + const std::string ts = aare::Logger::Timestamp(); + REQUIRE_THAT(ts, Matches(R"(\d{2}:\d{2}:\d{2}\.\d{3})")); +} + +TEST_CASE("short_file_name strips both kinds of directory separators", + "[logger]") { + REQUIRE(std::string(aare::short_file_name("/a/b/File.cpp")) == "File.cpp"); + REQUIRE(std::string(aare::short_file_name("C:\\a\\b\\File.cpp")) == + "File.cpp"); + REQUIRE(std::string(aare::short_file_name("File.cpp")) == "File.cpp"); +} + +TEST_CASE("LOG writes one coloured line with timestamp and level", "[logger]") { + std::ostringstream captured; + auto *previous = std::clog.rdbuf(captured.rdbuf()); + LOG(aare::logWARNING) << "hello " << 42; + std::clog.rdbuf(previous); + + const std::string line = captured.str(); + // - HH:MM:SS.mmm WARNING: hello 42\n + REQUIRE_THAT(line, StartsWith(std::string(YELLOW BOLD) + "- ")); + REQUIRE_THAT(line, Matches(std::string("\x1b\\[33m\x1b\\[1m- ") + + R"(\d{2}:\d{2}:\d{2}\.\d{3} WARNING: hello 42)" + + "\x1b\\[0m\n")); + REQUIRE_THAT(line, EndsWith(std::string(RESET) + "\n")); +} diff --git a/src/to_string.cpp b/src/to_string.cpp index 8776c501..2753695f 100644 --- a/src/to_string.cpp +++ b/src/to_string.cpp @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #include "aare/to_string.hpp" namespace aare { diff --git a/tests/test_config.hpp.in b/tests/test_config.hpp.in index e314b8fe..8030213e 100644 --- a/tests/test_config.hpp.in +++ b/tests/test_config.hpp.in @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MPL-2.0 #pragma once #include #include